From 64a3787c2414cc5f2e07ff10477c824efc2e9a5d Mon Sep 17 00:00:00 2001 From: darkag Date: Fri, 30 Jun 2023 09:55:23 +0200 Subject: [PATCH 001/533] Add various Vertica connection parameters (#32089) --------- Co-authored-by: ivascot --- airflow/providers/vertica/hooks/vertica.py | 51 ++++++++++++ .../connections/vertica.rst | 83 +++++++++++++++++++ .../index.rst | 7 ++ tests/providers/vertica/hooks/test_vertica.py | 76 +++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 docs/apache-airflow-providers-vertica/connections/vertica.rst diff --git a/airflow/providers/vertica/hooks/vertica.py b/airflow/providers/vertica/hooks/vertica.py index 92a74ea36901..06b2e3cf179b 100644 --- a/airflow/providers/vertica/hooks/vertica.py +++ b/airflow/providers/vertica/hooks/vertica.py @@ -46,5 +46,56 @@ def get_conn(self) -> connect: else: conn_config["port"] = int(conn.port) + bool_options = [ + "connection_load_balance", + "binary_transfer", + "disable_copy_local", + "request_complex_types", + "use_prepared_statements", + ] + std_options = [ + "session_label", + "backup_server_node", + "kerberos_host_name", + "kerberos_service_name", + "unicode_error", + "workload", + "ssl", + ] + conn_extra = conn.extra_dejson + + for bo in bool_options: + if bo in conn_extra: + conn_config[bo] = str(conn_extra[bo]).lower() in ["true", "on"] + + for so in std_options: + if so in conn_extra: + conn_config[so] = conn_extra[so] + + if "connection_timeout" in conn_extra: + conn_config["connection_timeout"] = float(conn_extra["connection_timeout"]) + + if "log_level" in conn_extra: + import logging + + log_lvl = conn_extra["log_level"] + conn_config["log_path"] = None + if isinstance(log_lvl, str): + log_lvl = log_lvl.lower() + if log_lvl == "critical": + conn_config["log_level"] = logging.CRITICAL + elif log_lvl == "error": + conn_config["log_level"] = logging.ERROR + elif log_lvl == "warning": + conn_config["log_level"] = logging.WARNING + elif log_lvl == "info": + conn_config["log_level"] = logging.INFO + elif log_lvl == "debug": + conn_config["log_level"] = logging.DEBUG + elif log_lvl == "notset": + conn_config["log_level"] = logging.NOTSET + else: + conn_config["log_level"] = int(conn_extra["log_level"]) + conn = connect(**conn_config) return conn diff --git a/docs/apache-airflow-providers-vertica/connections/vertica.rst b/docs/apache-airflow-providers-vertica/connections/vertica.rst new file mode 100644 index 000000000000..86f583a54812 --- /dev/null +++ b/docs/apache-airflow-providers-vertica/connections/vertica.rst @@ -0,0 +1,83 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:vertica: + +Vertica Connection +================== +The Vertica connection type provides connection to a Vertica database. + +Configuring the Connection +-------------------------- +Host (required) + The host to connect to. + +Schema (optional) + Specify the schema name to be used in the database. + +Login (required) + Specify the user name to connect. + +Password (required) + Specify the password to connect. + +Extra (optional) + Specify the extra parameters (as json dictionary) that can be used in Vertica + connection. + + The following extras are supported: + + * ``backup_server_node``: See `Connection Failover `_. + * ``binary_transfer``: See `Data Transfer Format `_. + * ``connection_load_balance``: See `Connection Load Balancing `_. + * ``connection_timeout``: The number of seconds (can be a nonnegative floating point number) the client + waits for a socket operation (Establishing a TCP connection or read/write operation). + * ``disable_copy_local``: See `COPY FROM LOCAL `_. + * ``kerberos_host_name``: See `Kerberos Authentication `_. + * ``kerberos_service_name``: See `Kerberos Authentication `_. + * ``log_level``: Enable vertica client logging. Traces will be visible in tasks log. See `Logging `_. + * ``request_complex_types:``: See `SQL Data conversion to Python objects `_. + * ``session_label``: Sets a label for the connection on the server. + * ``ssl``: Support only True or False. See `TLS/SSL `_. + * ``unicode_error``: See `UTF-8 encoding issues `_. + * ``use_prepared_statements``: See `Passing parameters to SQL queries `_. + * ``workload``: Sets the workload name associated with this session. + + See `vertica-python docs `_ for details. + + + Example "extras" field: + + .. code-block:: json + + { + "connection_load_balance": true, + "log_level": "error", + "ssl": true + } + + or + + .. code-block:: json + + { + "session_label": "airflow-session", + "connection_timeout": 30, + "backup_server_node": ["bck_server_1", "bck_server_2"] + } diff --git a/docs/apache-airflow-providers-vertica/index.rst b/docs/apache-airflow-providers-vertica/index.rst index db09f1924c4d..ae7c2457b811 100644 --- a/docs/apache-airflow-providers-vertica/index.rst +++ b/docs/apache-airflow-providers-vertica/index.rst @@ -29,6 +29,13 @@ Changelog Security +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + .. toctree:: :hidden: :maxdepth: 1 diff --git a/tests/providers/vertica/hooks/test_vertica.py b/tests/providers/vertica/hooks/test_vertica.py index e78c2a0c5c81..146c3bcd1136 100644 --- a/tests/providers/vertica/hooks/test_vertica.py +++ b/tests/providers/vertica/hooks/test_vertica.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import json from unittest import mock from unittest.mock import patch @@ -47,6 +48,81 @@ def test_get_conn(self, mock_connect): host="host", port=5433, database="vertica", user="login", password="password" ) + @patch("airflow.providers.vertica.hooks.vertica.connect") + def test_get_conn_extra_parameters_no_cast(self, mock_connect): + """Test if parameters are correctly passed to connection""" + extra_dict = self.connection.extra_dejson + bool_options = [ + "connection_load_balance", + "binary_transfer", + "disable_copy_local", + "use_prepared_statements", + ] + for bo in bool_options: + extra_dict.update({bo: True}) + extra_dict.update({"request_complex_types": False}) + + std_options = [ + "session_label", + "kerberos_host_name", + "kerberos_service_name", + "unicode_error", + "workload", + "ssl", + ] + for so in std_options: + extra_dict.update({so: so}) + bck_server_node = ["1.2.3.4", "4.3.2.1"] + conn_timeout = 30 + log_lvl = 40 + extra_dict.update({"backup_server_node": bck_server_node}) + extra_dict.update({"connection_timeout": conn_timeout}) + extra_dict.update({"log_level": log_lvl}) + self.connection.extra = json.dumps(extra_dict) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + for bo in bool_options: + assert kwargs[bo] is True + assert kwargs["request_complex_types"] is False + for so in std_options: + assert kwargs[so] == so + assert bck_server_node[0] in kwargs["backup_server_node"] + assert bck_server_node[1] in kwargs["backup_server_node"] + assert kwargs["connection_timeout"] == conn_timeout + assert kwargs["log_level"] == log_lvl + assert kwargs["log_path"] is None + + @patch("airflow.providers.vertica.hooks.vertica.connect") + def test_get_conn_extra_parameters_cast(self, mock_connect): + """Test if parameters that can be passed either as string or int/bool + like log_level are correctly converted when passed as string + (while test_get_conn_extra_parameters_no_cast tests them passed as int/bool)""" + import logging + + extra_dict = self.connection.extra_dejson + bool_options = [ + "connection_load_balance", + "binary_transfer", + "disable_copy_local", + "use_prepared_statements", + ] + for bo in bool_options: + extra_dict.update({bo: "True"}) + extra_dict.update({"request_complex_types": "False"}) + extra_dict.update({"log_level": "Error"}) + self.connection.extra = json.dumps(extra_dict) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + for bo in bool_options: + assert kwargs[bo] is True + assert kwargs["request_complex_types"] is False + assert kwargs["log_level"] == logging.ERROR + assert kwargs["log_path"] is None + class TestVerticaHook: def setup_method(self): From dd937e51fe1ae3cd36a6993bd42e425960644e1d Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 30 Jun 2023 10:01:13 +0200 Subject: [PATCH 002/533] Add `on_finish_action` to `KubernetesPodOperator` (#30718) * Add a new arg for KPO to only delete the pod when it doesn't fail * deprecate is_delete_operator_pod and add on_finish_action * Add deprecated properties and fix unit tests * add missing attribute * Apply suggestions from code review Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> * update GKEStartPodOperator to be consistent with KPO * update EksPodOperator to be consistent with KPO * update unit tests and the method used to check the kpo compatibility * Fix a bug and add a new unit test for each provider * warn with AirflowProviderDeprecationWarning instead of DeprecationWarning * Bump KPO min version in GCP provider and add a new one to AWS provider * Add the new param to the GKE trigger * Apply suggestions from code review Co-authored-by: Jarek Potiuk --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Co-authored-by: Jarek Potiuk --- airflow/providers/amazon/aws/operators/eks.py | 34 ++++++-- airflow/providers/amazon/provider.yaml | 3 + .../cncf/kubernetes/operators/pod.py | 43 ++++++++-- .../providers/cncf/kubernetes/triggers/pod.py | 34 ++++++-- .../cncf/kubernetes/utils/pod_manager.py | 9 ++ .../cloud/operators/kubernetes_engine.py | 38 ++++++--- .../cloud/triggers/kubernetes_engine.py | 34 ++++++-- airflow/providers/google/provider.yaml | 2 +- .../amazon/aws/operators/test_eks.py | 58 ++++++++++++- .../cncf/kubernetes/operators/test_pod.py | 84 +++++++++++++++---- .../cncf/kubernetes/triggers/test_pod.py | 74 ++++++++++++++-- .../cloud/operators/test_kubernetes_engine.py | 61 ++++++++++++++ .../cloud/triggers/test_kubernetes_engine.py | 4 +- .../amazon/aws/example_eks_templated.py | 2 +- .../example_eks_with_fargate_in_one_step.py | 2 +- .../aws/example_eks_with_fargate_profile.py | 2 +- .../example_eks_with_nodegroup_in_one_step.py | 2 +- .../cncf/kubernetes/example_kubernetes.py | 6 +- .../kubernetes/example_kubernetes_async.py | 6 +- .../example_kubernetes_engine.py | 4 +- .../example_kubernetes_engine_async.py | 4 +- 21 files changed, 430 insertions(+), 76 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/eks.py b/airflow/providers/amazon/aws/operators/eks.py index 9c27e88350c4..bea422398726 100644 --- a/airflow/providers/amazon/aws/operators/eks.py +++ b/airflow/providers/amazon/aws/operators/eks.py @@ -34,6 +34,7 @@ EksNodegroupTrigger, ) from airflow.providers.amazon.aws.utils.waiter_with_logging import wait +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction try: from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator @@ -854,10 +855,15 @@ class EksPodOperator(KubernetesPodOperator): running Airflow in a distributed manner and aws_conn_id is None or empty, then the default boto3 configuration would be used (and must be maintained on each worker node). + :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. + If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. + Current default is `keep_pod`, but this will be changed in the next major release of this provider. :param is_delete_operator_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True, delete the - pod; if False, leave the pod. Current default is False, but this will be + pod; if False, leave the pod. Current default is False, but this will be changed in the next major release of this provider. + Deprecated - use `on_finish_action` instead. """ @@ -885,19 +891,32 @@ def __init__( pod_username: str | None = None, aws_conn_id: str = DEFAULT_CONN_ID, region: str | None = None, + on_finish_action: str | None = None, is_delete_operator_pod: bool | None = None, **kwargs, ) -> None: - if is_delete_operator_pod is None: + if is_delete_operator_pod is not None: warnings.warn( - f"You have not set parameter `is_delete_operator_pod` in class {self.__class__.__name__}. " - "Currently the default for this parameter is `False` but in a future release the default " - "will be changed to `True`. To ensure pods are not deleted in the future you will need to " - "set `is_delete_operator_pod=False` explicitly.", + "`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`", AirflowProviderDeprecationWarning, stacklevel=2, ) - is_delete_operator_pod = False + kwargs["on_finish_action"] = ( + OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD + ) + else: + if on_finish_action is not None: + kwargs["on_finish_action"] = OnFinishAction(on_finish_action) + else: + warnings.warn( + f"You have not set parameter `on_finish_action` in class {self.__class__.__name__}. " + "Currently the default for this parameter is `keep_pod` but in a future release" + " the default will be changed to `delete_pod`. To ensure pods are not deleted in" + " the future you will need to set `on_finish_action=keep_pod` explicitly.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + kwargs["on_finish_action"] = OnFinishAction.KEEP_POD self.cluster_name = cluster_name self.in_cluster = in_cluster @@ -909,7 +928,6 @@ def __init__( in_cluster=self.in_cluster, namespace=self.namespace, name=self.pod_name, - is_delete_operator_pod=is_delete_operator_pod, **kwargs, ) # There is no need to manage the kube_config file, as it will be generated automatically. diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 0c74094e8193..a7dac9fad481 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -667,3 +667,6 @@ additional-extras: - name: aiobotocore dependencies: - aiobotocore[boto3]>=2.2.0 + - name: cncf.kubernetes + dependencies: + - apache-airflow-providers-cncf-kubernetes>=7.2.0 diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 87bfc41301b8..696611c6c2e9 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -23,6 +23,7 @@ import re import secrets import string +import warnings from collections.abc import Container from contextlib import AbstractContextManager from functools import cached_property @@ -32,7 +33,7 @@ from slugify import slugify from urllib3.exceptions import HTTPError -from airflow.exceptions import AirflowException, AirflowSkipException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException from airflow.kubernetes import pod_generator from airflow.kubernetes.pod_generator import PodGenerator from airflow.kubernetes.secret import Secret @@ -52,6 +53,7 @@ from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger from airflow.providers.cncf.kubernetes.utils import xcom_sidecar # type: ignore[attr-defined] from airflow.providers.cncf.kubernetes.utils.pod_manager import ( + OnFinishAction, PodLaunchFailedException, PodManager, PodOperatorHookProtocol, @@ -188,9 +190,6 @@ class KubernetesPodOperator(BaseOperator): If more than one secret is required, provide a comma separated list: secret_a,secret_b :param service_account_name: Name of the service account - :param is_delete_operator_pod: What to do when the pod reaches its final - state, or the execution is interrupted. If True (default), delete the - pod; if False, leave the pod. :param hostnetwork: If True enable host networking on the pod. :param tolerations: A list of kubernetes tolerations. :param security_context: security options the pod should run with (PodSecurityContext). @@ -226,6 +225,13 @@ class KubernetesPodOperator(BaseOperator): :param deferrable: Run operator in the deferrable mode. :param poll_interval: Polling period in seconds to check for the status. Used only in deferrable mode. :param log_pod_spec_on_failure: Log the pod's specification if a failure occurs + :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. + If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. + :param is_delete_operator_pod: What to do when the pod reaches its final + state, or the execution is interrupted. If True (default), delete the + pod; if False, leave the pod. + Deprecated - use `on_finish_action` instead. """ # This field can be overloaded at the instance level via base_container_name @@ -279,7 +285,6 @@ def __init__( node_selector: dict | None = None, image_pull_secrets: list[k8s.V1LocalObjectReference] | None = None, service_account_name: str | None = None, - is_delete_operator_pod: bool = True, hostnetwork: bool = False, tolerations: list[k8s.V1Toleration] | None = None, security_context: dict | None = None, @@ -303,6 +308,8 @@ def __init__( deferrable: bool = False, poll_interval: float = 2, log_pod_spec_on_failure: bool = True, + on_finish_action: str = "delete_pod", + is_delete_operator_pod: None | bool = None, **kwargs, ) -> None: # TODO: remove in provider 6.0.0 release. This is a mitigate step to advise users to switch to the @@ -350,7 +357,6 @@ def __init__( self.config_file = config_file self.image_pull_secrets = convert_image_pull_secrets(image_pull_secrets) if image_pull_secrets else [] self.service_account_name = service_account_name - self.is_delete_operator_pod = is_delete_operator_pod self.hostnetwork = hostnetwork self.tolerations = ( [convert_toleration(toleration) for toleration in tolerations] if tolerations else [] @@ -384,6 +390,20 @@ def __init__( self.poll_interval = poll_interval self.remote_pod: k8s.V1Pod | None = None self.log_pod_spec_on_failure = log_pod_spec_on_failure + if is_delete_operator_pod is not None: + warnings.warn( + "`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + self.on_finish_action = ( + OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD + ) + self.is_delete_operator_pod = is_delete_operator_pod + else: + self.on_finish_action = OnFinishAction(on_finish_action) + self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD + self._config_dict: dict | None = None # TODO: remove it when removing convert_config_file_to_dict @cached_property @@ -595,10 +615,10 @@ def invoke_defer_method(self): config_file=self.config_file, in_cluster=self.in_cluster, poll_interval=self.poll_interval, - should_delete_pod=self.is_delete_operator_pod, get_logs=self.get_logs, startup_timeout=self.startup_timeout_seconds, base_container_name=self.base_container_name, + on_finish_action=self.on_finish_action.value, ), method_name="execute_complete", ) @@ -669,7 +689,8 @@ def post_complete_action(self, *, pod, remote_pod, **kwargs): def cleanup(self, pod: k8s.V1Pod, remote_pod: k8s.V1Pod): pod_phase = remote_pod.status.phase if hasattr(remote_pod, "status") else None - if pod_phase != PodPhase.SUCCEEDED or not self.is_delete_operator_pod: + # if the pod fails or success, but we don't want to delete it + if pod_phase != PodPhase.SUCCEEDED or self.on_finish_action == OnFinishAction.KEEP_POD: self.patch_already_checked(remote_pod, reraise=False) if pod_phase != PodPhase.SUCCEEDED: @@ -722,7 +743,11 @@ def _read_pod_events(self, pod, *, reraise=True): def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True): with _optionally_suppress(reraise=reraise): if pod is not None: - if self.is_delete_operator_pod: + should_delete_pod = (self.on_finish_action == OnFinishAction.DELETE_POD) or ( + self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD + and pod.status.phase == PodPhase.SUCCEEDED + ) + if should_delete_pod: self.log.info("Deleting pod: %s", pod.metadata.name) self.pod_manager.delete_pod(pod) else: diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py b/airflow/providers/cncf/kubernetes/triggers/pod.py index 6cfb6c523a31..6fdf763eceda 100644 --- a/airflow/providers/cncf/kubernetes/triggers/pod.py +++ b/airflow/providers/cncf/kubernetes/triggers/pod.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import warnings from asyncio import CancelledError from datetime import datetime from enum import Enum @@ -25,8 +26,9 @@ import pytz from kubernetes_asyncio.client.models import V1Pod +from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook -from airflow.providers.cncf.kubernetes.utils.pod_manager import PodPhase +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction, PodPhase from airflow.triggers.base import BaseTrigger, TriggerEvent @@ -57,11 +59,15 @@ class KubernetesPodTrigger(BaseTrigger): :param poll_interval: Polling period in seconds to check for the status. :param trigger_start_time: time in Datetime format when the trigger was started :param in_cluster: run kubernetes client with in_cluster configuration. + :param get_logs: get the stdout of the container as logs of the tasks. + :param startup_timeout: timeout in seconds to start up the pod. + :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. + If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. :param should_delete_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True (default), delete the pod; if False, leave the pod. - :param get_logs: get the stdout of the container as logs of the tasks. - :param startup_timeout: timeout in seconds to start up the pod. + Deprecated - use `on_finish_action` instead. """ def __init__( @@ -75,9 +81,10 @@ def __init__( cluster_context: str | None = None, config_file: str | None = None, in_cluster: bool | None = None, - should_delete_pod: bool = True, get_logs: bool = True, startup_timeout: int = 120, + on_finish_action: str = "delete_pod", + should_delete_pod: bool | None = None, ): super().__init__() self.pod_name = pod_name @@ -89,10 +96,22 @@ def __init__( self.cluster_context = cluster_context self.config_file = config_file self.in_cluster = in_cluster - self.should_delete_pod = should_delete_pod self.get_logs = get_logs self.startup_timeout = startup_timeout + if should_delete_pod is not None: + warnings.warn( + "`should_delete_pod` parameter is deprecated, please use `on_finish_action`", + AirflowProviderDeprecationWarning, + ) + self.on_finish_action = ( + OnFinishAction.DELETE_POD if should_delete_pod else OnFinishAction.KEEP_POD + ) + self.should_delete_pod = should_delete_pod + else: + self.on_finish_action = OnFinishAction(on_finish_action) + self.should_delete_pod = self.on_finish_action == OnFinishAction.DELETE_POD + self._hook: AsyncKubernetesHook | None = None self._since_time = None @@ -109,10 +128,11 @@ def serialize(self) -> tuple[str, dict[str, Any]]: "cluster_context": self.cluster_context, "config_file": self.config_file, "in_cluster": self.in_cluster, - "should_delete_pod": self.should_delete_pod, "get_logs": self.get_logs, "startup_timeout": self.startup_timeout, "trigger_start_time": self.trigger_start_time, + "should_delete_pod": self.should_delete_pod, + "on_finish_action": self.on_finish_action.value, }, ) @@ -191,7 +211,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] name=self.pod_name, namespace=self.pod_namespace, ) - if self.should_delete_pod: + if self.on_finish_action == OnFinishAction.DELETE_POD: self.log.info("Deleting pod...") await self._get_async_hook().delete_pod( name=self.pod_name, diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 3bacb95f4ff7..71b5e171271b 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -17,6 +17,7 @@ """Launches PODs.""" from __future__ import annotations +import enum import json import logging import math @@ -585,3 +586,11 @@ def _exec_pod_command(self, resp, command: str) -> str | None: if res: return res return res + + +class OnFinishAction(enum.Enum): + """Action to take when the pod finishes.""" + + KEEP_POD = "keep_pod" + DELETE_POD = "delete_pod" + DELETE_SUCCEEDED_POD = "delete_succeeded_pod" diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 55971f52b786..bf14828d8702 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -27,6 +27,7 @@ from kubernetes.client.models import V1Pod from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction try: from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator @@ -45,7 +46,6 @@ if TYPE_CHECKING: from airflow.utils.context import Context - KUBE_CONFIG_ENV_VAR = "KUBECONFIG" @@ -427,11 +427,16 @@ class GKEStartPodOperator(KubernetesPodOperator): Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). :param regional: The location param is region name. + :param deferrable: Run operator in the deferrable mode. + :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. + If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. + Current default is `keep_pod`, but this will be changed in the next major release of this provider. :param is_delete_operator_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True, delete the pod; if False, leave the pod. Current default is False, but this will be changed in the next major release of this provider. - :param deferrable: Run operator in the deferrable mode. + Deprecated - use `on_finish_action` instead. """ template_fields: Sequence[str] = tuple( @@ -449,19 +454,32 @@ def __init__( gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, regional: bool | None = None, + on_finish_action: str | None = None, is_delete_operator_pod: bool | None = None, **kwargs, ) -> None: - if is_delete_operator_pod is None: + if is_delete_operator_pod is not None: warnings.warn( - f"You have not set parameter `is_delete_operator_pod` in class {self.__class__.__name__}. " - "Currently the default for this parameter is `False` but in a future release the default " - "will be changed to `True`. To ensure pods are not deleted in the future you will need to " - "set `is_delete_operator_pod=False` explicitly.", + "`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`", AirflowProviderDeprecationWarning, stacklevel=2, ) - is_delete_operator_pod = False + kwargs["on_finish_action"] = ( + OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD + ) + else: + if on_finish_action is not None: + kwargs["on_finish_action"] = OnFinishAction(on_finish_action) + else: + warnings.warn( + f"You have not set parameter `on_finish_action` in class {self.__class__.__name__}. " + "Currently the default for this parameter is `keep_pod` but in a future release" + " the default will be changed to `delete_pod`. To ensure pods are not deleted in" + " the future you will need to set `on_finish_action=keep_pod` explicitly.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + kwargs["on_finish_action"] = OnFinishAction.KEEP_POD if regional is not None: warnings.warn( @@ -472,7 +490,7 @@ def __init__( stacklevel=2, ) - super().__init__(is_delete_operator_pod=is_delete_operator_pod, **kwargs) + super().__init__(**kwargs) self.project_id = project_id self.location = location self.cluster_name = cluster_name @@ -560,8 +578,8 @@ def invoke_defer_method(self): cluster_context=self.cluster_context, poll_interval=self.poll_interval, in_cluster=self.in_cluster, - should_delete_pod=self.is_delete_operator_pod, base_container_name=self.base_container_name, + on_finish_action=self.on_finish_action, ), method_name="execute_complete", kwargs={"cluster_url": self._cluster_url, "ssl_ca_cert": self._ssl_ca_cert}, diff --git a/airflow/providers/google/cloud/triggers/kubernetes_engine.py b/airflow/providers/google/cloud/triggers/kubernetes_engine.py index 1ec0e420f93a..ba0df0fc153c 100644 --- a/airflow/providers/google/cloud/triggers/kubernetes_engine.py +++ b/airflow/providers/google/cloud/triggers/kubernetes_engine.py @@ -18,11 +18,15 @@ from __future__ import annotations import asyncio +import warnings from datetime import datetime from typing import Any, AsyncIterator, Sequence from google.cloud.container_v1.types import Operation +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction + try: from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger except ImportError: @@ -44,15 +48,19 @@ class GKEStartPodTrigger(KubernetesPodTrigger): :param poll_interval: Polling period in seconds to check for the status. :param trigger_start_time: time in Datetime format when the trigger was started :param in_cluster: run kubernetes client with in_cluster configuration. - :param should_delete_pod: What to do when the pod reaches its final - state, or the execution is interrupted. If True (default), delete the - pod; if False, leave the pod. :param get_logs: get the stdout of the container as logs of the tasks. :param startup_timeout: timeout in seconds to start up the pod. :param base_container_name: The name of the base container in the pod. This container's logs will appear as part of this task's logs if get_logs is True. Defaults to None. If None, will consult the class variable BASE_CONTAINER_NAME (which defaults to "base") for the base container name to use. + :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. + If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. + :param should_delete_pod: What to do when the pod reaches its final + state, or the execution is interrupted. If True (default), delete the + pod; if False, leave the pod. + Deprecated - use `on_finish_action` instead. """ def __init__( @@ -66,9 +74,10 @@ def __init__( cluster_context: str | None = None, poll_interval: float = 2, in_cluster: bool | None = None, - should_delete_pod: bool = True, get_logs: bool = True, startup_timeout: int = 120, + on_finish_action: str = "delete_pod", + should_delete_pod: bool | None = None, *args, **kwargs, ): @@ -87,10 +96,22 @@ def __init__( self.poll_interval = poll_interval self.cluster_context = cluster_context self.in_cluster = in_cluster - self.should_delete_pod = should_delete_pod self.get_logs = get_logs self.startup_timeout = startup_timeout + if should_delete_pod is not None: + warnings.warn( + "`should_delete_pod` parameter is deprecated, please use `on_finish_action`", + AirflowProviderDeprecationWarning, + ) + self.on_finish_action = ( + OnFinishAction.DELETE_POD if should_delete_pod else OnFinishAction.KEEP_POD + ) + self.should_delete_pod = should_delete_pod + else: + self.on_finish_action = OnFinishAction(on_finish_action) + self.should_delete_pod = self.on_finish_action == OnFinishAction.DELETE_POD + self._cluster_url = cluster_url self._ssl_ca_cert = ssl_ca_cert @@ -105,11 +126,12 @@ def serialize(self) -> tuple[str, dict[str, Any]]: "poll_interval": self.poll_interval, "cluster_context": self.cluster_context, "in_cluster": self.in_cluster, - "should_delete_pod": self.should_delete_pod, "get_logs": self.get_logs, "startup_timeout": self.startup_timeout, "trigger_start_time": self.trigger_start_time, "base_container_name": self.base_container_name, + "should_delete_pod": self.should_delete_pod, + "on_finish_action": self.on_finish_action.value, }, ) diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 3fc2c90b9b52..f22ff420821d 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -1130,7 +1130,7 @@ additional-extras: - apache-beam[gcp] - name: cncf.kubernetes dependencies: - - apache-airflow-providers-cncf-kubernetes>=6.2.0 + - apache-airflow-providers-cncf-kubernetes>=7.2.0 - name: leveldb dependencies: - plyvel diff --git a/tests/providers/amazon/aws/operators/test_eks.py b/tests/providers/amazon/aws/operators/test_eks.py index 5534f635d8f3..9ea8ab72d72f 100644 --- a/tests/providers/amazon/aws/operators/test_eks.py +++ b/tests/providers/amazon/aws/operators/test_eks.py @@ -39,6 +39,7 @@ EksDeleteFargateProfileTrigger, EksNodegroupTrigger, ) +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction from airflow.typing_compat import TypedDict from tests.providers.amazon.aws.utils.eks_test_constants import ( NODEROLE_ARN, @@ -641,7 +642,7 @@ def test_existing_nodegroup( labels={"demo": "hello_world"}, get_logs=True, # Delete the pod when it reaches its final state, or the execution is interrupted. - is_delete_operator_pod=True, + on_finish_action="delete_pod", ) op_return_value = op.execute(ti_context) mock_k8s_pod_operator_execute.assert_called_once_with(ti_context) @@ -651,3 +652,58 @@ def test_existing_nodegroup( ) assert mock_k8s_pod_operator_execute.return_value == op_return_value assert mock_generate_config_file.return_value.__enter__.return_value == op.config_file + + @pytest.mark.parametrize( + "compatible_kpo, kwargs, expected_attributes", + [ + ( + True, + {"on_finish_action": "delete_succeeded_pod"}, + {"on_finish_action": OnFinishAction.DELETE_SUCCEEDED_POD}, + ), + ( + # test that priority for deprecated param + True, + {"on_finish_action": "keep_pod", "is_delete_operator_pod": True}, + {"on_finish_action": OnFinishAction.DELETE_POD, "is_delete_operator_pod": True}, + ), + ( + # test default + True, + {}, + {"on_finish_action": OnFinishAction.KEEP_POD, "is_delete_operator_pod": False}, + ), + ( + False, + {"is_delete_operator_pod": True}, + {"is_delete_operator_pod": True}, + ), + ( + False, + {"is_delete_operator_pod": False}, + {"is_delete_operator_pod": False}, + ), + ( + # test default + False, + {}, + {"is_delete_operator_pod": False}, + ), + ], + ) + def test_on_finish_action_handler(self, compatible_kpo, kwargs, expected_attributes): + kpo_init_args_mock = mock.MagicMock(**{"parameters": ["on_finish_action"] if compatible_kpo else []}) + + with mock.patch("inspect.signature", return_value=kpo_init_args_mock): + op = EksPodOperator( + task_id="run_pod", + pod_name="run_pod", + cluster_name=CLUSTER_NAME, + image="amazon/aws-cli:latest", + cmds=["sh", "-c", "ls"], + labels={"demo": "hello_world"}, + get_logs=True, + **kwargs, + ) + for expected_attr in expected_attributes: + assert op.__getattribute__(expected_attr) == expected_attributes[expected_attr] diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 40feb3078928..1ce25190ba4c 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -589,9 +589,22 @@ def test_image_pull_policy_correctly_set(self): pod = k.build_pod_request_obj(create_context(k)) assert pod.spec.containers[0].image_pull_policy == "Always" + @pytest.mark.parametrize( + "task_kwargs, should_be_deleted", + [ + ({}, True), # default values + ({"is_delete_operator_pod": True}, True), # check b/c of is_delete_operator_pod + ({"is_delete_operator_pod": False}, False), # check b/c of is_delete_operator_pod + ({"on_finish_action": "delete_pod"}, True), + ({"on_finish_action": "delete_succeeded_pod"}, False), + ({"on_finish_action": "keep_pod"}, False), + ], + ) @patch(f"{POD_MANAGER_CLASS}.delete_pod") @patch(f"{KPO_MODULE}.KubernetesPodOperator.find_pod") - def test_pod_delete_after_await_container_error(self, find_pod_mock, delete_pod_mock): + def test_pod_delete_after_await_container_error( + self, find_pod_mock, delete_pod_mock, task_kwargs, should_be_deleted + ): """ When KPO fails unexpectedly during await_container, we should still try to delete the pod, and the pod we try to delete should be the one returned from find_pod earlier. @@ -600,13 +613,16 @@ def test_pod_delete_after_await_container_error(self, find_pod_mock, delete_pod_ cont_status.name = "base" cont_status.state.terminated.message = "my-failure" find_pod_mock.return_value.status.container_statuses = [cont_status] - k = KubernetesPodOperator(task_id="task") + k = KubernetesPodOperator(task_id="task", **task_kwargs) self.await_pod_mock.side_effect = AirflowException("fake failure") with pytest.raises(AirflowException, match="my-failure"): context = create_context(k) context["ti"].xcom_push = MagicMock() k.execute(context=context) - delete_pod_mock.assert_called_with(find_pod_mock.return_value) + if should_be_deleted: + delete_pod_mock.assert_called_with(find_pod_mock.return_value) + else: + delete_pod_mock.assert_not_called() @pytest.mark.parametrize("should_fail", [True, False]) @patch(f"{POD_MANAGER_CLASS}.delete_pod") @@ -618,7 +634,7 @@ def test_pod_delete_not_called_when_creation_fails(self, await_pod_mock, delete_ """ k = KubernetesPodOperator( task_id="task", - is_delete_operator_pod=True, + on_finish_action="delete_pod", ) if should_fail: @@ -1019,7 +1035,7 @@ def test_mark_checked_unexpected_exception(self, mock_patch_already_checked, moc """If we aren't deleting pods and have an exception, mark it so we don't reattach to it""" k = KubernetesPodOperator( task_id="task", - is_delete_operator_pod=False, + on_finish_action="keep_pod", ) self.await_pod_mock.side_effect = AirflowException("oops") context = create_context(k) @@ -1045,16 +1061,50 @@ def test_wait_for_xcom_sidecar_iff_push_xcom(self, mock_await, mock_extract_xcom else: mock_await.assert_not_called() - @pytest.mark.parametrize("should_fail", [True, False]) + @pytest.mark.parametrize( + "task_kwargs, should_fail, should_be_deleted", + [ + ({}, False, True), + ({}, True, True), + ( + {"is_delete_operator_pod": True, "on_finish_action": "keep_pod"}, + False, + True, + ), # check backcompat of is_delete_operator_pod + ( + {"is_delete_operator_pod": True, "on_finish_action": "keep_pod"}, + True, + True, + ), # check b/c of is_delete_operator_pod + ( + {"is_delete_operator_pod": False, "on_finish_action": "delete_pod"}, + False, + False, + ), # check b/c of is_delete_operator_pod + ( + {"is_delete_operator_pod": False, "on_finish_action": "delete_pod"}, + True, + False, + ), # check b/c of is_delete_operator_pod + ({"on_finish_action": "keep_pod"}, False, False), + ({"on_finish_action": "keep_pod"}, True, False), + ({"on_finish_action": "delete_pod"}, False, True), + ({"on_finish_action": "delete_pod"}, True, True), + ({"on_finish_action": "delete_succeeded_pod"}, False, True), + ({"on_finish_action": "delete_succeeded_pod"}, True, False), + ], + ) @patch(f"{POD_MANAGER_CLASS}.delete_pod") @patch(f"{KPO_MODULE}.KubernetesPodOperator.patch_already_checked") - def test_mark_checked_if_not_deleted(self, mock_patch_already_checked, mock_delete_pod, should_fail): + def test_mark_checked_if_not_deleted( + self, mock_patch_already_checked, mock_delete_pod, task_kwargs, should_fail, should_be_deleted + ): """If we aren't deleting pods mark "checked" if the task completes (successful or otherwise)""" dag = DAG("hello2", start_date=pendulum.now()) k = KubernetesPodOperator( task_id="task", - is_delete_operator_pod=False, dag=dag, + **task_kwargs, ) remote_pod_mock = MagicMock() remote_pod_mock.status.phase = "Failed" if should_fail else "Succeeded" @@ -1065,8 +1115,14 @@ def test_mark_checked_if_not_deleted(self, mock_patch_already_checked, mock_dele k.execute(context=context) else: k.execute(context=context) - mock_patch_already_checked.assert_called_once() - mock_delete_pod.assert_not_called() + if should_fail or not should_be_deleted: + mock_patch_already_checked.assert_called_once() + else: + mock_patch_already_checked.assert_not_called() + if should_be_deleted: + mock_delete_pod.assert_called_once() + else: + mock_delete_pod.assert_not_called() @patch(HOOK_CLASS, new=MagicMock) def test_patch_already_checked(self): @@ -1141,7 +1197,7 @@ def test_task_skip_when_pod_exit_with_certain_code( ): """Tests that an AirflowSkipException is raised when the container exits with the skip_on_exit_code""" k = KubernetesPodOperator( - task_id="task", is_delete_operator_pod=True, **(extra_kwargs if extra_kwargs else {}) + task_id="task", on_finish_action="delete_pod", **(extra_kwargs if extra_kwargs else {}) ) base_container = MagicMock() @@ -1282,7 +1338,7 @@ def test_async_create_pod_should_execute_successfully(self, mocked_pod, mocked_p arguments=TEST_ARGS, labels=TEST_LABELS, name=TEST_NAME, - is_delete_operator_pod=False, + on_finish_action="keep_pod", in_cluster=True, get_logs=True, deferrable=True, @@ -1306,7 +1362,7 @@ def test_async_create_pod_should_throw_exception(self, mocked_hook, mocked_clean arguments=TEST_ARGS, labels=TEST_LABELS, name=TEST_NAME, - is_delete_operator_pod=False, + on_finish_action="keep_pod", in_cluster=True, get_logs=True, deferrable=True, @@ -1353,7 +1409,7 @@ def test_async_create_pod_with_skip_on_exit_code_should_skip( arguments=TEST_ARGS, labels=TEST_LABELS, name=TEST_NAME, - is_delete_operator_pod=False, + on_finish_action="keep_pod", in_cluster=True, get_logs=True, deferrable=True, diff --git a/tests/providers/cncf/kubernetes/triggers/test_pod.py b/tests/providers/cncf/kubernetes/triggers/test_pod.py index 6d5d18d028c1..4ed731b42578 100644 --- a/tests/providers/cncf/kubernetes/triggers/test_pod.py +++ b/tests/providers/cncf/kubernetes/triggers/test_pod.py @@ -39,12 +39,12 @@ CLUSTER_CONTEXT = "test-context" CONFIG_FILE = "/path/to/config/file" IN_CLUSTER = False -SHOULD_DELETE_POD = True GET_LOGS = True STARTUP_TIMEOUT_SECS = 120 TRIGGER_START_TIME = datetime.now(tz=pytz.UTC) FAILED_RESULT_MSG = "Test message that appears when trigger have failed event." BASE_CONTAINER_NAME = "base" +ON_FINISH_ACTION = "delete_pod" @pytest.fixture @@ -58,10 +58,10 @@ def trigger(): cluster_context=CLUSTER_CONTEXT, config_file=CONFIG_FILE, in_cluster=IN_CLUSTER, - should_delete_pod=SHOULD_DELETE_POD, get_logs=GET_LOGS, startup_timeout=STARTUP_TIMEOUT_SECS, trigger_start_time=TRIGGER_START_TIME, + on_finish_action=ON_FINISH_ACTION, ) @@ -85,10 +85,11 @@ def test_serialize(self, trigger): "cluster_context": CLUSTER_CONTEXT, "config_file": CONFIG_FILE, "in_cluster": IN_CLUSTER, - "should_delete_pod": SHOULD_DELETE_POD, "get_logs": GET_LOGS, "startup_timeout": STARTUP_TIMEOUT_SECS, "trigger_start_time": TRIGGER_START_TIME, + "on_finish_action": ON_FINISH_ACTION, + "should_delete_pod": ON_FINISH_ACTION == "delete_pod", } @pytest.mark.asyncio @@ -237,8 +238,10 @@ async def test_logging_in_trigger_when_fail_should_execute_successfully( @pytest.mark.asyncio @mock.patch(f"{TRIGGER_PATH}._get_async_hook") - async def test_logging_in_trigger_when_cancelled_should_execute_successfully( - self, mock_hook, trigger, caplog + async def test_logging_in_trigger_when_cancelled_should_execute_successfully_and_delete_pod( + self, + mock_hook, + caplog, ): """ Test that KubernetesPodTrigger fires the correct event in case if the task was cancelled. @@ -248,6 +251,21 @@ async def test_logging_in_trigger_when_cancelled_should_execute_successfully( mock_hook.return_value.read_logs.return_value = self._mock_pod_result(mock.MagicMock()) mock_hook.return_value.delete_pod.return_value = self._mock_pod_result(mock.MagicMock()) + trigger = KubernetesPodTrigger( + pod_name=POD_NAME, + pod_namespace=NAMESPACE, + base_container_name=BASE_CONTAINER_NAME, + kubernetes_conn_id=CONN_ID, + poll_interval=POLL_INTERVAL, + cluster_context=CLUSTER_CONTEXT, + config_file=CONFIG_FILE, + in_cluster=IN_CLUSTER, + get_logs=GET_LOGS, + startup_timeout=STARTUP_TIMEOUT_SECS, + trigger_start_time=TRIGGER_START_TIME, + on_finish_action="delete_pod", + ) + generator = trigger.run() actual = await generator.asend(None) assert ( @@ -264,6 +282,52 @@ async def test_logging_in_trigger_when_cancelled_should_execute_successfully( assert "Outputting container logs..." in caplog.text assert "Deleting pod..." in caplog.text + @pytest.mark.asyncio + @mock.patch(f"{TRIGGER_PATH}._get_async_hook") + async def test_logging_in_trigger_when_cancelled_should_execute_successfully_without_delete_pod( + self, + mock_hook, + caplog, + ): + """ + Test that KubernetesPodTrigger fires the correct event if the task was cancelled. + """ + + mock_hook.return_value.get_pod.side_effect = CancelledError() + mock_hook.return_value.read_logs.return_value = self._mock_pod_result(mock.MagicMock()) + mock_hook.return_value.delete_pod.return_value = self._mock_pod_result(mock.MagicMock()) + + trigger = KubernetesPodTrigger( + pod_name=POD_NAME, + pod_namespace=NAMESPACE, + base_container_name=BASE_CONTAINER_NAME, + kubernetes_conn_id=CONN_ID, + poll_interval=POLL_INTERVAL, + cluster_context=CLUSTER_CONTEXT, + config_file=CONFIG_FILE, + in_cluster=IN_CLUSTER, + get_logs=GET_LOGS, + startup_timeout=STARTUP_TIMEOUT_SECS, + trigger_start_time=TRIGGER_START_TIME, + on_finish_action="delete_succeeded_pod", + ) + + generator = trigger.run() + actual = await generator.asend(None) + assert ( + TriggerEvent( + { + "name": POD_NAME, + "namespace": NAMESPACE, + "status": "cancelled", + "message": "Pod execution was cancelled", + } + ) + == actual + ) + assert "Outputting container logs..." in caplog.text + assert "Deleting pod..." not in caplog.text + @pytest.mark.parametrize( "container_state, expected_state", [ diff --git a/tests/providers/google/cloud/operators/test_kubernetes_engine.py b/tests/providers/google/cloud/operators/test_kubernetes_engine.py index 5d6acb196ad2..07f0c34c7a14 100644 --- a/tests/providers/google/cloud/operators/test_kubernetes_engine.py +++ b/tests/providers/google/cloud/operators/test_kubernetes_engine.py @@ -26,6 +26,7 @@ from airflow.exceptions import AirflowException, TaskDeferred from airflow.models import Connection from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator +from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction from airflow.providers.google.cloud.operators.kubernetes_engine import ( GKECreateClusterOperator, GKEDeleteClusterOperator, @@ -319,6 +320,66 @@ def test_cluster_info(self, get_cluster_mock, use_internal_ip): assert cluster_url == CLUSTER_PRIVATE_URL if use_internal_ip else CLUSTER_URL assert ssl_ca_cert == SSL_CA_CERT + @pytest.mark.parametrize( + "compatible_kpo, kwargs, expected_attributes", + [ + ( + True, + {"on_finish_action": "delete_succeeded_pod"}, + {"on_finish_action": OnFinishAction.DELETE_SUCCEEDED_POD}, + ), + ( + # test that priority for deprecated param + True, + {"on_finish_action": "keep_pod", "is_delete_operator_pod": True}, + {"on_finish_action": OnFinishAction.DELETE_POD, "is_delete_operator_pod": True}, + ), + ( + # test default + True, + {}, + {"on_finish_action": OnFinishAction.KEEP_POD, "is_delete_operator_pod": False}, + ), + ( + False, + {"is_delete_operator_pod": True}, + {"is_delete_operator_pod": True}, + ), + ( + False, + {"is_delete_operator_pod": False}, + {"is_delete_operator_pod": False}, + ), + ( + # test default + False, + {}, + {"is_delete_operator_pod": False}, + ), + ], + ) + def test_on_finish_action_handler( + self, + compatible_kpo, + kwargs, + expected_attributes, + ): + kpo_init_args_mock = mock.MagicMock(**{"parameters": ["on_finish_action"] if compatible_kpo else []}) + + with mock.patch("inspect.signature", return_value=kpo_init_args_mock): + op = GKEStartPodOperator( + project_id=TEST_GCP_PROJECT_ID, + location=PROJECT_LOCATION, + cluster_name=CLUSTER_NAME, + task_id=PROJECT_TASK_ID, + name=TASK_NAME, + namespace=NAMESPACE, + image=IMAGE, + **kwargs, + ) + for expected_attr in expected_attributes: + assert op.__getattribute__(expected_attr) == expected_attributes[expected_attr] + class TestGKEPodOperatorAsync: def setup_method(self): diff --git a/tests/providers/google/cloud/triggers/test_kubernetes_engine.py b/tests/providers/google/cloud/triggers/test_kubernetes_engine.py index e957767e3eea..154908a6c4b9 100644 --- a/tests/providers/google/cloud/triggers/test_kubernetes_engine.py +++ b/tests/providers/google/cloud/triggers/test_kubernetes_engine.py @@ -48,6 +48,7 @@ SSL_CA_CERT = "TEST_SSL_CA_CERT_CONTENT" FAILED_RESULT_MSG = "Test message that appears when trigger have failed event." BASE_CONTAINER_NAME = "base" +ON_FINISH_ACTION = "delete_pod" OPERATION_NAME = "test-operation-name" PROJECT_ID = "test-project-id" @@ -93,13 +94,14 @@ def test_serialize_should_execute_successfully(self, trigger): "poll_interval": POLL_INTERVAL, "cluster_context": CLUSTER_CONTEXT, "in_cluster": IN_CLUSTER, - "should_delete_pod": SHOULD_DELETE_POD, "get_logs": GET_LOGS, "startup_timeout": STARTUP_TIMEOUT_SECS, "trigger_start_time": TRIGGER_START_TIME, "cluster_url": CLUSTER_URL, "ssl_ca_cert": SSL_CA_CERT, "base_container_name": BASE_CONTAINER_NAME, + "on_finish_action": ON_FINISH_ACTION, + "should_delete_pod": SHOULD_DELETE_POD, } @pytest.mark.asyncio diff --git a/tests/system/providers/amazon/aws/example_eks_templated.py b/tests/system/providers/amazon/aws/example_eks_templated.py index d09eabf95982..e51783b53e76 100644 --- a/tests/system/providers/amazon/aws/example_eks_templated.py +++ b/tests/system/providers/amazon/aws/example_eks_templated.py @@ -107,7 +107,7 @@ labels={"demo": "hello_world"}, get_logs=True, # Delete the pod when it reaches its final state, or the execution is interrupted. - is_delete_operator_pod=True, + on_finish_action="delete_pod", ) delete_nodegroup = EksDeleteNodegroupOperator( diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py index d14e7a6885a4..ae67a26588bd 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -101,7 +101,7 @@ get_logs=True, startup_timeout_seconds=600, # Keep the pod alive, so we can describe it in case of trouble. It's deleted with the cluster anyway. - is_delete_operator_pod=False, + on_finish_action="keep_pod", ) describe_pod = get_describe_pod_operator( diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py b/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py index 0c19fda2d518..9e19c7594ffb 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py +++ b/tests/system/providers/amazon/aws/example_eks_with_fargate_profile.py @@ -119,7 +119,7 @@ labels={"demo": "hello_world"}, get_logs=True, # Keep the pod alive, so we can describe it in case of trouble. It's deleted with the cluster anyway. - is_delete_operator_pod=False, + on_finish_action="keep_pod", startup_timeout_seconds=200, ) diff --git a/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py b/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py index 76aa01bf5c96..9bfce2cc1696 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py +++ b/tests/system/providers/amazon/aws/example_eks_with_nodegroup_in_one_step.py @@ -111,7 +111,7 @@ def delete_launch_template(template_name: str): labels={"demo": "hello_world"}, get_logs=True, # Keep the pod alive, so we can describe it in case of trouble. It's deleted with the cluster anyway. - is_delete_operator_pod=False, + on_finish_action="keep_pod", ) describe_pod = get_describe_pod_operator( diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes.py b/tests/system/providers/cncf/kubernetes/example_kubernetes.py index 2f1f791d7882..74078541bdcf 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes.py @@ -122,7 +122,7 @@ name="airflow-test-pod", task_id="task", affinity=affinity, - is_delete_operator_pod=True, + on_finish_action="delete_pod", hostnetwork=False, tolerations=tolerations, init_containers=[init_container], @@ -138,7 +138,7 @@ arguments=["echo", "10", "echo pwd"], labels={"foo": "bar"}, name="airflow-private-image-pod", - is_delete_operator_pod=True, + on_finish_action="delete_pod", in_cluster=True, task_id="task-two", get_logs=True, @@ -152,7 +152,7 @@ cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="write-xcom", do_xcom_push=True, - is_delete_operator_pod=True, + on_finish_action="delete_pod", in_cluster=True, task_id="write-xcom", get_logs=True, diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py index f44e8637ffc4..36e143c096b9 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py @@ -123,7 +123,7 @@ env_from=configmaps, name="airflow-test-pod", affinity=affinity, - is_delete_operator_pod=True, + on_finish_action="delete_pod", hostnetwork=False, tolerations=tolerations, init_containers=[init_container], @@ -141,7 +141,7 @@ arguments=["echo", "10", "echo pwd"], labels={"foo": "bar"}, name="airflow-private-image-pod", - is_delete_operator_pod=True, + on_finish_action="delete_pod", in_cluster=True, get_logs=True, deferrable=True, @@ -156,7 +156,7 @@ cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="write-xcom", do_xcom_push=True, - is_delete_operator_pod=True, + on_finish_action="delete_pod", in_cluster=True, get_logs=True, deferrable=True, diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py index d2a6161ed8da..bf155499f294 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -67,7 +67,7 @@ image="perl", name="test-pod", in_cluster=False, - is_delete_operator_pod=True, + on_finish_action="delete_pod", ) # [START howto_operator_gke_start_pod_xcom] @@ -82,7 +82,7 @@ cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="test-pod-xcom", in_cluster=False, - is_delete_operator_pod=True, + on_finish_action="delete_pod", ) # [END howto_operator_gke_start_pod_xcom] diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index 13c310d88021..73430a39385a 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -66,7 +66,7 @@ image="perl", name="test-pod-async", in_cluster=False, - is_delete_operator_pod=True, + on_finish_action="delete_pod", get_logs=True, deferrable=True, ) @@ -82,7 +82,7 @@ cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="test-pod-xcom-async", in_cluster=False, - is_delete_operator_pod=True, + on_finish_action="delete_pod", do_xcom_push=True, deferrable=True, get_logs=True, From 7e06c80702a16db466e6321f7a46cd94c19058c8 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Jun 2023 10:49:40 +0200 Subject: [PATCH 003/533] Add licence preamble also in hidden files (#32276) Licence preamble adding has beeen skipped for markdown files in hidden/.github folders. Technically the licence is not needed there as we do not distribute those files in sources, but it does not hurt to have it for consistency --- .github/PULL_REQUEST_TEMPLATE.md | 19 +++++++++++++++++++ .pre-commit-config.yaml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 021313b68eb7..011b4caa5e71 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,22 @@ + + " From d6e254db689db070f2f181006e7d6bc593482300 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 30 Jun 2023 11:52:14 +0300 Subject: [PATCH 004/533] Deprecate `delimiter` param and source object's wildcards in GCS, introduce `match_glob` param. (#31261) * Deprecate `delimiter` param and source object's wildcards in GCS, introduce `match_glob` param. --------- Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- .../amazon/aws/transfers/gcs_to_s3.py | 18 +- airflow/providers/google/cloud/hooks/gcs.py | 126 +++++++++-- .../providers/google/cloud/operators/gcs.py | 48 +++-- .../google/cloud/transfers/gcs_to_gcs.py | 53 ++++- .../google/cloud/transfers/gcs_to_sftp.py | 5 +- .../google/suite/transfers/gcs_to_gdrive.py | 4 + .../operators/transfer/gcs_to_gcs.rst | 15 +- .../amazon/aws/transfers/test_gcs_to_s3.py | 195 +++++++++++------- .../providers/google/cloud/hooks/test_gcs.py | 60 +++++- .../google/cloud/operators/test_gcs.py | 16 +- .../google/cloud/transfers/test_gcs_to_gcs.py | 26 ++- .../cloud/transfers/test_gcs_to_sftp.py | 2 + .../suite/transfers/test_gcs_to_gdrive.py | 6 + .../google/cloud/gcs/example_gcs_to_gcs.py | 12 ++ 14 files changed, 448 insertions(+), 138 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 4004eea8c2c2..2213de2b60c1 100644 --- a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -19,8 +19,10 @@ from __future__ import annotations import os +import warnings from typing import TYPE_CHECKING, Sequence +from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.google.cloud.hooks.gcs import GCSHook @@ -40,7 +42,7 @@ class GCSToS3Operator(BaseOperator): :param bucket: The Google Cloud Storage bucket to find the objects. (templated) :param prefix: Prefix string which filters objects whose name begin with this prefix. (templated) - :param delimiter: The delimiter by which you want to filter the objects. (templated) + :param delimiter: (Deprecated) The delimiter by which you want to filter the objects. (templated) For e.g to lists the CSV files from in a directory in GCS you would use delimiter='.csv'. :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. @@ -76,6 +78,8 @@ class GCSToS3Operator(BaseOperator): object to be uploaded in S3 :param keep_directory_structure: (Optional) When set to False the path of the file on the bucket is recreated within path passed in dest_s3_key. + :param match_glob: (Optional) filters objects based on the glob pattern given by the string + (e.g, ``'**/*/.json'``) """ template_fields: Sequence[str] = ( @@ -102,12 +106,19 @@ def __init__( dest_s3_extra_args: dict | None = None, s3_acl_policy: str | None = None, keep_directory_structure: bool = True, + match_glob: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) self.bucket = bucket self.prefix = prefix + if delimiter: + warnings.warn( + "Usage of 'delimiter' is deprecated, please use 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.delimiter = delimiter self.gcp_conn_id = gcp_conn_id self.dest_aws_conn_id = dest_aws_conn_id @@ -118,6 +129,7 @@ def __init__( self.dest_s3_extra_args = dest_s3_extra_args or {} self.s3_acl_policy = s3_acl_policy self.keep_directory_structure = keep_directory_structure + self.match_glob = match_glob def execute(self, context: Context) -> list[str]: # list all files in an Google Cloud Storage bucket @@ -133,7 +145,9 @@ def execute(self, context: Context) -> list[str]: self.prefix, ) - files = hook.list(bucket_name=self.bucket, prefix=self.prefix, delimiter=self.delimiter) + files = hook.list( + bucket_name=self.bucket, prefix=self.prefix, delimiter=self.delimiter, match_glob=self.match_glob + ) s3_hook = S3Hook( aws_conn_id=self.dest_aws_conn_id, verify=self.dest_verify, extra_args=self.dest_s3_extra_args diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py index b17af1d9b917..42a27e80f898 100644 --- a/airflow/providers/google/cloud/hooks/gcs.py +++ b/airflow/providers/google/cloud/hooks/gcs.py @@ -24,6 +24,7 @@ import os import shutil import time +import warnings from contextlib import contextmanager from datetime import datetime from functools import partial @@ -44,7 +45,7 @@ from google.cloud.storage.retry import DEFAULT_RETRY from requests import Session -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.utils.helpers import normalize_directory_path from airflow.providers.google.common.consts import CLIENT_INFO from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook @@ -709,6 +710,7 @@ def list( max_results: int | None = None, prefix: str | List[str] | None = None, delimiter: str | None = None, + match_glob: str | None = None, ): """ List all objects from the bucket with the given a single prefix or multiple prefixes. @@ -717,9 +719,19 @@ def list( :param versions: if true, list all versions of the objects :param max_results: max count of items to return in a single page of responses :param prefix: string or list of strings which filter objects whose name begin with it/them - :param delimiter: filters objects based on the delimiter (for e.g '.csv') + :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv') + :param match_glob: (Optional) filters objects based on the glob pattern given by the string + (e.g, ``'**/*/.json'``). :return: a stream of object names matching the filtering criteria """ + if delimiter and delimiter != "/": + warnings.warn( + "Usage of 'delimiter' param is deprecated, please use 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + if match_glob and delimiter and delimiter != "/": + raise AirflowException("'match_glob' param cannot be used with 'delimiter' that differs than '/'") objects = [] if isinstance(prefix, list): for prefix_item in prefix: @@ -730,6 +742,7 @@ def list( max_results=max_results, prefix=prefix_item, delimiter=delimiter, + match_glob=match_glob, ) ) else: @@ -740,6 +753,7 @@ def list( max_results=max_results, prefix=prefix, delimiter=delimiter, + match_glob=match_glob, ) ) return objects @@ -751,6 +765,7 @@ def _list( max_results: int | None = None, prefix: str | None = None, delimiter: str | None = None, + match_glob: str | None = None, ) -> List: """ List all objects from the bucket with the give string prefix in name. @@ -759,7 +774,9 @@ def _list( :param versions: if true, list all versions of the objects :param max_results: max count of items to return in a single page of responses :param prefix: string which filters objects whose name begin with it - :param delimiter: filters objects based on the delimiter (for e.g '.csv') + :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv') + :param match_glob: (Optional) filters objects based on the glob pattern given by the string + (e.g, ``'**/*/.json'``). :return: a stream of object names matching the filtering criteria """ client = self.get_conn() @@ -768,13 +785,25 @@ def _list( ids = [] page_token = None while True: - blobs = bucket.list_blobs( - max_results=max_results, - page_token=page_token, - prefix=prefix, - delimiter=delimiter, - versions=versions, - ) + if match_glob: + blobs = self._list_blobs_with_match_glob( + bucket=bucket, + client=client, + match_glob=match_glob, + max_results=max_results, + page_token=page_token, + path=bucket.path + "/o", + prefix=prefix, + versions=versions, + ) + else: + blobs = bucket.list_blobs( + max_results=max_results, + page_token=page_token, + prefix=prefix, + delimiter=delimiter, + versions=versions, + ) blob_names = [] for blob in blobs: @@ -792,6 +821,52 @@ def _list( break return ids + @staticmethod + def _list_blobs_with_match_glob( + bucket, + client, + path: str, + max_results: int | None = None, + page_token: str | None = None, + match_glob: str | None = None, + prefix: str | None = None, + versions: bool | None = None, + ) -> Any: + """ + List blobs when match_glob param is given. + This method is a patched version of google.cloud.storage Client.list_blobs(). + It is used as a temporary workaround to support "match_glob" param, + as it isn't officially supported by GCS Python client. + (follow `issue #1035`__). + """ + from google.api_core import page_iterator + from google.cloud.storage.bucket import _blobs_page_start, _item_to_blob + + extra_params: Any = {} + if prefix is not None: + extra_params["prefix"] = prefix + if match_glob is not None: + extra_params["matchGlob"] = match_glob + if versions is not None: + extra_params["versions"] = versions + api_request = functools.partial( + client._connection.api_request, timeout=DEFAULT_TIMEOUT, retry=DEFAULT_RETRY + ) + + blobs: Any = page_iterator.HTTPIterator( + client=client, + api_request=api_request, + path=path, + item_to_value=_item_to_blob, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_start=_blobs_page_start, + ) + blobs.prefixes = set() + blobs.bucket = bucket + return blobs + def list_by_timespan( self, bucket_name: str, @@ -801,6 +876,7 @@ def list_by_timespan( max_results: int | None = None, prefix: str | None = None, delimiter: str | None = None, + match_glob: str | None = None, ) -> List[str]: """ List all objects from the bucket with the give string prefix in name that were @@ -813,7 +889,9 @@ def list_by_timespan( :param max_results: max count of items to return in a single page of responses :param prefix: prefix string which filters objects whose name begin with this prefix - :param delimiter: filters objects based on the delimiter (for e.g '.csv') + :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv') + :param match_glob: (Optional) filters objects based on the glob pattern given by the string + (e.g, ``'**/*/.json'``). :return: a stream of object names matching the filtering criteria """ client = self.get_conn() @@ -823,13 +901,25 @@ def list_by_timespan( page_token = None while True: - blobs = bucket.list_blobs( - max_results=max_results, - page_token=page_token, - prefix=prefix, - delimiter=delimiter, - versions=versions, - ) + if match_glob: + blobs = self._list_blobs_with_match_glob( + bucket=bucket, + client=client, + match_glob=match_glob, + max_results=max_results, + page_token=page_token, + path=bucket.path + "/o", + prefix=prefix, + versions=versions, + ) + else: + blobs = bucket.list_blobs( + max_results=max_results, + page_token=page_token, + prefix=prefix, + delimiter=delimiter, + versions=versions, + ) blob_names = [] for blob in blobs: diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py index e2ac68c90d65..6e50cbad2d16 100644 --- a/airflow/providers/google/cloud/operators/gcs.py +++ b/airflow/providers/google/cloud/operators/gcs.py @@ -21,6 +21,7 @@ import datetime import subprocess import sys +import warnings from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory from typing import TYPE_CHECKING, Sequence @@ -33,7 +34,7 @@ from google.api_core.exceptions import Conflict from google.cloud.exceptions import GoogleCloudError -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator from airflow.providers.google.common.links.storage import FileDetailsLink, StorageLink @@ -157,16 +158,17 @@ def execute(self, context: Context) -> None: class GCSListObjectsOperator(GoogleCloudBaseOperator): """ - List all objects from the bucket with the given string prefix and delimiter in name. + List all objects from the bucket filtered by given string prefix and delimiter in name, + or match_glob. This operator returns a python list with the name of objects which can be used by XCom in the downstream task. :param bucket: The Google Cloud Storage bucket to find the objects. (templated) - :param prefix: String or list of strings, which filter objects whose name begin with + :param prefix: String or list of strings, which filter objects whose name begins with it/them. (templated) - :param delimiter: The delimiter by which you want to filter the objects. (templated) - For example, to lists the CSV files from in a directory in GCS you would use + :param delimiter: (Deprecated) The delimiter by which you want to filter the objects. (templated) + For example, to list the CSV files from in a directory in GCS you would use delimiter='.csv'. :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. :param impersonation_chain: Optional service account to impersonate using short-term @@ -177,6 +179,8 @@ class GCSListObjectsOperator(GoogleCloudBaseOperator): If set as a sequence, the identities from the list must grant Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). + :param match_glob: (Optional) filters objects based on the glob pattern given by the string + (e.g, ``'**/*/.json'``) **Example**: The following Operator would list all the Avro files from ``sales/sales-2017`` @@ -186,7 +190,7 @@ class GCSListObjectsOperator(GoogleCloudBaseOperator): task_id='GCS_Files', bucket='data', prefix='sales/sales-2017/', - delimiter='.avro', + match_glob='**/*/.avro', gcp_conn_id=google_cloud_conn_id ) """ @@ -210,14 +214,22 @@ def __init__( delimiter: str | None = None, gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, + match_glob: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) self.bucket = bucket self.prefix = prefix + if delimiter: + warnings.warn( + "Usage of 'delimiter' is deprecated, please use 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.delimiter = delimiter self.gcp_conn_id = gcp_conn_id self.impersonation_chain = impersonation_chain + self.match_glob = match_glob def execute(self, context: Context) -> list: hook = GCSHook( @@ -225,12 +237,20 @@ def execute(self, context: Context) -> list: impersonation_chain=self.impersonation_chain, ) - self.log.info( - "Getting list of the files. Bucket: %s; Delimiter: %s; Prefix(es): %s", - self.bucket, - self.delimiter, - self.prefix, - ) + if self.match_glob: + self.log.info( + "Getting list of the files. Bucket: %s; MatchGlob: %s; Prefix(es): %s", + self.bucket, + self.match_glob, + self.prefix, + ) + else: + self.log.info( + "Getting list of the files. Bucket: %s; Delimiter: %s; Prefix(es): %s", + self.bucket, + self.delimiter, + self.prefix, + ) StorageLink.persist( context=context, @@ -238,7 +258,9 @@ def execute(self, context: Context) -> list: uri=self.bucket, project_id=hook.project_id, ) - return hook.list(bucket_name=self.bucket, prefix=self.prefix, delimiter=self.delimiter) + return hook.list( + bucket_name=self.bucket, prefix=self.prefix, delimiter=self.delimiter, match_glob=self.match_glob + ) class GCSDeleteObjectsOperator(GoogleCloudBaseOperator): diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py index c8f811c6e7d8..2b39df1c6a79 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py @@ -18,9 +18,10 @@ """This module contains a Google Cloud Storage operator.""" from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Sequence -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook @@ -66,8 +67,8 @@ class GCSToGCSOperator(BaseOperator): of copied to the new location. This is the equivalent of a mv command as opposed to a cp command. :param replace: Whether you want to replace existing destination files or not. - :param delimiter: This is used to restrict the result to only the 'files' in a given 'folder'. - If source_objects = ['foo/bah/'] and delimiter = '.avro', then only the 'files' in the + :param delimiter: (Deprecated) This is used to restrict the result to only the 'files' in a given + 'folder'. If source_objects = ['foo/bah/'] and delimiter = '.avro', then only the 'files' in the folder 'foo/bah/' with '.avro' delimiter will be copied to the destination object. :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. :param last_modified_time: When specified, the objects will be copied or moved, @@ -90,6 +91,8 @@ class GCSToGCSOperator(BaseOperator): doesn't exist. It doesn't have any effect when the source objects are folders or patterns. :param exact_match: When specified, only exact match of the source object (filename) will be copied. + :param match_glob: (Optional) filters objects based on the glob pattern given by the string ( + e.g, ``'**/*/.json'``) :Example: @@ -116,7 +119,7 @@ class GCSToGCSOperator(BaseOperator): source_objects=['sales/sales-2017'], destination_bucket='data_backup', destination_object='copied_sales/2017/', - delimiter='.avro' + match_glob='**/*.avro' gcp_conn_id=google_cloud_conn_id ) @@ -190,15 +193,34 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, source_object_required=False, exact_match=False, + match_glob: str | None = None, **kwargs, ): super().__init__(**kwargs) self.source_bucket = source_bucket + if source_object and WILDCARD in source_object: + warnings.warn( + "Usage of wildcard (*) in 'source_object' is deprecated, utilize 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.source_object = source_object + if source_objects and any([WILDCARD in obj for obj in source_objects]): + warnings.warn( + "Usage of wildcard (*) in 'source_objects' is deprecated, utilize 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.source_objects = source_objects self.destination_bucket = destination_bucket self.destination_object = destination_object + if delimiter: + warnings.warn( + "Usage of 'delimiter' is deprecated, please use 'match_glob' instead", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.delimiter = delimiter self.move_object = move_object self.replace = replace @@ -209,6 +231,7 @@ def __init__( self.impersonation_chain = impersonation_chain self.source_object_required = source_object_required self.exact_match = exact_match + self.match_glob = match_glob def execute(self, context: Context): @@ -251,6 +274,7 @@ def execute(self, context: Context): for prefix in self.source_objects: # Check if prefix contains wildcard if WILDCARD in prefix: + self._copy_source_with_wildcard(hook=hook, prefix=prefix) # Now search with prefix using provided delimiter if any else: @@ -261,15 +285,19 @@ def _ignore_existing_files(self, hook, prefix, **kwargs): # and only keep those files which are present in # Source GCS bucket and not in Destination GCS bucket delimiter = kwargs.get("delimiter") + match_glob = kwargs.get("match_glob") objects = kwargs.get("objects") if self.destination_object is None: - existing_objects = hook.list(self.destination_bucket, prefix=prefix, delimiter=delimiter) + existing_objects = hook.list( + self.destination_bucket, prefix=prefix, delimiter=delimiter, match_glob=match_glob + ) else: self.log.info("Replaced destination_object with source_object prefix.") destination_objects = hook.list( self.destination_bucket, prefix=self.destination_object, delimiter=delimiter, + match_glob=match_glob, ) existing_objects = [ dest_object.replace(self.destination_object, prefix, 1) for dest_object in destination_objects @@ -338,11 +366,15 @@ def _copy_source_without_wildcard(self, hook, prefix): gcp_conn_id=google_cloud_conn_id ) """ - objects = hook.list(self.source_bucket, prefix=prefix, delimiter=self.delimiter) + objects = hook.list( + self.source_bucket, prefix=prefix, delimiter=self.delimiter, match_glob=self.match_glob + ) if not self.replace: # If we are not replacing, ignore files already existing in source buckets - objects = self._ignore_existing_files(hook, prefix, objects=objects, delimiter=self.delimiter) + objects = self._ignore_existing_files( + hook, prefix, objects=objects, delimiter=self.delimiter, match_glob=self.match_glob + ) # If objects is empty, and we have prefix, let's check if prefix is a blob # and copy directly @@ -397,11 +429,18 @@ def _copy_source_with_wildcard(self, hook, prefix): self.log.info("Delimiter ignored because wildcard is in prefix") prefix_, delimiter = prefix.split(WILDCARD, 1) objects = hook.list(self.source_bucket, prefix=prefix_, delimiter=delimiter) + # TODO: After deprecating delimiter and wildcards in source objects, + # remove previous line and uncomment the following: + # match_glob = f"**/*{delimiter}" if delimiter else None + # objects = hook.list(self.source_bucket, prefix=prefix_, match_glob=match_glob) if not self.replace: # If we are not replacing, list all files in the Destination GCS bucket # and only keep those files which are present in # Source GCS bucket and not in Destination GCS bucket objects = self._ignore_existing_files(hook, prefix_, delimiter=delimiter, objects=objects) + # TODO: After deprecating delimiter and wildcards in source objects, + # remove previous line and uncomment the following: + # objects = self._ignore_existing_files(hook, prefix_, match_glob=match_glob, objects=objects) for source_object in objects: if self.destination_object is None: diff --git a/airflow/providers/google/cloud/transfers/gcs_to_sftp.py b/airflow/providers/google/cloud/transfers/gcs_to_sftp.py index f4942311da9d..150c801861ee 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_sftp.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_sftp.py @@ -145,8 +145,11 @@ def execute(self, context: Context): prefix, delimiter = self.source_object.split(WILDCARD, 1) prefix_dirname = os.path.dirname(prefix) - objects = gcs_hook.list(self.source_bucket, prefix=prefix, delimiter=delimiter) + # TODO: After deprecating delimiter and wildcards in source objects, + # remove the previous line and uncomment the following: + # match_glob = f"**/*{delimiter}" if delimiter else None + # objects = gcs_hook.list(self.source_bucket, prefix=prefix, match_glob=match_glob) for source_object in objects: destination_path = self._resolve_destination_path(source_object, prefix=prefix_dirname) diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py index 7f8568688c7e..c1e796258f53 100644 --- a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py +++ b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py @@ -132,6 +132,10 @@ def execute(self, context: Context): prefix, delimiter = self.source_object.split(WILDCARD, 1) objects = self.gcs_hook.list(self.source_bucket, prefix=prefix, delimiter=delimiter) + # TODO: After deprecating delimiter and wildcards in source objects, + # remove the previous line and uncomment the following: + # match_glob = f"**/*{delimiter}" if delimiter else None + # objects = self.gcs_hook.list(self.source_bucket, prefix=prefix, match_glob=match_glob) for source_object in objects: if self.destination_object is None: diff --git a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst index b3fa2ac19115..ef805355c4c9 100644 --- a/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst +++ b/docs/apache-airflow-providers-google/operators/transfer/gcs_to_gcs.rst @@ -83,6 +83,10 @@ When you use this operator, you can specify whether objects should be deleted fr they are transferred to the sink. Source objects can be specified using a single wildcard, as well as based on the file modification date. +Filtering objects according to their path could be done by using the `match_glob field `__. +You should avoid using the ``delimiter`` field nor a wildcard in the path of the source object(s), as both practices are deprecated. +Additionally, filtering could be achieved based on the file's creation date (``is_older_than``) or modification date (``last_modified_time`` and ``maximum_modified_time``). + The way this operator works by default can be compared to the ``cp`` command. When the file move option is active, this operator functions like the ``mv`` command. @@ -124,6 +128,15 @@ folder in ``BUCKET_1_DST``, with file names unchanged. For source_objects with no wildcard, all files in source_objects would be listed, using provided delimiter if any. Then copy files from source_objects to destination_object and rename each source file. +As previously stated, the ``delimiter`` field, as well as utilizing a wildcard (``*``) in the source object(s), +are both deprecated. Thus, it is not recommended to use them - but to utilize ``match_glob`` instead, as follows: + +.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_gcs_to_gcs_match_glob] + :end-before: [END howto_operator_gcs_to_gcs_match_glob] + The following example would copy all the files in ``subdir/`` folder (i.e subdir/a.csv, subdir/b.csv, subdir/c.csv) from the ``BUCKET_1_SRC`` GCS bucket to the ``backup/`` folder in ``BUCKET_1_DST`` bucket. (i.e backup/a.csv, backup/b.csv, backup/c.csv) @@ -133,7 +146,7 @@ the ``BUCKET_1_SRC`` GCS bucket to the ``backup/`` folder in ``BUCKET_1_DST`` bu :start-after: [START howto_operator_gcs_to_gcs_without_wildcard] :end-before: [END howto_operator_gcs_to_gcs_without_wildcard] -The delimiter filed may be specified to select any source files starting with ``source_object`` and ending with the +The delimiter field may be specified to select any source files starting with ``source_object`` and ending with the value supplied to ``delimiter``. This example uses the ``delimiter`` value to implement the same functionality as the prior example. diff --git a/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py b/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py index dce115551b6e..a7a0b2e4305a 100644 --- a/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py +++ b/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py @@ -20,6 +20,7 @@ from tempfile import NamedTemporaryFile from unittest import mock +import pytest from moto import mock_s3 from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -47,9 +48,9 @@ def _create_test_bucket(): @mock_s3 class TestGCSToS3Operator: - # Test1: incremental behaviour (just some files missing) + # Test0: match_glob @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") - def test_execute_incremental(self, mock_hook): + def test_execute__match_glob(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES with NamedTemporaryFile() as f: gcs_provide_file = mock_hook.return_value.provide_file @@ -59,14 +60,40 @@ def test_execute_incremental(self, mock_hook): task_id=TASK_ID, bucket=GCS_BUCKET, prefix=PREFIX, - delimiter=DELIMITER, dest_aws_conn_id="aws_default", dest_s3_key=S3_BUCKET, replace=False, + match_glob=f"**/*{DELIMITER}", ) hook, bucket = _create_test_bucket() bucket.put_object(Key=MOCK_FILES[0], Body=b"testing") + operator.execute(None) + mock_hook.return_value.list.assert_called_once_with( + bucket_name=GCS_BUCKET, delimiter=None, match_glob=f"**/*{DELIMITER}", prefix=PREFIX + ) + + # Test1: incremental behaviour (just some files missing) + @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") + def test_execute_incremental(self, mock_hook): + mock_hook.return_value.list.return_value = MOCK_FILES + with NamedTemporaryFile() as f: + gcs_provide_file = mock_hook.return_value.provide_file + gcs_provide_file.return_value.__enter__.return_value.name = f.name + + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=False, + ) + hook, bucket = _create_test_bucket() + bucket.put_object(Key=MOCK_FILES[0], Body=b"testing") + # we expect all except first file in MOCK_FILES to be uploaded # and all the MOCK_FILES to be present at the S3 bucket uploaded_files = operator.execute(None) @@ -81,15 +108,16 @@ def test_execute_without_replace(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=False, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=False, + ) hook, bucket = _create_test_bucket() for mock_file in MOCK_FILES: bucket.put_object(Key=mock_file, Body=b"testing") @@ -108,15 +136,16 @@ def test_execute(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=False, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=False, + ) hook, _ = _create_test_bucket() # we expect all MOCK_FILES to be uploaded @@ -133,15 +162,16 @@ def test_execute_with_replace(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=True, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=True, + ) hook, bucket = _create_test_bucket() for mock_file in MOCK_FILES: bucket.put_object(Key=mock_file, Body=b"testing") @@ -160,15 +190,16 @@ def test_execute_incremental_with_replace(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=True, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=True, + ) hook, bucket = _create_test_bucket() for mock_file in MOCK_FILES[:2]: bucket.put_object(Key=mock_file, Body=b"testing") @@ -187,15 +218,16 @@ def test_execute_should_handle_with_default_dest_s3_extra_args(self, s3_mock_hoo s3_mock_hook.return_value = mock.Mock() s3_mock_hook.parse_s3_url.return_value = mock.Mock() - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=True, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=True, + ) operator.execute(None) s3_mock_hook.assert_called_once_with(aws_conn_id="aws_default", extra_args={}, verify=None) @@ -209,18 +241,19 @@ def test_execute_should_pass_dest_s3_extra_args_to_s3_hook(self, s3_mock_hook, m s3_mock_hook.return_value = mock.Mock() s3_mock_hook.parse_s3_url.return_value = mock.Mock() - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=True, - dest_s3_extra_args={ - "ContentLanguage": "value", - }, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=True, + dest_s3_extra_args={ + "ContentLanguage": "value", + }, + ) operator.execute(None) s3_mock_hook.assert_called_once_with( aws_conn_id="aws_default", extra_args={"ContentLanguage": "value"}, verify=None @@ -235,16 +268,17 @@ def test_execute_with_s3_acl_policy(self, mock_load_file, mock_gcs_hook): gcs_provide_file = mock_gcs_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=False, - s3_acl_policy=S3_ACL_POLICY, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=False, + s3_acl_policy=S3_ACL_POLICY, + ) _create_test_bucket() operator.execute(None) @@ -259,16 +293,17 @@ def test_execute_without_keep_director_structure(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - operator = GCSToS3Operator( - task_id=TASK_ID, - bucket=GCS_BUCKET, - prefix=PREFIX, - delimiter=DELIMITER, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - replace=False, - keep_directory_structure=False, - ) + with pytest.deprecated_call(): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + replace=False, + keep_directory_structure=False, + ) hook, _ = _create_test_bucket() # we expect all except first file in MOCK_FILES to be uploaded diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/tests/providers/google/cloud/hooks/test_gcs.py index 926ad36431b2..9e7ff971d2ba 100644 --- a/tests/providers/google/cloud/hooks/test_gcs.py +++ b/tests/providers/google/cloud/hooks/test_gcs.py @@ -817,14 +817,66 @@ def test_provide_file_upload(self, mock_upload, mock_temp_file): ), ) @mock.patch(GCS_STRING.format("GCSHook.get_conn")) - def test_list(self, mock_service, prefix, result): + def test_list__delimiter(self, mock_service, prefix, result): mock_service.return_value.bucket.return_value.list_blobs.return_value.next_page_token = None + with pytest.deprecated_call(): + self.gcs_hook.list( + bucket_name="test_bucket", + prefix=prefix, + delimiter=",", + ) + assert mock_service.return_value.bucket.return_value.list_blobs.call_args_list == result + + @mock.patch(GCS_STRING.format("GCSHook.get_conn")) + @mock.patch("airflow.providers.google.cloud.hooks.gcs.functools") + @mock.patch("google.cloud.storage.bucket._item_to_blob") + @mock.patch("google.cloud.storage.bucket._blobs_page_start") + @mock.patch("google.api_core.page_iterator.HTTPIterator") + def test_list__match_glob( + self, http_iterator, _blobs_page_start, _item_to_blob, mocked_functools, mock_service + ): + http_iterator.return_value.next_page_token = None self.gcs_hook.list( bucket_name="test_bucket", - prefix=prefix, - delimiter=",", + prefix="prefix", + match_glob="**/*.json", + ) + http_iterator.assert_has_calls( + [ + mock.call( + api_request=mocked_functools.partial.return_value, + client=mock_service.return_value, + extra_params={"prefix": "prefix", "matchGlob": "**/*.json"}, + item_to_value=_item_to_blob, + max_results=None, + page_start=_blobs_page_start, + page_token=None, + path=mock_service.return_value.bucket.return_value.path.__add__.return_value, + ) + ] + ) + + @mock.patch(GCS_STRING.format("GCSHook.get_conn")) + def test_list__error_match_glob_and_invalid_delimiter(self, _): + with pytest.raises(AirflowException): + self.gcs_hook.list( + bucket_name="test_bucket", + prefix="prefix", + delimiter=",", + match_glob="**/*.json", + ) + + @pytest.mark.parametrize("delimiter", [None, "", "/"]) + @mock.patch("google.api_core.page_iterator.HTTPIterator") + @mock.patch(GCS_STRING.format("GCSHook.get_conn")) + def test_list__error_match_glob_and_valid_delimiter(self, mock_service, http_iterator, delimiter): + http_iterator.return_value.next_page_token = None + self.gcs_hook.list( + bucket_name="test_bucket", + prefix="prefix", + delimiter="/", + match_glob="**/*.json", ) - assert mock_service.return_value.bucket.return_value.list_blobs.call_args_list == result @mock.patch(GCS_STRING.format("GCSHook.get_conn")) def test_list_by_timespans(self, mock_service): diff --git a/tests/providers/google/cloud/operators/test_gcs.py b/tests/providers/google/cloud/operators/test_gcs.py index bf9a4f5d7ea5..b048aa0c8e7c 100644 --- a/tests/providers/google/cloud/operators/test_gcs.py +++ b/tests/providers/google/cloud/operators/test_gcs.py @@ -159,14 +159,26 @@ def test_delete_prefix_as_empty_string(self, mock_hook): class TestGoogleCloudStorageListOperator: @mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook") - def test_execute(self, mock_hook): + def test_execute__delimiter(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES operator = GCSListObjectsOperator( task_id=TASK_ID, bucket=TEST_BUCKET, prefix=PREFIX, delimiter=DELIMITER ) files = operator.execute(context=mock.MagicMock()) mock_hook.return_value.list.assert_called_once_with( - bucket_name=TEST_BUCKET, prefix=PREFIX, delimiter=DELIMITER + bucket_name=TEST_BUCKET, prefix=PREFIX, delimiter=DELIMITER, match_glob=None + ) + assert sorted(files) == sorted(MOCK_FILES) + + @mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook") + def test_execute__match_glob(self, mock_hook): + mock_hook.return_value.list.return_value = MOCK_FILES + operator = GCSListObjectsOperator( + task_id=TASK_ID, bucket=TEST_BUCKET, prefix=PREFIX, match_glob=f"**/*{DELIMITER}", delimiter=None + ) + files = operator.execute(context=mock.MagicMock()) + mock_hook.return_value.list.assert_called_once_with( + bucket_name=TEST_BUCKET, prefix=PREFIX, match_glob=f"**/*{DELIMITER}", delimiter=None ) assert sorted(files) == sorted(MOCK_FILES) diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py index d8fa94c0c690..1cf7be116679 100644 --- a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py @@ -54,6 +54,8 @@ MOD_TIME_2 = datetime(2019, 1, 1) +# TODO: After deprecating delimiter and wildcards in source objects, +# implement reverted changes from the first commit of PR #31261 class TestGoogleCloudStorageToCloudStorageOperator: """ Tests the three use-cases for the wildcard operator. These are @@ -100,7 +102,7 @@ def test_execute_wildcard_with_replace_flag_false(self, mock_hook): operator.execute(None) mock_calls = [ mock.call(TEST_BUCKET, prefix="test_object", delimiter=""), - mock.call(DESTINATION_BUCKET, prefix="test_object", delimiter=""), + mock.call(DESTINATION_BUCKET, prefix="test_object", delimiter="", match_glob=None), ] mock_hook.return_value.list.assert_has_calls(mock_calls) @@ -117,8 +119,8 @@ def test_execute_no_wildcard_with_replace_flag_false(self, mock_hook): operator.execute(None) mock_calls = [ - mock.call(TEST_BUCKET, prefix=SOURCE_OBJECT_NO_WILDCARD, delimiter=None), - mock.call(DESTINATION_BUCKET, prefix=SOURCE_OBJECT_NO_WILDCARD, delimiter=None), + mock.call(TEST_BUCKET, prefix=SOURCE_OBJECT_NO_WILDCARD, delimiter=None, match_glob=None), + mock.call(DESTINATION_BUCKET, prefix=SOURCE_OBJECT_NO_WILDCARD, delimiter=None, match_glob=None), ] mock_hook.return_value.list.assert_has_calls(mock_calls) @@ -140,7 +142,7 @@ def test_copy_file_with_exact_match(self, mock_hook): operator.execute(None) mock_calls = [ - mock.call(TEST_BUCKET, prefix="test_object.txt", delimiter=None), + mock.call(TEST_BUCKET, prefix="test_object.txt", delimiter=None, match_glob=None), ] mock_hook.return_value.list.assert_has_calls(mock_calls) @@ -450,7 +452,9 @@ def test_executes_with_empty_source_objects(self, mock_hook): ) operator.execute(None) - mock_hook.return_value.list.assert_called_once_with(TEST_BUCKET, prefix="", delimiter=None) + mock_hook.return_value.list.assert_called_once_with( + TEST_BUCKET, prefix="", delimiter=None, match_glob=None + ) @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") def test_raises_exception_with_two_empty_list_inside_source_objects(self, mock_hook): @@ -469,7 +473,7 @@ def test_executes_with_single_item_in_source_objects(self, mock_hook): ) operator.execute(None) mock_hook.return_value.list.assert_called_once_with( - TEST_BUCKET, prefix=SOURCE_OBJECTS_SINGLE_FILE[0], delimiter=None + TEST_BUCKET, prefix=SOURCE_OBJECTS_SINGLE_FILE[0], delimiter=None, match_glob=None ) @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") @@ -480,8 +484,8 @@ def test_executes_with_multiple_items_in_source_objects(self, mock_hook): operator.execute(None) mock_hook.return_value.list.assert_has_calls( [ - mock.call(TEST_BUCKET, prefix="test_object/file1.txt", delimiter=None), - mock.call(TEST_BUCKET, prefix="test_object/file2.txt", delimiter=None), + mock.call(TEST_BUCKET, prefix="test_object/file1.txt", delimiter=None, match_glob=None), + mock.call(TEST_BUCKET, prefix="test_object/file2.txt", delimiter=None, match_glob=None), ], any_order=True, ) @@ -495,7 +499,9 @@ def test_executes_with_a_delimiter(self, mock_hook): delimiter=DELIMITER, ) operator.execute(None) - mock_hook.return_value.list.assert_called_once_with(TEST_BUCKET, prefix="", delimiter=DELIMITER) + mock_hook.return_value.list.assert_called_once_with( + TEST_BUCKET, prefix="", delimiter=DELIMITER, match_glob=None + ) # COPY @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") @@ -593,7 +599,7 @@ def test_execute_wildcard_with_replace_flag_false_with_destination_object(self, operator.execute(None) mock_calls = [ mock.call(TEST_BUCKET, prefix="test_object", delimiter=""), - mock.call(DESTINATION_BUCKET, prefix="foo/bar", delimiter=""), + mock.call(DESTINATION_BUCKET, prefix="foo/bar", delimiter="", match_glob=None), ] mock_hook.return_value.list.assert_has_calls(mock_calls) diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_sftp.py b/tests/providers/google/cloud/transfers/test_gcs_to_sftp.py index 9c6884766bea..f2ede74a8a73 100644 --- a/tests/providers/google/cloud/transfers/test_gcs_to_sftp.py +++ b/tests/providers/google/cloud/transfers/test_gcs_to_sftp.py @@ -34,6 +34,8 @@ DESTINATION_SFTP = "destination_path" +# TODO: After deprecating delimiter and wildcards in source objects, +# implement reverted changes from the first commit of PR #31261 class TestGoogleCloudStorageToSFTPOperator: @pytest.mark.parametrize( "source_object, target_object, keep_directory_structure", diff --git a/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py b/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py index 5730a4b66ed5..525f20398ab3 100644 --- a/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py +++ b/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py @@ -95,6 +95,9 @@ def test_should_copy_files(self, mock_named_temporary_file, mock_gdrive, mock_gc impersonation_chain=IMPERSONATION_CHAIN, ), mock.call().list("data", delimiter=".avro", prefix="sales/sales-2017/"), + # TODO: After deprecating delimiter and wildcards in source objects, + # remove previous line and uncomment the following: + # mock.call().list("data", match_glob="**/*.avro", prefix="sales/sales-2017/"), mock.call().download(bucket_name="data", filename="TMP1", object_name="sales/A.avro"), mock.call().download(bucket_name="data", filename="TMP2", object_name="sales/B.avro"), mock.call().download(bucket_name="data", filename="TMP3", object_name="sales/C.avro"), @@ -137,6 +140,9 @@ def test_should_move_files(self, mock_named_temporary_file, mock_gdrive, mock_gc impersonation_chain=IMPERSONATION_CHAIN, ), mock.call().list("data", delimiter=".avro", prefix="sales/sales-2017/"), + # TODO: After deprecating delimiter and wildcards in source objects, + # remove previous line and uncomment the following: + # mock.call().list("data", match_glob="**/*.avro", prefix="sales/sales-2017/"), mock.call().download(bucket_name="data", filename="TMP1", object_name="sales/A.avro"), mock.call().delete("data", "sales/A.avro"), mock.call().download(bucket_name="data", filename="TMP2", object_name="sales/B.avro"), diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py index 7931295d23e7..e13f76ebc469 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py @@ -173,6 +173,17 @@ ) # [END howto_operator_gcs_to_gcs_delimiter] + # [START howto_operator_gcs_to_gcs_match_glob] + copy_files_with_match_glob = GCSToGCSOperator( + task_id="copy_files_with_match_glob", + source_bucket=BUCKET_NAME_SRC, + source_object="data/", + destination_bucket=BUCKET_NAME_DST, + destination_object="backup/", + match_glob="**/*.txt", + ) + # [END howto_operator_gcs_to_gcs_match_glob] + # [START howto_operator_gcs_to_gcs_list] copy_files_with_list = GCSToGCSOperator( task_id="copy_files_with_list", @@ -226,6 +237,7 @@ copy_files_with_wildcard, copy_files_without_wildcard, copy_files_with_delimiter, + copy_files_with_match_glob, copy_files_with_list, move_single_file, move_files_with_list, From b6ca28e55026acb98fb50aa098c7f621a165cd62 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Jun 2023 11:14:38 +0200 Subject: [PATCH 005/533] Update git-sync description in Helm Chart documentation (#32181) There are quite a few recurring themes when it comes to using git-sync for DAG synchronisation and this documentation is an attempt to capture results of a number of discussions and conversations. It adds some notes that might make it possible to make more informed decisions by our users and Deployment managers who want to make decisions on how they should synchronize their DAGs. The changes include: * notes on potential side-effects one has to be aware when using both git-sync and persistence together (there are some unobvious operations performed by git-sync that might affect performances of persistence solutions) * notes on how you can use multiple git repositories with git-sync using submodule approach - including link to a real-life use case from Airflow summit where it has been used in production for 100s of repositories. --- docs/helm-chart/manage-dags-files.rst | 80 ++++++++++++++++++++++++++- docs/spelling_wordlist.txt | 2 +- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/helm-chart/manage-dags-files.rst b/docs/helm-chart/manage-dags-files.rst index 05b283410ff7..9d74c2c852a9 100644 --- a/docs/helm-chart/manage-dags-files.rst +++ b/docs/helm-chart/manage-dags-files.rst @@ -98,8 +98,11 @@ If you are deploying an image from a private repository, you need to create a se --set images.airflow.pullPolicy=Always \ --set registry.secretName=gitlab-registry-credentials +Using Git-sync +-------------- + Mounting DAGs using Git-Sync sidecar with Persistence enabled -------------------------------------------------------------- +............................................................. This option will use a Persistent Volume Claim with an access mode of ``ReadWriteMany``. The scheduler pod will sync DAGs from a git repository onto the PVC every configured number of @@ -117,8 +120,9 @@ for details. # by setting the dags.persistence.* and dags.gitSync.* values # Please refer to values.yaml for details + Mounting DAGs using Git-Sync sidecar without Persistence --------------------------------------------------------- +........................................................ This option will use an always running Git-Sync sidecar on every scheduler, webserver (if ``airflowVersion < 2.0.0``) and worker pods. @@ -137,6 +141,78 @@ seconds. If you are using the ``KubernetesExecutor``, Git-sync will run as an in When using ``apache-airflow >= 2.0.0``, :ref:`DAG Serialization ` is enabled by default, hence Webserver does not need access to DAG files, so ``git-sync`` sidecar is not run on Webserver. +Notes for combining git-sync and persistence +............................................ + +While using both git-sync and persistence for DAGs is possible, it is generally not recommended unless the +deployment manager carefully considered the trade-offs it brings. There are cases when git-sync without +persistence has other trade-offs (for example delays in synchronization of DAGS vs. rate-limiting of Git +servers) that can often be mitigated (for example by sending signals to git-sync containers via web-hooks +when new commits are pushed to the repository) but there might be cases where you still might want to choose +git-sync and Persistence together, but as a Deployment Manager you should be aware of some consequences it has. + +git-sync solution is primarily designed to be used for local, POSIX-compliant volumes to checkout Git +repositories into. Part of the process of synchronization of commits from git-sync involves checking out +new version of files in a freshly created folder and swapping symbolic links to the new folder, after the +checkout is complete. This is done to ensure that the whole DAGs folder is consistent at all times. The way +git-sync works with symbolic-link swaps, makes sure that Parsing the DAGs always work on a consistent +(single-commit-based) set of files in the whole DAG folder. + +This approach, however might have undesirable side effects when the folder that git-sync works on is not +a local volume, but is a persistent volume (so effectively a networked, distributed volume). Depending on +the technology behind the persistent volumes might handle git-sync approach differently and with non-obvious +consequences. There are a lot of persistence solutions available for various K8S installations and each of +them has different characteristics, so you need to carefully test and monitor your filesystem to make sure +those undesired side effects do not affect you. Those effects might change over time or depend on parameters +like how often the files are being scanned by the Dag File Processor, the number and complexity of your +DAGs, how remote and how distributed your persistent volumes are, how many IOPS you allocate for some of +the filesystem (usually highly paid feature of such filesystems is how many IOPS you can get) and many other +factors. + +The way git-sync works with symbolic links swapping generally causes a linear growth of the throughput and +potential delays in synchronization. The networking traffic from checkouts comes in bursts and the bursts +are linearly proportional to the number and size of files you have in the repository, makes it vulnerable +to pretty sudden and unexpected demand increase. Most of the persistence solution work "good enough" for +smaller/shorter burst of traffic, but when they outgrow certain thresholds, you need to upgrade the +networking to a much more capable and expensive options. This is difficult to control and impossible to +mitigate, so you might be suddenly faced with situation to pay a lot more for IOPS/persistence option to +keep your DAGs sufficiently synchronized to avoid inconsistencies and delays in synchronization. + +The side-effects that you might observe: + +* burst of networking/communication at the moment when new commit is checked out (because of the quick + succession of deleting old files, creating new files, symbolic link swapping. +* temporary lack of consistency between files in DAG folders while DAGS are being synced (because of delays + in distributing changes to individual files for various nodes in the cluster) +* visible drops of performance of the persistence solution when your DAG number grows, drops that might + amplify the side effects described above. +* some of persistence solutions might lack filesystem functionality that git-sync needs to perform the sync + (for example changing permissions or creating symbolic links). While those can often be mitigated it is + only recommended to use git-sync with fully POSIX-filesystem compliant persistence filesystems. + +General recommendation to use git-sync with local volumes only, and if you want to also use persistence, you +need to make sure that the persistence solution you use is POSIX-compliant and you monitor the side-effects +it might have. + +Synchronizing multiple Git repositories with git-sync +..................................................... + +Airflow git-sync integration in the Helm Chart, does not allow to configure multiple repositories to be +synchronized at the same time. The DAG folder must come from single git repository. However it is possible +to use `submodules `_ to create an "umbrella" repository +that you can use to bring a number of git repositories checked out together (with ``--submodules recursive`` +option). There are success stories of Airflow users using such approach with 100s of repositories put +together as submodules via such "umbrella" repo approach. When you choose this solution, however, +you need to work out the way how to ling the submodules, when to updated the umbrella repo when "submodule" +repository change and work out versioning approach and automate it. This might be as simple as always +using latest versions of all the submodule repositories, or as complex as managing versioning of shared +libraries, DAGs and code across multiple teams and doing that following your release process. + +An example of such complex approach can found in this +`Manage DAGs at scale `_ presentation from the Airflow +Summit. + + Mounting DAGs from an externally populated PVC ---------------------------------------------- diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index f51dc1215fc0..d3725c6e4db4 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1426,7 +1426,7 @@ Subdirectory subdirectory subfolder subfolders -Submodules +submodule submodules subnet subnets From ead2530d3500dd27df54383a0802b6c94828c359 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Jun 2023 11:15:17 +0200 Subject: [PATCH 006/533] Separate out advanced logging configuration (#32131) The "advanced logging configuration" applies not only to task logs but also to component logs and you can use it not only to configure custom way how task logs are created but also custom way how "regular" component logs are created. This has been a source of confusion for those who wanted to configure (for example) elasticsearch or opensearch for the whole airflow deployment, because the "advanced configuration" and how to modify standard configuration chapter was a small section in "task logging". This change extracts "advanced logging configuration" to separate page right under the "logging and monitoring" and directs the user from the "task" logging section to this page. It also adds a bit more explanation on how standard Python logging framework is leveraged here and links to Python logging documentation for those who never used it before, to understand more about Loggers, Handlers and Formatters. Co-authored-by: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Co-authored-by: Tzu-ping Chung --- .../logging/stackdriver.rst | 3 +- .../core-extensions/logging.rst | 2 +- .../core-extensions/secrets-backends.rst | 2 +- .../advanced-logging-configuration.rst | 90 +++++++++++++++++++ .../logging-monitoring/index.rst | 1 + .../logging-architecture.rst | 10 ++- .../logging-monitoring/logging-tasks.rst | 53 ++--------- docs/spelling_wordlist.txt | 1 + 8 files changed, 113 insertions(+), 49 deletions(-) create mode 100644 docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst diff --git a/docs/apache-airflow-providers-google/logging/stackdriver.rst b/docs/apache-airflow-providers-google/logging/stackdriver.rst index fa3f3391af6c..12be70cf65e4 100644 --- a/docs/apache-airflow-providers-google/logging/stackdriver.rst +++ b/docs/apache-airflow-providers-google/logging/stackdriver.rst @@ -66,7 +66,8 @@ be used. Make sure that with those credentials, you can read and write the logs. the logs. For security reasons, limiting the access of the log reader to only allow log reading and writing is an important security measure. -By using the ``logging_config_class`` option you can get :ref:`advanced features ` of +By using the ``logging_config_class`` option you can get +:doc:`advanced features ` of this handler. Details are available in the handler's documentation - :class:`~airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler`. diff --git a/docs/apache-airflow-providers/core-extensions/logging.rst b/docs/apache-airflow-providers/core-extensions/logging.rst index 822cf54d2b2f..0fa10d98c412 100644 --- a/docs/apache-airflow-providers/core-extensions/logging.rst +++ b/docs/apache-airflow-providers/core-extensions/logging.rst @@ -20,7 +20,7 @@ Writing logs This is a summary of all Apache Airflow Community provided implementations of writing task logs exposed via community-managed providers. You can also see logging options available in the core Airflow in -:doc:`apache-airflow:administration-and-deployment/logging-monitoring/logging-tasks` and here you can see those +:doc:`/administration-and-deployment/logging-monitoring/logging-tasks` and here you can see those provided by the community-managed providers: .. airflow-logging:: diff --git a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst index 22fae4b8a6c4..c1cc5a0c0508 100644 --- a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst +++ b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst @@ -28,7 +28,7 @@ via providers that implement secrets backends for services Airflow integrates wi You can also take a look at Secret backends available in the core Airflow in -:doc:`apache-airflow:administration-and-deployment/security/secrets/secrets-backend/index` and here you can see the ones +:doc:`/administration-and-deployment/security/secrets/secrets-backend/index` and here you can see the ones provided by the community-managed providers: .. airflow-secrets-backends:: diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst new file mode 100644 index 000000000000..739b5380e3c6 --- /dev/null +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst @@ -0,0 +1,90 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Advanced logging configuration +------------------------------ + +Not all configuration options are available from the ``airflow.cfg`` file. The config file describes +how to configure logging for tasks, because the logs generated by tasks are not only logged in separate +files by default but has to be also accessible via the webserver. + +By default standard airflow component logs are written to the ``$AIRFLOW_HOME/logs`` directory, but you +can also customize it and configure it as you want by overriding Python logger configuration that can +be configured by providing custom logging configuration object. Some configuration options require +that the logging config class be overwritten. You can do it by copying the default +configuration of Airflow and modifying it to suit your needs. The default configuration can be seen in the +`airflow_local_settings.py template `_ +and you can see the loggers and handlers used there. Except the custom loggers and handlers configurable there +via the ``airflow.cfg``, the logging methods in Airflow follow the usual Python logging convention, +that Python objects log to loggers that follow naming convention of ``.``. + +You can read more about standard python logging classes (Loggers, Handlers, Formatters) in the +`Python logging documentation `_. + +Configuring your logging classes can be done via the ``logging_config_class`` option in ``airflow.cfg`` file. +This configuration should specify the import path to a configuration compatible with +:func:`logging.config.dictConfig`. If your file is a standard import location, then you should set a +:envvar:`PYTHONPATH` environment variable. + +Follow the steps below to enable custom logging config class: + +#. Start by setting environment variable to known directory e.g. ``~/airflow/`` + + .. code-block:: bash + + export PYTHONPATH=~/airflow/ + +#. Create a directory to store the config file e.g. ``~/airflow/config`` +#. Create file called ``~/airflow/config/log_config.py`` with following the contents: + + .. code-block:: python + + from copy import deepcopy + from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG + + LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG) + +#. At the end of the file, add code to modify the default dictionary configuration. +#. Update ``$AIRFLOW_HOME/airflow.cfg`` to contain: + + .. code-block:: ini + + [logging] + logging_config_class = log_config.LOGGING_CONFIG + +You can also use the ``logging_config_class`` together with remote logging if you plan to just extend/update +the configuration with remote logging enabled. Then the deep-copied dictionary will contain the remote logging +configuration generated for you and your modification will apply after remote logging configuration has +been added: + + .. code-block:: ini + + [logging] + remote_logging = True + logging_config_class = log_config.LOGGING_CONFIG + + +#. Restart the application. + +See :doc:`../modules_management` for details on how Python and Airflow manage modules. + + +.. note:: + + You can override the way both standard logs of the components and "task" logs are handled. diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/index.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/index.rst index 58edec4be0a9..850bfa77a933 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/index.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/index.rst @@ -30,6 +30,7 @@ In addition to the standard logging and metrics capabilities, Airflow supports t logging-architecture logging-tasks + advanced-logging-configuration metrics callbacks diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-architecture.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-architecture.rst index 07cd3f42cc2e..ae000ddfe0ba 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-architecture.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-architecture.rst @@ -26,9 +26,15 @@ Airflow supports a variety of logging and monitoring mechanisms as shown below. By default, Airflow supports logging into the local file system. These include logs from the Web server, the Scheduler, and the Workers running tasks. This is suitable for development environments and for quick debugging. -For cloud deployments, Airflow also has handlers contributed by the Community for logging to cloud storage such as AWS, Google Cloud, and Azure. +For cloud deployments, Airflow also has task handlers contributed by the Community for +logging to cloud storage such as AWS, Google Cloud, and Azure. -The logging settings and options can be specified in the Airflow Configuration file, which as usual needs to be available to all the Airflow process: Web server, Scheduler, and Workers. +The logging settings and options can be specified in the Airflow Configuration file, +which as usual needs to be available to all the Airflow process: Web server, Scheduler, and Workers. + +You can customize the logging settings for each of the Airflow components by specifying the logging settings +in the Airflow Configuration file, or for advanced configuration by using +:doc:`advanced features `. For production deployments, we recommend using FluentD to capture logs and send it to destinations such as ElasticSearch or Splunk. diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-tasks.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-tasks.rst index 0b7ef9a472e2..2b1d93176521 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-tasks.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/logging-tasks.rst @@ -117,53 +117,11 @@ the example below. The output of ``airflow info`` above is truncated to only display the section that pertains to the logging configuration. You can also run ``airflow config list`` to check that the logging configuration options have valid values. -.. _write-logs-advanced: - Advanced configuration ---------------------- -Not all configuration options are available from the ``airflow.cfg`` file. Some configuration options require -that the logging config class be overwritten. This can be done via the ``logging_config_class`` option -in ``airflow.cfg`` file. This option should specify the import path to a configuration compatible with -:func:`logging.config.dictConfig`. If your file is a standard import location, then you should set a :envvar:`PYTHONPATH` environment variable. - -Follow the steps below to enable custom logging config class: - -#. Start by setting environment variable to known directory e.g. ``~/airflow/`` - - .. code-block:: bash - - export PYTHONPATH=~/airflow/ - -#. Create a directory to store the config file e.g. ``~/airflow/config`` -#. Create file called ``~/airflow/config/log_config.py`` with following the contents: - - .. code-block:: python - - from copy import deepcopy - from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG - - LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG) - -#. At the end of the file, add code to modify the default dictionary configuration. -#. Update ``$AIRFLOW_HOME/airflow.cfg`` to contain: - - .. code-block:: ini - - [logging] - remote_logging = True - logging_config_class = log_config.LOGGING_CONFIG - -#. Restart the application. - -See :doc:`../modules_management` for details on how Python and Airflow manage modules. - -External Links --------------- - -When using remote logging, you can configure Airflow to show a link to an external UI within the Airflow Web UI. Clicking the link redirects you to the external UI. - -Some external systems require specific configuration in Airflow for redirection to work but others do not. +You can configure :doc:`advanced features ` +- including adding your own custom task log handlers (but also log handlers for all airflow components). .. _serving-worker-trigger-logs: @@ -197,3 +155,10 @@ To accomplish this we have a few attributes that may be set on the handler, eith - ``trigger_should_queue``: Controls whether the triggerer should put a QueueListener between the event loop and the handler, to ensure blocking IO in the handler does not disrupt the event loop. - ``trigger_send_end_marker``: Controls whether an END signal should be sent to the logger when trigger completes. It is used to tell the wrapper to close and remove the individual file handler specific to the trigger that just completed. - ``trigger_supported``: If ``trigger_should_wrap`` and ``trigger_should_queue`` are not True, we generally assume that the handler does not support triggers. But if in this case the handler has ``trigger_supported`` set to True, then we'll still move the handler to root at triggerer start so that it will process trigger messages. Essentially, this should be true for handlers that "natively" support triggers. One such example of this is the StackdriverTaskHandler. + +External Links +-------------- + +When using remote logging, you can configure Airflow to show a link to an external UI within the Airflow Web UI. Clicking the link redirects you to the external UI. + +Some external systems require specific configuration in Airflow for redirection to work but others do not. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d3725c6e4db4..af0a3ff5dcbf 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -599,6 +599,7 @@ fn fo followsa formatter +formatters Formaturas forwardability forwardable From 1b599c9fbfb6151a41a588edaa786745f50eec38 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 30 Jun 2023 11:26:46 +0200 Subject: [PATCH 007/533] Break AwaitMessageTrigger execution when finding a message with the desired format (#31803) Signed-off-by: Hussein Awala Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- airflow/providers/apache/kafka/triggers/await_message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/apache/kafka/triggers/await_message.py b/airflow/providers/apache/kafka/triggers/await_message.py index 5445579e125f..f2c1bb81ca6b 100644 --- a/airflow/providers/apache/kafka/triggers/await_message.py +++ b/airflow/providers/apache/kafka/triggers/await_message.py @@ -113,6 +113,7 @@ async def run(self): if rv: await async_commit(asynchronous=False) yield TriggerEvent(rv) + break else: await async_commit(asynchronous=False) await asyncio.sleep(self.poll_interval) From ef49e595f1ab729cdb72198a3ee47c7dcd182da2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Jun 2023 13:28:16 +0200 Subject: [PATCH 008/533] Update references to restructured documentation from airflow core (#32282) The change #32131 restructured some of the code for documentation and it broke references to apache-airlfow from "providers" doc package - it has not been visible in selective PR (not sure the reason) but this PR fixes it. --- docs/apache-airflow-providers-google/logging/stackdriver.rst | 2 +- docs/apache-airflow-providers/core-extensions/logging.rst | 2 +- .../core-extensions/secrets-backends.rst | 2 +- .../logging-monitoring/advanced-logging-configuration.rst | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow-providers-google/logging/stackdriver.rst b/docs/apache-airflow-providers-google/logging/stackdriver.rst index 12be70cf65e4..60943bda70ef 100644 --- a/docs/apache-airflow-providers-google/logging/stackdriver.rst +++ b/docs/apache-airflow-providers-google/logging/stackdriver.rst @@ -67,7 +67,7 @@ be used. Make sure that with those credentials, you can read and write the logs. an important security measure. By using the ``logging_config_class`` option you can get -:doc:`advanced features ` of +:ref:`advanced features ` of this handler. Details are available in the handler's documentation - :class:`~airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler`. diff --git a/docs/apache-airflow-providers/core-extensions/logging.rst b/docs/apache-airflow-providers/core-extensions/logging.rst index 0fa10d98c412..822cf54d2b2f 100644 --- a/docs/apache-airflow-providers/core-extensions/logging.rst +++ b/docs/apache-airflow-providers/core-extensions/logging.rst @@ -20,7 +20,7 @@ Writing logs This is a summary of all Apache Airflow Community provided implementations of writing task logs exposed via community-managed providers. You can also see logging options available in the core Airflow in -:doc:`/administration-and-deployment/logging-monitoring/logging-tasks` and here you can see those +:doc:`apache-airflow:administration-and-deployment/logging-monitoring/logging-tasks` and here you can see those provided by the community-managed providers: .. airflow-logging:: diff --git a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst index c1cc5a0c0508..22fae4b8a6c4 100644 --- a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst +++ b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst @@ -28,7 +28,7 @@ via providers that implement secrets backends for services Airflow integrates wi You can also take a look at Secret backends available in the core Airflow in -:doc:`/administration-and-deployment/security/secrets/secrets-backend/index` and here you can see the ones +:doc:`apache-airflow:administration-and-deployment/security/secrets/secrets-backend/index` and here you can see the ones provided by the community-managed providers: .. airflow-secrets-backends:: diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst index 739b5380e3c6..21ae801b64ba 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/advanced-logging-configuration.rst @@ -16,6 +16,7 @@ under the License. +.. _write-logs-advanced: Advanced logging configuration ------------------------------ From ba05a1e8706abffe057fa83d891b44125d5913e6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Jun 2023 14:41:09 +0200 Subject: [PATCH 009/533] Add apache-airflow-providers to filter in selective checks if needed (#32284) When provider docs in "docs/apache-airflow-providers" changed, and selective docs build was used, the "apache-airflow-providers" package was not added to the filter - thus errors in those docs were not checked. This caused #32131 to fail the main build (fixed in the #32282). Added logic to handle this case. --- .../src/airflow_breeze/utils/selective_checks.py | 2 ++ dev/breeze/tests/test_selective_checks.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index c6e414472d06..7c7d622dab61 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -727,6 +727,8 @@ def docs_filter_list_as_string(self) -> str | None: [file.startswith("airflow/") or file.startswith("docs/apache-airflow/") for file in self._files] ): packages.append("apache-airflow") + if any([file.startswith("docs/apache-airflow-providers/") for file in self._files]): + packages.append("apache-airflow-providers") if any([file.startswith("chart/") or file.startswith("docs/helm-chart") for file in self._files]): packages.append("helm-chart") if any([file.startswith("docs/docker-stack/") for file in self._files]): diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 0dc12501b1c3..61e35ce20f59 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1024,6 +1024,20 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: }, id="Airbyte provider and airflow core docs changed", ), + pytest.param( + ( + "docs/apache-airflow-providers-airbyte/docs.rst", + "docs/apache-airflow/docs.rst", + "docs/apache-airflow-providers/docs.rst", + ), + { + "docs-filter-list-as-string": "--package-filter apache-airflow " + "--package-filter apache-airflow-providers " + "--package-filter apache-airflow-providers-airbyte " + "--package-filter apache-airflow-providers-http", + }, + id="Airbyte provider and airflow core and common provider docs changed", + ), pytest.param( ("docs/apache-airflow/docs.rst",), { From 4501f8b352aee9c2cd29126a64cab62fa19fc49d Mon Sep 17 00:00:00 2001 From: Matthias Vongerichten Date: Fri, 30 Jun 2023 15:51:55 +0200 Subject: [PATCH 010/533] Update gcp.rst (#32281) got an "TypeError: __init__() got an unexpected keyword argument 'key_path'" error message when not wrapped in the "extra" object. --- docs/apache-airflow-providers-google/connections/gcp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-google/connections/gcp.rst b/docs/apache-airflow-providers-google/connections/gcp.rst index b2f7243fe71e..c8374d455604 100644 --- a/docs/apache-airflow-providers-google/connections/gcp.rst +++ b/docs/apache-airflow-providers-google/connections/gcp.rst @@ -154,7 +154,7 @@ Number of Retries .. code-block:: bash - export AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT='{"conn_type": "google-cloud-platform", "key_path": "/keys/key.json", "scope": "https://www.googleapis.com/auth/cloud-platform", "project": "airflow", "num_retries": 5}' + export AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT='{"conn_type": "google_cloud_platform", "extra": {"key_path": "/keys/key.json", "scope": "https://www.googleapis.com/auth/cloud-platform", "project": "airflow", "num_retries": 5}}' .. _howto/connection:gcp:impersonation: From d117728cd6f337266bebcf4916325d5de815fe03 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Fri, 30 Jun 2023 22:19:06 +0200 Subject: [PATCH 011/533] Fix triggerers alive check and add a new conf for triggerer heartbeat rate (#32123) Signed-off-by: Hussein Awala --- airflow/cli/commands/triggerer_command.py | 3 +- airflow/config_templates/config.yml | 7 +++ airflow/config_templates/default_airflow.cfg | 3 + airflow/jobs/triggerer_job_runner.py | 2 +- airflow/models/trigger.py | 13 ++-- tests/models/test_trigger.py | 64 ++++++++++++++++++-- 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/airflow/cli/commands/triggerer_command.py b/airflow/cli/commands/triggerer_command.py index 2da482187726..aa06d641c760 100644 --- a/airflow/cli/commands/triggerer_command.py +++ b/airflow/cli/commands/triggerer_command.py @@ -55,7 +55,8 @@ def triggerer(args): """Starts Airflow Triggerer.""" settings.MASK_SECRETS_IN_LOGS = True print(settings.HEADER) - triggerer_job_runner = TriggererJobRunner(job=Job(), capacity=args.capacity) + triggerer_heartrate = conf.getfloat("triggerer", "JOB_HEARTBEAT_SEC") + triggerer_job_runner = TriggererJobRunner(job=Job(heartrate=triggerer_heartrate), capacity=args.capacity) if args.daemon: pid, stdout, stderr, log_file = setup_locations( diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index b6b160326523..a252716ed1d0 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2565,6 +2565,13 @@ triggerer: type: string example: ~ default: "1000" + job_heartbeat_sec: + description: | + How often to heartbeat the Triggerer job to ensure it hasn't been killed. + version_added: 2.6.3 + type: float + example: ~ + default: "5" kerberos: description: ~ options: diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 1fbf58f7bccd..4eaab9ae95db 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -1329,6 +1329,9 @@ task_queued_timeout_check_interval = 120.0 # How many triggers a single Triggerer will run at once, by default. default_capacity = 1000 +# How often to heartbeat the Triggerer job to ensure it hasn't been killed. +job_heartbeat_sec = 5 + [kerberos] ccache = /tmp/airflow_krb5_ccache diff --git a/airflow/jobs/triggerer_job_runner.py b/airflow/jobs/triggerer_job_runner.py index 32435cc1e1da..633e90944cd7 100644 --- a/airflow/jobs/triggerer_job_runner.py +++ b/airflow/jobs/triggerer_job_runner.py @@ -374,7 +374,7 @@ def load_triggers(self): adds them to our runner, and then removes ones from it we no longer need. """ - Trigger.assign_unassigned(self.job.id, self.capacity) + Trigger.assign_unassigned(self.job.id, self.capacity, self.job.heartrate) ids = Trigger.ids_for_triggerer(self.job.id) self.trigger_runner.update_triggers(set(ids)) diff --git a/airflow/models/trigger.py b/airflow/models/trigger.py index 603ff95157ca..c0d749eb5905 100644 --- a/airflow/models/trigger.py +++ b/airflow/models/trigger.py @@ -200,10 +200,11 @@ def ids_for_triggerer(cls, triggerer_id, session: Session = NEW_SESSION) -> list @classmethod @internal_api_call @provide_session - def assign_unassigned(cls, triggerer_id, capacity, session: Session = NEW_SESSION) -> None: + def assign_unassigned(cls, triggerer_id, capacity, heartrate, session: Session = NEW_SESSION) -> None: """ - Takes a triggerer_id and the capacity for that triggerer and assigns unassigned - triggers until that capacity is reached, or there are no more unassigned triggers. + Takes a triggerer_id, the capacity for that triggerer and the Triggerer job heartrate, + and assigns unassigned triggers until that capacity is reached, or there are no more + unassigned triggers. """ from airflow.jobs.job import Job # To avoid circular import @@ -212,12 +213,14 @@ def assign_unassigned(cls, triggerer_id, capacity, session: Session = NEW_SESSIO if capacity <= 0: return - + # we multiply heartrate by a grace_multiplier to give the triggerer + # a chance to heartbeat before we consider it dead + health_check_threshold = heartrate * 2.1 alive_triggerer_ids = [ row[0] for row in session.query(Job.id).filter( Job.end_date.is_(None), - Job.latest_heartbeat > timezone.utcnow() - datetime.timedelta(seconds=30), + Job.latest_heartbeat > timezone.utcnow() - datetime.timedelta(seconds=health_check_threshold), Job.job_type == "TriggererJob", ) ] diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 4abd7e9e42b8..0c5e5eeadcb8 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -141,16 +141,17 @@ def test_assign_unassigned(session, create_task_instance): """ Tests that unassigned triggers of all appropriate states are assigned. """ - finished_triggerer = Job(heartrate=10, state=State.SUCCESS) + triggerer_heartrate = 10 + finished_triggerer = Job(heartrate=triggerer_heartrate, state=State.SUCCESS) TriggererJobRunner(finished_triggerer) finished_triggerer.end_date = timezone.utcnow() - datetime.timedelta(hours=1) session.add(finished_triggerer) assert not finished_triggerer.is_alive() - healthy_triggerer = Job(heartrate=10, state=State.RUNNING) + healthy_triggerer = Job(heartrate=triggerer_heartrate, state=State.RUNNING) TriggererJobRunner(healthy_triggerer) session.add(healthy_triggerer) assert healthy_triggerer.is_alive() - new_triggerer = Job(heartrate=10, state=State.RUNNING) + new_triggerer = Job(heartrate=triggerer_heartrate, state=State.RUNNING) TriggererJobRunner(new_triggerer) session.add(new_triggerer) assert new_triggerer.is_alive() @@ -169,7 +170,7 @@ def test_assign_unassigned(session, create_task_instance): session.add(trigger_unassigned_to_triggerer) session.commit() assert session.query(Trigger).count() == 3 - Trigger.assign_unassigned(new_triggerer.id, 100, session=session) + Trigger.assign_unassigned(new_triggerer.id, 100, session=session, heartrate=triggerer_heartrate) session.expire_all() # Check that trigger on killed triggerer and unassigned trigger are assigned to new triggerer assert ( @@ -187,6 +188,61 @@ def test_assign_unassigned(session, create_task_instance): ) +@pytest.mark.parametrize("check_triggerer_heartrate", [10, 60, 300]) +def test_assign_unassigned_missing_heartbeat(session, create_task_instance, check_triggerer_heartrate): + """ + Tests that the triggers assigned to a dead triggers are considered as unassigned + and they are assigned to an alive triggerer. + """ + import time_machine + + block_triggerer_heartrate = 9999 + with time_machine.travel(datetime.datetime.utcnow(), tick=False) as t: + first_triggerer = Job(heartrate=block_triggerer_heartrate, state=State.RUNNING) + TriggererJobRunner(first_triggerer) + session.add(first_triggerer) + assert first_triggerer.is_alive() + second_triggerer = Job(heartrate=block_triggerer_heartrate, state=State.RUNNING) + TriggererJobRunner(second_triggerer) + session.add(second_triggerer) + assert second_triggerer.is_alive() + session.commit() + trigger_on_first_triggerer = Trigger(classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}) + trigger_on_first_triggerer.id = 1 + trigger_on_first_triggerer.triggerer_id = first_triggerer.id + trigger_on_second_triggerer = Trigger(classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}) + trigger_on_second_triggerer.id = 2 + trigger_on_second_triggerer.triggerer_id = second_triggerer.id + session.add(trigger_on_first_triggerer) + session.add(trigger_on_second_triggerer) + session.commit() + assert session.query(Trigger).count() == 2 + triggers_ids = [ + (first_triggerer.id, second_triggerer.id), + (first_triggerer.id, second_triggerer.id), + (first_triggerer.id, second_triggerer.id), + # Check that after more than 2.1 heartrates, the first triggerer is considered dead + # and the first trigger is assigned to the second triggerer + (second_triggerer.id, second_triggerer.id), + ] + for i in range(4): + Trigger.assign_unassigned( + second_triggerer.id, 100, session=session, heartrate=check_triggerer_heartrate + ) + session.expire_all() + # Check that trigger on killed triggerer and unassigned trigger are assigned to new triggerer + assert ( + session.query(Trigger).filter(Trigger.id == trigger_on_first_triggerer.id).one().triggerer_id + == triggers_ids[i][0] + ) + assert ( + session.query(Trigger).filter(Trigger.id == trigger_on_second_triggerer.id).one().triggerer_id + == triggers_ids[i][1] + ) + t.shift(datetime.timedelta(seconds=check_triggerer_heartrate)) + second_triggerer.latest_heartbeat += datetime.timedelta(seconds=check_triggerer_heartrate) + + def test_get_sorted_triggers(session, create_task_instance): """ Tests that triggers are sorted by the creation_date. From df4c8837d022e66921bc0cf33f3249b235de6fdd Mon Sep 17 00:00:00 2001 From: Ashwin Agate <87091342+aagateuip@users.noreply.github.com> Date: Sat, 1 Jul 2023 02:43:48 -0400 Subject: [PATCH 012/533] Fix KubernetesPodOperator validate xcom json and add retries (#32113) * Fix KubernetesPodOperator validate xcom json and add retries --- .../cncf/kubernetes/utils/pod_manager.py | 42 +++++++++++++++- .../cncf/kubernetes/utils/test_pod_manager.py | 48 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 71b5e171271b..2251f1d438c0 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -545,6 +545,19 @@ def await_xcom_sidecar_container_start(self, pod: V1Pod) -> None: def extract_xcom(self, pod: V1Pod) -> str: """Retrieves XCom value and kills xcom sidecar container.""" + try: + result = self.extract_xcom_json(pod) + return result + finally: + self.extract_xcom_kill(pod) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + ) + def extract_xcom_json(self, pod: V1Pod) -> str: + """Retrieves XCom value and also checks if xcom json is valid.""" with closing( kubernetes_stream( self._client.connect_get_namespaced_pod_exec, @@ -563,11 +576,38 @@ def extract_xcom(self, pod: V1Pod) -> str: resp, f"if [ -s {PodDefaults.XCOM_MOUNT_PATH}/return.json ]; then cat {PodDefaults.XCOM_MOUNT_PATH}/return.json; else echo __airflow_xcom_result_empty__; fi", # noqa ) - self._exec_pod_command(resp, "kill -s SIGINT 1") + if result and result.rstrip() != "__airflow_xcom_result_empty__": + # Note: result string is parsed to check if its valid json. + # This function still returns a string which is converted into json in the calling method. + json.loads(result) + if result is None: raise AirflowException(f"Failed to extract xcom from pod: {pod.metadata.name}") return result + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + ) + def extract_xcom_kill(self, pod: V1Pod): + """Kills xcom sidecar container.""" + with closing( + kubernetes_stream( + self._client.connect_get_namespaced_pod_exec, + pod.metadata.name, + pod.metadata.namespace, + container=PodDefaults.SIDECAR_CONTAINER_NAME, + command=["/bin/sh"], + stdin=True, + stdout=True, + stderr=True, + tty=False, + _preload_content=False, + ) + ) as resp: + self._exec_pod_command(resp, "kill -s SIGINT 1") + def _exec_pod_command(self, resp, command: str) -> str | None: res = None if resp.is_open(): diff --git a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py index a55be38a5e22..8f28d33dfdea 100644 --- a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py +++ b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py @@ -18,6 +18,7 @@ import logging from datetime import datetime +from json.decoder import JSONDecodeError from unittest import mock from unittest.mock import MagicMock @@ -370,6 +371,53 @@ def test_container_is_terminated_with_waiting_state(self, container_state, expec pod_info.status.container_statuses = [container_status] assert container_is_terminated(pod_info, "base") == expected_is_terminated + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.kubernetes_stream") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager._exec_pod_command") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager.extract_xcom_kill") + def test_extract_xcom_success(self, mock_exec_xcom_kill, mock_exec_pod_command, mock_kubernetes_stream): + """test when valid json is retrieved from xcom sidecar container.""" + xcom_json = """{"a": "true"}""" + mock_pod = MagicMock() + mock_exec_pod_command.return_value = xcom_json + ret = self.pod_manager.extract_xcom(pod=mock_pod) + assert ret == xcom_json + assert mock_exec_xcom_kill.call_count == 1 + + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.kubernetes_stream") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager._exec_pod_command") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager.extract_xcom_kill") + def test_extract_xcom_failure(self, mock_exec_xcom_kill, mock_exec_pod_command, mock_kubernetes_stream): + """test when invalid json is retrieved from xcom sidecar container.""" + with pytest.raises(JSONDecodeError): + xcom_json = """{"a": "tru""" + mock_pod = MagicMock() + mock_exec_pod_command.return_value = xcom_json + self.pod_manager.extract_xcom(pod=mock_pod) + assert mock_exec_xcom_kill.call_count == 1 + + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.kubernetes_stream") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager._exec_pod_command") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager.extract_xcom_kill") + def test_extract_xcom_empty(self, mock_exec_xcom_kill, mock_exec_pod_command, mock_kubernetes_stream): + """test when __airflow_xcom_result_empty__ is retrieved from xcom sidecar container.""" + mock_pod = MagicMock() + xcom_result = "__airflow_xcom_result_empty__" + mock_exec_pod_command.return_value = xcom_result + ret = self.pod_manager.extract_xcom(pod=mock_pod) + assert ret == xcom_result + assert mock_exec_xcom_kill.call_count == 1 + + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.kubernetes_stream") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager._exec_pod_command") + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager.extract_xcom_kill") + def test_extract_xcom_none(self, mock_exec_xcom_kill, mock_exec_pod_command, mock_kubernetes_stream): + """test when None is retrieved from xcom sidecar container.""" + with pytest.raises(AirflowException): + mock_pod = MagicMock() + mock_exec_pod_command.return_value = None + self.pod_manager.extract_xcom(pod=mock_pod) + assert mock_exec_xcom_kill.call_count == 1 + def params_for_test_container_is_running(): """The `container_is_running` method is designed to handle an assortment of bad objects From 4d84e304b86c97d0437fddbc6b6757b5201eefcc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 1 Jul 2023 23:41:59 +0200 Subject: [PATCH 013/533] Fix Pydantic 2 pickiness about model definition (#32307) The new Pydantic 2 has been released on 30th of June and it is a bit more picky about model definition, thus causing Airflow to fail because there were wrong definitions of models (previously corretly validated). Fixes: #32301 --- airflow/serialization/pydantic/dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py index 5ee0d18128be..659e5a1899ca 100644 --- a/airflow/serialization/pydantic/dataset.py +++ b/airflow/serialization/pydantic/dataset.py @@ -38,10 +38,10 @@ class TaskOutletDatasetReferencePydantic(BaseModelPydantic): """Serializable version of the TaskOutletDatasetReference ORM SqlAlchemyModel used by internal API.""" dataset_id: int - dag_id = str - task_id = str - created_at = datetime - updated_at = datetime + dag_id: str + task_id: str + created_at: datetime + updated_at: datetime class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" From f6db66e16374e504665972feba0831d4148c6d50 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 1 Jul 2023 23:56:03 +0200 Subject: [PATCH 014/533] Add information for users who ask for requirements (#32262) * Add information for users who ask for requirements This change is based on a number of discussions with the users asking what are the minimum requirements for Airflow to run. While we cannot give precise answer, we should also make the users aware that simple answers are not possible, and that when they are deciding to install airflow and manage it on their own, they also take the responsibility to monitor and adjust the resources they need based on the monitoring they have to run. * Apply suggestions from code review Co-authored-by: Pankaj Koti * Update docs/apache-airflow/installation/index.rst --------- Co-authored-by: Pankaj Koti --- .../scheduler.rst | 1 + docs/apache-airflow/installation/index.rst | 72 ++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/apache-airflow/administration-and-deployment/scheduler.rst b/docs/apache-airflow/administration-and-deployment/scheduler.rst index dfc97e6120d7..1a2a41b136ec 100644 --- a/docs/apache-airflow/administration-and-deployment/scheduler.rst +++ b/docs/apache-airflow/administration-and-deployment/scheduler.rst @@ -154,6 +154,7 @@ The following databases are fully supported and provide an "optimal" experience: Microsoft SQLServer has not been tested with HA. +.. _fine-tuning-scheduler: Fine-tuning your Scheduler performance -------------------------------------- diff --git a/docs/apache-airflow/installation/index.rst b/docs/apache-airflow/installation/index.rst index 1ddbf5d66b6e..8f37ca208d78 100644 --- a/docs/apache-airflow/installation/index.rst +++ b/docs/apache-airflow/installation/index.rst @@ -77,6 +77,9 @@ More details: :doc:`installing-from-sources` * You should develop and handle the deployment for all components of Airflow. * You are responsible for setting up database, creating and managing database schema with ``airflow db`` commands, automated startup and recovery, maintenance, cleanup and upgrades of Airflow and the Airflow Providers. +* You need to setup monitoring of your system allowing you to observe resources and react to problems. +* You are expected to configure and manage appropriate resources for the installation (memory, CPU, etc) based + on the monitoring of your installation and feedback loop. See the notes about requirements. **What Apache Airflow Community provides for that method** @@ -123,6 +126,9 @@ More details: :doc:`/installation/installing-from-pypi` * You should develop and handle the deployment for all components of Airflow. * You are responsible for setting up database, creating and managing database schema with ``airflow db`` commands, automated startup and recovery, maintenance, cleanup and upgrades of Airflow and Airflow Providers. +* You need to setup monitoring of your system allowing you to observe resources and react to problems. +* You are expected to configure and manage appropriate resources for the installation (memory, CPU, etc) based + on the monitoring of your installation and feedback loop. **What Apache Airflow Community provides for that method** @@ -181,6 +187,9 @@ and official constraint files- same that are used for installing Airflow from Py deployments of containers. You can use your own custom mechanism, custom Kubernetes deployments, custom Docker Compose, custom Helm charts etc., and you should choose it based on your experience and expectations. +* You need to setup monitoring of your system allowing you to observe resources and react to problems. +* You are expected to configure and manage appropriate resources for the installation (memory, CPU, etc) based + on the monitoring of your installation and feedback loop. **What Apache Airflow Community provides for that method** @@ -238,6 +247,9 @@ More details: :doc:`helm-chart:index` those changes when released by upgrading the base image. However, you are responsible in creating a pipeline of building your own custom images with your own added dependencies and Providers and need to repeat the customization step and building your own image when new version of Airflow image is released. +* You need to setup monitoring of your system allowing you to observe resources and react to problems. +* You are expected to configure and manage appropriate resources for the installation (memory, CPU, etc) based + on the monitoring of your installation and feedback loop. **What Apache Airflow Community provides for that method** @@ -256,7 +268,6 @@ More details: :doc:`helm-chart:index` * If you can provide description of a reproducible problem with Airflow software, you can open issue at `GitHub issues `__ - Using Managed Airflow Services '''''''''''''''''''''''''''''' @@ -316,3 +327,62 @@ Follow the `Ecosystem `__ page to find a **Where to ask for help** * Depends on what the 3rd-party provides. Look at the documentation of the 3rd-party deployment you use. + + +Notes about minimum requirements +'''''''''''''''''''''''''''''''' + +There are often questions about minimum requirements for Airflow for production systems, but it is +not possible to give a simple answer to that question. + +The requirements that Airflow might need depend on many factors, including (but not limited to): + * The deployment your Airflow is installed with (see above ways of installing Airflow) + * The requirements of the deployment environment (for example Kubernetes, Docker, Helm, etc.) that + are completely independent from Airflow (for example DNS resources, sharing the nodes/resources) + with more (or less) pods and containers that are needed that might depend on particular choice of + the technology/cloud/integration of monitoring etc. + * Technical details of database, hardware, network, etc. that your deployment is running on + * The complexity of the code you add to your DAGS, configuration, plugins, settings etc. (note, that + Airflow runs the code that DAG author and Deployment Manager provide) + * The number and choice of providers you install and use (Airflow has more than 80 providers) that can + be installed by choice of the Deployment Manager and using them might require more resources. + * The choice of parameters that you use when tuning Airflow. Airflow has many configuration parameters + that can fine-tuned to your needs + * The number of DagRuns and tasks instances you run with parallel instances of each in consideration + * How complex are the tasks you run + +The above "DAG" characteristics will change over time and even will change depending on the time of the day +or week, so you have to be prepared to continuously monitor the system and adjust the parameters to make +it works smoothly. + +While we can provide some specific minimum requirements for some development "quick start" - such as +in case of our :ref:`running-airflow-in-docker` quick-start guide, it is not possible to provide any minimum +requirements for production systems. + +The best way to think of resource allocation for Airflow instance is to think of it in terms of process +control theory - where there are two types of systems: + +1. Fully predictable, with few knobs and variables, where you can reliably set the values for the + knobs and have an easy way to determine the behaviour of the system + +2. Complex systems with multiple variables, that are hard to predict and where you need to monitor + the system and adjust the knobs continuously to make sure the system is running smoothly. + +Airflow (and generally any modern system running usually on cloud services, with multiple layers responsible +for resources as well multiple parameters to control their behaviour) is a complex system and they fall +much more in the second category. If you decide to run Airflow in production on your own, you should be +prepared for the monitor/observe/adjust feedback loop to make sure the system is running smoothly. + +Having a good monitoring system that will allow you to monitor the system and adjust the parameters +is a must to put that in practice. + +There are few guidelines that you can use for optimizing your resource usage as well. The +:ref:`fine-tuning-scheduler` is a good starting point to fine-tune your scheduler, you can also follow +the :ref:`best_practice` guide to make sure you are using Airflow in the most efficient way. + +Also, one of the important things that Managed Services for Airflow provide is that they make a lot +of opinionated choices and fine-tune the system for you, so you don't have to worry about it too much. +With such managed services, there are usually far less number of knobs to turn and choices to make and one +of the things you pay for is that the Managed Service provider manages the system for you and provides +paid support and allows you to scale the system as needed and allocate the right resources - following the +choices made there when it comes to the kinds of deployment you might have. From 4efbcdc8fc48e929f18fbe75c61a5125072f71fc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 1 Jul 2023 23:56:33 +0200 Subject: [PATCH 015/533] Update security model of Airflow (#32098) * Update security model of Airflow This change updates the security model of Airflow to better explain what are the capabilities of various kinds of users in Airflow deployments and give both users and security researchers a way to understand what security measures they can take and whether they can qualify potential security issues in Airflow properly - taking into account that various users of Airflow have various capabilities and behaviours considered by some of the users as security vulnerabilities, are standard capabilities of the users. It also splits the security information of ours in two separate pages: * .github/SECURITY.md where we explain how to report the issues to Apache Airflow security team by the researchers * documentation security/index.html which is available via Airflow Website where we explain what our security model is and the different kinds of users we have. Both serve slightly different purpose and both contain cross-reference links to each other in order to be able to redirect people who read about the security model to find out how they can report the issues but also to guide security researchers who want to assess whether their findings are real vulnerabilities, or rather normal behaviours following the Airflow Security model. Security has been also moved to be a top level topic, so that it is easier to find and navigate to. Old links have been redirected to the new locations. Also chapters were added explaining Airflow vs. Providers security releases, what is the relation between Airflow and Providers security issues and how users should treat security announcements in providers. * Update .github/SECURITY.md Co-authored-by: Pankaj Koti * Apply Niko's suggestions from code review Co-authored-by: Niko Oliveira * fixup! Apply Niko's suggestions from code review --------- Co-authored-by: Pankaj Koti Co-authored-by: Niko Oliveira --- .github/{SECURITY.rst => SECURITY.md} | 115 ++++++------ .github/boring-cyborg.yml | 2 +- .pre-commit-config.yaml | 2 +- .../core-extensions/auth-backends.rst | 2 +- .../core-extensions/secrets-backends.rst | 2 +- .../administration-and-deployment/index.rst | 1 - .../security/index.rst | 28 --- docs/apache-airflow/howto/connection.rst | 6 +- docs/apache-airflow/howto/variable.rst | 2 +- docs/apache-airflow/index.rst | 1 + .../installation/installing-from-pypi.rst | 1 + docs/apache-airflow/integration.rst | 8 +- .../public-airflow-interface.rst | 2 +- docs/apache-airflow/redirects.txt | 35 ++-- .../security/access-control.rst | 0 .../security/api.rst | 2 +- .../security/audit_logs.rst | 0 .../security/flower.rst | 0 docs/apache-airflow/security/index.rst | 164 ++++++++++++++++++ .../security/kerberos.rst | 0 .../security/secrets/fernet.rst | 0 .../security/secrets/index.rst | 0 .../secrets/mask-sensitive-values.rst | 0 .../secrets/secrets-backend/index.rst | 0 .../local-filesystem-secrets-backend.rst | 0 .../security/webserver.rst | 2 +- .../security/workload.rst | 0 27 files changed, 254 insertions(+), 121 deletions(-) rename .github/{SECURITY.rst => SECURITY.md} (53%) delete mode 100644 docs/apache-airflow/administration-and-deployment/security/index.rst rename docs/apache-airflow/{administration-and-deployment => }/security/access-control.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/api.rst (98%) rename docs/apache-airflow/{administration-and-deployment => }/security/audit_logs.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/flower.rst (100%) create mode 100644 docs/apache-airflow/security/index.rst rename docs/apache-airflow/{administration-and-deployment => }/security/kerberos.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/secrets/fernet.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/secrets/index.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/secrets/mask-sensitive-values.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/secrets/secrets-backend/index.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst (100%) rename docs/apache-airflow/{administration-and-deployment => }/security/webserver.rst (99%) rename docs/apache-airflow/{administration-and-deployment => }/security/workload.rst (100%) diff --git a/.github/SECURITY.rst b/.github/SECURITY.md similarity index 53% rename from .github/SECURITY.rst rename to .github/SECURITY.md index e7f5a40366db..950ab7f44521 100644 --- a/.github/SECURITY.rst +++ b/.github/SECURITY.md @@ -1,37 +1,28 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Security Model --------------- - -In the Airflow security model, the system administrators are fully trusted. -They are the only ones who can upload new DAGs, which gives them the ability -to execute any code on the server. - -Authenticated web interface and API users with Admin/Op permissions are trusted, -but to a lesser extent: they can configure the DAGs which gives them some control, -but not arbitrary code execution. - -Authenticated Web interface and API users with 'regular' permissions are trusted -to the point where they can impact resource consumption and pause/unpause configured DAGs, -but not otherwise influence their functionality. - -Reporting Vulnerabilities -------------------------- + + +This document contains information on how to report security vulnerabilities in Apache Airflow and +how the security issues reported to Apache Airflow security team are handled. If you would like +to learn about the security model of Airflow head to +[Airflow Security](https://airflow.apache.org/docs/apache-airflow/stable/security/) + +## Reporting Vulnerabilities **⚠️ Please do not file GitHub issues for security vulnerabilities as they are public! ⚠️** @@ -39,9 +30,9 @@ The Apache Software Foundation takes security issues very seriously. Apache Airflow specifically offers security features and is responsive to issues around its features. If you have any concern around Airflow Security or believe you have uncovered a vulnerability, we suggest that you get in touch via the -e-mail address security@airflow.apache.org. In the message, try to provide a -description of the issue and ideally a way of reproducing it. The security team -will get back to you after assessing the description. +e-mail address [security@airflow.apache.org](mailto:security@airflow.apache.org). +In the message, try to provide a description of the issue and ideally a way of +reproducing it. The security team will get back to you after assessing the report. Note that this security address should be used only for undisclosed vulnerabilities. Dealing with fixed issues or general questions on how to use @@ -49,13 +40,30 @@ the security features should be handled regularly via the user and the dev lists. Please report any security problems to the project security address before disclosing it publicly. -The `ASF Security team's page `_ describes -how vulnerability reports are handled, and includes PGP keys if you wish to use -that. +Before reporting vulnerabilities, please make sure to read and understand the +[security model](https://airflow.apache.org/docs/apache-airflow/stable/security/) of Airflow, because +some of the potential security vulnerabilities that are valid for projects that are publicly accessible +from the Internet, are not valid for Airflow. Airflow is not designed to be used by untrusted users, and some +trusted users are trusted enough to do a variety of operations that could be considered as vulnerabilities +in other products/circumstances. Therefore, some potential security vulnerabilities do not +apply to Airflow, or have a different severity than some generic scoring systems (for example `CVSS`) +calculation suggests. +The [ASF Security team's page](https://www.apache.org/security/) describes +how vulnerability reports are handled in general by all ASF projects, and includes PGP keys if +you wish to use them when you report the issues. -Handling security issues in Airflow ------------------------------------ +## Security vulnerabilities in Airflow and Airflow community managed providers + +Airflow core package is released separately from provider packages. While Airflow comes with ``constraints`` +which describe which version of providers have been tested when the version of Airflow was released, the +users of Airflow are advised to install providers independently from Airflow core when they want to apply +security fixes found and released in providers. Therefore, the issues found and fixed in providers do +not apply to the Airflow core package. There are also Airflow providers released by 3rd-parties, but the +Airflow community is not responsible for releasing and announcing security vulnerabilities in them, this +is handled entirely by the 3rd-parties that release their own providers. + +## Handling security issues in Airflow The security issues in Airflow are handled by the Airflow Security Team. The team consists of selected PMC members that are interested in looking at, discussing about and fixing the @@ -80,7 +88,7 @@ There are certain expectations from the members of the security team: experts that are available through Airflow stakeholders. The intent about involving 3rd parties has to be discussed and agreed up at security@airflow.apache.org. -* They have to have an `ICLA `_ signed with +* They have to have an [ICLA](https://www.apache.org/licenses/contributor-agreements.html) signed with Apache Software Foundation. * The security team members might inform 3rd parties about fixes, for example in order to assess if the fix @@ -92,7 +100,7 @@ There are certain expectations from the members of the security team: with the intent of minimizing the time between the fix being available and the fix being released. In this case the PR might be sent to review and comment to the PMC members on private list, in order to request an expedited voting on the release. The voting for such release might be done on the - ``private@airflow.apache.org`` mailing list and should be made public at the ``dev@apache.airflow.org`` + `private@airflow.apache.org` mailing list and should be made public at the `dev@apache.airflow.org` mailing list as soon as the release is ready to be announced. * The security team members working on the fix might be mentioned as remediation developers in the CVE @@ -104,22 +112,5 @@ There are certain expectations from the members of the security team: release process. This is facilitated by the security tool provided by the Apache Software Foundation. * Severity of the issue is determined based on the criteria described in the - `Severity Rating blog post `_ by the Apache Software + [Severity Rating blog post](https://security.apache.org/blog/severityrating/) by the Apache Software Foundation Security team - -Releasing Airflow with security patches ---------------------------------------- - -Apache Airflow uses strict `SemVer `_ versioning policy, which means that we strive for -any release of a given ``MAJOR`` Version (version "2" currently) to be backwards compatible. When we -release ``MINOR`` version, the development continues in the ``main`` branch where we prepare the next -``MINOR`` version, but we release ``PATCHLEVEL`` releases with selected bugfixes (including security -bugfixes) cherry-picked to the latest released ``MINOR`` line of Apache Airflow. At the moment, when we -release a new ``MINOR`` version, we stop releasing ``PATCHLEVEL`` releases for the previous ``MINOR`` version. - -For example, when we released ``2.6.0`` version on April 30, 2023, until we release ``2.7.0`` version, -all the security patches will be cherry-picked and released in ``2.6.*`` versions only. There will be no -``2.5.*`` versions released after ``2.6.0`` has been released. - -This means that in order to apply security fixes with Apache Airflow software released by us, you -MUST upgrade to the latest ``MINOR`` version of Airflow. diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index b55d145ad6a5..e364b37984a5 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -270,7 +270,7 @@ labelPRBasedOnFilePath: - airflow/providers/**/secrets/* - tests/secrets/**/* - tests/providers/**/secrets/* - - docs/apache-airflow/administration-and-deployment/security/secrets/**/* + - docs/apache-airflow/security/secrets/**/* area:Triggerer: - airflow/cli/commands/triggerer_command.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f8371431aa53..55129ceaa117 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -495,7 +495,7 @@ repos: ^airflow/providers/microsoft/winrm/hooks/winrm.py$| ^airflow/www/fab_security/manager.py$| ^docs/.*commits.rst$| - ^docs/apache-airflow/administration-and-deployment/security/webserver.rst$| + ^docs/apache-airflow/security/webserver.rst$| ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$| ^airflow/providers/microsoft/winrm/operators/winrm.py$| ^airflow/providers/opsgenie/hooks/opsgenie.py$| diff --git a/docs/apache-airflow-providers/core-extensions/auth-backends.rst b/docs/apache-airflow-providers/core-extensions/auth-backends.rst index edb38bb882f4..325b0c2819a2 100644 --- a/docs/apache-airflow-providers/core-extensions/auth-backends.rst +++ b/docs/apache-airflow-providers/core-extensions/auth-backends.rst @@ -26,7 +26,7 @@ capabilities. You can read more about those in `FAB security docs `_. You can also -take a look at Auth backends available in the core Airflow in :doc:`apache-airflow:administration-and-deployment/security/webserver` +take a look at Auth backends available in the core Airflow in :doc:`apache-airflow:security/webserver` or see those provided by the community-managed providers: .. airflow-auth-backends:: diff --git a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst index 22fae4b8a6c4..26ee3ce882a0 100644 --- a/docs/apache-airflow-providers/core-extensions/secrets-backends.rst +++ b/docs/apache-airflow-providers/core-extensions/secrets-backends.rst @@ -28,7 +28,7 @@ via providers that implement secrets backends for services Airflow integrates wi You can also take a look at Secret backends available in the core Airflow in -:doc:`apache-airflow:administration-and-deployment/security/secrets/secrets-backend/index` and here you can see the ones +:doc:`apache-airflow:security/secrets/secrets-backend/index` and here you can see the ones provided by the community-managed providers: .. airflow-secrets-backends:: diff --git a/docs/apache-airflow/administration-and-deployment/index.rst b/docs/apache-airflow/administration-and-deployment/index.rst index 1acb9d8e3a90..5ba79a5439ba 100644 --- a/docs/apache-airflow/administration-and-deployment/index.rst +++ b/docs/apache-airflow/administration-and-deployment/index.rst @@ -24,7 +24,6 @@ This section contains information about deploying DAGs into production and the a :maxdepth: 2 production-deployment - security/index logging-monitoring/index kubernetes lineage diff --git a/docs/apache-airflow/administration-and-deployment/security/index.rst b/docs/apache-airflow/administration-and-deployment/security/index.rst deleted file mode 100644 index 65d02f7f536b..000000000000 --- a/docs/apache-airflow/administration-and-deployment/security/index.rst +++ /dev/null @@ -1,28 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Security -======== - -.. toctree:: - :maxdepth: 1 - :glob: - - * - secrets/index - -.. include:: /../../.github/SECURITY.rst diff --git a/docs/apache-airflow/howto/connection.rst b/docs/apache-airflow/howto/connection.rst index b43094a88bb3..0db100fd2974 100644 --- a/docs/apache-airflow/howto/connection.rst +++ b/docs/apache-airflow/howto/connection.rst @@ -27,7 +27,7 @@ Airflow's :class:`~airflow.models.connection.Connection` object is used for stor Connections may be defined in the following ways: - in :ref:`environment variables ` - - in an external :doc:`/administration-and-deployment/security/secrets/secrets-backend/index` + - in an external :doc:`/security/secrets/secrets-backend/index` - in the :ref:`Airflow metadata database ` (using the :ref:`CLI ` or :ref:`web UI `) @@ -86,7 +86,7 @@ See :ref:`Connection URI format ` for more details on how Storing connections in a Secrets Backend ---------------------------------------- -You can store Airflow connections in external secrets backends like HashiCorp Vault, AWS SSM Parameter Store, and other such services. For more details see :doc:`/administration-and-deployment/security/secrets/secrets-backend/index`. +You can store Airflow connections in external secrets backends like HashiCorp Vault, AWS SSM Parameter Store, and other such services. For more details see :doc:`/security/secrets/secrets-backend/index`. .. _connections-in-database: @@ -94,7 +94,7 @@ Storing connections in the database ----------------------------------- .. seealso:: - Connections can alternatively be stored in :ref:`environment variables ` or an :doc:`external secrets backend ` such as HashiCorp Vault, AWS SSM Parameter Store, etc. + Connections can alternatively be stored in :ref:`environment variables ` or an :doc:`external secrets backend ` such as HashiCorp Vault, AWS SSM Parameter Store, etc. When storing connections in the database, you may manage them using either the web UI or the Airflow CLI. diff --git a/docs/apache-airflow/howto/variable.rst b/docs/apache-airflow/howto/variable.rst index a9e0d82023f9..b4b395dd63c2 100644 --- a/docs/apache-airflow/howto/variable.rst +++ b/docs/apache-airflow/howto/variable.rst @@ -73,4 +73,4 @@ It guarantees that without the encryption password, content cannot be manipulate without the key. For information on configuring Fernet, look at :ref:`security/fernet`. In addition to retrieving variables from environment variables or the metastore database, you can enable -a secrets backend to retrieve variables. For more details see :doc:`/administration-and-deployment/security/secrets/secrets-backend/index`. +a secrets backend to retrieve variables. For more details see :doc:`/security/secrets/secrets-backend/index`. diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index 548ced7b076a..feaf2d388b50 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -133,6 +133,7 @@ so coding will always be required. Overview start installation/index + security/index tutorial/index howto/index ui diff --git a/docs/apache-airflow/installation/installing-from-pypi.rst b/docs/apache-airflow/installation/installing-from-pypi.rst index 975edb5d3f16..ed345f0cc1f3 100644 --- a/docs/apache-airflow/installation/installing-from-pypi.rst +++ b/docs/apache-airflow/installation/installing-from-pypi.rst @@ -272,6 +272,7 @@ released and tested together when the version of Airflow you are installing was CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" pip install "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" +.. _installing-from-pypi-managing-providers-separately-from-airflow-core: Managing providers separately from Airflow core =============================================== diff --git a/docs/apache-airflow/integration.rst b/docs/apache-airflow/integration.rst index 3f477911048a..2e9c450f1fa8 100644 --- a/docs/apache-airflow/integration.rst +++ b/docs/apache-airflow/integration.rst @@ -20,18 +20,18 @@ Integration Airflow has a mechanism that allows you to expand its functionality and integrate with other systems. -* :doc:`API Authentication backends ` +* :doc:`API Authentication backends ` * :doc:`Email backends ` * :doc:`Executor ` -* :doc:`Kerberos ` +* :doc:`Kerberos ` * :doc:`Logging ` * :doc:`Metrics (statsd) ` * :doc:`Operators and hooks ` * :doc:`Plugins ` * :doc:`Listeners ` -* :doc:`Secrets backends ` +* :doc:`Secrets backends ` * :doc:`Tracking systems ` -* :doc:`Web UI Authentication backends ` +* :doc:`Web UI Authentication backends ` * :doc:`Serialization ` It also has integration with :doc:`Sentry ` service for error tracking. Other applications can also integrate using diff --git a/docs/apache-airflow/public-airflow-interface.rst b/docs/apache-airflow/public-airflow-interface.rst index 74d86572446d..5f0f51163873 100644 --- a/docs/apache-airflow/public-airflow-interface.rst +++ b/docs/apache-airflow/public-airflow-interface.rst @@ -339,7 +339,7 @@ All Secrets Backend implementations are public. You can extend their functionali _api/airflow/secrets/index -You can read more about Secret Backends in :doc:`administration-and-deployment/security/secrets/secrets-backend/index`. +You can read more about Secret Backends in :doc:`security/secrets/secrets-backend/index`. You can also find all the available Secrets Backends implemented in community providers in :doc:`apache-airflow-providers:core-extensions/secrets-backends`. diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt index b99a5b253cbc..9d8a5527fbbd 100644 --- a/docs/apache-airflow/redirects.txt +++ b/docs/apache-airflow/redirects.txt @@ -15,9 +15,26 @@ # specific language governing permissions and limitations # under the License. +# Administration and deployment security -> security + +administration-and-deployment/security/index.rst security/index.rst +administration-and-deployment/security/kerberos.rst security/kerberos.rst +administration-and-deployment/security/access-control.rst security/access-control.rst +administration-and-deployment/security/access-control/index.rst security/access-control.rst +administration-and-deployment/security/api.rst security/api.rst +administration-and-deployment/security/audit_logs.rst security/audit_logs.rst +administration-and-deployment/security/flower.rst security/flower.rst +administration-and-deployment/security/webserver.rst security/webserver.rst +administration-and-deployment/security/workload.rst security/workload.rst +administration-and-deployment/security/secrets/secrets-backends/index.rst security/secrets/secrets-backends/index.rst +administration-and-deployment/security/secrets/secrets-backends/local-filesystem-secrets-backend.rst security/secrets/secrets-backends/local-filesystem-secrets-backend.rst +administration-and-deployment/security/secrets/fernet.rst security/secrets/fernet.rst +administration-and-deployment/security/secrets/index.rst security/secrets/index.rst +administration-and-deployment/security/secrets/mask-sensitive-values.rst security/secrets/mask-sensitive-values.rst + # Security -howto/use-alternative-secrets-backend.rst administration-and-deployment/security/secrets/secrets-backend/index.rst -security.rst administration-and-deployment/security/index.rst +howto/use-alternative-secrets-backend.rst security/secrets/secrets-backend/index.rst +security.rst security/index.rst # Operators guides howto/operator/external.rst howto/operator/external_task_sensor.rst @@ -27,7 +44,7 @@ howto/customize-dag-ui-page-instance-name.rst howto/customize-ui.rst#customizing howto/customize-state-colors-ui.rst howto/customize-ui.rst#customizing-state-colours # Web UI -howto/add-new-role.rst administration-and-deployment/security/access-control.rst +howto/add-new-role.rst security/access-control.rst # Set up a database howto/initialize-database.rst howto/set-up-database.rst @@ -68,9 +85,6 @@ tutorial_taskflow_api.rst tutorial/taskflow.rst ## Docs Structure Refactor # indexes -security/index.rst administration-and-deployment/security/index.rst -security/secrets/index.rst administration-and-deployment/security/secrets/index.rst -security/secrets/secrets-backend/index.rst administration-and-deployment/security/secrets/secrets-backend/index.rst logging-monitoring/index.rst administration-and-deployment/logging-monitoring/index.rst concepts/index.rst core-concepts/index.rst executor/index.rst core-concepts/executor/index.rst @@ -88,15 +102,6 @@ usage-cli.rst howto/usage-cli.rst lineage.rst administration-and-deployment/lineage.rst dag-run.rst core-concepts/dag-run.rst concepts/taskflow.rst core-concepts/taskflow.rst -security/kerberos.rst administration-and-deployment/security/kerberos.rst -security/workload.rst administration-and-deployment/security/workload.rst -security/flower.rst administration-and-deployment/security/flower.rst -security/webserver.rst administration-and-deployment/security/webserver.rst -security/api.rst administration-and-deployment/security/api.rst -security/access-control.rst administration-and-deployment/security/access-control.rst -security/secrets/fernet.rst administration-and-deployment/security/secrets/fernet.rst -security/secrets/mask-sensitive-values.rst administration-and-deployment/security/secrets/mask-sensitive-values.rst -security/secrets/secrets-backend/local-filesystem-secrets-backend.rst administration-and-deployment/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst logging-monitoring/tracking-user-activity.rst administration-and-deployment/logging-monitoring/tracking-user-activity.rst logging-monitoring/errors.rst administration-and-deployment/logging-monitoring/errors.rst logging-monitoring/logging-architecture.rst administration-and-deployment/logging-monitoring/logging-architecture.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/access-control.rst b/docs/apache-airflow/security/access-control.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/access-control.rst rename to docs/apache-airflow/security/access-control.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/api.rst b/docs/apache-airflow/security/api.rst similarity index 98% rename from docs/apache-airflow/administration-and-deployment/security/api.rst rename to docs/apache-airflow/security/api.rst index 6551ef928ff5..61319e21c62b 100644 --- a/docs/apache-airflow/administration-and-deployment/security/api.rst +++ b/docs/apache-airflow/security/api.rst @@ -63,7 +63,7 @@ If you wish to have the experimental API work, and aware of the risks of enablin You can only disable authentication for experimental API, not the stable REST API. -See :doc:`../modules_management` for details on how Python and Airflow manage modules. +See :doc:`../administration-and-deployment/modules_management` for details on how Python and Airflow manage modules. Kerberos authentication ''''''''''''''''''''''' diff --git a/docs/apache-airflow/administration-and-deployment/security/audit_logs.rst b/docs/apache-airflow/security/audit_logs.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/audit_logs.rst rename to docs/apache-airflow/security/audit_logs.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/flower.rst b/docs/apache-airflow/security/flower.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/flower.rst rename to docs/apache-airflow/security/flower.rst diff --git a/docs/apache-airflow/security/index.rst b/docs/apache-airflow/security/index.rst new file mode 100644 index 000000000000..3573348f1006 --- /dev/null +++ b/docs/apache-airflow/security/index.rst @@ -0,0 +1,164 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Security +======== + +.. toctree:: + :maxdepth: 1 + :glob: + + * + secrets/index + + +This document describes Airflow's security model from the perspective of +the Airflow user. It is intended to help users understand the security +model and make informed decisions about how to deploy and manage Airflow. +If you would like to know how to report security vulnerabilities and how +security reports are handled by the security team of Airflow, head to +`Airflow's Security Policy `_. + +Airflow security model - user types +----------------------------------- + +The Airflow security model involves different types of users with +varying access and capabilities: + +1. **Deployment Managers**: They have the highest level of access and + control. They install and configure Airflow, make decisions about + technologies and permissions. They can potentially delete the entire + installation and have access to all credentials. Deployment Managers + can also decide to keep audits, backups and copies of information + outside of Airflow, which are not covered by Airflow's security + model. + +2. **DAG Authors**: They can upload, modify, and delete DAG files. The + code in DAG files is executed on workers. Therefore, DAG authors can create + and change code executed on workers and potentially access the credentials + that DAG code uses to access external systems. DAG Authors have full access + to the metadata database and internal audit logs. + +3. **Authenticated UI users**: They have access to the UI and API. Admin + users can manage permissions and execute code on workers. Connection + configuration users can configure connections and execute code on + workers. Operations users have access to DAG execution status. Trust + is crucial to prevent abuse and Denial of Service attacks. + +4. **Non-authenticated UI users**: Airflow doesn't support + unauthenticated users by default. If allowed, vulnerabilities must be + addressed by the Deployment Manager. + +Capabilities of authenticated UI users +-------------------------------------- + +The capabilities of **Authenticated UI users** can vary depending on +what roles have been configured by the Deployment Manager or Admin users as well as what permissions those roles have. Permissions on roles can be scoped as tightly as a single DAG, for example, or as broad as Admin. Below are three general categories to help conceptualize some of the capabilities authenticated users may have: + +1. **Admin users**: They manage and grant permissions to other users, + with full access to all UI capabilities. They can potentially execute + code on workers by configuring connections and need to be trusted not + to abuse these privileges. They have access to sensitive credentials + and can modify them. By default, they don't have access to + system-level configuration. They should be trusted not to misuse + sensitive information accessible through connection configuration. + They also have the ability to create a Webserver Denial of Service + situation and should be trusted not to misuse this capability. + +2. **Connection configuration users**: They configure connections and + potentially execute code on workers during DAG execution. Trust is + required to prevent misuse of these privileges. They have full access + to sensitive credentials stored in connections and can modify them. + Access to sensitive information through connection configuration + should be trusted not to be abused. They also have the ability to + create a Webserver Denial of Service situation and should be trusted + not to misuse this capability. + +3. **Operations users**: They have access to DAG execution status via + the UI. Currently, Airflow lacks full protection for accessing groups + of DAGs' history and execution. They can perform actions such as + clearing, re-running, triggering DAGs, and changing parameters. + Depending on access restrictions, they may also have access to + editing variables and viewing Airflow configuration. They should not + have access to sensitive system-level information or connections, and + they should not be able to access sensitive task information unless + deliberately exposed in logs by DAG authors. They should be trusted + not to abuse their privileges, as they can potentially overload the + server and cause Denial of Service situations. + +Responsibilities of Deployment Managers +--------------------------------------- + +Deployment Managers determine access levels and understand the potential +damage users can cause. Some Deployment Managers may further limit +access through fine-grained privileges for the **Authenticated UI +users**. However, these limitations are outside the basic Airflow's +security model and are at the discretion of Deployment Managers. +Examples of fine-grained access control include (but are not limited +to): + +- Limiting login permissions: Restricting the accounts that users can + log in with, allowing only specific accounts or roles belonging to + access the Airflow system. + +- Access restrictions to views or DAGs: Controlling user access to + certain views or specific DAGs, ensuring that users can only view or + interact with authorized components. + +- Implementing static code analysis and code review: Introducing + processes such as static code analysis and code review as part of the + DAG submission pipeline. This helps enforce code quality standards, + security checks, and adherence to best practices before DAGs are + deployed. + +These examples showcase ways in which administrators can refine and +limit user privileges within Airflow, providing tighter control and +ensuring that users have access only to the necessary components and +functionalities based on their roles and responsibilities, however the +fine-grained access control does not provide full isolation and +separation of access allowing isolation of different user groups in a +multi-tenant fashion yet. In the future versions of Airflow some of the +fine-grained access features might become part of the Airflow security +model. The Airflow community is working on a multi-tenant model that might +address some of the fine-grained access components. + + +Releasing Airflow with security patches +--------------------------------------- + +Apache Airflow uses a strict [SemVer](https://semver.org) versioning policy, which means that we strive for +any release of a given ``MAJOR`` Version (version "2" currently) to be backwards compatible. When we +release a ``MINOR`` version, the development continues in the ``main`` branch where we prepare the next +``MINOR`` version, but we release ``PATCHLEVEL`` releases with selected bugfixes (including security +bugfixes) cherry-picked to the latest released ``MINOR`` line of Apache Airflow. At the moment, when we +release a new ``MINOR`` version, we stop releasing ``PATCHLEVEL`` releases for the previous ``MINOR`` version. + +For example, once we released ``2.6.0`` version on April 30, 2023 all the security patches will be cherry-picked and released in ``2.6.*`` versions until we release ``2.7.0`` version. There will be no +``2.5.*`` versions released after ``2.6.0`` has been released. + +This means that in order to apply security fixes with Apache Airflow software released by us, you +MUST upgrade to the latest ``MINOR`` version of Airflow. + +Releasing Airflow providers with security patches +------------------------------------------------- + +Similarly to Airflow, providers use strict [SemVer](https://semver.org) versioning policy, and the same +policies apply for providers as for Airflow itself. This means that you need to upgrade to the latest +``MINOR`` version of the provider to get the latest security fixes. +Airflow providers are released independently from Airflow itself and the information about vulnerabilities +is published separately. You can upgrade providers independently from Airflow itself, following the +instructions found in :ref:`installing-from-pypi-managing-providers-separately-from-airflow-core`. diff --git a/docs/apache-airflow/administration-and-deployment/security/kerberos.rst b/docs/apache-airflow/security/kerberos.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/kerberos.rst rename to docs/apache-airflow/security/kerberos.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/secrets/fernet.rst b/docs/apache-airflow/security/secrets/fernet.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/secrets/fernet.rst rename to docs/apache-airflow/security/secrets/fernet.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/secrets/index.rst b/docs/apache-airflow/security/secrets/index.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/secrets/index.rst rename to docs/apache-airflow/security/secrets/index.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/secrets/mask-sensitive-values.rst b/docs/apache-airflow/security/secrets/mask-sensitive-values.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/secrets/mask-sensitive-values.rst rename to docs/apache-airflow/security/secrets/mask-sensitive-values.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/secrets/secrets-backend/index.rst b/docs/apache-airflow/security/secrets/secrets-backend/index.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/secrets/secrets-backend/index.rst rename to docs/apache-airflow/security/secrets/secrets-backend/index.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst diff --git a/docs/apache-airflow/administration-and-deployment/security/webserver.rst b/docs/apache-airflow/security/webserver.rst similarity index 99% rename from docs/apache-airflow/administration-and-deployment/security/webserver.rst rename to docs/apache-airflow/security/webserver.rst index d5e551f8b15f..71ad85fb3a9b 100644 --- a/docs/apache-airflow/administration-and-deployment/security/webserver.rst +++ b/docs/apache-airflow/security/webserver.rst @@ -74,7 +74,7 @@ user will have by default: AUTH_ROLE_PUBLIC = 'Admin' -Be sure to checkout :doc:`/administration-and-deployment/security/api` for securing the API. +Be sure to checkout :doc:`/security/api` for securing the API. .. note:: diff --git a/docs/apache-airflow/administration-and-deployment/security/workload.rst b/docs/apache-airflow/security/workload.rst similarity index 100% rename from docs/apache-airflow/administration-and-deployment/security/workload.rst rename to docs/apache-airflow/security/workload.rst From 2811ba710991d71466afa21c500ee55aeda5d8a1 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 2 Jul 2023 08:40:57 +0200 Subject: [PATCH 016/533] Limit Pydantic to < 2.0.0 until we solve 2.0.0 incompatibilities (#32312) The Pydantic 2.0.0 release on 30th of June broke a number of CI tests (including some K8S tests). We limit it temporarily until the incompatibilities are fixed. Related: #32311 --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d5d91d3d8ff6..91d3b86331de 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,7 +126,9 @@ install_requires = pendulum>=2.0 pluggy>=1.0 psutil>=4.2.0 - pydantic>=1.10.0 + # Limit Pydantic to <2.0.0 because it breaks Kubernetes tests and building providers + # This limit should be removed after https://github.com/apache/airflow/issues/32311 is fixed + pydantic>=1.10.0,<2.0.0 pygments>=2.0.1 pyjwt>=2.0.0 python-daemon>=3.0.0 From 05bd90f563649f2e9c8f0c85cf5838315a665a02 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Sun, 2 Jul 2023 12:31:13 +0100 Subject: [PATCH 017/533] Sanitize `DagRun.run_id` and allow flexibility (#32293) This commit sanitizes the DagRun.run_id parameter by introducing a configurable option. Users now have the ability to select a specific run_id pattern for their runs, ensuring stricter control over the values used. This update does not impact the default run_id generation performed by the scheduler for scheduled DAG runs or for Dag runs triggered without modifying the run_id parameter in the run configuration page. The configuration flexibility empowers users to align the run_id pattern with their specific requirements. --- airflow/config_templates/config.yml | 9 ++++ airflow/config_templates/default_airflow.cfg | 5 ++ airflow/models/dag.py | 55 +++++++++++--------- airflow/models/dagrun.py | 16 +++++- airflow/www/views.py | 32 ++++++++---- tests/models/test_dagrun.py | 32 ++++++++++++ tests/www/views/test_views_trigger_dag.py | 30 +++++++++++ 7 files changed, 143 insertions(+), 36 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index a252716ed1d0..f48a569b545f 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2555,6 +2555,15 @@ scheduler: type: float example: ~ default: "120.0" + allowed_run_id_pattern: + description: | + The run_id pattern used to verify the validity of user input to the run_id parameter when + triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id + for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. + version_added: 2.6.3 + type: string + example: ~ + default: "^[A-Za-z0-9_.~:+-]+$" triggerer: description: ~ options: diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 4eaab9ae95db..c28155ac2c40 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -1325,6 +1325,11 @@ task_queued_timeout = 600.0 # longer than `[scheduler] task_queued_timeout`. task_queued_timeout_check_interval = 120.0 +# The run_id pattern used to verify the validity of user input to the run_id parameter when +# triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id +# for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. +allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$ + [triggerer] # How many triggers a single Triggerer will run at once, by default. default_capacity = 1000 diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 31070a7b7c53..11468b59ea70 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -80,7 +80,7 @@ import airflow.templates from airflow import settings, utils from airflow.api_internal.internal_api_call import internal_api_call -from airflow.configuration import conf, secrets_backend_list +from airflow.configuration import conf as airflow_conf, secrets_backend_list from airflow.exceptions import ( AirflowDagInconsistent, AirflowException, @@ -96,7 +96,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.dagcode import DagCode from airflow.models.dagpickle import DagPickle -from airflow.models.dagrun import DagRun +from airflow.models.dagrun import RUN_ID_REGEX, DagRun from airflow.models.operator import Operator from airflow.models.param import DagParam, ParamsDict from airflow.models.taskinstance import Context, TaskInstance, TaskInstanceKey, clear_task_instances @@ -422,13 +422,13 @@ def __init__( user_defined_filters: dict | None = None, default_args: dict | None = None, concurrency: int | None = None, - max_active_tasks: int = conf.getint("core", "max_active_tasks_per_dag"), - max_active_runs: int = conf.getint("core", "max_active_runs_per_dag"), + max_active_tasks: int = airflow_conf.getint("core", "max_active_tasks_per_dag"), + max_active_runs: int = airflow_conf.getint("core", "max_active_runs_per_dag"), dagrun_timeout: timedelta | None = None, sla_miss_callback: None | SLAMissCallback | list[SLAMissCallback] = None, - default_view: str = conf.get_mandatory_value("webserver", "dag_default_view").lower(), - orientation: str = conf.get_mandatory_value("webserver", "dag_orientation"), - catchup: bool = conf.getboolean("scheduler", "catchup_by_default"), + default_view: str = airflow_conf.get_mandatory_value("webserver", "dag_default_view").lower(), + orientation: str = airflow_conf.get_mandatory_value("webserver", "dag_orientation"), + catchup: bool = airflow_conf.getboolean("scheduler", "catchup_by_default"), on_success_callback: None | DagStateChangeCallback | list[DagStateChangeCallback] = None, on_failure_callback: None | DagStateChangeCallback | list[DagStateChangeCallback] = None, doc_md: str | None = None, @@ -2588,7 +2588,7 @@ def run( mark_success=False, local=False, executor=None, - donot_pickle=conf.getboolean("core", "donot_pickle"), + donot_pickle=airflow_conf.getboolean("core", "donot_pickle"), ignore_task_deps=False, ignore_first_depends_on_past=True, pool=None, @@ -2826,13 +2826,14 @@ def create_dagrun( "Creating DagRun needs either `run_id` or both `run_type` and `execution_date`" ) - if run_id and "/" in run_id: - warnings.warn( - "Using forward slash ('/') in a DAG run ID is deprecated. Note that this character " - "also makes the run impossible to retrieve via Airflow's REST API.", - RemovedInAirflow3Warning, - stacklevel=3, - ) + regex = airflow_conf.get("scheduler", "allowed_run_id_pattern") + + if run_id and not re.match(RUN_ID_REGEX, run_id): + if not regex.strip() or not re.match(regex.strip(), run_id): + raise AirflowException( + f"The provided run ID '{run_id}' is invalid. It does not match either " + f"the configured pattern: '{regex}' or the built-in pattern: '{RUN_ID_REGEX}'" + ) # create a copy of params before validating copied_params = copy.deepcopy(self.params) @@ -3125,7 +3126,7 @@ def sync_to_db(self, processor_subdir: str | None = None, session=NEW_SESSION): def get_default_view(self): """This is only there for backward compatible jinja2 templates.""" if self.default_view is None: - return conf.get("webserver", "dag_default_view").lower() + return airflow_conf.get("webserver", "dag_default_view").lower() else: return self.default_view @@ -3342,7 +3343,7 @@ class DagModel(Base): root_dag_id = Column(StringID()) # A DAG can be paused from the UI / DB # Set this default value of is_paused based on a configuration value! - is_paused_at_creation = conf.getboolean("core", "dags_are_paused_at_creation") + is_paused_at_creation = airflow_conf.getboolean("core", "dags_are_paused_at_creation") is_paused = Column(Boolean, default=is_paused_at_creation) # Whether the DAG is a subdag is_subdag = Column(Boolean, default=False) @@ -3416,7 +3417,9 @@ class DagModel(Base): "TaskOutletDatasetReference", cascade="all, delete, delete-orphan", ) - NUM_DAGS_PER_DAGRUN_QUERY = conf.getint("scheduler", "max_dagruns_to_create_per_loop", fallback=10) + NUM_DAGS_PER_DAGRUN_QUERY = airflow_conf.getint( + "scheduler", "max_dagruns_to_create_per_loop", fallback=10 + ) def __init__(self, concurrency=None, **kwargs): super().__init__(**kwargs) @@ -3429,10 +3432,10 @@ def __init__(self, concurrency=None, **kwargs): ) self.max_active_tasks = concurrency else: - self.max_active_tasks = conf.getint("core", "max_active_tasks_per_dag") + self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag") if self.max_active_runs is None: - self.max_active_runs = conf.getint("core", "max_active_runs_per_dag") + self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag") if self.has_task_concurrency_limits is None: # Be safe -- this will be updated later once the DAG is parsed @@ -3510,7 +3513,7 @@ def get_default_view(self) -> str: have a value. """ # This is for backwards-compatibility with old dags that don't have None as default_view - return self.default_view or conf.get_mandatory_value("webserver", "dag_default_view").lower() + return self.default_view or airflow_conf.get_mandatory_value("webserver", "dag_default_view").lower() @property def safe_dag_id(self): @@ -3699,13 +3702,13 @@ def dag( user_defined_filters: dict | None = None, default_args: dict | None = None, concurrency: int | None = None, - max_active_tasks: int = conf.getint("core", "max_active_tasks_per_dag"), - max_active_runs: int = conf.getint("core", "max_active_runs_per_dag"), + max_active_tasks: int = airflow_conf.getint("core", "max_active_tasks_per_dag"), + max_active_runs: int = airflow_conf.getint("core", "max_active_runs_per_dag"), dagrun_timeout: timedelta | None = None, sla_miss_callback: None | SLAMissCallback | list[SLAMissCallback] = None, - default_view: str = conf.get_mandatory_value("webserver", "dag_default_view").lower(), - orientation: str = conf.get_mandatory_value("webserver", "dag_orientation"), - catchup: bool = conf.getboolean("scheduler", "catchup_by_default"), + default_view: str = airflow_conf.get_mandatory_value("webserver", "dag_default_view").lower(), + orientation: str = airflow_conf.get_mandatory_value("webserver", "dag_orientation"), + catchup: bool = airflow_conf.getboolean("scheduler", "catchup_by_default"), on_success_callback: None | DagStateChangeCallback | list[DagStateChangeCallback] = None, on_failure_callback: None | DagStateChangeCallback | list[DagStateChangeCallback] = None, doc_md: str | None = None, diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index dab89c12cd9d..8f3b3a33018a 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -24,6 +24,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, NamedTuple, Sequence, TypeVar, overload +import re2 as re from sqlalchemy import ( Boolean, Column, @@ -44,7 +45,7 @@ ) from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm import Query, Session, declared_attr, joinedload, relationship, synonym +from sqlalchemy.orm import Query, Session, declared_attr, joinedload, relationship, synonym, validates from sqlalchemy.sql.expression import false, select, true from airflow import settings @@ -76,6 +77,8 @@ CreatedTasks = TypeVar("CreatedTasks", Iterator["dict[str, Any]"], Iterator[TI]) TaskCreator = Callable[[Operator, Iterable[int]], CreatedTasks] +RUN_ID_REGEX = r"^(?:manual|scheduled|dataset_triggered)__(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)$" + class TISchedulingDecision(NamedTuple): """Type of return for DagRun.task_instance_scheduling_decisions.""" @@ -240,6 +243,17 @@ def __repr__(self): external_trigger=self.external_trigger, ) + @validates("run_id") + def validate_run_id(self, key: str, run_id: str) -> str | None: + if not run_id: + return None + regex = airflow_conf.get("scheduler", "allowed_run_id_pattern") + if not re.match(regex, run_id) and not re.match(RUN_ID_REGEX, run_id): + raise ValueError( + f"The run_id provided '{run_id}' does not match the pattern '{regex}' or '{RUN_ID_REGEX}'" + ) + return run_id + @property def stats_tags(self) -> dict[str, str]: return prune_dict({"dag_id": self.dag_id, "run_type": self.run_type}) diff --git a/airflow/www/views.py b/airflow/www/views.py index 607927650bb1..026f8b533499 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -99,7 +99,7 @@ from airflow.models.abstractoperator import AbstractOperator from airflow.models.dag import DAG, get_dataset_triggered_next_run_info from airflow.models.dagcode import DagCode -from airflow.models.dagrun import DagRun, DagRunType +from airflow.models.dagrun import RUN_ID_REGEX, DagRun, DagRunType from airflow.models.dataset import DagScheduleDatasetReference, DatasetDagRunQueue, DatasetEvent, DatasetModel from airflow.models.mappedoperator import MappedOperator from airflow.models.operator import Operator @@ -1975,7 +1975,7 @@ def delete(self): @provide_session def trigger(self, dag_id: str, session: Session = NEW_SESSION): """Triggers DAG Run.""" - run_id = request.values.get("run_id", "") + run_id = request.values.get("run_id", "").replace(" ", "+") origin = get_safe_url(request.values.get("origin")) unpause = request.values.get("unpause") request_conf = request.values.get("conf") @@ -2096,13 +2096,27 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): flash(message, "error") return redirect(origin) - # Flash a warning when slash is used, but still allow it to continue on. - if run_id and "/" in run_id: - flash( - "Using forward slash ('/') in a DAG run ID is deprecated. Note that this character " - "also makes the run impossible to retrieve via Airflow's REST API.", - "warning", - ) + regex = conf.get("scheduler", "allowed_run_id_pattern") + if run_id and not re.match(RUN_ID_REGEX, run_id): + if not regex.strip() or not re.match(regex.strip(), run_id): + flash( + f"The provided run ID '{run_id}' is invalid. It does not match either " + f"the configured pattern: '{regex}' or the built-in pattern: '{RUN_ID_REGEX}'", + "error", + ) + + form = DateTimeForm(data={"execution_date": execution_date}) + return self.render_template( + "airflow/trigger.html", + form_fields=form_fields, + dag=dag, + dag_id=dag_id, + origin=origin, + conf=request_conf, + form=form, + is_dag_run_conf_overrides_params=is_dag_run_conf_overrides_params, + recent_confs=recent_confs, + ) run_conf = {} if request_conf: diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 024b9df3a1f3..5d7db547d31c 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -30,6 +30,7 @@ from airflow import settings from airflow.callbacks.callback_requests import DagCallbackRequest from airflow.decorators import setup, task, task_group, teardown +from airflow.exceptions import AirflowException from airflow.models import ( DAG, DagBag, @@ -54,6 +55,7 @@ from airflow.utils.types import DagRunType from tests.models import DEFAULT_DATE as _DEFAULT_DATE from tests.test_utils import db +from tests.test_utils.config import conf_vars from tests.test_utils.mock_operators import MockOperator DEFAULT_DATE = pendulum.instance(_DEFAULT_DATE) @@ -2541,3 +2543,33 @@ def make_task(task_id, dag): tis = dr.task_instance_scheduling_decisions(session).tis tis_for_state = {x.task_id for x in dr._tis_for_dagrun_state(dag=dag, tis=tis)} assert tis_for_state == expected + + +@pytest.mark.parametrize( + "pattern, run_id, result", + [ + ["^[A-Z]", "ABC", True], + ["^[A-Z]", "abc", False], + ["^[0-9]", "123", True], + # The below params tests that user configuration does not affect internally generated + # run_ids + ["", "scheduled__2023-01-01T00:00:00+00:00", True], + ["", "manual__2023-01-01T00:00:00+00:00", True], + ["", "dataset_triggered__2023-01-01T00:00:00+00:00", True], + ["", "scheduled_2023-01-01T00", False], + ["", "manual_2023-01-01T00", False], + ["", "dataset_triggered_2023-01-01T00", False], + ["^[0-9]", "scheduled__2023-01-01T00:00:00+00:00", True], + ["^[0-9]", "manual__2023-01-01T00:00:00+00:00", True], + ["^[a-z]", "dataset_triggered__2023-01-01T00:00:00+00:00", True], + ], +) +def test_dag_run_id_config(session, dag_maker, pattern, run_id, result): + with conf_vars({("scheduler", "allowed_run_id_pattern"): pattern}): + with dag_maker(): + ... + if result: + dag_maker.create_dagrun(run_id=run_id) + else: + with pytest.raises(AirflowException): + dag_maker.create_dagrun(run_id=run_id) diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index 7cf140ed7ca6..12f090bc4f4e 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -30,6 +30,7 @@ from airflow.utils.session import create_session from airflow.utils.types import DagRunType from tests.test_utils.api_connexion_utils import create_test_client +from tests.test_utils.config import conf_vars from tests.test_utils.www import check_content_in_response @@ -287,3 +288,32 @@ def test_trigger_dag_params_array_value_none_render(admin_client, dag_maker, ses f'', resp, ) + + +@pytest.mark.parametrize( + "pattern, run_id, result", + [ + ["^[A-Z]", "ABC", True], + ["^[A-Z]", "abc", False], + ["^[0-9]", "123", True], + # The below params tests that user configuration does not affect internally generated + # run_ids. We use manual__ as a prefix for manually triggered DAGs due to a restriction + # in manually triggered DAGs that the run_id must not start with scheduled__. + ["", "manual__2023-01-01T00:00:00+00:00", True], + ["", "scheduled_2023-01-01T00", False], + ["", "manual_2023-01-01T00", False], + ["", "dataset_triggered_2023-01-01T00", False], + ["^[0-9]", "manual__2023-01-01T00:00:00+00:00", True], + ["^[a-z]", "manual__2023-01-01T00:00:00+00:00", True], + ], +) +def test_dag_run_id_pattern(session, admin_client, pattern, run_id, result): + with conf_vars({("scheduler", "allowed_run_id_pattern"): pattern}): + test_dag_id = "example_bash_operator" + admin_client.post(f"dags/{test_dag_id}/trigger?&run_id={run_id}") + run = session.query(DagRun).filter(DagRun.dag_id == test_dag_id).first() + if result: + assert run is not None + assert run.run_type == DagRunType.MANUAL + else: + assert run is None From 00bf2ab2191fd7ebce34583007a4c06b7561b360 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Sun, 2 Jul 2023 11:47:29 -0400 Subject: [PATCH 018/533] Use reactflow for datasets graph (#31775) --- airflow/www/package.json | 2 - .../static/js/api/useDatasetDependencies.ts | 5 +- airflow/www/static/js/api/useGraphData.ts | 7 +- .../graph => components/Graph}/Edge.tsx | 20 +- .../www/static/js/dag/details/graph/index.tsx | 2 +- .../www/static/js/dag/details/graph/utils.ts | 3 +- .../www/static/js/datasets/Graph/DagNode.tsx | 5 +- airflow/www/static/js/datasets/Graph/Edge.tsx | 63 ------ .../www/static/js/datasets/Graph/Legend.tsx | 70 ++---- airflow/www/static/js/datasets/Graph/Node.tsx | 115 +++++----- .../www/static/js/datasets/Graph/index.tsx | 213 ++++++++++-------- airflow/www/static/js/datasets/index.tsx | 31 ++- airflow/www/static/js/types/index.ts | 12 + airflow/www/static/js/utils/graph.ts | 9 +- airflow/www/yarn.lock | 48 +--- 15 files changed, 252 insertions(+), 353 deletions(-) rename airflow/www/static/js/{dag/details/graph => components/Graph}/Edge.tsx (78%) delete mode 100644 airflow/www/static/js/datasets/Graph/Edge.tsx diff --git a/airflow/www/package.json b/airflow/www/package.json index 5eac3744663a..cf1727b45abc 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -92,9 +92,7 @@ "@emotion/react": "^11.9.3", "@emotion/styled": "^11", "@visx/group": "^2.10.0", - "@visx/marker": "^2.12.2", "@visx/shape": "^2.12.2", - "@visx/zoom": "^2.10.0", "axios": "^0.26.0", "bootstrap-3-typeahead": "^4.0.2", "camelcase-keys": "^7.0.0", diff --git a/airflow/www/static/js/api/useDatasetDependencies.ts b/airflow/www/static/js/api/useDatasetDependencies.ts index 3760cbfded13..2680ee9a782d 100644 --- a/airflow/www/static/js/api/useDatasetDependencies.ts +++ b/airflow/www/static/js/api/useDatasetDependencies.ts @@ -22,11 +22,10 @@ import { useQuery } from "react-query"; import ELK, { ElkShape, ElkExtendedEdge } from "elkjs"; import { getMetaValue } from "src/utils"; -import type { DepEdge, DepNode } from "src/types"; -import type { NodeType } from "src/datasets/Graph/Node"; - import { getTextWidth } from "src/utils/graph"; +import type { NodeType, DepEdge, DepNode } from "src/types"; + interface DatasetDependencies { edges: DepEdge[]; nodes: DepNode[]; diff --git a/airflow/www/static/js/api/useGraphData.ts b/airflow/www/static/js/api/useGraphData.ts index 9c0f089e9c75..e7a94156ef26 100644 --- a/airflow/www/static/js/api/useGraphData.ts +++ b/airflow/www/static/js/api/useGraphData.ts @@ -21,12 +21,12 @@ import { useQuery } from "react-query"; import axios, { AxiosResponse } from "axios"; import { getMetaValue } from "src/utils"; -import type { DepNode } from "src/types"; import useFilters, { FILTER_DOWNSTREAM_PARAM, FILTER_UPSTREAM_PARAM, ROOT_PARAM, } from "src/dag/useFilters"; +import type { WebserverEdge, DepNode } from "src/types"; const DAG_ID_PARAM = "dag_id"; @@ -38,11 +38,6 @@ interface GraphData { nodes: DepNode; arrange: string; } -export interface WebserverEdge { - label?: string; - sourceId: string; - targetId: string; -} const useGraphData = () => { const { diff --git a/airflow/www/static/js/dag/details/graph/Edge.tsx b/airflow/www/static/js/components/Graph/Edge.tsx similarity index 78% rename from airflow/www/static/js/dag/details/graph/Edge.tsx rename to airflow/www/static/js/components/Graph/Edge.tsx index 561048f79821..0205bf49cd3f 100644 --- a/airflow/www/static/js/dag/details/graph/Edge.tsx +++ b/airflow/www/static/js/components/Graph/Edge.tsx @@ -18,11 +18,11 @@ */ import React from "react"; -import { Text } from "@chakra-ui/react"; +import { Text, useTheme } from "@chakra-ui/react"; import type { ElkEdgeSection, ElkLabel, ElkPoint, LayoutOptions } from "elkjs"; -import Edge from "src/datasets/Graph/Edge"; import { Group } from "@visx/group"; +import { LinePath } from "@visx/shape"; interface EdgeProps { data?: { @@ -40,6 +40,7 @@ interface EdgeProps { } const CustomEdge = ({ data }: EdgeProps) => { + const { colors } = useTheme(); if (!data) return null; const { rest } = data; return ( @@ -54,11 +55,16 @@ const CustomEdge = ({ data }: EdgeProps) => { ); })} - + {(rest.sections || []).map((s) => ( + d.x || 0} + y={(d) => d.y || 0} + data={[s.startPoint, ...(s.bendPoints || []), s.endPoint]} + /> + ))} ); }; diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index 31578869cf2f..4dae9477bdc5 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -38,8 +38,8 @@ import { useGraphLayout } from "src/utils/graph"; import Tooltip from "src/components/Tooltip"; import { useContainerRef } from "src/context/containerRef"; import useFilters from "src/dag/useFilters"; +import Edge from "src/components/Graph/Edge"; -import Edge from "./Edge"; import Node, { CustomNodeProps } from "./Node"; import { buildEdges, nodeStrokeColor, nodeColor, flattenNodes } from "./utils"; diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index 703fb38b9ca9..998451e05ba1 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -23,8 +23,7 @@ import type { ElkExtendedEdge } from "elkjs"; import type { SelectionProps } from "src/dag/useSelection"; import { getTask } from "src/utils"; -import type { Task, TaskInstance } from "src/types"; -import type { NodeType } from "src/datasets/Graph/Node"; +import type { Task, TaskInstance, NodeType } from "src/types"; import type { CustomNodeProps } from "./Node"; diff --git a/airflow/www/static/js/datasets/Graph/DagNode.tsx b/airflow/www/static/js/datasets/Graph/DagNode.tsx index cb4900a5c31c..aa12c575221f 100644 --- a/airflow/www/static/js/datasets/Graph/DagNode.tsx +++ b/airflow/www/static/js/datasets/Graph/DagNode.tsx @@ -33,6 +33,7 @@ import { useTheme, } from "@chakra-ui/react"; import { MdOutlineAccountTree } from "react-icons/md"; + import { useContainerRef } from "src/context/containerRef"; import { getMetaValue } from "src/utils"; @@ -41,7 +42,7 @@ const DagNode = ({ isHighlighted, }: { dagId: string; - isHighlighted: boolean; + isHighlighted?: boolean; }) => { const { colors } = useTheme(); const containerRef = useContainerRef(); @@ -63,7 +64,7 @@ const DagNode = ({ alignItems="center" > - {dagId} + {dagId} diff --git a/airflow/www/static/js/datasets/Graph/Edge.tsx b/airflow/www/static/js/datasets/Graph/Edge.tsx deleted file mode 100644 index a5ef5fbfed3d..000000000000 --- a/airflow/www/static/js/datasets/Graph/Edge.tsx +++ /dev/null @@ -1,63 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import React from "react"; -import { LinePath } from "@visx/shape"; -import { MarkerArrow } from "@visx/marker"; -import { useTheme } from "@chakra-ui/react"; - -interface Props { - edge: { - id: string; - sources: string[]; - targets: string[]; - sections: Record[]; - }; - isSelected?: boolean; - showMarker?: boolean; -} - -const Edge = ({ edge, isSelected = false, showMarker = true }: Props) => { - const { colors } = useTheme(); - return ( - <> - {showMarker && ( - - )} - {(edge.sections || []).map((s) => ( - d.x || 0} - y={(d) => d.y || 0} - data={[s.startPoint, ...(s.bendPoints || []), s.endPoint]} - markerEnd="url(#marker-arrow)" - /> - ))} - - ); -}; - -export default Edge; diff --git a/airflow/www/static/js/datasets/Graph/Legend.tsx b/airflow/www/static/js/datasets/Graph/Legend.tsx index 250d6852b9b4..3ecbed766473 100644 --- a/airflow/www/static/js/datasets/Graph/Legend.tsx +++ b/airflow/www/static/js/datasets/Graph/Legend.tsx @@ -18,59 +18,29 @@ */ import React from "react"; -import { Flex, Box, IconButton, Text } from "@chakra-ui/react"; -import { - MdOutlineZoomOutMap, - MdFilterCenterFocus, - MdOutlineAccountTree, -} from "react-icons/md"; +import { Flex, Box, Text } from "@chakra-ui/react"; +import { MdOutlineAccountTree } from "react-icons/md"; import { HiDatabase } from "react-icons/hi"; -interface Props { - zoom: any; - center: () => void; -} - -const Legend = ({ zoom, center }: Props) => ( - - - } - /> - } - /> - - - Legend - - - - DAG - - - - Dataset - +const Legend = () => ( + + + + + DAG + + + + Dataset - - + + ); export default Legend; diff --git a/airflow/www/static/js/datasets/Graph/Node.tsx b/airflow/www/static/js/datasets/Graph/Node.tsx index d692861b66b4..ed02653139c4 100644 --- a/airflow/www/static/js/datasets/Graph/Node.tsx +++ b/airflow/www/static/js/datasets/Graph/Node.tsx @@ -18,71 +18,76 @@ */ import React from "react"; -import { Flex, Text, useTheme } from "@chakra-ui/react"; -import { Group } from "@visx/group"; +import { Box, Text, Flex, useTheme } from "@chakra-ui/react"; +import { Handle, NodeProps, Position } from "reactflow"; import { MdPlayArrow, MdSensors } from "react-icons/md"; import { HiDatabase } from "react-icons/hi"; -import type { ElkShape } from "elkjs"; -import type { DepNode } from "src/types"; - import DagNode from "./DagNode"; -export interface NodeType extends ElkShape { - value: DepNode["value"]; - children?: NodeType[]; -} - -interface Props { - node: NodeType; - onSelect: (datasetId: string) => void; - isSelected: boolean; - isHighlighted: boolean; +export interface CustomNodeProps { + label: string; + type?: string; + height?: number; + width?: number; + isSelected?: boolean; + isHighlighted?: boolean; + onSelect: (datasetUri: string) => void; + isOpen?: boolean; + isActive?: boolean; } -const Node = ({ - node: { height, width, x, y, value }, - onSelect, - isSelected, - isHighlighted, -}: Props) => { +const BaseNode = ({ + data: { label, type, isSelected, isHighlighted, onSelect }, +}: NodeProps) => { const { colors } = useTheme(); + return ( - - - {value.class === "dag" && ( - - )} - {value.class !== "dag" && ( - { - e.preventDefault(); - e.stopPropagation(); - if (value.class === "dataset") onSelect(value.label); - }} - cursor="pointer" - fontSize={16} - justifyContent="space-between" - alignItems="center" - > - {value.class === "dataset" && } - {value.class === "sensor" && } - {value.class === "trigger" && } - {value.label} - - )} - - + + {type === "dag" && ( + + )} + {type !== "dag" && ( + { + e.preventDefault(); + e.stopPropagation(); + if (type === "dataset") onSelect(label); + }} + cursor="pointer" + fontSize={16} + justifyContent="space-between" + alignItems="center" + > + {type === "dataset" && } + {type === "sensor" && } + {type === "trigger" && } + {label} + + )} + ); }; +const Node = (props: NodeProps) => ( + <> + + + + +); + export default Node; diff --git a/airflow/www/static/js/datasets/Graph/index.tsx b/airflow/www/static/js/datasets/Graph/index.tsx index f8693659451a..f137be36d107 100644 --- a/airflow/www/static/js/datasets/Graph/index.tsx +++ b/airflow/www/static/js/datasets/Graph/index.tsx @@ -17,28 +17,50 @@ * under the License. */ -import React, { RefObject } from "react"; -import { Box, Spinner } from "@chakra-ui/react"; -import { Zoom } from "@visx/zoom"; -import { Group } from "@visx/group"; +import React, { useEffect } from "react"; +import ReactFlow, { + ReactFlowProvider, + Controls, + Background, + MiniMap, + Node as ReactFlowNode, + useReactFlow, + ControlButton, + Panel, +} from "reactflow"; +import { Box, Tooltip, useTheme } from "@chakra-ui/react"; +import { RiFocus3Line } from "react-icons/ri"; import { useDatasetDependencies } from "src/api"; +import Edge from "src/components/Graph/Edge"; +import { useContainerRef } from "src/context/containerRef"; -import Node from "./Node"; -import Edge from "./Edge"; +import Node, { CustomNodeProps } from "./Node"; import Legend from "./Legend"; interface Props { onSelect: (datasetId: string) => void; selectedUri: string | null; - height: number; - width: number; } -const Graph = ({ onSelect, selectedUri, height, width }: Props) => { - const { data, isLoading } = useDatasetDependencies(); +const nodeTypes = { custom: Node }; +const edgeTypes = { custom: Edge }; + +const Graph = ({ onSelect, selectedUri }: Props) => { + const { data } = useDatasetDependencies(); + const { colors } = useTheme(); + const { setCenter, setViewport } = useReactFlow(); + const containerRef = useContainerRef(); + + useEffect(() => { + setViewport({ x: 0, y: 0, zoom: 1 }); + }, [selectedUri, setViewport]); + + const nodeColor = ({ + data: { isSelected }, + }: ReactFlowNode) => + isSelected ? colors.blue["300"] : colors.gray["300"]; - if (isLoading && !data) return ; if (!data || !data.fullGraph || !data.subGraphs) return null; const graph = selectedUri ? data.subGraphs.find((g) => @@ -46,91 +68,102 @@ const Graph = ({ onSelect, selectedUri, height, width }: Props) => { ) : data.fullGraph; if (!graph) return null; - const { edges, children, width: graphWidth, height: graphHeight } = graph; - const initialTransform = { - scaleX: 1, - scaleY: 1, - translateX: 0, - translateY: 0, - skewX: 0, - skewY: 0, - }; + const edges = graph.edges.map((e) => ({ + id: e.id, + source: e.sources[0], + target: e.targets[0], + type: "custom", + data: { + rest: { + ...e, + isSelected: selectedUri && e.id.includes(selectedUri), + }, + }, + })); - const selectedEdges = selectedUri - ? edges?.filter( - ({ sources, targets }) => - sources[0].includes(selectedUri) || targets[0].includes(selectedUri) - ) - : []; - const highlightedNodes = children.filter((n) => - selectedEdges.some( - ({ sources, targets }) => sources[0] === n.id || targets[0] === n.id - ) - ); + const nodes: ReactFlowNode[] = graph.children.map((c) => ({ + id: c.id, + data: { + label: c.value.label, + type: c.value.class, + width: c.width, + height: c.height, + onSelect, + isSelected: selectedUri === c.value.label, + isHighlighted: edges.some( + (e) => e.data.rest.isSelected && e.id.includes(c.id) + ), + }, + type: "custom", + position: { + x: c.x || 0, + y: c.y || 0, + }, + })); + + const focusNode = () => { + if (selectedUri) { + const node = nodes.find((n) => n.data.label === selectedUri); + if (!node || !node.position) return; + const { x, y } = node.position; + setCenter( + x + (node.data.width || 0) / 2, + y + (node.data.height || 0) / 2, + { + duration: 1000, + } + ); + } + }; return ( - - {(zoom) => ( - - } - style={{ - cursor: zoom.isDragging ? "grabbing" : "grab", - touchAction: "none", - }} + + + + - - - {edges.map((edge) => ( - e.id === edge.id)} - /> - ))} - {children.map((node) => ( - n.id === node.id - )} - /> - ))} - - - - - - zoom.translateTo({ - x: (width - (graphWidth ?? 0)) / 2, - y: (height - (graphHeight ?? 0)) / 2, - }) - } - /> - - - - - )} - + + + + + + + + + + + ); }; -export default Graph; +const GraphWrapper = (props: Props) => ( + + + +); + +export default GraphWrapper; diff --git a/airflow/www/static/js/datasets/index.tsx b/airflow/www/static/js/datasets/index.tsx index bd21ca26f426..166b3f26ec99 100644 --- a/airflow/www/static/js/datasets/index.tsx +++ b/airflow/www/static/js/datasets/index.tsx @@ -23,9 +23,11 @@ import React, { useRef } from "react"; import { createRoot } from "react-dom/client"; import createCache from "@emotion/cache"; import { useSearchParams } from "react-router-dom"; -import { Flex, Box, useDimensions } from "@chakra-ui/react"; +import { Flex, Box } from "@chakra-ui/react"; +import reactFlowStyle from "reactflow/dist/style.css"; import App from "src/App"; +import { useOffsetTop } from "src/utils"; import DatasetsList from "./List"; import DatasetDetails from "./Details"; @@ -45,8 +47,9 @@ const DATASET_URI = "uri"; const Datasets = () => { const [searchParams, setSearchParams] = useSearchParams(); const contentRef = useRef(null); - const graphRef = useRef(null); - const dimensions = useDimensions(graphRef, true); + const offsetTop = useOffsetTop(contentRef); + // 60px for footer height + const height = `calc(100vh - ${offsetTop + 60}px)`; const onBack = () => { searchParams.delete(DATASET_URI); @@ -66,26 +69,15 @@ const Datasets = () => { justifyContent="space-between" ref={contentRef} > - + {datasetUri ? ( ) : ( )} - - + + ); @@ -93,6 +85,11 @@ const Datasets = () => { if (mainElement) { shadowRoot?.appendChild(mainElement); + const styleTag = document.createElement("style"); + const style = reactFlowStyle.toString(); + styleTag.innerHTML = style; + shadowRoot?.appendChild(styleTag); + const reactRoot = createRoot(mainElement); reactRoot.render( diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts index 45b34c924fa7..08929d33c400 100644 --- a/airflow/www/static/js/types/index.ts +++ b/airflow/www/static/js/types/index.ts @@ -18,6 +18,7 @@ */ import type { CamelCase } from "type-fest"; +import type { ElkShape } from "elkjs"; import type * as API from "./api-generated"; type RunState = "success" | "running" | "queued" | "failed"; @@ -127,6 +128,17 @@ interface DepEdge { target: string; } +export interface NodeType extends ElkShape { + value: DepNode["value"]; + children?: NodeType[]; +} + +export interface WebserverEdge { + label?: string; + sourceId: string; + targetId: string; +} + interface DatasetListItem extends API.Dataset { lastDatasetUpdate: string | null; totalUpdates: number; diff --git a/airflow/www/static/js/utils/graph.ts b/airflow/www/static/js/utils/graph.ts index d2d2923c5ab8..e69a3045d945 100644 --- a/airflow/www/static/js/utils/graph.ts +++ b/airflow/www/static/js/utils/graph.ts @@ -19,8 +19,7 @@ import ELK, { ElkExtendedEdge, ElkShape } from "elkjs"; -import type { DepNode } from "src/types"; -import type { NodeType } from "src/datasets/Graph/Node"; +import type { NodeType, DepNode, WebserverEdge } from "src/types"; import { useQuery } from "react-query"; import useFilters from "src/dag/useFilters"; @@ -32,12 +31,6 @@ interface GenerateProps { arrange: string; } -interface WebserverEdge { - label?: string; - sourceId: string; - targetId: string; -} - interface Graph extends ElkShape { children: NodeType[]; edges: ElkExtendedEdge[]; diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 6e1fce53343c..1916d9c9001a 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -3575,18 +3575,6 @@ "@typescript-eslint/types" "5.27.1" eslint-visitor-keys "^3.3.0" -"@use-gesture/core@10.2.17": - version "10.2.17" - resolved "https://registry.yarnpkg.com/@use-gesture/core/-/core-10.2.17.tgz#dc78913cd5d105217c3f1d1c16a32ad6426a00ba" - integrity sha512-62hCybe4x6oGZ1/JA9gSYIdghV1FqxCdvYWt9SqCEAAikwT1OmVl2Q/Uu8CP636L57D+DfXtw6PWM+fdhr4oJQ== - -"@use-gesture/react@^10.0.0-beta.22": - version "10.2.17" - resolved "https://registry.yarnpkg.com/@use-gesture/react/-/react-10.2.17.tgz#00bc413da42a358dd3f9173c0631b54522e76614" - integrity sha512-Vfrp1KgdYn/kOEUAYNXtGBCl2dr38s3G6rru1TOPs+cVUjfNyNxvJK56grUyJ336N3rQLK8F9G7+FfrHuc3g/Q== - dependencies: - "@use-gesture/core" "10.2.17" - "@visx/curve@2.1.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@visx/curve/-/curve-2.1.0.tgz#f614bfe3db66df7db7382db7a75ced1506b94602" @@ -3595,14 +3583,6 @@ "@types/d3-shape" "^1.3.1" d3-shape "^1.0.6" -"@visx/event@2.6.0": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@visx/event/-/event-2.6.0.tgz#0718eb1efabd5305cf659a153779c94ba4038996" - integrity sha512-WGp91g82s727g3NAnENF1ppC3ZAlvWg+Y+GG0WFg34NmmOZbvPI/PTOqTqZE3x6B8EUn8NJiMxRjxIMbi+IvRw== - dependencies: - "@types/react" "*" - "@visx/point" "2.6.0" - "@visx/group@2.10.0", "@visx/group@^2.10.0": version "2.10.0" resolved "https://registry.yarnpkg.com/@visx/group/-/group-2.10.0.tgz#95839851832545621eb0d091866a61dafe552ae1" @@ -3612,22 +3592,6 @@ classnames "^2.3.1" prop-types "^15.6.2" -"@visx/marker@^2.12.2": - version "2.12.2" - resolved "https://registry.yarnpkg.com/@visx/marker/-/marker-2.12.2.tgz#b81cea1a5d2b61c065aa97e4baccf9d0f17cab51" - integrity sha512-yvJDMBw9oKQDD2gX5q7O+raR9qk/NYqKFDZ0GtS4ZVH87PfNe0ZyTXt0vWbIaDaix/r58SMpv38GluIOxWE7jg== - dependencies: - "@types/react" "*" - "@visx/group" "2.10.0" - "@visx/shape" "2.12.2" - classnames "^2.3.1" - prop-types "^15.6.2" - -"@visx/point@2.6.0": - version "2.6.0" - resolved "https://registry.yarnpkg.com/@visx/point/-/point-2.6.0.tgz#c4316ca409b5b829c5455f07118d8c14a92cc633" - integrity sha512-amBi7yMz4S2VSchlPdliznN41TuES64506ySI22DeKQ+mc1s1+BudlpnY90sM1EIw4xnqbKmrghTTGfy6SVqvQ== - "@visx/scale@2.2.2": version "2.2.2" resolved "https://registry.yarnpkg.com/@visx/scale/-/scale-2.2.2.tgz#b8eafabdcf92bb45ab196058fe184772ad80fd25" @@ -3640,7 +3604,7 @@ d3-scale "^3.3.0" d3-time "^2.1.1" -"@visx/shape@2.12.2", "@visx/shape@^2.12.2": +"@visx/shape@^2.12.2": version "2.12.2" resolved "https://registry.yarnpkg.com/@visx/shape/-/shape-2.12.2.tgz#81ed88bf823aa84a4f5f32a9c9daf8371a606897" integrity sha512-4gN0fyHWYXiJ+Ck8VAazXX0i8TOnLJvOc5jZBnaJDVxgnSIfCjJn0+Nsy96l9Dy/bCMTh4DBYUBv9k+YICBUOA== @@ -3658,16 +3622,6 @@ lodash "^4.17.21" prop-types "^15.5.10" -"@visx/zoom@^2.10.0": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@visx/zoom/-/zoom-2.10.0.tgz#143248813a35d2057eaf1a6336011d8650955533" - integrity sha512-sId1kuO3NvlzQTOorjeMWXRR3J44zQm8sofwKEt3O9IgaBZ49WzuPUm/owSdVT+YGsXnvxEr2qAdt26GRMzS7Q== - dependencies: - "@types/react" "*" - "@use-gesture/react" "^10.0.0-beta.22" - "@visx/event" "2.6.0" - prop-types "^15.6.2" - "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" From 1c07ef076108e2eb92a596ff27ea4eb6099893db Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 2 Jul 2023 20:38:08 +0200 Subject: [PATCH 019/533] Fix SemVer link in security documentation (#32320) The SemVer link was an md-style one, we should change it rst-style --- docs/apache-airflow/security/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/security/index.rst b/docs/apache-airflow/security/index.rst index 3573348f1006..4b80e06e8d2d 100644 --- a/docs/apache-airflow/security/index.rst +++ b/docs/apache-airflow/security/index.rst @@ -156,7 +156,7 @@ MUST upgrade to the latest ``MINOR`` version of Airflow. Releasing Airflow providers with security patches ------------------------------------------------- -Similarly to Airflow, providers use strict [SemVer](https://semver.org) versioning policy, and the same +Similarly to Airflow, providers use strict `SemVer `_ versioning policy, and the same policies apply for providers as for Airflow itself. This means that you need to upgrade to the latest ``MINOR`` version of the provider to get the latest security fixes. Airflow providers are released independently from Airflow itself and the information about vulnerabilities From 88da71ed1fdffc558de28d5c3fd78e5ae1ac4e8c Mon Sep 17 00:00:00 2001 From: Sai Pragna Etikyala Date: Sun, 2 Jul 2023 14:26:32 -0700 Subject: [PATCH 020/533] Add Twilio to the list of Apache Airflow platform users (#32314) --------- Co-authored-by: Pankaj Koti --- INTHEWILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INTHEWILD.md b/INTHEWILD.md index da58858618d4..d203db35c5ca 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -460,6 +460,7 @@ Currently, **officially** using Airflow: 1. [Travix](https://www.travix.com/) 1. [Trocafone](https://www.trocafone.com/) [[@idontdomath](https://github.com/idontdomath) & [@gseva](https://github.com/gseva) & [@ordonezf](https://github.com/ordonezf) & [@PalmaLeandro](https://github.com/PalmaLeandro)] 1. [TruFactor](https://trufactor.io/) [[@gholmes](https://github.com/gholmes) & [@angadsingh](https://github.com/angadsingh/)] +1. [Twilio](https://www.twilio.com/) [[@twilio](https://github.com/twilio)] 1. [Twine Labs](https://www.twinelabs.com/) [[@ivorpeles](https://github.com/ivorpeles)] 1. [Twitter](https://www.twitter.com/) [[@aoen](https://github.com/aoen)] 1. [Ubisoft](https://www.ubisoft.com/) [[@Walkoss](https://github.com/Walkoss)] From 5c72befcfde63ade2870491cfeb708675399d9d6 Mon Sep 17 00:00:00 2001 From: Elad Galili <33863800+eladi99@users.noreply.github.com> Date: Mon, 3 Jul 2023 09:45:24 +0300 Subject: [PATCH 021/533] Fix `LambdaInvokeFunctionOperator` payload parameter type (#32259) * Fixing issue - Fix payload parameter of amazon LambdaCreateFunctionOperator --------- Co-authored-by: Elad Galili --- .../amazon/aws/hooks/lambda_function.py | 5 +++- .../amazon/aws/operators/lambda_function.py | 2 +- .../amazon/aws/hooks/test_lambda_function.py | 11 ++++++--- .../aws/operators/test_lambda_function.py | 23 +++++++++++++------ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/lambda_function.py b/airflow/providers/amazon/aws/hooks/lambda_function.py index 2d61f0751f7c..58ecac8bcccb 100644 --- a/airflow/providers/amazon/aws/hooks/lambda_function.py +++ b/airflow/providers/amazon/aws/hooks/lambda_function.py @@ -48,7 +48,7 @@ def invoke_lambda( invocation_type: str | None = None, log_type: str | None = None, client_context: str | None = None, - payload: str | None = None, + payload: bytes | str | None = None, qualifier: str | None = None, ): """ @@ -65,6 +65,9 @@ def invoke_lambda( :param payload: The JSON that you want to provide to your Lambda function as input. :param qualifier: AWS Lambda Function Version or Alias Name """ + if isinstance(payload, str): + payload = payload.encode() + invoke_args = { "FunctionName": function_name, "InvocationType": invocation_type, diff --git a/airflow/providers/amazon/aws/operators/lambda_function.py b/airflow/providers/amazon/aws/operators/lambda_function.py index 93907634c12d..28b631320422 100644 --- a/airflow/providers/amazon/aws/operators/lambda_function.py +++ b/airflow/providers/amazon/aws/operators/lambda_function.py @@ -150,7 +150,7 @@ def __init__( qualifier: str | None = None, invocation_type: str | None = None, client_context: str | None = None, - payload: str | None = None, + payload: bytes | str | None = None, aws_conn_id: str = "aws_default", **kwargs, ): diff --git a/tests/providers/amazon/aws/hooks/test_lambda_function.py b/tests/providers/amazon/aws/hooks/test_lambda_function.py index f21c000ea6d9..caaf164be47b 100644 --- a/tests/providers/amazon/aws/hooks/test_lambda_function.py +++ b/tests/providers/amazon/aws/hooks/test_lambda_function.py @@ -26,6 +26,7 @@ FUNCTION_NAME = "test_function" PAYLOAD = '{"hello": "airflow"}' +BYTES_PAYLOAD = b'{"hello": "airflow"}' RUNTIME = "python3.9" ROLE = "role" HANDLER = "handler" @@ -48,13 +49,17 @@ def test_get_conn_returns_a_boto3_connection(self, hook): @mock.patch( "airflow.providers.amazon.aws.hooks.lambda_function.LambdaHook.conn", new_callable=mock.PropertyMock ) - def test_invoke_lambda(self, mock_conn): + @pytest.mark.parametrize( + "payload, invoke_payload", + [(PAYLOAD, BYTES_PAYLOAD), (BYTES_PAYLOAD, BYTES_PAYLOAD)], + ) + def test_invoke_lambda(self, mock_conn, payload, invoke_payload): hook = LambdaHook() - hook.invoke_lambda(function_name=FUNCTION_NAME, payload=PAYLOAD) + hook.invoke_lambda(function_name=FUNCTION_NAME, payload=payload) mock_conn().invoke.assert_called_once_with( FunctionName=FUNCTION_NAME, - Payload=PAYLOAD, + Payload=invoke_payload, ) @pytest.mark.parametrize( diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/tests/providers/amazon/aws/operators/test_lambda_function.py index 6f1d98e8ac8b..f0b4b834eb00 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/tests/providers/amazon/aws/operators/test_lambda_function.py @@ -17,7 +17,6 @@ # under the License. from __future__ import annotations -import json from unittest import mock from unittest.mock import Mock, patch @@ -30,6 +29,8 @@ ) FUNCTION_NAME = "function_name" +PAYLOAD = '{"hello": "airflow"}' +BYTES_PAYLOAD = b'{"hello": "airflow"}' ROLE_ARN = "role_arn" IMAGE_URI = "image_uri" @@ -70,29 +71,37 @@ def test_create_lambda_with_wait_for_completion(self, mock_hook_conn, mock_hook_ class TestLambdaInvokeFunctionOperator: - def test_init(self): + @pytest.mark.parametrize( + "payload", + [PAYLOAD, BYTES_PAYLOAD], + ) + def test_init(self, payload): lambda_operator = LambdaInvokeFunctionOperator( task_id="test", function_name="test", - payload=json.dumps({"TestInput": "Testdata"}), + payload=payload, log_type="None", aws_conn_id="aws_conn_test", ) assert lambda_operator.task_id == "test" assert lambda_operator.function_name == "test" - assert lambda_operator.payload == json.dumps({"TestInput": "Testdata"}) + assert lambda_operator.payload == payload assert lambda_operator.log_type == "None" assert lambda_operator.aws_conn_id == "aws_conn_test" @patch.object(LambdaInvokeFunctionOperator, "hook", new_callable=mock.PropertyMock) - def test_invoke_lambda(self, hook_mock): + @pytest.mark.parametrize( + "payload", + [PAYLOAD, BYTES_PAYLOAD], + ) + def test_invoke_lambda(self, hook_mock, payload): operator = LambdaInvokeFunctionOperator( task_id="task_test", function_name="a", invocation_type="b", log_type="c", client_context="d", - payload="e", + payload=payload, qualifier="f", ) returned_payload = Mock() @@ -111,7 +120,7 @@ def test_invoke_lambda(self, hook_mock): invocation_type="b", log_type="c", client_context="d", - payload="e", + payload=payload, qualifier="f", ) From b6eeccdfb67409f5035adc3ab8af97c4c03a40dc Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 3 Jul 2023 09:42:26 +0200 Subject: [PATCH 022/533] Fix ts/eslint warnings (#32321) --- airflow/www/alias-rest-types.js | 1 + airflow/www/babel.config.js | 2 +- airflow/www/jest-setup.js | 2 ++ airflow/www/package.json | 2 +- airflow/www/static/js/api/useDatasets.ts | 2 +- airflow/www/static/js/api/useExtraLinks.ts | 7 +++++-- airflow/www/static/js/callModal.js | 3 +++ .../static/js/cluster-activity/index.test.tsx | 12 +++++++----- .../www/static/js/components/Table/Cells.tsx | 2 ++ .../static/js/components/Table/Table.test.tsx | 2 +- .../www/static/js/components/Table/index.tsx | 6 ++++-- airflow/www/static/js/components/Tooltip.tsx | 3 ++- airflow/www/static/js/connection_form.js | 4 ++++ .../www/static/js/dag/details/graph/utils.ts | 2 +- .../static/js/dag/grid/dagRuns/index.test.tsx | 1 + airflow/www/static/js/dag/grid/index.test.tsx | 1 + airflow/www/static/js/dag_dependencies.js | 2 ++ airflow/www/static/js/dags.js | 1 + airflow/www/static/js/datasets/List.test.tsx | 12 +++++++++--- airflow/www/static/js/datasets/List.tsx | 2 ++ airflow/www/static/js/datetime_utils.js | 1 + airflow/www/static/js/gantt.js | 13 ++++++++++++- airflow/www/static/js/graph.js | 1 + airflow/www/static/js/main.js | 19 ++++++++++--------- airflow/www/static/js/task.js | 2 ++ airflow/www/static/js/task_instances.js | 2 ++ airflow/www/static/js/ti_log.js | 9 ++++++--- .../static/js/types/react-table-config.d.ts | 4 ++-- .../static/js/utils/URLSearchParamWrapper.ts | 3 ++- airflow/www/static/js/utils/graph.ts | 14 +++++++++++--- 30 files changed, 100 insertions(+), 37 deletions(-) diff --git a/airflow/www/alias-rest-types.js b/airflow/www/alias-rest-types.js index 83ddd89aaee8..c2767acca6e8 100644 --- a/airflow/www/alias-rest-types.js +++ b/airflow/www/alias-rest-types.js @@ -225,6 +225,7 @@ function generate(file) { fs.writeFileSync(file, finalText, (err) => { if (err) { + // eslint-disable-next-line no-console console.error(err); } }); diff --git a/airflow/www/babel.config.js b/airflow/www/babel.config.js index f38c60b91fb1..7f40a9fdf6b3 100644 --- a/airflow/www/babel.config.js +++ b/airflow/www/babel.config.js @@ -17,7 +17,7 @@ * under the License. */ -module.exports = function (api) { +module.exports = (api) => { api.cache(true); const presets = [ diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js index fcab19bc8f41..b2e403c25ef9 100644 --- a/airflow/www/jest-setup.js +++ b/airflow/www/jest-setup.js @@ -32,8 +32,10 @@ axios.defaults.adapter = require("axios/lib/adapters/http"); axios.interceptors.response.use((res) => res.data || res); setLogger({ + /* eslint-disable no-console */ log: console.log, warn: console.warn, + /* eslint-enable no-console */ // ✅ no more errors on the console error: () => {}, }); diff --git a/airflow/www/package.json b/airflow/www/package.json index cf1727b45abc..c5a9c69e6c1a 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -7,7 +7,7 @@ "dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development", "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress", "build": "NODE_ENV=production webpack --progress --mode production", - "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc", + "lint": "eslint --ignore-path=.eslintignore --max-warnings=0 --ext .js,.jsx,.ts,.tsx . && tsc", "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc", "format": "yarn prettier --write .", "generate-api-types": "npx openapi-typescript \"../api_connexion/openapi/v1.yaml\" --output static/js/types/api-generated.ts && node alias-rest-types.js static/js/types/api-generated.ts" diff --git a/airflow/www/static/js/api/useDatasets.ts b/airflow/www/static/js/api/useDatasets.ts index 379d3c3e61b2..62d2583d905a 100644 --- a/airflow/www/static/js/api/useDatasets.ts +++ b/airflow/www/static/js/api/useDatasets.ts @@ -24,7 +24,7 @@ import { getMetaValue } from "src/utils"; import type { DatasetListItem } from "src/types"; import type { unitOfTime } from "moment"; -interface DatasetsData { +export interface DatasetsData { datasets: DatasetListItem[]; totalEntries: number; } diff --git a/airflow/www/static/js/api/useExtraLinks.ts b/airflow/www/static/js/api/useExtraLinks.ts index 302d58aa536c..3d654e8cbc75 100644 --- a/airflow/www/static/js/api/useExtraLinks.ts +++ b/airflow/www/static/js/api/useExtraLinks.ts @@ -46,14 +46,16 @@ export default function useExtraLinks({ async () => { const data = await Promise.all( extraLinks.map(async (link) => { - mapIndex ??= -1; + const definedMapIndex = mapIndex ?? -1; const url = `${extraLinksUrl}?task_id=${encodeURIComponent( taskId )}&dag_id=${encodeURIComponent( dagId )}&execution_date=${encodeURIComponent( executionDate - )}&link_name=${encodeURIComponent(link)}&map_index=${mapIndex}`; + )}&link_name=${encodeURIComponent( + link + )}&map_index=${definedMapIndex}`; try { const datum = await axios.get(url); return { @@ -61,6 +63,7 @@ export default function useExtraLinks({ url: datum.url, }; } catch (e) { + // eslint-disable-next-line no-console console.error(e); return { name: link, diff --git a/airflow/www/static/js/callModal.js b/airflow/www/static/js/callModal.js index a2cac9f26b42..050c7eebffa2 100644 --- a/airflow/www/static/js/callModal.js +++ b/airflow/www/static/js/callModal.js @@ -56,6 +56,7 @@ const showExternalLogRedirect = const buttons = Array.from( document.querySelectorAll('a[id^="btn_"][data-base-url]') ).reduce((obj, elm) => { + // eslint-disable-next-line no-param-reassign obj[elm.id.replace("btn_", "")] = elm; return obj; }, {}); @@ -64,12 +65,14 @@ function updateButtonUrl(elm, params) { let url = elm.dataset.baseUrl; if (params.dag_id && elm.dataset.baseUrl.indexOf(dagId) !== -1) { url = url.replace(dagId, params.dag_id); + // eslint-disable-next-line no-param-reassign delete params.dag_id; } if ( Object.prototype.hasOwnProperty.call(params, "map_index") && params.map_index === undefined ) { + // eslint-disable-next-line no-param-reassign delete params.map_index; } elm.setAttribute("href", `${url}?${$.param(params)}`); diff --git a/airflow/www/static/js/cluster-activity/index.test.tsx b/airflow/www/static/js/cluster-activity/index.test.tsx index f12c4c159abc..13f6d4234434 100644 --- a/airflow/www/static/js/cluster-activity/index.test.tsx +++ b/airflow/www/static/js/cluster-activity/index.test.tsx @@ -29,6 +29,8 @@ import * as useHealthModule from "src/api/useHealth"; import { render } from "@testing-library/react"; import { Wrapper } from "src/utils/testUtils"; +import type { UseQueryResult } from "react-query"; +import type { API, HistoricalMetricsData } from "src/types"; import ClusterActivity from "."; const mockHistoricalMetricsData = { @@ -100,7 +102,7 @@ describe("Test ToggleGroups", () => { ({ data: mockHistoricalMetricsData, isSuccess: true, - } as any) + } as never as UseQueryResult) ); jest.spyOn(useHealthModule, "default").mockImplementation( @@ -108,7 +110,7 @@ describe("Test ToggleGroups", () => { ({ data: mockHealthData, isSuccess: true, - } as any) + } as never as UseQueryResult) ); jest.spyOn(useDagsModule, "default").mockImplementation( @@ -116,7 +118,7 @@ describe("Test ToggleGroups", () => { ({ data: mockDagsData, isSuccess: true, - } as any) + } as never as UseQueryResult) ); jest.spyOn(useDagRunsModule, "default").mockImplementation( @@ -124,7 +126,7 @@ describe("Test ToggleGroups", () => { ({ data: mockDagRunsData, isSuccess: true, - } as any) + } as never as UseQueryResult) ); jest.spyOn(usePoolsModule, "default").mockImplementation( @@ -132,7 +134,7 @@ describe("Test ToggleGroups", () => { ({ data: mockPoolsData, isSuccess: true, - } as any) + } as never as UseQueryResult) ); }); diff --git a/airflow/www/static/js/components/Table/Cells.tsx b/airflow/www/static/js/components/Table/Cells.tsx index 8084eb917c68..9e86a74935f1 100644 --- a/airflow/www/static/js/components/Table/Cells.tsx +++ b/airflow/www/static/js/components/Table/Cells.tsx @@ -41,8 +41,10 @@ import { SimpleStatus } from "src/dag/StatusBox"; interface CellProps { cell: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any value: any; row: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any original: Record; }; }; diff --git a/airflow/www/static/js/components/Table/Table.test.tsx b/airflow/www/static/js/components/Table/Table.test.tsx index a3664d49a237..36d9b22456e2 100644 --- a/airflow/www/static/js/components/Table/Table.test.tsx +++ b/airflow/www/static/js/components/Table/Table.test.tsx @@ -28,7 +28,7 @@ import type { SortingRule } from "react-table"; import { ChakraWrapper } from "src/utils/testUtils"; import { Table } from "."; -const data: Record[] = [ +const data: Record[] = [ { firstName: "Lamont", lastName: "Grimes", country: "United States" }, { firstName: "Alysa", lastName: "Armstrong", country: "Spain" }, { firstName: "Petra", lastName: "Blick", country: "France" }, diff --git a/airflow/www/static/js/components/Table/index.tsx b/airflow/www/static/js/components/Table/index.tsx index a2450707b385..753035da0723 100644 --- a/airflow/www/static/js/components/Table/index.tsx +++ b/airflow/www/static/js/components/Table/index.tsx @@ -89,7 +89,7 @@ interface TableProps { pageSize?: number; isLoading?: boolean; selectRows?: (selectedRows: number[]) => void; - onRowClicked?: (row: Row, e: any) => void; + onRowClicked?: (row: Row, e: unknown) => void; } export const Table = ({ @@ -244,7 +244,9 @@ export const Table = ({ } } onClick={ - onRowClicked ? (e: any) => onRowClicked(row, e) : undefined + onRowClicked + ? (e: unknown) => onRowClicked(row, e) + : undefined } > {row.cells.map((cell) => ( diff --git a/airflow/www/static/js/components/Tooltip.tsx b/airflow/www/static/js/components/Tooltip.tsx index fc09cefc1f88..72a55b0862da 100644 --- a/airflow/www/static/js/components/Tooltip.tsx +++ b/airflow/www/static/js/components/Tooltip.tsx @@ -136,7 +136,7 @@ const Tooltip = forwardRef((props, ref) => { * Ensure tooltip has only one child node */ const child = React.Children.only(children) as React.ReactElement & { - ref?: React.Ref; + ref?: React.Ref; }; const trigger: React.ReactElement = React.cloneElement( child, @@ -168,6 +168,7 @@ const Tooltip = forwardRef((props, ref) => { > { + // eslint-disable-next-line no-param-reassign elem.innerText = elem.dataset.origText; + // eslint-disable-next-line no-param-reassign delete elem.dataset.origText; } ); Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); @@ -259,6 +262,7 @@ $(document).ready(() => { - All other custom form fields (i.e. fields that are named ``extra__...``) in alphabetical order */ + // eslint-disable-next-line func-names $.each(inArray, function () { if (this.name === "conn_id") { outObj.connection_id = this.value; diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index 998451e05ba1..459632024692 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -131,7 +131,7 @@ export const nodeColor = ({ export const nodeStrokeColor = ( { data: { isSelected } }: ReactFlowNode, - colors: any + colors: Record ) => (isSelected ? colors.blue[500] : ""); interface BuildEdgesProps { diff --git a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx index d77f30af322d..2ac20ea3ccbb 100644 --- a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx +++ b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx @@ -48,6 +48,7 @@ const generateRuns = (length: number): DagRun[] => note: "someRandomValue", })); +/* eslint-disable @typescript-eslint/no-explicit-any */ describe("Test DagRuns", () => { test("Durations and manual run arrow render correctly, but without any date ticks", () => { const dagRuns: DagRun[] = [ diff --git a/airflow/www/static/js/dag/grid/index.test.tsx b/airflow/www/static/js/dag/grid/index.test.tsx index 1b3888ffae3f..66e1cafaeb40 100644 --- a/airflow/www/static/js/dag/grid/index.test.tsx +++ b/airflow/www/static/js/dag/grid/index.test.tsx @@ -135,6 +135,7 @@ describe("Test ToggleGroups", () => { const returnValue = { data: mockGridData, isSuccess: true, + // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; jest diff --git a/airflow/www/static/js/dag_dependencies.js b/airflow/www/static/js/dag_dependencies.js index ddd40c248a59..a3c4a713eac0 100644 --- a/airflow/www/static/js/dag_dependencies.js +++ b/airflow/www/static/js/dag_dependencies.js @@ -97,6 +97,7 @@ function setUpZoomSupport() { } function setUpNodeHighlighting(focusItem = null) { + // eslint-disable-next-line func-names d3.selectAll("g.node").on("mouseover", function (d) { d3.select(this).selectAll("rect").style("stroke", highlightColor); highlightNodes(g.predecessors(d), upstreamColor, highlightStrokeWidth); @@ -114,6 +115,7 @@ function setUpNodeHighlighting(focusItem = null) { }); }); + // eslint-disable-next-line func-names d3.selectAll("g.node").on("mouseout", function (d) { d3.select(this).selectAll("rect,circle").style("stroke", null); highlightNodes(g.predecessors(d), null, initialStrokeWidth); diff --git a/airflow/www/static/js/dags.js b/airflow/www/static/js/dags.js index 1854ae53a292..ea53be18ac02 100644 --- a/airflow/www/static/js/dags.js +++ b/airflow/www/static/js/dags.js @@ -320,6 +320,7 @@ function getDagIds({ activeDagsOnly = false } = {}) { dagIds = dagIds.filter(":checked"); } dagIds = dagIds + // eslint-disable-next-line func-names .map(function () { return $(this).data("dag-id"); }) diff --git a/airflow/www/static/js/datasets/List.test.tsx b/airflow/www/static/js/datasets/List.test.tsx index 34e70f70b027..c1cd0da344db 100644 --- a/airflow/www/static/js/datasets/List.test.tsx +++ b/airflow/www/static/js/datasets/List.test.tsx @@ -25,6 +25,8 @@ import { render } from "@testing-library/react"; import * as useDatasetsModule from "src/api/useDatasets"; import { Wrapper } from "src/utils/testUtils"; +import type { UseQueryResult } from "react-query"; +import type { DatasetListItem } from "src/types"; import DatasetsList from "./List"; const datasets = [ @@ -57,22 +59,26 @@ const datasets = [ }, ]; +type UseDatasetsReturn = UseQueryResult & { + data: useDatasetsModule.DatasetsData; +}; + const returnValue = { data: { datasets, totalEntries: datasets.length, }, isSuccess: true, -} as any; +} as UseDatasetsReturn; const emptyReturnValue = { data: { - datasets: [], + datasets: [] as DatasetListItem[], totalEntries: 0, }, isSuccess: true, isLoading: false, -} as any; +} as UseDatasetsReturn; describe("Test Datasets List", () => { test("Displays a list of datasets", () => { diff --git a/airflow/www/static/js/datasets/List.tsx b/airflow/www/static/js/datasets/List.tsx index e201e1d2d4cc..6bbfd2649790 100644 --- a/airflow/www/static/js/datasets/List.tsx +++ b/airflow/www/static/js/datasets/List.tsx @@ -49,8 +49,10 @@ interface Props { interface CellProps { cell: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any value: any; row: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any original: Record; }; }; diff --git a/airflow/www/static/js/datetime_utils.js b/airflow/www/static/js/datetime_utils.js index b9e4b76eebe5..c23f8d665e83 100644 --- a/airflow/www/static/js/datetime_utils.js +++ b/airflow/www/static/js/datetime_utils.js @@ -104,6 +104,7 @@ export function updateAllDateTimes() { // Since we have set the default timezone for moment, it will automatically // convert it to the new target for us $(".datetime input").each((_, el) => { + // eslint-disable-next-line no-param-reassign el.value = moment(el.value).format(); }); } diff --git a/airflow/www/static/js/gantt.js b/airflow/www/static/js/gantt.js index 048ba741e6d2..2b7cf10c8191 100644 --- a/airflow/www/static/js/gantt.js +++ b/airflow/www/static/js/gantt.js @@ -74,7 +74,7 @@ const replacements = { "%": "%", }; -moment.fn.strftime = function (format) { +moment.fn.strftime = function formatTime(format) { // Break up format string based on strftime tokens const tokens = format.split(/(%-?.)/); const momentFormat = tokens @@ -173,12 +173,15 @@ d3.gantt = () => { tasks.forEach((a) => { if (!(a.start_date instanceof moment)) { + // eslint-disable-next-line no-param-reassign a.start_date = moment(a.start_date); } if (!(a.end_date instanceof moment)) { + // eslint-disable-next-line no-param-reassign a.end_date = moment(a.end_date); } if (a.queued_dttm && !(a.queued_dttm instanceof moment)) { + // eslint-disable-next-line no-param-reassign a.queued_dttm = moment(a.queued_dttm); } }); @@ -349,12 +352,14 @@ d3.gantt = () => { return gantt; }; + // eslint-disable-next-line func-names gantt.margin = function (value) { if (!arguments.length) return margin; margin = value; return gantt; }; + // eslint-disable-next-line func-names gantt.timeDomain = function (value) { if (!arguments.length) return [timeDomainStart, timeDomainEnd]; timeDomainStart = +value[0]; @@ -367,36 +372,42 @@ d3.gantt = () => { * vale The value can be "fit" - the domain fits the data or * "fixed" - fixed domain. */ + // eslint-disable-next-line func-names gantt.timeDomainMode = function (value) { if (!arguments.length) return timeDomainMode; timeDomainMode = value; return gantt; }; + // eslint-disable-next-line func-names gantt.taskTypes = function (value) { if (!arguments.length) return taskTypes; taskTypes = value; return gantt; }; + // eslint-disable-next-line func-names gantt.width = function (value) { if (!arguments.length) return width; width = +value; return gantt; }; + // eslint-disable-next-line func-names gantt.height = function (value) { if (!arguments.length) return height; height = +value; return gantt; }; + // eslint-disable-next-line func-names gantt.tickFormat = function (value) { if (!arguments.length) return tickFormat; tickFormat = value; return gantt; }; + // eslint-disable-next-line func-names gantt.selector = function (value) { if (!arguments.length) return selector; selector = value; diff --git a/airflow/www/static/js/graph.js b/airflow/www/static/js/graph.js index 38b1ae6211fb..b61639d3a86c 100644 --- a/airflow/www/static/js/graph.js +++ b/airflow/www/static/js/graph.js @@ -153,6 +153,7 @@ function collapseGroup(nodeId, node) { getChildrenIds(node).forEach((childId) => mapTaskToNode.set(childId, nodeId)); + // eslint-disable-next-line no-param-reassign node = g.node(nodeId); // Set children edges onto the group edge diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index 7350a5cfffa7..95861ea6c13a 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -69,19 +69,20 @@ export function escapeHtml(text) { window.escapeHtml = escapeHtml; export function convertSecsToHumanReadable(seconds) { + let processedSeconds = seconds; const oriSeconds = seconds; const floatingPart = oriSeconds - Math.floor(oriSeconds); - seconds = Math.floor(seconds); + processedSeconds = Math.floor(processedSeconds); const secondsPerHour = 60 * 60; const secondsPerMinute = 60; - const hours = Math.floor(seconds / secondsPerHour); - seconds -= hours * secondsPerHour; + const hours = Math.floor(processedSeconds / secondsPerHour); + processedSeconds -= hours * secondsPerHour; - const minutes = Math.floor(seconds / secondsPerMinute); - seconds -= minutes * secondsPerMinute; + const minutes = Math.floor(processedSeconds / secondsPerMinute); + processedSeconds -= minutes * secondsPerMinute; let readableFormat = ""; if (hours > 0) { @@ -90,12 +91,12 @@ export function convertSecsToHumanReadable(seconds) { if (minutes > 0) { readableFormat += `${minutes}Min `; } - if (seconds + floatingPart > 0) { + if (processedSeconds + floatingPart > 0) { if (Math.floor(oriSeconds) === oriSeconds) { - readableFormat += `${seconds}Sec`; + readableFormat += `${processedSeconds}Sec`; } else { - seconds += floatingPart; - readableFormat += `${seconds.toFixed(3)}Sec`; + processedSeconds += floatingPart; + readableFormat += `${processedSeconds.toFixed(3)}Sec`; } } return readableFormat; diff --git a/airflow/www/static/js/task.js b/airflow/www/static/js/task.js index df546027290b..d4c03d184894 100644 --- a/airflow/www/static/js/task.js +++ b/airflow/www/static/js/task.js @@ -27,6 +27,7 @@ document.addEventListener("DOMContentLoaded", () => { const value = attr.innerHTML; if (value.length === 32 && moment(value, "YYYY-MM-DD").isValid()) { // 32 is the length of our timestamps + // eslint-disable-next-line no-param-reassign attr.innerHTML = ""; const timeElement = document.createElement("time"); timeElement.setAttribute("datetime", value); @@ -35,6 +36,7 @@ document.addEventListener("DOMContentLoaded", () => { attr.appendChild(timeElement); } else if (validator.isURL(value)) { // very basic url detection + // eslint-disable-next-line no-param-reassign attr.innerHTML = ""; const linkElement = document.createElement("a"); linkElement.setAttribute("href", value); diff --git a/airflow/www/static/js/task_instances.js b/airflow/www/static/js/task_instances.js index 61ff19259975..c539d5c3ea16 100644 --- a/airflow/www/static/js/task_instances.js +++ b/airflow/www/static/js/task_instances.js @@ -119,12 +119,14 @@ export default function tiTooltip(ti, task, { includeTryNumber = false } = {}) { if (ti.state === "running") { const startDate = ti.start_date instanceof moment ? ti.start_date : moment(ti.start_date); + // eslint-disable-next-line no-param-reassign ti.duration = moment().diff(startDate, "second"); } else if (!ti.duration && ti.end_date) { const startDate = ti.start_date instanceof moment ? ti.start_date : moment(ti.start_date); const endDate = ti.end_date instanceof moment ? ti.end_date : moment(ti.end_date); + // eslint-disable-next-line no-param-reassign ti.duration = moment(endDate).diff(startDate, "second"); } diff --git a/airflow/www/static/js/ti_log.js b/airflow/www/static/js/ti_log.js index d2e223b103df..23fd0af76967 100644 --- a/airflow/www/static/js/ti_log.js +++ b/airflow/www/static/js/ti_log.js @@ -37,6 +37,7 @@ function recurse(delay = DELAY) { setTimeout(resolve, delay); }); } +/* eslint-disable no-console */ // Enable auto tailing only when users scroll down to the bottom // of the page. This prevent auto tailing the page if users want @@ -166,15 +167,16 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { } function setDownloadUrl(tryNumber) { - if (!tryNumber) { + let tryNumberData = tryNumber; + if (!tryNumberData) { // default to the currently selected tab - tryNumber = $("#ti_log_try_number_list .active a").data("try-number"); + tryNumberData = $("#ti_log_try_number_list .active a").data("try-number"); } const query = new URLSearchParams({ dag_id: dagId, task_id: taskId, execution_date: executionDate, - try_number: tryNumber, + try_number: tryNumberData, metadata: "null", format: "file", }); @@ -187,6 +189,7 @@ $(document).ready(() => { autoTailingLog(TOTAL_ATTEMPTS, null, true); setDownloadUrl(); + // eslint-disable-next-line func-names $("#ti_log_try_number_list a").click(function () { const tryNumber = $(this).data("try-number"); diff --git a/airflow/www/static/js/types/react-table-config.d.ts b/airflow/www/static/js/types/react-table-config.d.ts index 0b62ceb16f42..2bc503ed1829 100644 --- a/airflow/www/static/js/types/react-table-config.d.ts +++ b/airflow/www/static/js/types/react-table-config.d.ts @@ -87,7 +87,7 @@ declare module "react-table" { // this matches the spirit of the underlying js library, // but might be cleaner if it's replaced by a more specific type that matches your // feature set, this is a safe default. - Record {} + Record {} export interface Hooks< D extends Record = Record @@ -139,7 +139,7 @@ declare module "react-table" { export interface Cell< D extends Record = Record, // eslint-disable-next-line @typescript-eslint/no-unused-vars - V = any + V = unknown > extends UseGroupByCellProps, UseRowStateCellProps {} diff --git a/airflow/www/static/js/utils/URLSearchParamWrapper.ts b/airflow/www/static/js/utils/URLSearchParamWrapper.ts index befeadcaf5d9..476db61b9067 100644 --- a/airflow/www/static/js/utils/URLSearchParamWrapper.ts +++ b/airflow/www/static/js/utils/URLSearchParamWrapper.ts @@ -18,7 +18,8 @@ */ class URLSearchParamsWrapper extends URLSearchParams { - constructor(init?: { [keys: string]: any }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(init?: Record) { if (init) { const stringValues: { [keys: string]: string } = {}; Object.keys(init).forEach((key) => { diff --git a/airflow/www/static/js/utils/graph.ts b/airflow/www/static/js/utils/graph.ts index e69a3045d945..2dd970bf8c10 100644 --- a/airflow/www/static/js/utils/graph.ts +++ b/airflow/www/static/js/utils/graph.ts @@ -77,13 +77,21 @@ const generateGraph = ({ }: GenerateProps) => { const closedGroupIds: string[] = []; - const formatChildNode = (node: any) => { + const formatChildNode = ( + node: DepNode + ): DepNode & { + label: string; + layoutOptions?: Record; + width?: number; + height?: number; + } => { const { id, value, children } = node; const isOpen = openGroupIds?.includes(value.label); const childCount = - children?.filter((c: any) => !c.id.includes("join_id")).length || 0; - if (isOpen && children.length) { + children?.filter((c: DepNode) => !c.id.includes("join_id")).length || 0; + if (isOpen && children?.length) { return { + ...node, id, value: { ...value, From 1f22db7ed395ab88270faa3c1eee6c7ade53fcc7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 3 Jul 2023 15:57:28 +0800 Subject: [PATCH 023/533] Fixup config var types under the scheduler section (#32132) --- airflow/config_templates/config.yml | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index f48a569b545f..88aa055a3901 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2256,7 +2256,7 @@ scheduler: from the CLI or the UI), this defines the frequency at which they should listen (in seconds). version_added: ~ - type: string + type: float example: ~ default: "5" scheduler_heartbeat_sec: @@ -2265,7 +2265,7 @@ scheduler: scheduler section in the docs for more information). This defines how often the scheduler should run (in seconds). version_added: ~ - type: string + type: integer example: ~ default: "5" num_runs: @@ -2273,7 +2273,7 @@ scheduler: The number of times to try to schedule each DAG file -1 indicates unlimited number version_added: 1.10.6 - type: string + type: integer example: ~ default: "-1" scheduler_idle_sleep_time: @@ -2282,7 +2282,7 @@ scheduler: in the loop. i.e. if it scheduled something then it will start the next loop iteration straight away. version_added: 2.2.0 - type: string + type: float example: ~ default: "1" min_file_process_interval: @@ -2291,7 +2291,7 @@ scheduler: ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after this interval. Keeping this number low will increase CPU usage. version_added: ~ - type: string + type: integer example: ~ default: "30" parsing_cleanup_interval: @@ -2319,14 +2319,14 @@ scheduler: description: | How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. version_added: ~ - type: string + type: integer example: ~ default: "300" print_stats_interval: description: | How often should stats be printed to the logs. Setting to 0 will disable printing stats version_added: ~ - type: string + type: integer example: ~ default: "30" pool_metrics_interval: @@ -2343,7 +2343,7 @@ scheduler: This is used by the health check in the "/health" endpoint and in `airflow jobs check` CLI for SchedulerJob. version_added: 1.10.2 - type: string + type: integer example: ~ default: "30" enable_health_check: @@ -2359,7 +2359,7 @@ scheduler: When you start a scheduler, airflow starts a tiny web server subprocess to serve a health check on this port version_added: 2.4.0 - type: string + type: integer example: ~ default: "8974" orphaned_tasks_check_interval: @@ -2381,7 +2381,7 @@ scheduler: not heartbeat in this many seconds, the scheduler will mark the associated task instance as failed and will re-schedule the task. version_added: ~ - type: string + type: integer example: ~ default: "300" zombie_detection_interval: @@ -2400,7 +2400,7 @@ scheduler: however it can be set on a per DAG basis in the DAG definition (catchup) version_added: ~ - type: string + type: boolean example: ~ default: "True" ignore_first_depends_on_past_by_default: @@ -2411,7 +2411,7 @@ scheduler: in the DB with an execution_date earlier than it., i.e. no manual marking success will be needed for a newly added task to be scheduled. version_added: 2.3.0 - type: string + type: boolean example: ~ default: "True" max_tis_per_query: @@ -2422,7 +2422,7 @@ scheduler: Additionally, you may hit the maximum allowable query length for your db. Set this to 0 for no limit (not advised) version_added: ~ - type: string + type: integer example: ~ default: "512" use_row_level_locking: @@ -2439,7 +2439,7 @@ scheduler: Max number of DAGs to create DagRuns for per scheduler loop. example: ~ version_added: 2.0.0 - type: string + type: integer default: "10" see_also: ":ref:`scheduler:ha:tunables`" max_dagruns_per_loop_to_schedule: @@ -2448,7 +2448,7 @@ scheduler: and queuing tasks. example: ~ version_added: 2.0.0 - type: string + type: integer default: "20" see_also: ":ref:`scheduler:ha:tunables`" schedule_after_task_execution: @@ -2475,7 +2475,7 @@ scheduler: The scheduler can run multiple processes in parallel to parse dags. This defines how many processes will run. version_added: 1.10.14 - type: string + type: integer example: ~ default: "2" file_parsing_sort_mode: @@ -2522,7 +2522,7 @@ scheduler: Turn off scheduler use of cron intervals by setting this to False. DAGs submitted manually in the web UI or with trigger_dag will still run. version_added: 1.10.2 - type: string + type: boolean example: ~ default: "True" allow_trigger_in_future: @@ -2530,14 +2530,14 @@ scheduler: Allow externally triggered DagRuns for Execution Dates in the future Only has effect if schedule_interval is set to None in DAG version_added: 1.10.8 - type: string + type: boolean example: ~ default: "False" trigger_timeout_check_interval: description: | How often to check for expired trigger requests that have not run yet. version_added: 2.2.0 - type: string + type: float example: ~ default: "15" task_queued_timeout: From c03d7b7c337df7eee4cc7722a1c8da56abd7027a Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Mon, 3 Jul 2023 16:09:57 +0530 Subject: [PATCH 024/533] Adds Sensor section in the Azure providers docs (#32299) * added transfers section: * spell * spell * spell * static-checks * fixed static checks * doc added for local_to_wasb * added sensors section * added sensors section * fixed static check --- .../microsoft/azure/example_dag/__init__.py | 16 +++++ .../example_cosmos_document_sensor.py | 53 ++++++++++++++++ .../azure/example_dag/example_wasb_sensors.py | 61 +++++++++++++++++++ .../index.rst | 1 + .../sensors/cosmos_document_sesnor.rst | 35 +++++++++++ .../sensors/index.rst | 27 ++++++++ .../sensors/wasb_sensors.rst | 48 +++++++++++++++ docs/spelling_wordlist.txt | 1 + 8 files changed, 242 insertions(+) create mode 100644 airflow/providers/microsoft/azure/example_dag/__init__.py create mode 100644 airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py create mode 100644 airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py create mode 100644 docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst create mode 100644 docs/apache-airflow-providers-microsoft-azure/sensors/index.rst create mode 100644 docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst diff --git a/airflow/providers/microsoft/azure/example_dag/__init__.py b/airflow/providers/microsoft/azure/example_dag/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/airflow/providers/microsoft/azure/example_dag/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py b/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py new file mode 100644 index 000000000000..198420147eb0 --- /dev/null +++ b/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG that senses document in Azure Cosmos DB. + +This DAG relies on the following OS environment variables + +* DATABASE_NAME - Target CosmosDB database_name. +* COLLECTION_NAME - Target CosmosDB collection_name. +* DOCUMENT_ID - The ID of the target document. +""" +from __future__ import annotations + +import os +from datetime import datetime + +from airflow.models import DAG +from airflow.providers.microsoft.azure.sensors.cosmos import AzureCosmosDocumentSensor + +DATABASE_NAME = os.environ.get("DATABASE_NAME", "example-database-name") +COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "example-collection-name") +DOCUMENT_ID = os.environ.get("DOCUMENT_ID", "example-document-id") + + +with DAG( + "example_cosmos_document_sensor", + start_date=datetime(2022, 8, 8), + catchup=False, + tags=["example"], +) as dag: + # [START cosmos_document_sensor] + azure_wasb_sensor = AzureCosmosDocumentSensor( + database_name=DATABASE_NAME, + collection_name=COLLECTION_NAME, + document_id=DOCUMENT_ID, + task_id="cosmos_document_sensor", + ) + # [END cosmos_document_sensor] diff --git a/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py new file mode 100644 index 000000000000..349e09c44ad9 --- /dev/null +++ b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG that senses blob(s) in Azure Blob Storage. + +This DAG relies on the following OS environment variables + +* CONTAINER_NAME - The container under which to look for the blob. +* BLOB_NAME - The name of the blob to match. +* PREFIX - The blob with the specified prefix to match. +""" +from __future__ import annotations + +import os +from datetime import datetime + +from airflow.models import DAG +from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor, WasbPrefixSensor + +CONTAINER_NAME = os.environ.get("CONTAINER_NAME", "example-container-name") +BLOB_NAME = os.environ.get("BLOB_NAME", "example-blob-name") +PREFIX = os.environ.get("PREFIX", "example-prefix") + + +with DAG( + "example_wasb_sensors", + start_date=datetime(2022, 8, 8), + catchup=False, + tags=["example"], +) as dag: + # [START wasb_blob_sensor] + azure_wasb_sensor = WasbBlobSensor( + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + task_id="wasb_sense_blob", + ) + # [END wasb_blob_sensor] + + # [START wasb_prefix_sensor] + azure_wasb_prefix_sensor = WasbPrefixSensor( + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + prefix=PREFIX, + task_id="wasb_sense_prefix", + ) + # [END wasb_prefix_sensor] diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index dfc0bf56e14e..a10c5017314a 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -39,6 +39,7 @@ Transfers Secrets backends Logging for Tasks + Sensors .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst new file mode 100644 index 000000000000..d4b563120452 --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst @@ -0,0 +1,35 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Azure Cosmos DB +================== +Cosmos Database (DB) is a globally distributed, low latency, multi-model database for managing data at large scales. +It is a cloud-based NoSQL database offered as a PaaS (Platform as a Service) from Microsoft Azure. +It is a highly available, high throughput, reliable database and is often called a serverless database. +Cosmos database contains the Azure Document DB and is available everywhere. + +Azure Cosmos Document Sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Checks for the existence of a document which matches the given query in CosmosDB. +:class:`~airflow.providers.microsoft.azure.sensors.cosmos.AzureCosmosDocumentSensor` + +.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py + :language: python + :dedent: 4 + :start-after: [START cosmos_document_sensor] + :end-before: [END cosmos_document_sensor] diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst new file mode 100644 index 000000000000..439d2348972c --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst @@ -0,0 +1,27 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Microsoft Azure Sensors +======================= + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst new file mode 100644 index 000000000000..f901928d9b0f --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst @@ -0,0 +1,48 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Azure Blob Storage +================== +The Blob service stores text and binary data as objects in the cloud. +The Blob service offers the following three resources: the storage account, containers, and blobs. +Within your storage account, containers provide a way to organize sets of blobs. +For more information about the service visit `Azure Blob Storage API documentation `_. +This page shows how to check for blobs in a particular container. + +Wasb Blob Sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Waits for a blob to arrive on Azure Blob Storage. +:class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbBlobSensor` + +.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py + :language: python + :dedent: 4 + :start-after: [START wasb_blob_sensor] + :end-before: [END wasb_blob_sensor] + +Wasb Prefix Sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Waits for blobs matching a prefix to arrive on Azure Blob Storage. +:class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbPrefixSensor` + +.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py + :language: python + :dedent: 4 + :start-after: [START wasb_prefix_sensor] + :end-before: [END wasb_prefix_sensor] diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index af0a3ff5dcbf..6389ba181420 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1056,6 +1056,7 @@ ot otel overridable oversubscription +PaaS Pagerduty pagerduty pageviews From 1065687ec6df2b9b3557e38a67e71f835796427f Mon Sep 17 00:00:00 2001 From: Phani Kumar <94376113+phanikumv@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:49:51 +0530 Subject: [PATCH 025/533] Refactor Sqlalchemy queries to 2.0 style (Part 3) (#32177) --- airflow/utils/db.py | 38 ++-- airflow/www/utils.py | 11 +- airflow/www/views.py | 460 ++++++++++++++++++++++--------------------- 3 files changed, 265 insertions(+), 244 deletions(-) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index a76f0d4f675d..46ddcbfb3453 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -92,7 +92,7 @@ def _format_airflow_moved_table_name(source_table, version, category): @provide_session def merge_conn(conn, session: Session = NEW_SESSION): """Add new Connection.""" - if not session.query(conn.__class__).filter_by(conn_id=conn.conn_id).first(): + if not session.scalar(select(conn.__class__).filter_by(conn_id=conn.conn_id).limit(1)): session.add(conn) session.commit() @@ -959,7 +959,9 @@ def check_conn_id_duplicates(session: Session) -> Iterable[str]: dups = [] try: - dups = session.query(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1).all() + dups = session.execute( + select(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1) + ).all() except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -984,12 +986,11 @@ def check_username_duplicates(session: Session) -> Iterable[str]: for model in [User, RegisterUser]: dups = [] try: - dups = ( - session.query(model.username) # type: ignore[attr-defined] + dups = session.execute( + select(model.username) # type: ignore[attr-defined] .group_by(model.username) # type: ignore[attr-defined] .having(func.count() > 1) - .all() - ) + ).all() except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -1058,13 +1059,13 @@ def check_task_fail_for_duplicates(session): """ minimal_table_obj = table(table_name, *[column(x) for x in uniqueness]) try: - subquery = ( - session.query(minimal_table_obj, func.count().label("dupe_count")) + subquery = session.execute( + select(minimal_table_obj, func.count().label("dupe_count")) .group_by(*[text(x) for x in uniqueness]) .having(func.count() > text("1")) .subquery() ) - dupe_count = session.query(func.sum(subquery.c.dupe_count)).scalar() + dupe_count = session.scalar(select(func.sum(subquery.c.dupe_count))) if not dupe_count: # there are no duplicates; nothing to do. return @@ -1101,7 +1102,7 @@ def check_conn_type_null(session: Session) -> Iterable[str]: n_nulls = [] try: - n_nulls = session.query(Connection.conn_id).filter(Connection.conn_type.is_(None)).all() + n_nulls = session.scalars(select(Connection.conn_id).where(Connection.conn_type.is_(None))).all() except (exc.OperationalError, exc.ProgrammingError, exc.InternalError): # fallback if tables hasn't been created yet session.rollback() @@ -1143,7 +1144,7 @@ def check_run_id_null(session: Session) -> Iterable[str]: dagrun_table.c.run_id.is_(None), dagrun_table.c.execution_date.is_(None), ) - invalid_dagrun_count = session.query(func.count(dagrun_table.c.id)).filter(invalid_dagrun_filter).scalar() + invalid_dagrun_count = session.scalar(select(func.count(dagrun_table.c.id)).where(invalid_dagrun_filter)) if invalid_dagrun_count > 0: dagrun_dangling_table_name = _format_airflow_moved_table_name(dagrun_table.name, "2.2", "dangling") if dagrun_dangling_table_name in inspect(session.get_bind()).get_table_names(): @@ -1240,7 +1241,7 @@ def _move_dangling_data_to_new_table( pk_cols = source_table.primary_key.columns delete = source_table.delete().where( - tuple_(*pk_cols).in_(session.query(*target_table.primary_key.columns).subquery()) + tuple_(*pk_cols).in_(session.select(*target_table.primary_key.columns).subquery()) ) else: delete = source_table.delete().where( @@ -1262,10 +1263,11 @@ def _dangling_against_dag_run(session, source_table, dag_run): source_table.c.dag_id == dag_run.c.dag_id, source_table.c.execution_date == dag_run.c.execution_date, ) + return ( - session.query(*[c.label(c.name) for c in source_table.c]) + select(*[c.label(c.name) for c in source_table.c]) .join(dag_run, source_to_dag_run_join_cond, isouter=True) - .filter(dag_run.c.dag_id.is_(None)) + .where(dag_run.c.dag_id.is_(None)) ) @@ -1304,10 +1306,10 @@ def _dangling_against_task_instance(session, source_table, dag_run, task_instanc ) return ( - session.query(*[c.label(c.name) for c in source_table.c]) + select(*[c.label(c.name) for c in source_table.c]) .join(dag_run, dr_join_cond, isouter=True) .join(task_instance, ti_join_cond, isouter=True) - .filter(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) + .where(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) ) @@ -1331,9 +1333,9 @@ def _move_duplicate_data_to_new_table( """ bind = session.get_bind() dialect_name = bind.dialect.name + query = ( - session.query(source_table) - .with_entities(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) + select(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) .select_from(source_table) .join(subquery, and_(*[getattr(source_table.c, x) == getattr(subquery.c, x) for x in uniqueness])) ) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 25fc1a28f98f..46256ee3359e 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -38,6 +38,7 @@ from pygments.formatters import HtmlFormatter from sqlalchemy import delete, func, types from sqlalchemy.ext.associationproxy import AssociationProxy +from sqlalchemy.sql import Select from airflow.exceptions import RemovedInAirflow3Warning from airflow.models import errors @@ -52,7 +53,6 @@ from airflow.www.widgets import AirflowDateTimePickerWidget if TYPE_CHECKING: - from sqlalchemy.orm.query import Query from sqlalchemy.orm.session import Session from sqlalchemy.sql.operators import ColumnOperators @@ -517,18 +517,21 @@ def _get_run_ordering_expr(name: str) -> ColumnOperators: return expr.desc() -def sorted_dag_runs(query: Query, *, ordering: Sequence[str], limit: int) -> Sequence[DagRun]: +def sorted_dag_runs( + query: Select, *, ordering: Sequence[str], limit: int, session: Session +) -> Sequence[DagRun]: """Produce DAG runs sorted by specified columns. - :param query: An ORM query object against *DagRun*. + :param query: An ORM select object against *DagRun*. :param ordering: Column names to sort the runs. should generally come from a timetable's ``run_ordering``. :param limit: Number of runs to limit to. + :param session: SQLAlchemy ORM session object :return: A list of DagRun objects ordered by the specified columns. The list contains only the *last* objects, but in *ascending* order. """ ordering_exprs = (_get_run_ordering_expr(name) for name in ordering) - runs = query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit).all() + runs = session.scalars(query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit)).all() runs.reverse() return runs diff --git a/airflow/www/views.py b/airflow/www/views.py index 026f8b533499..d791eefc37c7 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -69,7 +69,7 @@ from pendulum.parsing.exceptions import ParserError from pygments import highlight, lexers from pygments.formatters import HtmlFormatter -from sqlalchemy import Date, and_, case, desc, func, inspect, or_, union_all +from sqlalchemy import Date, and_, case, desc, func, inspect, or_, select, union_all from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload from wtforms import SelectField, validators @@ -95,7 +95,7 @@ from airflow.jobs.job import Job from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner -from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, XCom, errors +from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, Trigger, XCom, errors from airflow.models.abstractoperator import AbstractOperator from airflow.models.dag import DAG, get_dataset_triggered_next_run_info from airflow.models.dagcode import DagCode @@ -231,16 +231,15 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): # loaded and the actual requested run would be excluded by the limit(). Once # the user has changed base date to be anything else we want to use that instead. query_date = base_date - if date_time < base_date and date_time + datetime.timedelta(seconds=1) >= base_date: + if date_time < base_date <= date_time + datetime.timedelta(seconds=1): query_date = date_time - drs = ( - session.query(DagRun) - .filter(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) + drs = session.scalars( + select(DagRun) + .where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) .order_by(desc(DagRun.execution_date)) .limit(num_runs) - .all() - ) + ).all() dr_choices = [] dr_state = None for dr in drs: @@ -291,8 +290,8 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): Create a nested dict representation of the DAG's TaskGroup and its children used to construct the Graph and Grid views. """ - query = ( - session.query( + query = session.execute( + select( TaskInstance.task_id, TaskInstance.run_id, TaskInstance.state, @@ -303,7 +302,7 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): func.max(TaskInstance.end_date).label("end_date"), ) .join(TaskInstance.task_instance_note, isouter=True) - .filter( + .where( TaskInstance.dag_id == dag.dag_id, TaskInstance.run_id.in_([dag_run.run_id for dag_run in dag_runs]), ) @@ -426,11 +425,9 @@ def get_summary(dag_run: DagRun): } def get_mapped_group_summaries(): - mapped_ti_query = ( - session.query( - TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index - ) - .filter( + mapped_ti_query = session.execute( + select(TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index) + .where( TaskInstance.dag_id == dag.dag_id, TaskInstance.task_id.in_(child["id"] for child in children), TaskInstance.run_id.in_(r.run_id for r in dag_runs), @@ -738,21 +735,20 @@ def index(self): with create_session() as session: # read orm_dags from the db - dags_query = session.query(DagModel).filter(~DagModel.is_subdag, DagModel.is_active) + dags_query = select(DagModel).where(~DagModel.is_subdag, DagModel.is_active) if arg_search_query: escaped_arg_search_query = arg_search_query.replace("_", r"\_") - dags_query = dags_query.filter( + dags_query = dags_query.where( DagModel.dag_id.ilike("%" + escaped_arg_search_query + "%", escape="\\") | DagModel.owners.ilike("%" + escaped_arg_search_query + "%", escape="\\") ) if arg_tags_filter: - dags_query = dags_query.filter(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) + dags_query = dags_query.where(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) - dags_query = dags_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - - filtered_dag_count = dags_query.count() + dags_query = dags_query.where(DagModel.dag_id.in_(filter_dag_ids)) + filtered_dag_count = session.scalar(select(func.count()).select_from(dags_query)) if filtered_dag_count == 0 and len(arg_tags_filter): flash( "No matching DAG tags found.", @@ -762,28 +758,28 @@ def index(self): return redirect(url_for("Airflow.index")) all_dags = dags_query - active_dags = dags_query.filter(~DagModel.is_paused) - paused_dags = dags_query.filter(DagModel.is_paused) + active_dags = dags_query.where(~DagModel.is_paused) + paused_dags = dags_query.where(DagModel.is_paused) # find DAGs which have a RUNNING DagRun - running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).filter( + running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).where( DagRun.state == State.RUNNING ) # find DAGs for which the latest DagRun is FAILED subq_all = ( - session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) .group_by(DagRun.dag_id) .subquery() ) subq_failed = ( - session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) - .filter(DagRun.state == State.FAILED) + select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + .where(DagRun.state == State.FAILED) .group_by(DagRun.dag_id) .subquery() ) subq_join = ( - session.query(subq_all.c.dag_id, subq_all.c.start_date) + select(subq_all.c.dag_id, subq_all.c.start_date) .join( subq_failed, and_( @@ -796,16 +792,18 @@ def index(self): failed_dags = dags_query.join(subq_join, DagModel.dag_id == subq_join.c.dag_id) is_paused_count = dict( - all_dags.with_entities(DagModel.is_paused, func.count(DagModel.dag_id)).group_by( - DagModel.is_paused - ) + session.execute( + select(DagModel.is_paused, func.count(DagModel.dag_id)) + .group_by(DagModel.is_paused) + .select_from(all_dags) + ).all() ) status_count_active = is_paused_count.get(False, 0) status_count_paused = is_paused_count.get(True, 0) - status_count_running = running_dags.count() - status_count_failed = failed_dags.count() + status_count_running = session.scalar(select(func.count()).select_from(running_dags)) + status_count_failed = session.scalar(select(func.count()).select_from(failed_dags)) all_dags_count = status_count_active + status_count_paused if arg_status_filter == "active": @@ -826,7 +824,7 @@ def index(self): if arg_sorting_key == "last_dagrun": dag_run_subquery = ( - session.query( + select( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) @@ -854,7 +852,13 @@ def index(self): else: current_dags = current_dags.order_by(null_case, sort_column) - dags = current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page).all() + dags = ( + session.scalars( + current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page) + ) + .unique() + .all() + ) user_permissions = g.user.perms can_create_dag_run = ( permissions.ACTION_CAN_CREATE, @@ -874,7 +878,7 @@ def index(self): dag.can_trigger = dag.can_edit and can_create_dag_run dag.can_delete = get_airflow_app().appbuilder.sm.can_delete_dag(dag.dag_id, g.user) - dagtags = session.query(func.distinct(DagTag.name)).order_by(DagTag.name).all() + dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() tags = [ {"name": name, "selected": bool(arg_tags_filter and name in arg_tags_filter)} for name, in dagtags @@ -882,14 +886,15 @@ def index(self): owner_links_dict = DagOwnerAttributes.get_all(session) - import_errors = session.query(errors.ImportError).order_by(errors.ImportError.id) + import_errors = select(errors.ImportError).order_by(errors.ImportError.id) if (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG) not in user_permissions: # if the user doesn't have access to all DAGs, only display errors from visible DAGs import_errors = import_errors.join( DagModel, DagModel.fileloc == errors.ImportError.filename - ).filter(DagModel.dag_id.in_(filter_dag_ids)) + ).where(DagModel.dag_id.in_(filter_dag_ids)) + import_errors = session.scalars(import_errors) for import_error in import_errors: flash( f"Broken DAG: [{import_error.filename}] {import_error.stacktrace}", @@ -933,10 +938,12 @@ def _iter_parsed_moved_data_table_names(): permissions.RESOURCE_ADMIN_MENU, ) in user_permissions and conf.getboolean("webserver", "warn_deployment_exposure"): robots_file_access_count = ( - session.query(Log) - .filter(Log.event == "robots") - .filter(Log.dttm > (utcnow() - datetime.timedelta(days=7))) - .count() + select(Log) + .where(Log.event == "robots") + .where(Log.dttm > (utcnow() - datetime.timedelta(days=7))) + ) + robots_file_access_count = session.scalar( + select(func.count()).select_from(robots_file_access_count) ) if robots_file_access_count > 0: flash( @@ -1038,9 +1045,11 @@ def next_run_datasets_summary(self, session: Session = NEW_SESSION): dataset_triggered_dag_ids = [ dag.dag_id for dag in ( - session.query(DagModel.dag_id) - .filter(DagModel.dag_id.in_(filter_dag_ids)) - .filter(DagModel.schedule_interval == "Dataset") + session.scalars( + select(DagModel.dag_id) + .where(DagModel.dag_id.in_(filter_dag_ids)) + .where(DagModel.schedule_interval == "Dataset") + ) ) ] @@ -1071,10 +1080,10 @@ def dag_stats(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify({}) - dag_state_stats = ( - session.query(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) + dag_state_stats = session.execute( + select(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) .group_by(DagRun.dag_id, DagRun.state) - .filter(DagRun.dag_id.in_(filter_dag_ids)) + .where(DagRun.dag_id.in_(filter_dag_ids)) ) dag_state_data = {(dag_id, state): count for dag_id, state, count in dag_state_stats} @@ -1112,17 +1121,17 @@ def task_stats(self, session: Session = NEW_SESSION): filter_dag_ids = allowed_dag_ids running_dag_run_query_result = ( - session.query(DagRun.dag_id, DagRun.run_id) + select(DagRun.dag_id, DagRun.run_id) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .filter(DagRun.state == State.RUNNING, DagModel.is_active) + .where(DagRun.state == State.RUNNING, DagModel.is_active) ) - running_dag_run_query_result = running_dag_run_query_result.filter(DagRun.dag_id.in_(filter_dag_ids)) + running_dag_run_query_result = running_dag_run_query_result.where(DagRun.dag_id.in_(filter_dag_ids)) running_dag_run_query_result = running_dag_run_query_result.subquery("running_dag_run") # Select all task_instances from active dag_runs. - running_task_instance_query_result = session.query( + running_task_instance_query_result = select( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(True).label("is_dag_running"), @@ -1136,19 +1145,19 @@ def task_stats(self, session: Session = NEW_SESSION): if conf.getboolean("webserver", "SHOW_RECENT_STATS_FOR_COMPLETED_RUNS", fallback=True): last_dag_run = ( - session.query(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) + select(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .filter(DagRun.state != State.RUNNING, DagModel.is_active) + .where(DagRun.state != State.RUNNING, DagModel.is_active) .group_by(DagRun.dag_id) ) - last_dag_run = last_dag_run.filter(DagRun.dag_id.in_(filter_dag_ids)) + last_dag_run = last_dag_run.where(DagRun.dag_id.in_(filter_dag_ids)) last_dag_run = last_dag_run.subquery("last_dag_run") # Select all task_instances from active dag_runs. # If no dag_run is active, return task instances from most recent dag_run. last_task_instance_query_result = ( - session.query( + select( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(False).label("is_dag_running"), @@ -1169,8 +1178,8 @@ def task_stats(self, session: Session = NEW_SESSION): else: final_task_instance_query_result = running_task_instance_query_result.subquery("final_ti") - qry = ( - session.query( + qry = session.execute( + select( final_task_instance_query_result.c.dag_id, final_task_instance_query_result.c.state, final_task_instance_query_result.c.is_dag_running, @@ -1186,7 +1195,6 @@ def task_stats(self, session: Session = NEW_SESSION): final_task_instance_query_result.c.is_dag_running.desc(), ) ) - data = get_task_stats_from_query(qry) payload: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) for dag_id in filter_dag_ids: @@ -1219,29 +1227,31 @@ def last_dagruns(self, session: Session = NEW_SESSION): return flask.json.jsonify({}) last_runs_subquery = ( - session.query( + select( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) .group_by(DagRun.dag_id) - .filter(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. + .where(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. .subquery("last_runs") ) - query = session.query( - DagRun.dag_id, - DagRun.start_date, - DagRun.end_date, - DagRun.state, - DagRun.execution_date, - DagRun.data_interval_start, - DagRun.data_interval_end, - ).join( - last_runs_subquery, - and_( - last_runs_subquery.c.dag_id == DagRun.dag_id, - last_runs_subquery.c.max_execution_date == DagRun.execution_date, - ), + query = session.execute( + select( + DagRun.dag_id, + DagRun.start_date, + DagRun.end_date, + DagRun.state, + DagRun.execution_date, + DagRun.data_interval_start, + DagRun.data_interval_end, + ).join( + last_runs_subquery, + and_( + last_runs_subquery.c.dag_id == DagRun.dag_id, + last_runs_subquery.c.max_execution_date == DagRun.execution_date, + ), + ) ) resp = { @@ -1340,19 +1350,18 @@ def dag_details(self, dag_id, session: Session = NEW_SESSION): title = "DAG Details" root = request.args.get("root", "") - states = ( - session.query(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) - .filter(TaskInstance.dag_id == dag_id) + states = session.execute( + select(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) + .where(TaskInstance.dag_id == dag_id) .group_by(TaskInstance.state) - .all() - ) + ).all() active_runs = models.DagRun.find(dag_id=dag_id, state=DagRunState.RUNNING, external_trigger=False) - tags = session.query(models.DagTag).filter(models.DagTag.dag_id == dag_id).all() + tags = session.scalars(select(models.DagTag).where(models.DagTag.dag_id == dag_id)).all() # TODO: convert this to a relationship - owner_links = session.query(DagOwnerAttributes).filter_by(dag_id=dag_id).all() + owner_links = session.execute(select(DagOwnerAttributes).filter_by(dag_id=dag_id)).all() attrs_to_avoid = [ "schedule_datasets", @@ -1617,18 +1626,17 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): "metadata": {"end_of_log": True}, } - ti = ( - session.query(models.TaskInstance) - .filter( + ti = session.scalar( + select(models.TaskInstance) + .where( TaskInstance.task_id == task_id, TaskInstance.dag_id == dag_id, TaskInstance.execution_date == execution_date, TaskInstance.map_index == map_index, ) .join(TaskInstance.dag_run) - .options(joinedload("trigger")) - .options(joinedload("trigger.triggerer_job")) - .first() + .options(joinedload(TaskInstance.trigger).joinedload(Trigger.triggerer_job)) + .limit(1) ) if ti is None: @@ -1686,10 +1694,10 @@ def log(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) dag_model = DagModel.get_dagmodel(dag_id) - ti = ( - session.query(models.TaskInstance) + ti = session.scalar( + select(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .first() + .limit(1) ) num_logs = 0 @@ -1730,10 +1738,10 @@ def redirect_to_external_log(self, session: Session = NEW_SESSION): map_index = request.args.get("map_index", -1, type=int) try_number = request.args.get("try_number", 1) - ti = ( - session.query(models.TaskInstance) + ti = session.scalar( + select(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .first() + .limit(1) ) if not ti: @@ -1775,8 +1783,8 @@ def task(self, session: Session = NEW_SESSION): task = copy.copy(dag.get_task(task_id)) task.resolve_template_files() - ti: TaskInstance | None = ( - session.query(TaskInstance) + ti: TaskInstance | None = session.scalar( + select(TaskInstance) .options( # HACK: Eager-load relationships. This is needed because # multiple properties mis-use provide_session() that destroys @@ -1785,7 +1793,6 @@ def task(self, session: Session = NEW_SESSION): joinedload(TaskInstance.trigger, innerjoin=False), ) .filter_by(execution_date=dttm, dag_id=dag_id, task_id=task_id, map_index=map_index) - .one_or_none() ) if ti is None: ti_attrs: list[tuple[str, Any]] | None = None @@ -1902,17 +1909,19 @@ def xcom(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) root = request.args.get("root", "") dag = DagModel.get_dagmodel(dag_id) - ti = session.query(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).first() + ti = session.scalar(select(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).limit(1)) if not ti: flash(f"Task [{dag_id}.{task_id}] doesn't seem to exist at the moment", "error") return redirect(url_for("Airflow.index")) - xcom_query = session.query(XCom.key, XCom.value).filter( - XCom.dag_id == dag_id, - XCom.task_id == task_id, - XCom.execution_date == dttm, - XCom.map_index == map_index, + xcom_query = session.execute( + select(XCom.key, XCom.value).where( + XCom.dag_id == dag_id, + XCom.task_id == task_id, + XCom.execution_date == dttm, + XCom.map_index == map_index, + ) ) attributes = [(k, v) for k, v in xcom_query if not k.startswith("_")] @@ -1982,7 +1991,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): request_execution_date = request.values.get("execution_date", default=timezone.utcnow().isoformat()) is_dag_run_conf_overrides_params = conf.getboolean("core", "dag_run_conf_overrides_params") dag = get_airflow_app().dag_bag.get_dag(dag_id) - dag_orm: DagModel = session.query(DagModel).filter(DagModel.dag_id == dag_id).first() + dag_orm: DagModel = session.scalar(select(DagModel).where(DagModel.dag_id == dag_id).limit(1)) # Prepare form fields with param struct details to render a proper form with schema information form_fields = {} @@ -2020,11 +2029,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): flash(f"Cannot create dagruns because the dag {dag_id} has import errors", "error") return redirect(origin) - recent_runs = ( - session.query( - DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date) - ) - .filter( + recent_runs = session.execute( + select(DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date)) + .where( DagRun.dag_id == dag_id, DagRun.run_type == DagRunType.MANUAL, DagRun.conf.isnot(None), @@ -2297,15 +2304,17 @@ def clear(self, *, session: Session = NEW_SESSION): # Lock the related dag runs to prevent from possible dead lock. # https://github.com/apache/airflow/pull/26658 - dag_runs_query = session.query(DagRun.id).filter(DagRun.dag_id == dag_id).with_for_update() + dag_runs_query = session.scalars( + select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() + ) if start_date is None and end_date is None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date == start_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) else: if start_date is not None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date >= start_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date >= start_date) if end_date is not None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date <= end_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) locked_dag_run_ids = dag_runs_query.all() elif task_id: @@ -2394,10 +2403,10 @@ def blocked(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify([]) - dags = ( - session.query(DagRun.dag_id, sqla.func.count(DagRun.id)) - .filter(DagRun.state == DagRunState.RUNNING) - .filter(DagRun.dag_id.in_(filter_dag_ids)) + dags = session.execute( + select(DagRun.dag_id, sqla.func.count(DagRun.id)) + .where(DagRun.state == DagRunState.RUNNING) + .where(DagRun.dag_id.in_(filter_dag_ids)) .group_by(DagRun.dag_id) ) @@ -2478,9 +2487,11 @@ def _mark_dagrun_state_as_queued( # Identify tasks that will be queued up to run when confirmed all_task_ids = [task.task_id for task in dag.tasks] - existing_tis = session.query(TaskInstance.task_id).filter( - TaskInstance.dag_id == dag.dag_id, - TaskInstance.run_id == dag_run_id, + existing_tis = session.execute( + select(TaskInstance.task_id).where( + TaskInstance.dag_id == dag.dag_id, + TaskInstance.run_id == dag_run_id, + ) ) completed_tis_ids = [task_id for task_id, in existing_tis] @@ -2962,19 +2973,18 @@ def _convert_to_date(session, column): if root: dag = dag.partial_subset(task_ids_or_regex=root, include_downstream=False, include_upstream=True) - dag_states = ( - session.query( - (_convert_to_date(session, DagRun.execution_date)).label("date"), + dag_states = session.execute( + select( + _convert_to_date(session, DagRun.execution_date).label("date"), DagRun.state, func.max(DagRun.data_interval_start).label("data_interval_start"), func.max(DagRun.data_interval_end).label("data_interval_end"), func.count("*").label("count"), ) - .filter(DagRun.dag_id == dag.dag_id) + .where(DagRun.dag_id == dag.dag_id) .group_by(_convert_to_date(session, DagRun.execution_date), DagRun.state) .order_by(_convert_to_date(session, DagRun.execution_date).asc()) - .all() - ) + ).all() data_dag_states = [ { @@ -3245,16 +3255,17 @@ def duration(self, dag_id: str, session: Session = NEW_SESSION): else: min_date = timezone.utc_epoch() ti_fails = ( - session.query(TaskFail) + select(TaskFail) .join(TaskFail.dag_run) - .filter( + .where( TaskFail.dag_id == dag.dag_id, DagRun.execution_date >= min_date, DagRun.execution_date <= base_date, ) ) if dag.partial: - ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = session.scalars(ti_fails) fails_totals: dict[tuple[str, str, str], int] = defaultdict(int) for failed_task_instance in ti_fails: dict_key = ( @@ -3595,9 +3606,9 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): form = DateTimeWithNumRunsWithDagRunsForm(data=dt_nr_dr_data) form.execution_date.choices = dt_nr_dr_data["dr_choices"] - tis = ( - session.query(TaskInstance) - .filter( + tis = session.scalars( + select(TaskInstance) + .where( TaskInstance.dag_id == dag_id, TaskInstance.run_id == dag_run_id, TaskInstance.start_date.is_not(None), @@ -3606,10 +3617,10 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): .order_by(TaskInstance.start_date) ) - ti_fails = session.query(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) + ti_fails = select(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) if dag.partial: - ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) - + ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = session.scalars(ti_fails) tasks = [] for ti in tis: if not dag.has_task(ti.task_id): @@ -3715,12 +3726,13 @@ def extra_links(self, *, session: Session = NEW_SESSION): if link_name is None: return {"url": None, "error": "Link name not passed"}, 400 - ti = ( - session.query(TaskInstance) + ti = session.scalar( + select(TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) .options(joinedload(TaskInstance.dag_run)) - .first() + .limit(1) ) + if not ti: return {"url": None, "error": "Task Instances not found"}, 404 try: @@ -3828,27 +3840,25 @@ def grid_data(self): base_date = dag.get_latest_execution_date() or timezone.utcnow() with create_session() as session: - query = session.query(DagRun).filter( - DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date - ) + query = select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date) - run_type = request.args.get("run_type") - if run_type: - query = query.filter(DagRun.run_type == run_type) + run_type = request.args.get("run_type") + if run_type: + query = query.where(DagRun.run_type == run_type) - run_state = request.args.get("run_state") - if run_state: - query = query.filter(DagRun.state == run_state) + run_state = request.args.get("run_state") + if run_state: + query = query.where(DagRun.state == run_state) - dag_runs = wwwutils.sorted_dag_runs(query, ordering=dag.timetable.run_ordering, limit=num_runs) - encoded_runs = [ - wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs - ] - data = { - "groups": dag_to_grid(dag, dag_runs, session), - "dag_runs": encoded_runs, - "ordering": dag.timetable.run_ordering, - } + dag_runs = wwwutils.sorted_dag_runs( + query, ordering=dag.timetable.run_ordering, limit=num_runs, session=session + ) + encoded_runs = [wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs] + data = { + "groups": dag_to_grid(dag, dag_runs, session), + "dag_runs": encoded_runs, + "ordering": dag.timetable.run_ordering, + } # avoid spaces to reduce payload size return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -3867,37 +3877,34 @@ def historical_metrics_data(self): end_date = _safe_parse_datetime(request.args.get("end_date")) with create_session() as session: # DagRuns - dag_runs_type = ( - session.query(DagRun.run_type, func.count(DagRun.run_id)) - .filter( + dag_runs_type = session.execute( + select(DagRun.run_type, func.count(DagRun.run_id)) + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.run_type) - .all() - ) + ).all() - dag_run_states = ( - session.query(DagRun.state, func.count(DagRun.run_id)) - .filter( + dag_run_states = session.execute( + select(DagRun.state, func.count(DagRun.run_id)) + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.state) - .all() - ) + ).all() # TaskInstances - task_instance_states = ( - session.query(TaskInstance.state, func.count(TaskInstance.run_id)) + task_instance_states = session.execute( + select(TaskInstance.state, func.count(TaskInstance.run_id)) .join(TaskInstance.dag_run) - .filter( + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(TaskInstance.state) - .all() - ) + ).all() data = { "dag_run_types": { @@ -3931,28 +3938,32 @@ def next_run_datasets(self, dag_id): with create_session() as session: data = [ dict(info) - for info in session.query( - DatasetModel.id, - DatasetModel.uri, - func.max(DatasetEvent.timestamp).label("lastUpdate"), - ) - .join(DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id) - .join( - DatasetDagRunQueue, - and_( - DatasetDagRunQueue.dataset_id == DatasetModel.id, - DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, - ), - isouter=True, - ) - .join( - DatasetEvent, - DatasetEvent.dataset_id == DatasetModel.id, - isouter=True, + for info in session.execute( + select( + DatasetModel.id, + DatasetModel.uri, + func.max(DatasetEvent.timestamp).label("lastUpdate"), + ) + .join( + DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id + ) + .join( + DatasetDagRunQueue, + and_( + DatasetDagRunQueue.dataset_id == DatasetModel.id, + DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, + ), + isouter=True, + ) + .join( + DatasetEvent, + DatasetEvent.dataset_id == DatasetModel.id, + isouter=True, + ) + .where(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) + .group_by(DatasetModel.id, DatasetModel.uri) + .order_by(DatasetModel.uri) ) - .filter(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) - .group_by(DatasetModel.id, DatasetModel.uri) - .order_by(DatasetModel.uri) ] return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -4050,12 +4061,12 @@ def datasets_summary(self): if session.bind.dialect.name == "postgresql": order_by = (order_by[0].nulls_first(), *order_by[1:]) - count_query = session.query(func.count(DatasetModel.id)) + count_query = select(func.count(DatasetModel.id)) has_event_filters = bool(updated_before or updated_after) query = ( - session.query( + select( DatasetModel.id, DatasetModel.uri, func.max(DatasetEvent.timestamp).label("last_dataset_update"), @@ -4080,11 +4091,12 @@ def datasets_summary(self): if updated_before: filters.append(DatasetEvent.timestamp <= updated_before) - query = query.filter(*filters).offset(offset).limit(limit) - count_query = count_query.filter(*filters) + query = query.where(*filters).offset(offset).limit(limit) + count_query = count_query.where(*filters) + query = session.execute(query) datasets = [dict(dataset) for dataset in query] - data = {"datasets": datasets, "total_entries": count_query.scalar()} + data = {"datasets": datasets, "total_entries": session.scalar(count_query)} return ( htmlsafe_json_dumps(data, separators=(",", ":"), cls=utils_json.WebEncoder), @@ -4130,20 +4142,20 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): included_events_raw = conf.get("webserver", "audit_view_included_events", fallback=None) excluded_events_raw = conf.get("webserver", "audit_view_excluded_events", fallback=None) - query = session.query(Log).filter(Log.dag_id == dag_id) + query = select(Log).where(Log.dag_id == dag_id) if included_events_raw: included_events = {event.strip() for event in included_events_raw.split(",")} - query = query.filter(Log.event.in_(included_events)) + query = query.where(Log.event.in_(included_events)) elif excluded_events_raw: excluded_events = {event.strip() for event in excluded_events_raw.split(",")} - query = query.filter(Log.event.notin_(excluded_events)) + query = query.where(Log.event.notin_(excluded_events)) current_page = request.args.get("page", default=0, type=int) arg_sorting_key = request.args.get("sorting_key", "dttm") arg_sorting_direction = request.args.get("sorting_direction", default="desc") logs_per_page = PAGE_SIZE - audit_logs_count = query.count() + audit_logs_count = session.scalar(select(func.count()).select_from(query)) num_of_pages = int(math.ceil(audit_logs_count / float(logs_per_page))) start = current_page * logs_per_page @@ -4155,7 +4167,7 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): sort_column = sort_column.desc() query = query.order_by(sort_column) - dag_audit_logs = query.offset(start).limit(logs_per_page).all() + dag_audit_logs = session.scalars(query.offset(start).limit(logs_per_page)).all() return self.render_template( "airflow/dag_audit_log.html", dag=dag, @@ -4276,7 +4288,7 @@ def apply(self, query, func): if get_airflow_app().appbuilder.sm.has_all_dags_access(g.user): return query filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - return query.filter(self.model.dag_id.in_(filter_dag_ids)) + return query.where(self.model.dag_id.in_(filter_dag_ids)) class AirflowModelView(ModelView): @@ -4721,9 +4733,11 @@ def action_mulduplicate(self, connections, session: Session = NEW_SESSION): potential_connection_ids = [f"{base_conn_id}_copy{i}" for i in range(1, 11)] - query = session.query(Connection.conn_id).filter(Connection.conn_id.in_(potential_connection_ids)) + query = session.scalars( + select(Connection.conn_id).where(Connection.conn_id.in_(potential_connection_ids)) + ) - found_conn_id_set = {conn_id for conn_id, in query} + found_conn_id_set = {conn_id for conn_id in query} possible_conn_id_iter = ( connection_id @@ -5393,7 +5407,7 @@ def _set_dag_runs_to_active_state(self, drs: list[DagRun], state: str, session: """This routine only supports Running and Queued state.""" try: count = 0 - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 if state == State.RUNNING: dr.start_date = timezone.utcnow() @@ -5419,7 +5433,7 @@ def action_set_failed(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 altered_tis += set_dag_run_state_to_failed( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5447,7 +5461,7 @@ def action_set_success(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 altered_tis += set_dag_run_state_to_success( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5471,7 +5485,7 @@ def action_clear(self, drs: list[DagRun], session: Session = NEW_SESSION): count = 0 cleared_ti_count = 0 dag_to_tis: dict[DAG, list[TaskInstance]] = {} - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 dag = get_airflow_app().dag_bag.get_dag(dr.dag_id) tis_to_clear = dag_to_tis.setdefault(dag, []) @@ -5887,35 +5901,37 @@ def autocomplete(self, session: Session = NEW_SESSION): return flask.json.jsonify([]) # Provide suggestions of dag_ids and owners - dag_ids_query = session.query( + dag_ids_query = select( sqla.literal("dag").label("type"), DagModel.dag_id.label("name"), - ).filter(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) + ).where(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) owners_query = ( - session.query( + select( sqla.literal("owner").label("type"), DagModel.owners.label("name"), ) .distinct() - .filter(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) + .where(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) ) # Hide DAGs if not showing status: "all" status = flask_session.get(FILTER_STATUS_COOKIE) if status == "active": - dag_ids_query = dag_ids_query.filter(~DagModel.is_paused) - owners_query = owners_query.filter(~DagModel.is_paused) + dag_ids_query = dag_ids_query.where(~DagModel.is_paused) + owners_query = owners_query.where(~DagModel.is_paused) elif status == "paused": - dag_ids_query = dag_ids_query.filter(DagModel.is_paused) - owners_query = owners_query.filter(DagModel.is_paused) + dag_ids_query = dag_ids_query.where(DagModel.is_paused) + owners_query = owners_query.where(DagModel.is_paused) filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - dag_ids_query = dag_ids_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - owners_query = owners_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - - payload = [row._asdict() for row in dag_ids_query.union(owners_query).order_by("name").limit(10)] + dag_ids_query = dag_ids_query.where(DagModel.dag_id.in_(filter_dag_ids)) + owners_query = owners_query.where(DagModel.dag_id.in_(filter_dag_ids)) + payload = [ + row._asdict() + for row in session.execute(dag_ids_query.union(owners_query).order_by("name").limit(10)) + ] return flask.json.jsonify(payload) From 985a6cd2a84daa4511649dfdc9e94b704de5c6d1 Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:54:59 +0530 Subject: [PATCH 026/533] invalid args fix (#32326) --- .../microsoft/azure/example_dag/example_wasb_sensors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py index 349e09c44ad9..8d449e52a1d3 100644 --- a/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py +++ b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py @@ -54,7 +54,6 @@ # [START wasb_prefix_sensor] azure_wasb_prefix_sensor = WasbPrefixSensor( container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, prefix=PREFIX, task_id="wasb_sense_prefix", ) From 42eb93b87b8877af5fbb3244cff786c636f1ec4f Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Mon, 3 Jul 2023 19:21:57 +0530 Subject: [PATCH 027/533] Fixing typo in LOCAL_VIRTUALENV.rst (#32327) --- LOCAL_VIRTUALENV.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 03b6d25e3927..987a0df4bfa5 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -73,7 +73,7 @@ Extra Packages There are known issues with ``bazel`` that might lead to circular dependencies when using it to install Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing the problem in `this PR `_ so it might be that - n ewer versions of ``bazel`` will handle it. + newer versions of ``bazel`` will handle it. If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. From 2b5d4e319b82267349aa20cb3230f3b2ad35adce Mon Sep 17 00:00:00 2001 From: Clemens Date: Tue, 4 Jul 2023 05:47:27 +0800 Subject: [PATCH 028/533] Render list items in rendered fields view (#32042) --------- Co-authored-by: clemens.valiente --- airflow/www/utils.py | 30 +++++++--- airflow/www/views.py | 6 +- tests/www/views/test_views_rendered.py | 78 +++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 14 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 46256ee3359e..b31f9326d988 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -20,7 +20,7 @@ import json import textwrap import time -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any, Callable, Sequence from urllib.parse import urlencode from flask import request, url_for @@ -36,6 +36,7 @@ from pendulum.datetime import DateTime from pygments import highlight, lexers from pygments.formatters import HtmlFormatter +from pygments.lexer import Lexer from sqlalchemy import delete, func, types from sqlalchemy.ext.associationproxy import AssociationProxy from sqlalchemy.sql import Select @@ -549,20 +550,35 @@ def pygment_html_render(s, lexer=lexers.TextLexer): return highlight(s, lexer(), HtmlFormatter(linenos=True)) -def render(obj, lexer): +def render(obj: Any, lexer: Lexer, handler: Callable[[Any], str] | None = None): """Render a given Python object with a given Pygments lexer.""" - out = "" if isinstance(obj, str): - out = Markup(pygment_html_render(obj, lexer)) + return Markup(pygment_html_render(obj, lexer)) + elif isinstance(obj, (tuple, list)): + out = "" for i, text_to_render in enumerate(obj): + if lexer is lexers.PythonLexer: + text_to_render = repr(text_to_render) out += Markup("
List item #{}
").format(i) out += Markup("
" + pygment_html_render(text_to_render, lexer) + "
") + return out + elif isinstance(obj, dict): + out = "" for k, v in obj.items(): + if lexer is lexers.PythonLexer: + v = repr(v) out += Markup('
Dict item "{}"
').format(k) out += Markup("
" + pygment_html_render(v, lexer) + "
") - return out + return out + + elif handler is not None and obj is not None: + return Markup(pygment_html_render(handler(obj), lexer)) + + else: + # Return empty string otherwise + return "" def json_render(obj, lexer): @@ -603,8 +619,8 @@ def get_attr_renderer(): "mysql": lambda x: render(x, lexers.MySqlLexer), "postgresql": lambda x: render(x, lexers.PostgresLexer), "powershell": lambda x: render(x, lexers.PowerShellLexer), - "py": lambda x: render(get_python_source(x), lexers.PythonLexer), - "python_callable": lambda x: render(get_python_source(x), lexers.PythonLexer), + "py": lambda x: render(x, lexers.PythonLexer, get_python_source), + "python_callable": lambda x: render(x, lexers.PythonLexer, get_python_source), "rst": lambda x: render(x, lexers.RstLexer), "sql": lambda x: render(x, lexers.SqlLexer), "tsql": lambda x: render(x, lexers.TransactSqlLexer), diff --git a/airflow/www/views.py b/airflow/www/views.py index d791eefc37c7..90d98a424c4a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -1484,11 +1484,7 @@ def rendered_templates(self, session): content = getattr(task, template_field) renderer = task.template_fields_renderers.get(template_field, template_field) if renderer in renderers: - if isinstance(content, (dict, list)): - json_content = json.dumps(content, sort_keys=True, indent=4) - html_dict[template_field] = renderers[renderer](json_content) - else: - html_dict[template_field] = renderers[renderer](content) + html_dict[template_field] = renderers[renderer](content) else: html_dict[template_field] = Markup("
{}
").format(pformat(content)) diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 6a2cb9717f7e..1557a083d50b 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -24,7 +24,9 @@ from markupsafe import escape from airflow.models import DAG, RenderedTaskInstanceFields, Variable +from airflow.models.baseoperator import BaseOperator from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.session import create_session @@ -64,6 +66,39 @@ def task2(dag): ) +@pytest.fixture() +def task3(dag): + class TestOperator(BaseOperator): + template_fields = ("sql",) + + def __init__(self, *, sql, **kwargs): + super().__init__(**kwargs) + self.sql = sql + + def execute(self, context): + pass + + return TestOperator( + task_id="task3", + sql=["SELECT 1;", "SELECT 2;"], + dag=dag, + ) + + +@pytest.fixture() +def task4(dag): + def func(*op_args): + pass + + return PythonOperator( + task_id="task4", + python_callable=func, + op_args=["{{ task_instance_key_str }}_args"], + op_kwargs={"0": "{{ task_instance_key_str }}_kwargs"}, + dag=dag, + ) + + @pytest.fixture() def task_secret(dag): return BashOperator( @@ -85,7 +120,7 @@ def init_blank_db(): @pytest.fixture(autouse=True) -def reset_db(dag, task1, task2, task_secret): +def reset_db(dag, task1, task2, task3, task4, task_secret): yield clear_db_dags() clear_db_runs() @@ -93,7 +128,7 @@ def reset_db(dag, task1, task2, task_secret): @pytest.fixture() -def create_dag_run(dag, task1, task2, task_secret): +def create_dag_run(dag, task1, task2, task3, task4, task_secret): def _create_dag_run(*, execution_date, session): dag_run = dag.create_dagrun( state=DagRunState.RUNNING, @@ -108,6 +143,10 @@ def _create_dag_run(*, execution_date, session): ti2.state = TaskInstanceState.SCHEDULED ti3 = dag_run.get_task_instance(task_secret.task_id, session=session) ti3.state = TaskInstanceState.QUEUED + ti4 = dag_run.get_task_instance(task3.task_id, session=session) + ti4.state = TaskInstanceState.SUCCESS + ti5 = dag_run.get_task_instance(task4.task_id, session=session) + ti5.state = TaskInstanceState.SUCCESS session.flush() return dag_run @@ -290,3 +329,38 @@ def test_rendered_task_detail_env_secret(patch_app, admin_client, request, env, if request.node.callspec.id.endswith("-tpld-var"): Variable.delete("plain_var") Variable.delete("secret_var") + + +@pytest.mark.usefixtures("patch_app") +def test_rendered_template_view_for_list_template_field_args(admin_client, create_dag_run, task3): + """ + Test that the Rendered View can show a list of syntax-highlighted SQL statements + """ + assert task3.sql == ["SELECT 1;", "SELECT 2;"] + + with create_session() as session: + create_dag_run(execution_date=DEFAULT_DATE, session=session) + + url = f"rendered-templates?task_id=task3&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + + resp = admin_client.get(url, follow_redirects=True) + check_content_in_response("List item #0", resp) + check_content_in_response("List item #1", resp) + + +@pytest.mark.usefixtures("patch_app") +def test_rendered_template_view_for_op_args(admin_client, create_dag_run, task4): + """ + Test that the Rendered View can show rendered values in op_args and op_kwargs + """ + assert task4.op_args == ["{{ task_instance_key_str }}_args"] + assert list(task4.op_kwargs.values()) == ["{{ task_instance_key_str }}_kwargs"] + + with create_session() as session: + create_dag_run(execution_date=DEFAULT_DATE, session=session) + + url = f"rendered-templates?task_id=task4&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + + resp = admin_client.get(url, follow_redirects=True) + check_content_in_response("testdag__task4__20200301_args", resp) + check_content_in_response("testdag__task4__20200301_kwargs", resp) From 17132ef4c65b842dab4ff311fd2b0e6d08969e1b Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Tue, 4 Jul 2023 03:44:45 +0530 Subject: [PATCH 029/533] Added `AzureBlobStorageToS3Operator` transfer operator (#32270) --- .../amazon/aws/transfers/azure_blob_to_s3.py | 167 ++++++++++++++ airflow/providers/amazon/provider.yaml | 4 + .../providers/microsoft/azure/hooks/wasb.py | 27 +++ dev/breeze/tests/test_selective_checks.py | 8 +- .../transfer/azure_blob_to_s3.rst | 52 +++++ generated/provider_dependencies.json | 1 + .../aws/transfers/test_azure_blob_to_s3.py | 218 ++++++++++++++++++ .../microsoft/azure/hooks/test_wasb.py | 25 ++ .../amazon/aws/example_azure_blob_to_s3.py | 83 +++++++ 9 files changed, 581 insertions(+), 4 deletions(-) create mode 100644 airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py create mode 100644 docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst create mode 100644 tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py create mode 100644 tests/system/providers/amazon/aws/example_azure_blob_to_s3.py diff --git a/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py b/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py new file mode 100644 index 000000000000..ebcd49bdc411 --- /dev/null +++ b/airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py @@ -0,0 +1,167 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import tempfile +from typing import TYPE_CHECKING, Sequence + +from airflow.models import BaseOperator +from airflow.providers.amazon.aws.hooks.s3 import S3Hook +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class AzureBlobStorageToS3Operator(BaseOperator): + """ + Operator transfers data from Azure Blob Storage to specified bucket in Amazon S3. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:AzureBlobStorageToGCSOperator` + + :param wasb_conn_id: Reference to the wasb connection. + :param container_name: Name of the container + :param prefix: Prefix string which filters objects whose name begin with + this prefix. (templated) + :param delimiter: The delimiter by which you want to filter the objects. (templated) + For e.g to lists the CSV files from in a directory in GCS you would use + delimiter='.csv'. + :param aws_conn_id: Connection id of the S3 connection to use + :param dest_s3_key: The base S3 key to be used to store the files. (templated) + :param dest_verify: Whether or not to verify SSL certificates for S3 connection. + By default SSL certificates are verified. + You can provide the following values: + + - ``False``: do not validate SSL certificates. SSL will still be used + (unless use_ssl is False), but SSL certificates will not be + verified. + - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses. + You can specify this argument if you want to use a different + CA cert bundle than the one used by botocore. + :param dest_s3_extra_args: Extra arguments that may be passed to the download/upload operations. + :param replace: Whether or not to verify the existence of the files in the + destination bucket. + By default is set to False + If set to True, will upload all the files replacing the existing ones in + the destination bucket. + If set to False, will upload only the files that are in the origin but not + in the destination bucket. + :param s3_acl_policy: Optional The string to specify the canned ACL policy for the + object to be uploaded in S3 + :param wasb_extra_kargs: kwargs to pass to WasbHook + :param s3_extra_kargs: kwargs to pass to S3Hook + """ + + template_fields: Sequence[str] = ( + "container_name", + "prefix", + "delimiter", + "dest_s3_key", + ) + + def __init__( + self, + *, + wasb_conn_id: str = "wasb_default", + container_name: str, + prefix: str | None = None, + delimiter: str = "", + aws_conn_id: str = "aws_default", + dest_s3_key: str, + dest_verify: str | bool | None = None, + dest_s3_extra_args: dict | None = None, + replace: bool = False, + s3_acl_policy: str | None = None, + wasb_extra_args: dict = {}, + s3_extra_args: dict = {}, + **kwargs, + ) -> None: + super().__init__(**kwargs) + + self.wasb_conn_id = wasb_conn_id + self.container_name = container_name + self.prefix = prefix + self.delimiter = delimiter + self.aws_conn_id = aws_conn_id + self.dest_s3_key = dest_s3_key + self.dest_verify = dest_verify + self.dest_s3_extra_args = dest_s3_extra_args or {} + self.replace = replace + self.s3_acl_policy = s3_acl_policy + self.wasb_extra_args = wasb_extra_args + self.s3_extra_args = s3_extra_args + + def execute(self, context: Context) -> list[str]: + # list all files in the Azure Blob Storage container + wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id, **self.wasb_extra_args) + s3_hook = S3Hook( + aws_conn_id=self.aws_conn_id, + verify=self.dest_verify, + extra_args=self.dest_s3_extra_args, + **self.s3_extra_args, + ) + + self.log.info( + f"Getting list of the files in Container: {self.container_name}; " + f"Prefix: {self.prefix}; Delimiter: {self.delimiter};" + ) + + files = wasb_hook.get_blobs_list_recursive( + container_name=self.container_name, prefix=self.prefix, endswith=self.delimiter + ) + + if not self.replace: + # if we are not replacing -> list all files in the S3 bucket + # and only keep those files which are present in + # Azure Blob Storage and not in S3 + bucket_name, prefix = S3Hook.parse_s3_url(self.dest_s3_key) + # look for the bucket and the prefix to avoid look into + # parent directories/keys + existing_files = s3_hook.list_keys(bucket_name, prefix=prefix) + # in case that no files exists, return an empty array to avoid errors + existing_files = existing_files if existing_files is not None else [] + # remove the prefix for the existing files to allow the match + existing_files = [file.replace(f"{prefix}/", "", 1) for file in existing_files] + files = list(set(files) - set(existing_files)) + + if files: + for file in files: + with tempfile.NamedTemporaryFile() as temp_file: + + dest_key = os.path.join(self.dest_s3_key, file) + self.log.info("Downloading data from blob: %s", file) + wasb_hook.get_file( + file_path=temp_file.name, + container_name=self.container_name, + blob_name=file, + ) + + self.log.info("Uploading data to s3: %s", dest_key) + s3_hook.load_file( + filename=temp_file.name, + key=dest_key, + replace=self.replace, + acl_policy=self.s3_acl_policy, + ) + self.log.info("All done, uploaded %d files to S3", len(files)) + else: + self.log.info("All files are already in sync!") + return files diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index a7dac9fad481..575a2b5cab1b 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -625,6 +625,10 @@ transfers: - source-integration-name: Amazon Web Services target-integration-name: Amazon Web Services python-module: airflow.providers.amazon.aws.transfers.base + - source-integration-name: Microsoft Azure Blob Storage + target-integration-name: Amazon Simple Storage Service (S3) + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst + python-module: airflow.providers.amazon.aws.transfers.azure_blob_to_s3 extra-links: - airflow.providers.amazon.aws.links.batch.BatchJobDefinitionLink diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index c1d615810b96..ad9c4754c525 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -292,6 +292,33 @@ def get_blobs_list( blob_list.append(blob.name) return blob_list + def get_blobs_list_recursive( + self, + container_name: str, + prefix: str | None = None, + include: list[str] | None = None, + endswith: str = "", + **kwargs, + ) -> list: + """ + List blobs in a given container. + + :param container_name: The name of the container + :param prefix: Filters the results to return only blobs whose names + begin with the specified prefix. + :param include: Specifies one or more additional datasets to include in the + response. Options include: ``snapshots``, ``metadata``, ``uncommittedblobs``, + ``copy`, ``deleted``. + :param delimiter: filters objects based on the delimiter (for e.g '.csv') + """ + container = self._get_container_client(container_name) + blob_list = [] + blobs = container.list_blobs(name_starts_with=prefix, include=include, **kwargs) + for blob in blobs: + if blob.name.endswith(endswith): + blob_list.append(blob.name) + return blob_list + def load_file( self, file_path: str, diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 61e35ce20f59..7437642e4e9b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google http imap " + "common.sql exasol ftp google http imap microsoft.azure " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap,microsoft.azure," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google http imap " + "common.sql exasol ftp google http imap microsoft.azure " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -368,7 +368,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," - "http,imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", + "http,imap,microsoft.azure,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", ), diff --git a/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst new file mode 100644 index 000000000000..a3b9df5eb1ee --- /dev/null +++ b/docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +=================================================== +Azure Blob Storage to Amazon S3 transfer operator +=================================================== + +Use the ``AzureBlobStorageToS3Operator`` transfer to copy the data from Azure Blob Storage to Amazon Simple Storage Service (S3). + +Prerequisite Tasks +------------------ + +.. include:: ../_partials/prerequisite_tasks.rst + +Operators +--------- + +.. _howto/operator:AzureBlobStorageToS3Operator: + +Azure Blob Storage to Amazon S3 +================================= + +To copy data from an Azure Blob Storage container to an Amazon S3 bucket you can use +:class:`~airflow.providers.amazon.aws.transfers.azure_blob_to_s3.AzureBlobStorageToS3Operator` + +Example usage: + +.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_azure_blob_to_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_transfer_azure_blob_to_s3] + :end-before: [END howto_transfer_azure_blob_to_s3] + +Reference +--------- + +* `Azure Blob Storage client library `__ +* `AWS boto3 library documentation for Amazon S3 `__ diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index de1f387add83..d80c3d084e01 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -45,6 +45,7 @@ "google", "http", "imap", + "microsoft.azure", "mongo", "salesforce", "ssh" diff --git a/tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py b/tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py new file mode 100644 index 000000000000..f34877e18722 --- /dev/null +++ b/tests/providers/amazon/aws/transfers/test_azure_blob_to_s3.py @@ -0,0 +1,218 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from io import RawIOBase +from unittest import mock + +from moto import mock_s3 + +from airflow.providers.amazon.aws.hooks.s3 import S3Hook +from airflow.providers.amazon.aws.transfers.azure_blob_to_s3 import AzureBlobStorageToS3Operator + +TASK_ID = "test-gcs-list-operator" +CONTAINER_NAME = "test-container" +DELIMITER = ".csv" +PREFIX = "TEST" +S3_BUCKET = "s3://bucket/" +MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv"] +S3_ACL_POLICY = "private-read" + + +def _create_test_bucket(): + hook = S3Hook(aws_conn_id="airflow_gcs_test") + # We're mocking all actual AWS calls and don't need a connection. + # This avoids an Airflow warning about connection cannot be found. + hook.get_connection = lambda _: None + bucket = hook.get_bucket("bucket") + bucket.create() + return hook, bucket + + +@mock_s3 +class TestAzureBlobToS3Operator: + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + def test_operator_all_file_upload(self, mock_hook): + """ + Destination bucket has no file (of interest) common with origin bucket i.e + Azure - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + S3 - [] + """ + mock_hook.return_value.get_blobs_list_recursive.return_value = MOCK_FILES + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + replace=False, + ) + + hook, _ = _create_test_bucket() + uploaded_files = operator.execute(None) + + assert sorted(MOCK_FILES) == sorted(uploaded_files) + assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) + + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + def test_operator_incremental_file_upload_without_replace(self, mock_hook): + """ + Destination bucket has subset of files common with origin bucket i.e + Azure - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + S3 - ["TEST1.csv"] + """ + mock_hook.return_value.get_blobs_list_recursive.return_value = MOCK_FILES + get_file = mock_hook.return_value.get_file + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + # without replace + replace=False, + ) + + hook, bucket = _create_test_bucket() + # uploading only first file + bucket.put_object(Key=MOCK_FILES[0], Body=b"testing") + + uploaded_files = operator.execute(None) + + assert sorted(MOCK_FILES[1:]) == sorted(uploaded_files) + assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) + assert get_file.call_count == len(MOCK_FILES[1:]) + + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + def test_operator_incremental_file_upload_with_replace(self, mock_hook): + """ + Destination bucket has subset of files common with origin bucket i.e + Azure - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + S3 - ["TEST1.csv"] + """ + mock_hook.return_value.get_blobs_list_recursive.return_value = MOCK_FILES + get_file = mock_hook.return_value.get_file + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + # with replace + replace=True, + ) + + hook, bucket = _create_test_bucket() + # uploading only first file + bucket.put_object(Key=MOCK_FILES[0], Body=b"testing") + + uploaded_files = operator.execute(None) + + assert sorted(MOCK_FILES) == sorted(uploaded_files) + assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) + assert get_file.call_count == len(MOCK_FILES) + + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + def test_operator_no_file_upload_without_replace(self, mock_hook): + """ + Destination bucket has all the files common with origin bucket i.e + Azure - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + S3 - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + """ + mock_hook.return_value.get_blobs_list_recursive.return_value = MOCK_FILES + get_file = mock_hook.return_value.get_file + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + replace=False, + ) + + hook, bucket = _create_test_bucket() + # uploading all the files + for mock_file in MOCK_FILES: + bucket.put_object(Key=mock_file, Body=b"testing") + + uploaded_files = operator.execute(None) + + assert sorted([]) == sorted(uploaded_files) + assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) + assert get_file.call_count == 0 + + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + def test_operator_no_file_upload_with_replace(self, mock_hook): + """ + Destination bucket has all the files common with origin bucket i.e + Azure - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + S3 - ["TEST1.csv", "TEST2.csv", "TEST3.csv"] + """ + mock_hook.return_value.get_blobs_list_recursive.return_value = MOCK_FILES + get_file = mock_hook.return_value.get_file + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + replace=True, + ) + + hook, bucket = _create_test_bucket() + # uploading all the files + for mock_file in MOCK_FILES: + bucket.put_object(Key=mock_file, Body=b"testing") + + uploaded_files = operator.execute(None) + + assert sorted(MOCK_FILES) == sorted(uploaded_files) + assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) + # this ensures that upload happened + assert get_file.call_count == len(MOCK_FILES) + + @mock.patch("tempfile.NamedTemporaryFile") + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.WasbHook") + @mock.patch("airflow.providers.amazon.aws.transfers.azure_blob_to_s3.S3Hook") + def test_operator_should_pass_dest_s3_extra_args_and_s3_acl_policy( + self, s3_hook_mock, wasb_hook_mock, mock_tempfile + ): + wasb_blob_name = "test_file" + s3_acl_policy = "test policy" + s3_extra_args = {"ContentLanguage": "value"} + + wasb_hook_mock.return_value.get_blobs_list_recursive.return_value = [wasb_blob_name] + wasb_hook_mock.return_value.download.return_value = RawIOBase(b"testing") + mock_tempfile.return_value.__enter__.return_value.name = "test_temp_file" + + # with current S3_BUCKET url, parse_s3_url would complain + s3_hook_mock.parse_s3_url.return_value = ("bucket", wasb_blob_name) + mock_load_files = s3_hook_mock.return_value.load_file + + operator = AzureBlobStorageToS3Operator( + task_id=TASK_ID, + container_name=CONTAINER_NAME, + dest_s3_key=S3_BUCKET, + replace=False, + dest_s3_extra_args=s3_extra_args, + s3_acl_policy=s3_acl_policy, + ) + + operator.execute(None) + s3_hook_mock.assert_called_once_with(aws_conn_id="aws_default", extra_args=s3_extra_args, verify=None) + mock_load_files.assert_called_once_with( + filename="test_temp_file", + key=f"{S3_BUCKET}{wasb_blob_name}", + replace=False, + acl_policy=s3_acl_policy, + ) diff --git a/tests/providers/microsoft/azure/hooks/test_wasb.py b/tests/providers/microsoft/azure/hooks/test_wasb.py index 464db0f39f43..672d20d3a65e 100644 --- a/tests/providers/microsoft/azure/hooks/test_wasb.py +++ b/tests/providers/microsoft/azure/hooks/test_wasb.py @@ -23,6 +23,7 @@ import pytest from azure.identity import ClientSecretCredential, DefaultAzureCredential from azure.storage.blob import BlobServiceClient +from azure.storage.blob._models import BlobProperties from airflow.exceptions import AirflowException from airflow.models import Connection @@ -299,6 +300,30 @@ def test_get_blobs_list(self, mock_service): name_starts_with="my", include=None, delimiter="/" ) + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.BlobServiceClient") + def test_get_blobs_list_recursive(self, mock_service): + hook = WasbHook(wasb_conn_id=self.shared_key_conn_id) + hook.get_blobs_list_recursive( + container_name="mycontainer", prefix="test", include=None, endswith="file_extension" + ) + mock_service.return_value.get_container_client.assert_called_once_with("mycontainer") + mock_service.return_value.get_container_client.return_value.list_blobs.assert_called_once_with( + name_starts_with="test", include=None + ) + + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.BlobServiceClient") + def test_get_blobs_list_recursive_endswith(self, mock_service): + hook = WasbHook(wasb_conn_id=self.shared_key_conn_id) + mock_service.return_value.get_container_client.return_value.list_blobs.return_value = [ + BlobProperties(name="test/abc.py"), + BlobProperties(name="test/inside_test/abc.py"), + BlobProperties(name="test/abc.csv"), + ] + blob_list_output = hook.get_blobs_list_recursive( + container_name="mycontainer", prefix="test", include=None, endswith=".py" + ) + assert blob_list_output == ["test/abc.py", "test/inside_test/abc.py"] + @pytest.mark.parametrize(argnames="create_container", argvalues=[True, False]) @mock.patch.object(WasbHook, "upload") def test_load_file(self, mock_upload, create_container): diff --git a/tests/system/providers/amazon/aws/example_azure_blob_to_s3.py b/tests/system/providers/amazon/aws/example_azure_blob_to_s3.py new file mode 100644 index 000000000000..79b49966ca54 --- /dev/null +++ b/tests/system/providers/amazon/aws/example_azure_blob_to_s3.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime + +from airflow import DAG +from airflow.models.baseoperator import chain +from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator +from airflow.providers.amazon.aws.transfers.azure_blob_to_s3 import AzureBlobStorageToS3Operator +from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +sys_test_context_task = SystemTestContextBuilder().build() + +DAG_ID = "example_azure_blob_to_s3" + +with DAG( + dag_id=DAG_ID, + schedule="@once", + start_date=datetime(2021, 1, 1), + tags=["example"], + catchup=False, +) as dag: + test_context = sys_test_context_task() + env_id = test_context["ENV_ID"] + + s3_bucket = f"{env_id}-azure_blob-to-s3-bucket" + s3_key = f"{env_id}-azure_blob-to-s3-key" + s3_key_url = f"s3://{s3_bucket}/{s3_key}" + azure_container_name = f"{env_id}-azure_blob-to-s3-container" + + create_s3_bucket = S3CreateBucketOperator(task_id="create_s3_bucket", bucket_name=s3_bucket) + + # [START howto_transfer_azure_blob_to_s3] + azure_blob_to_s3 = AzureBlobStorageToS3Operator( + task_id="azure_blob_to_s3", + container_name=azure_container_name, + dest_s3_key=s3_key_url, + ) + # [END howto_transfer_azure_blob_to_s3] + + delete_s3_bucket = S3DeleteBucketOperator( + task_id="delete_s3_bucket", + bucket_name=s3_bucket, + force_delete=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + + chain( + # TEST SETUP + test_context, + create_s3_bucket, + # TEST BODY + azure_blob_to_s3, + # TEST TEARDOWN + delete_s3_bucket, + ) + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From bc212f80e89e3d460e6320db0983126cea9300e0 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 4 Jul 2023 08:02:21 +0200 Subject: [PATCH 030/533] Exckude not-releaseable providers from extras not only from releasing (#32325) Since we are not releasing some providers yet, we want to exclude them also from being referenced in cross-provider dependencies. --- dev/provider_packages/prepare_provider_packages.py | 14 +++++++++----- generated/provider_dependencies.json | 4 +--- .../pre_commit_update_providers_dependencies.py | 9 ++++++++- setup.py | 4 ++++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 8bf6e24852dc..7cf37d382bde 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -123,7 +123,11 @@ # those imports need to come after the above sys.path.insert to make sure that Airflow # sources are importable without having to add the airflow sources to the PYTHONPATH before # running the script -from setup import PREINSTALLED_PROVIDERS, ALL_PROVIDERS # type: ignore[attr-defined] # isort:skip # noqa +from setup import ( # type: ignore[attr-defined] # isort:skip # noqa + ALL_PROVIDERS, + NOT_RELEASED_YET_PROVIDERS, + PREINSTALLED_PROVIDERS, +) # Note - we do not test protocols as they are not really part of the official API of # Apache Airflow @@ -1675,10 +1679,10 @@ def verify_changelog_exists(package: str) -> str: def list_providers_packages(): """List all provider packages.""" providers = get_all_providers() - # For now we should exclude open-lineage from being consider for releasing until it is ready to - # be released - if "openlineage" in providers: - providers.remove("openlineage") + # exclude providers we do not release yet + for provider in NOT_RELEASED_YET_PROVIDERS: + if provider in providers: + providers.remove(provider) for provider in providers: console.print(provider) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d80c3d084e01..aaa5033b9a25 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -272,9 +272,7 @@ "apache-airflow>=2.4.0", "sqlparse>=0.4.2" ], - "cross-providers-deps": [ - "openlineage" - ], + "cross-providers-deps": [], "excluded-python-versions": [] }, "databricks": { diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index 0c489bad63e3..b5a366499190 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -52,6 +52,7 @@ ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict()) ALL_PROVIDER_FILES: list[Path] = [] + # Allow AST to parse the files. sys.path.append(str(AIRFLOW_SOURCES_ROOT)) @@ -168,9 +169,15 @@ def check_if_different_provider_used(file_path: Path) -> None: imports = get_imports_from_file(file_path) for import_name in imports: imported_provider = get_provider_id_from_import(import_name, file_path) + from setup import NOT_RELEASED_YET_PROVIDERS + if imported_provider is not None and imported_provider not in ALL_PROVIDERS: warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.") - elif imported_provider and file_provider != imported_provider: + elif ( + imported_provider + and file_provider != imported_provider + and imported_provider not in NOT_RELEASED_YET_PROVIDERS + ): ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) diff --git a/setup.py b/setup.py index 547e1221d37e..32d5c5444737 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,10 @@ DEPS = "deps" CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" +# we do not yet release some providers so we should skip them when releasing and when referring to +# in cross-provider-dependencies. +NOT_RELEASED_YET_PROVIDERS = ["openlineage"] + # # NOTE! IN Airflow 2.4.+ dependencies for providers are maintained in `provider.yaml` files for each From d7c966441ee57da6f612d9c0d41a37a9ff323de7 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Tue, 4 Jul 2023 09:08:16 +0300 Subject: [PATCH 031/533] Revert "Exckude not-releaseable providers from extras not only from releasing (#32325)" (#32338) This reverts commit bc212f80e89e3d460e6320db0983126cea9300e0. --- dev/provider_packages/prepare_provider_packages.py | 14 +++++--------- generated/provider_dependencies.json | 4 +++- .../pre_commit_update_providers_dependencies.py | 9 +-------- setup.py | 4 ---- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 7cf37d382bde..8bf6e24852dc 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -123,11 +123,7 @@ # those imports need to come after the above sys.path.insert to make sure that Airflow # sources are importable without having to add the airflow sources to the PYTHONPATH before # running the script -from setup import ( # type: ignore[attr-defined] # isort:skip # noqa - ALL_PROVIDERS, - NOT_RELEASED_YET_PROVIDERS, - PREINSTALLED_PROVIDERS, -) +from setup import PREINSTALLED_PROVIDERS, ALL_PROVIDERS # type: ignore[attr-defined] # isort:skip # noqa # Note - we do not test protocols as they are not really part of the official API of # Apache Airflow @@ -1679,10 +1675,10 @@ def verify_changelog_exists(package: str) -> str: def list_providers_packages(): """List all provider packages.""" providers = get_all_providers() - # exclude providers we do not release yet - for provider in NOT_RELEASED_YET_PROVIDERS: - if provider in providers: - providers.remove(provider) + # For now we should exclude open-lineage from being consider for releasing until it is ready to + # be released + if "openlineage" in providers: + providers.remove("openlineage") for provider in providers: console.print(provider) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index aaa5033b9a25..d80c3d084e01 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -272,7 +272,9 @@ "apache-airflow>=2.4.0", "sqlparse>=0.4.2" ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "openlineage" + ], "excluded-python-versions": [] }, "databricks": { diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index b5a366499190..0c489bad63e3 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -52,7 +52,6 @@ ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict()) ALL_PROVIDER_FILES: list[Path] = [] - # Allow AST to parse the files. sys.path.append(str(AIRFLOW_SOURCES_ROOT)) @@ -169,15 +168,9 @@ def check_if_different_provider_used(file_path: Path) -> None: imports = get_imports_from_file(file_path) for import_name in imports: imported_provider = get_provider_id_from_import(import_name, file_path) - from setup import NOT_RELEASED_YET_PROVIDERS - if imported_provider is not None and imported_provider not in ALL_PROVIDERS: warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.") - elif ( - imported_provider - and file_provider != imported_provider - and imported_provider not in NOT_RELEASED_YET_PROVIDERS - ): + elif imported_provider and file_provider != imported_provider: ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) diff --git a/setup.py b/setup.py index 32d5c5444737..547e1221d37e 100644 --- a/setup.py +++ b/setup.py @@ -60,10 +60,6 @@ DEPS = "deps" CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" -# we do not yet release some providers so we should skip them when releasing and when referring to -# in cross-provider-dependencies. -NOT_RELEASED_YET_PROVIDERS = ["openlineage"] - # # NOTE! IN Airflow 2.4.+ dependencies for providers are maintained in `provider.yaml` files for each From 702209898eeb3061024bdedc5ac7b4d48c13108c Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:22:43 +0530 Subject: [PATCH 032/533] typo fix (#32317) --- docs/apache-airflow-providers-amazon/redirects.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow-providers-amazon/redirects.txt b/docs/apache-airflow-providers-amazon/redirects.txt index bf35801e962e..fad1f914880a 100644 --- a/docs/apache-airflow-providers-amazon/redirects.txt +++ b/docs/apache-airflow-providers-amazon/redirects.txt @@ -28,9 +28,9 @@ operators/redshift_data.rst operators/redshift/redshift_data.rst operators/redshift_cluster.rst operators/redshift/redshift_cluster.rst operators/s3.rst operators/s3/s3.rst operators/glacier.rst operators/s3/glacier.rst -operators/transfer/dynamodb_to_s3.rst.rst transfer/dynamodb_to_s3.rst.rst -operators/transfer/ftp_to_s3.rst.rst transfer/ftp_to_s3.rst.rst -operators/transfer/gcs_to_s3.rst.rst transfer/gcs_to_s3.rst.rst +operators/transfer/dynamodb_to_s3.rst transfer/dynamodb_to_s3.rst +operators/transfer/ftp_to_s3.rst transfer/ftp_to_s3.rst +operators/transfer/gcs_to_s3.rst transfer/gcs_to_s3.rst operators/transfer/glacier_to_gcs.rst transfer/glacier_to_gcs.rst operators/transfer/google_api_to_s3.rst transfer/google_api_to_s3.rst operators/transfer/hive_to_dynamodb.rst transfer/hive_to_dynamodb.rst From 1c1dbd872d5378856b4242961dcdf77c7f0dd712 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 4 Jul 2023 13:48:59 +0200 Subject: [PATCH 033/533] Add proxy_user template check (#32334) --- airflow/providers/apache/hive/hooks/hive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index 4ad3b9d12c69..89fcda466ffc 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -152,6 +152,8 @@ def _prepare_cli_cmd(self) -> list[Any]: proxy_user = self._get_proxy_user() if ";" in template: raise RuntimeError("The principal should not contain the ';' character") + if ";" in proxy_user: + raise RuntimeError("The proxy_user should not contain the ';' character") jdbc_url += f";principal={template};{proxy_user}" elif self.auth: jdbc_url += ";auth=" + self.auth From 3c3c337edd6a6905c958206dc8f9fe4303c856eb Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 4 Jul 2023 14:58:29 +0200 Subject: [PATCH 034/533] Revert "Refactor Sqlalchemy queries to 2.0 style (Part 3) (#32177)" (#32343) This reverts commit 1065687ec6df2b9b3557e38a67e71f835796427f. --- airflow/utils/db.py | 38 ++-- airflow/www/utils.py | 11 +- airflow/www/views.py | 460 +++++++++++++++++++++---------------------- 3 files changed, 244 insertions(+), 265 deletions(-) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 46ddcbfb3453..a76f0d4f675d 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -92,7 +92,7 @@ def _format_airflow_moved_table_name(source_table, version, category): @provide_session def merge_conn(conn, session: Session = NEW_SESSION): """Add new Connection.""" - if not session.scalar(select(conn.__class__).filter_by(conn_id=conn.conn_id).limit(1)): + if not session.query(conn.__class__).filter_by(conn_id=conn.conn_id).first(): session.add(conn) session.commit() @@ -959,9 +959,7 @@ def check_conn_id_duplicates(session: Session) -> Iterable[str]: dups = [] try: - dups = session.execute( - select(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1) - ).all() + dups = session.query(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1).all() except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -986,11 +984,12 @@ def check_username_duplicates(session: Session) -> Iterable[str]: for model in [User, RegisterUser]: dups = [] try: - dups = session.execute( - select(model.username) # type: ignore[attr-defined] + dups = ( + session.query(model.username) # type: ignore[attr-defined] .group_by(model.username) # type: ignore[attr-defined] .having(func.count() > 1) - ).all() + .all() + ) except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -1059,13 +1058,13 @@ def check_task_fail_for_duplicates(session): """ minimal_table_obj = table(table_name, *[column(x) for x in uniqueness]) try: - subquery = session.execute( - select(minimal_table_obj, func.count().label("dupe_count")) + subquery = ( + session.query(minimal_table_obj, func.count().label("dupe_count")) .group_by(*[text(x) for x in uniqueness]) .having(func.count() > text("1")) .subquery() ) - dupe_count = session.scalar(select(func.sum(subquery.c.dupe_count))) + dupe_count = session.query(func.sum(subquery.c.dupe_count)).scalar() if not dupe_count: # there are no duplicates; nothing to do. return @@ -1102,7 +1101,7 @@ def check_conn_type_null(session: Session) -> Iterable[str]: n_nulls = [] try: - n_nulls = session.scalars(select(Connection.conn_id).where(Connection.conn_type.is_(None))).all() + n_nulls = session.query(Connection.conn_id).filter(Connection.conn_type.is_(None)).all() except (exc.OperationalError, exc.ProgrammingError, exc.InternalError): # fallback if tables hasn't been created yet session.rollback() @@ -1144,7 +1143,7 @@ def check_run_id_null(session: Session) -> Iterable[str]: dagrun_table.c.run_id.is_(None), dagrun_table.c.execution_date.is_(None), ) - invalid_dagrun_count = session.scalar(select(func.count(dagrun_table.c.id)).where(invalid_dagrun_filter)) + invalid_dagrun_count = session.query(func.count(dagrun_table.c.id)).filter(invalid_dagrun_filter).scalar() if invalid_dagrun_count > 0: dagrun_dangling_table_name = _format_airflow_moved_table_name(dagrun_table.name, "2.2", "dangling") if dagrun_dangling_table_name in inspect(session.get_bind()).get_table_names(): @@ -1241,7 +1240,7 @@ def _move_dangling_data_to_new_table( pk_cols = source_table.primary_key.columns delete = source_table.delete().where( - tuple_(*pk_cols).in_(session.select(*target_table.primary_key.columns).subquery()) + tuple_(*pk_cols).in_(session.query(*target_table.primary_key.columns).subquery()) ) else: delete = source_table.delete().where( @@ -1263,11 +1262,10 @@ def _dangling_against_dag_run(session, source_table, dag_run): source_table.c.dag_id == dag_run.c.dag_id, source_table.c.execution_date == dag_run.c.execution_date, ) - return ( - select(*[c.label(c.name) for c in source_table.c]) + session.query(*[c.label(c.name) for c in source_table.c]) .join(dag_run, source_to_dag_run_join_cond, isouter=True) - .where(dag_run.c.dag_id.is_(None)) + .filter(dag_run.c.dag_id.is_(None)) ) @@ -1306,10 +1304,10 @@ def _dangling_against_task_instance(session, source_table, dag_run, task_instanc ) return ( - select(*[c.label(c.name) for c in source_table.c]) + session.query(*[c.label(c.name) for c in source_table.c]) .join(dag_run, dr_join_cond, isouter=True) .join(task_instance, ti_join_cond, isouter=True) - .where(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) + .filter(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) ) @@ -1333,9 +1331,9 @@ def _move_duplicate_data_to_new_table( """ bind = session.get_bind() dialect_name = bind.dialect.name - query = ( - select(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) + session.query(source_table) + .with_entities(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) .select_from(source_table) .join(subquery, and_(*[getattr(source_table.c, x) == getattr(subquery.c, x) for x in uniqueness])) ) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index b31f9326d988..76914dd9cdcd 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -39,7 +39,6 @@ from pygments.lexer import Lexer from sqlalchemy import delete, func, types from sqlalchemy.ext.associationproxy import AssociationProxy -from sqlalchemy.sql import Select from airflow.exceptions import RemovedInAirflow3Warning from airflow.models import errors @@ -54,6 +53,7 @@ from airflow.www.widgets import AirflowDateTimePickerWidget if TYPE_CHECKING: + from sqlalchemy.orm.query import Query from sqlalchemy.orm.session import Session from sqlalchemy.sql.operators import ColumnOperators @@ -518,21 +518,18 @@ def _get_run_ordering_expr(name: str) -> ColumnOperators: return expr.desc() -def sorted_dag_runs( - query: Select, *, ordering: Sequence[str], limit: int, session: Session -) -> Sequence[DagRun]: +def sorted_dag_runs(query: Query, *, ordering: Sequence[str], limit: int) -> Sequence[DagRun]: """Produce DAG runs sorted by specified columns. - :param query: An ORM select object against *DagRun*. + :param query: An ORM query object against *DagRun*. :param ordering: Column names to sort the runs. should generally come from a timetable's ``run_ordering``. :param limit: Number of runs to limit to. - :param session: SQLAlchemy ORM session object :return: A list of DagRun objects ordered by the specified columns. The list contains only the *last* objects, but in *ascending* order. """ ordering_exprs = (_get_run_ordering_expr(name) for name in ordering) - runs = session.scalars(query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit)).all() + runs = query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit).all() runs.reverse() return runs diff --git a/airflow/www/views.py b/airflow/www/views.py index 90d98a424c4a..3f0965da3889 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -69,7 +69,7 @@ from pendulum.parsing.exceptions import ParserError from pygments import highlight, lexers from pygments.formatters import HtmlFormatter -from sqlalchemy import Date, and_, case, desc, func, inspect, or_, select, union_all +from sqlalchemy import Date, and_, case, desc, func, inspect, or_, union_all from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload from wtforms import SelectField, validators @@ -95,7 +95,7 @@ from airflow.jobs.job import Job from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner -from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, Trigger, XCom, errors +from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, XCom, errors from airflow.models.abstractoperator import AbstractOperator from airflow.models.dag import DAG, get_dataset_triggered_next_run_info from airflow.models.dagcode import DagCode @@ -231,15 +231,16 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): # loaded and the actual requested run would be excluded by the limit(). Once # the user has changed base date to be anything else we want to use that instead. query_date = base_date - if date_time < base_date <= date_time + datetime.timedelta(seconds=1): + if date_time < base_date and date_time + datetime.timedelta(seconds=1) >= base_date: query_date = date_time - drs = session.scalars( - select(DagRun) - .where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) + drs = ( + session.query(DagRun) + .filter(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) .order_by(desc(DagRun.execution_date)) .limit(num_runs) - ).all() + .all() + ) dr_choices = [] dr_state = None for dr in drs: @@ -290,8 +291,8 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): Create a nested dict representation of the DAG's TaskGroup and its children used to construct the Graph and Grid views. """ - query = session.execute( - select( + query = ( + session.query( TaskInstance.task_id, TaskInstance.run_id, TaskInstance.state, @@ -302,7 +303,7 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): func.max(TaskInstance.end_date).label("end_date"), ) .join(TaskInstance.task_instance_note, isouter=True) - .where( + .filter( TaskInstance.dag_id == dag.dag_id, TaskInstance.run_id.in_([dag_run.run_id for dag_run in dag_runs]), ) @@ -425,9 +426,11 @@ def get_summary(dag_run: DagRun): } def get_mapped_group_summaries(): - mapped_ti_query = session.execute( - select(TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index) - .where( + mapped_ti_query = ( + session.query( + TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index + ) + .filter( TaskInstance.dag_id == dag.dag_id, TaskInstance.task_id.in_(child["id"] for child in children), TaskInstance.run_id.in_(r.run_id for r in dag_runs), @@ -735,20 +738,21 @@ def index(self): with create_session() as session: # read orm_dags from the db - dags_query = select(DagModel).where(~DagModel.is_subdag, DagModel.is_active) + dags_query = session.query(DagModel).filter(~DagModel.is_subdag, DagModel.is_active) if arg_search_query: escaped_arg_search_query = arg_search_query.replace("_", r"\_") - dags_query = dags_query.where( + dags_query = dags_query.filter( DagModel.dag_id.ilike("%" + escaped_arg_search_query + "%", escape="\\") | DagModel.owners.ilike("%" + escaped_arg_search_query + "%", escape="\\") ) if arg_tags_filter: - dags_query = dags_query.where(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) + dags_query = dags_query.filter(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) - dags_query = dags_query.where(DagModel.dag_id.in_(filter_dag_ids)) - filtered_dag_count = session.scalar(select(func.count()).select_from(dags_query)) + dags_query = dags_query.filter(DagModel.dag_id.in_(filter_dag_ids)) + + filtered_dag_count = dags_query.count() if filtered_dag_count == 0 and len(arg_tags_filter): flash( "No matching DAG tags found.", @@ -758,28 +762,28 @@ def index(self): return redirect(url_for("Airflow.index")) all_dags = dags_query - active_dags = dags_query.where(~DagModel.is_paused) - paused_dags = dags_query.where(DagModel.is_paused) + active_dags = dags_query.filter(~DagModel.is_paused) + paused_dags = dags_query.filter(DagModel.is_paused) # find DAGs which have a RUNNING DagRun - running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).where( + running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).filter( DagRun.state == State.RUNNING ) # find DAGs for which the latest DagRun is FAILED subq_all = ( - select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) .group_by(DagRun.dag_id) .subquery() ) subq_failed = ( - select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) - .where(DagRun.state == State.FAILED) + session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + .filter(DagRun.state == State.FAILED) .group_by(DagRun.dag_id) .subquery() ) subq_join = ( - select(subq_all.c.dag_id, subq_all.c.start_date) + session.query(subq_all.c.dag_id, subq_all.c.start_date) .join( subq_failed, and_( @@ -792,18 +796,16 @@ def index(self): failed_dags = dags_query.join(subq_join, DagModel.dag_id == subq_join.c.dag_id) is_paused_count = dict( - session.execute( - select(DagModel.is_paused, func.count(DagModel.dag_id)) - .group_by(DagModel.is_paused) - .select_from(all_dags) - ).all() + all_dags.with_entities(DagModel.is_paused, func.count(DagModel.dag_id)).group_by( + DagModel.is_paused + ) ) status_count_active = is_paused_count.get(False, 0) status_count_paused = is_paused_count.get(True, 0) - status_count_running = session.scalar(select(func.count()).select_from(running_dags)) - status_count_failed = session.scalar(select(func.count()).select_from(failed_dags)) + status_count_running = running_dags.count() + status_count_failed = failed_dags.count() all_dags_count = status_count_active + status_count_paused if arg_status_filter == "active": @@ -824,7 +826,7 @@ def index(self): if arg_sorting_key == "last_dagrun": dag_run_subquery = ( - select( + session.query( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) @@ -852,13 +854,7 @@ def index(self): else: current_dags = current_dags.order_by(null_case, sort_column) - dags = ( - session.scalars( - current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page) - ) - .unique() - .all() - ) + dags = current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page).all() user_permissions = g.user.perms can_create_dag_run = ( permissions.ACTION_CAN_CREATE, @@ -878,7 +874,7 @@ def index(self): dag.can_trigger = dag.can_edit and can_create_dag_run dag.can_delete = get_airflow_app().appbuilder.sm.can_delete_dag(dag.dag_id, g.user) - dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() + dagtags = session.query(func.distinct(DagTag.name)).order_by(DagTag.name).all() tags = [ {"name": name, "selected": bool(arg_tags_filter and name in arg_tags_filter)} for name, in dagtags @@ -886,15 +882,14 @@ def index(self): owner_links_dict = DagOwnerAttributes.get_all(session) - import_errors = select(errors.ImportError).order_by(errors.ImportError.id) + import_errors = session.query(errors.ImportError).order_by(errors.ImportError.id) if (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG) not in user_permissions: # if the user doesn't have access to all DAGs, only display errors from visible DAGs import_errors = import_errors.join( DagModel, DagModel.fileloc == errors.ImportError.filename - ).where(DagModel.dag_id.in_(filter_dag_ids)) + ).filter(DagModel.dag_id.in_(filter_dag_ids)) - import_errors = session.scalars(import_errors) for import_error in import_errors: flash( f"Broken DAG: [{import_error.filename}] {import_error.stacktrace}", @@ -938,12 +933,10 @@ def _iter_parsed_moved_data_table_names(): permissions.RESOURCE_ADMIN_MENU, ) in user_permissions and conf.getboolean("webserver", "warn_deployment_exposure"): robots_file_access_count = ( - select(Log) - .where(Log.event == "robots") - .where(Log.dttm > (utcnow() - datetime.timedelta(days=7))) - ) - robots_file_access_count = session.scalar( - select(func.count()).select_from(robots_file_access_count) + session.query(Log) + .filter(Log.event == "robots") + .filter(Log.dttm > (utcnow() - datetime.timedelta(days=7))) + .count() ) if robots_file_access_count > 0: flash( @@ -1045,11 +1038,9 @@ def next_run_datasets_summary(self, session: Session = NEW_SESSION): dataset_triggered_dag_ids = [ dag.dag_id for dag in ( - session.scalars( - select(DagModel.dag_id) - .where(DagModel.dag_id.in_(filter_dag_ids)) - .where(DagModel.schedule_interval == "Dataset") - ) + session.query(DagModel.dag_id) + .filter(DagModel.dag_id.in_(filter_dag_ids)) + .filter(DagModel.schedule_interval == "Dataset") ) ] @@ -1080,10 +1071,10 @@ def dag_stats(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify({}) - dag_state_stats = session.execute( - select(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) + dag_state_stats = ( + session.query(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) .group_by(DagRun.dag_id, DagRun.state) - .where(DagRun.dag_id.in_(filter_dag_ids)) + .filter(DagRun.dag_id.in_(filter_dag_ids)) ) dag_state_data = {(dag_id, state): count for dag_id, state, count in dag_state_stats} @@ -1121,17 +1112,17 @@ def task_stats(self, session: Session = NEW_SESSION): filter_dag_ids = allowed_dag_ids running_dag_run_query_result = ( - select(DagRun.dag_id, DagRun.run_id) + session.query(DagRun.dag_id, DagRun.run_id) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .where(DagRun.state == State.RUNNING, DagModel.is_active) + .filter(DagRun.state == State.RUNNING, DagModel.is_active) ) - running_dag_run_query_result = running_dag_run_query_result.where(DagRun.dag_id.in_(filter_dag_ids)) + running_dag_run_query_result = running_dag_run_query_result.filter(DagRun.dag_id.in_(filter_dag_ids)) running_dag_run_query_result = running_dag_run_query_result.subquery("running_dag_run") # Select all task_instances from active dag_runs. - running_task_instance_query_result = select( + running_task_instance_query_result = session.query( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(True).label("is_dag_running"), @@ -1145,19 +1136,19 @@ def task_stats(self, session: Session = NEW_SESSION): if conf.getboolean("webserver", "SHOW_RECENT_STATS_FOR_COMPLETED_RUNS", fallback=True): last_dag_run = ( - select(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) + session.query(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .where(DagRun.state != State.RUNNING, DagModel.is_active) + .filter(DagRun.state != State.RUNNING, DagModel.is_active) .group_by(DagRun.dag_id) ) - last_dag_run = last_dag_run.where(DagRun.dag_id.in_(filter_dag_ids)) + last_dag_run = last_dag_run.filter(DagRun.dag_id.in_(filter_dag_ids)) last_dag_run = last_dag_run.subquery("last_dag_run") # Select all task_instances from active dag_runs. # If no dag_run is active, return task instances from most recent dag_run. last_task_instance_query_result = ( - select( + session.query( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(False).label("is_dag_running"), @@ -1178,8 +1169,8 @@ def task_stats(self, session: Session = NEW_SESSION): else: final_task_instance_query_result = running_task_instance_query_result.subquery("final_ti") - qry = session.execute( - select( + qry = ( + session.query( final_task_instance_query_result.c.dag_id, final_task_instance_query_result.c.state, final_task_instance_query_result.c.is_dag_running, @@ -1195,6 +1186,7 @@ def task_stats(self, session: Session = NEW_SESSION): final_task_instance_query_result.c.is_dag_running.desc(), ) ) + data = get_task_stats_from_query(qry) payload: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) for dag_id in filter_dag_ids: @@ -1227,31 +1219,29 @@ def last_dagruns(self, session: Session = NEW_SESSION): return flask.json.jsonify({}) last_runs_subquery = ( - select( + session.query( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) .group_by(DagRun.dag_id) - .where(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. + .filter(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. .subquery("last_runs") ) - query = session.execute( - select( - DagRun.dag_id, - DagRun.start_date, - DagRun.end_date, - DagRun.state, - DagRun.execution_date, - DagRun.data_interval_start, - DagRun.data_interval_end, - ).join( - last_runs_subquery, - and_( - last_runs_subquery.c.dag_id == DagRun.dag_id, - last_runs_subquery.c.max_execution_date == DagRun.execution_date, - ), - ) + query = session.query( + DagRun.dag_id, + DagRun.start_date, + DagRun.end_date, + DagRun.state, + DagRun.execution_date, + DagRun.data_interval_start, + DagRun.data_interval_end, + ).join( + last_runs_subquery, + and_( + last_runs_subquery.c.dag_id == DagRun.dag_id, + last_runs_subquery.c.max_execution_date == DagRun.execution_date, + ), ) resp = { @@ -1350,18 +1340,19 @@ def dag_details(self, dag_id, session: Session = NEW_SESSION): title = "DAG Details" root = request.args.get("root", "") - states = session.execute( - select(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) - .where(TaskInstance.dag_id == dag_id) + states = ( + session.query(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) + .filter(TaskInstance.dag_id == dag_id) .group_by(TaskInstance.state) - ).all() + .all() + ) active_runs = models.DagRun.find(dag_id=dag_id, state=DagRunState.RUNNING, external_trigger=False) - tags = session.scalars(select(models.DagTag).where(models.DagTag.dag_id == dag_id)).all() + tags = session.query(models.DagTag).filter(models.DagTag.dag_id == dag_id).all() # TODO: convert this to a relationship - owner_links = session.execute(select(DagOwnerAttributes).filter_by(dag_id=dag_id)).all() + owner_links = session.query(DagOwnerAttributes).filter_by(dag_id=dag_id).all() attrs_to_avoid = [ "schedule_datasets", @@ -1622,17 +1613,18 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): "metadata": {"end_of_log": True}, } - ti = session.scalar( - select(models.TaskInstance) - .where( + ti = ( + session.query(models.TaskInstance) + .filter( TaskInstance.task_id == task_id, TaskInstance.dag_id == dag_id, TaskInstance.execution_date == execution_date, TaskInstance.map_index == map_index, ) .join(TaskInstance.dag_run) - .options(joinedload(TaskInstance.trigger).joinedload(Trigger.triggerer_job)) - .limit(1) + .options(joinedload("trigger")) + .options(joinedload("trigger.triggerer_job")) + .first() ) if ti is None: @@ -1690,10 +1682,10 @@ def log(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) dag_model = DagModel.get_dagmodel(dag_id) - ti = session.scalar( - select(models.TaskInstance) + ti = ( + session.query(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .limit(1) + .first() ) num_logs = 0 @@ -1734,10 +1726,10 @@ def redirect_to_external_log(self, session: Session = NEW_SESSION): map_index = request.args.get("map_index", -1, type=int) try_number = request.args.get("try_number", 1) - ti = session.scalar( - select(models.TaskInstance) + ti = ( + session.query(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .limit(1) + .first() ) if not ti: @@ -1779,8 +1771,8 @@ def task(self, session: Session = NEW_SESSION): task = copy.copy(dag.get_task(task_id)) task.resolve_template_files() - ti: TaskInstance | None = session.scalar( - select(TaskInstance) + ti: TaskInstance | None = ( + session.query(TaskInstance) .options( # HACK: Eager-load relationships. This is needed because # multiple properties mis-use provide_session() that destroys @@ -1789,6 +1781,7 @@ def task(self, session: Session = NEW_SESSION): joinedload(TaskInstance.trigger, innerjoin=False), ) .filter_by(execution_date=dttm, dag_id=dag_id, task_id=task_id, map_index=map_index) + .one_or_none() ) if ti is None: ti_attrs: list[tuple[str, Any]] | None = None @@ -1905,19 +1898,17 @@ def xcom(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) root = request.args.get("root", "") dag = DagModel.get_dagmodel(dag_id) - ti = session.scalar(select(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).limit(1)) + ti = session.query(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).first() if not ti: flash(f"Task [{dag_id}.{task_id}] doesn't seem to exist at the moment", "error") return redirect(url_for("Airflow.index")) - xcom_query = session.execute( - select(XCom.key, XCom.value).where( - XCom.dag_id == dag_id, - XCom.task_id == task_id, - XCom.execution_date == dttm, - XCom.map_index == map_index, - ) + xcom_query = session.query(XCom.key, XCom.value).filter( + XCom.dag_id == dag_id, + XCom.task_id == task_id, + XCom.execution_date == dttm, + XCom.map_index == map_index, ) attributes = [(k, v) for k, v in xcom_query if not k.startswith("_")] @@ -1987,7 +1978,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): request_execution_date = request.values.get("execution_date", default=timezone.utcnow().isoformat()) is_dag_run_conf_overrides_params = conf.getboolean("core", "dag_run_conf_overrides_params") dag = get_airflow_app().dag_bag.get_dag(dag_id) - dag_orm: DagModel = session.scalar(select(DagModel).where(DagModel.dag_id == dag_id).limit(1)) + dag_orm: DagModel = session.query(DagModel).filter(DagModel.dag_id == dag_id).first() # Prepare form fields with param struct details to render a proper form with schema information form_fields = {} @@ -2025,9 +2016,11 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): flash(f"Cannot create dagruns because the dag {dag_id} has import errors", "error") return redirect(origin) - recent_runs = session.execute( - select(DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date)) - .where( + recent_runs = ( + session.query( + DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date) + ) + .filter( DagRun.dag_id == dag_id, DagRun.run_type == DagRunType.MANUAL, DagRun.conf.isnot(None), @@ -2300,17 +2293,15 @@ def clear(self, *, session: Session = NEW_SESSION): # Lock the related dag runs to prevent from possible dead lock. # https://github.com/apache/airflow/pull/26658 - dag_runs_query = session.scalars( - select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() - ) + dag_runs_query = session.query(DagRun.id).filter(DagRun.dag_id == dag_id).with_for_update() if start_date is None and end_date is None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) + dag_runs_query = dag_runs_query.filter(DagRun.execution_date == start_date) else: if start_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date >= start_date) + dag_runs_query = dag_runs_query.filter(DagRun.execution_date >= start_date) if end_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) + dag_runs_query = dag_runs_query.filter(DagRun.execution_date <= end_date) locked_dag_run_ids = dag_runs_query.all() elif task_id: @@ -2399,10 +2390,10 @@ def blocked(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify([]) - dags = session.execute( - select(DagRun.dag_id, sqla.func.count(DagRun.id)) - .where(DagRun.state == DagRunState.RUNNING) - .where(DagRun.dag_id.in_(filter_dag_ids)) + dags = ( + session.query(DagRun.dag_id, sqla.func.count(DagRun.id)) + .filter(DagRun.state == DagRunState.RUNNING) + .filter(DagRun.dag_id.in_(filter_dag_ids)) .group_by(DagRun.dag_id) ) @@ -2483,11 +2474,9 @@ def _mark_dagrun_state_as_queued( # Identify tasks that will be queued up to run when confirmed all_task_ids = [task.task_id for task in dag.tasks] - existing_tis = session.execute( - select(TaskInstance.task_id).where( - TaskInstance.dag_id == dag.dag_id, - TaskInstance.run_id == dag_run_id, - ) + existing_tis = session.query(TaskInstance.task_id).filter( + TaskInstance.dag_id == dag.dag_id, + TaskInstance.run_id == dag_run_id, ) completed_tis_ids = [task_id for task_id, in existing_tis] @@ -2969,18 +2958,19 @@ def _convert_to_date(session, column): if root: dag = dag.partial_subset(task_ids_or_regex=root, include_downstream=False, include_upstream=True) - dag_states = session.execute( - select( - _convert_to_date(session, DagRun.execution_date).label("date"), + dag_states = ( + session.query( + (_convert_to_date(session, DagRun.execution_date)).label("date"), DagRun.state, func.max(DagRun.data_interval_start).label("data_interval_start"), func.max(DagRun.data_interval_end).label("data_interval_end"), func.count("*").label("count"), ) - .where(DagRun.dag_id == dag.dag_id) + .filter(DagRun.dag_id == dag.dag_id) .group_by(_convert_to_date(session, DagRun.execution_date), DagRun.state) .order_by(_convert_to_date(session, DagRun.execution_date).asc()) - ).all() + .all() + ) data_dag_states = [ { @@ -3251,17 +3241,16 @@ def duration(self, dag_id: str, session: Session = NEW_SESSION): else: min_date = timezone.utc_epoch() ti_fails = ( - select(TaskFail) + session.query(TaskFail) .join(TaskFail.dag_run) - .where( + .filter( TaskFail.dag_id == dag.dag_id, DagRun.execution_date >= min_date, DagRun.execution_date <= base_date, ) ) if dag.partial: - ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) - ti_fails = session.scalars(ti_fails) + ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) fails_totals: dict[tuple[str, str, str], int] = defaultdict(int) for failed_task_instance in ti_fails: dict_key = ( @@ -3602,9 +3591,9 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): form = DateTimeWithNumRunsWithDagRunsForm(data=dt_nr_dr_data) form.execution_date.choices = dt_nr_dr_data["dr_choices"] - tis = session.scalars( - select(TaskInstance) - .where( + tis = ( + session.query(TaskInstance) + .filter( TaskInstance.dag_id == dag_id, TaskInstance.run_id == dag_run_id, TaskInstance.start_date.is_not(None), @@ -3613,10 +3602,10 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): .order_by(TaskInstance.start_date) ) - ti_fails = select(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) + ti_fails = session.query(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) if dag.partial: - ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) - ti_fails = session.scalars(ti_fails) + ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + tasks = [] for ti in tis: if not dag.has_task(ti.task_id): @@ -3722,13 +3711,12 @@ def extra_links(self, *, session: Session = NEW_SESSION): if link_name is None: return {"url": None, "error": "Link name not passed"}, 400 - ti = session.scalar( - select(TaskInstance) + ti = ( + session.query(TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) .options(joinedload(TaskInstance.dag_run)) - .limit(1) + .first() ) - if not ti: return {"url": None, "error": "Task Instances not found"}, 404 try: @@ -3836,25 +3824,27 @@ def grid_data(self): base_date = dag.get_latest_execution_date() or timezone.utcnow() with create_session() as session: - query = select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date) + query = session.query(DagRun).filter( + DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date + ) - run_type = request.args.get("run_type") - if run_type: - query = query.where(DagRun.run_type == run_type) + run_type = request.args.get("run_type") + if run_type: + query = query.filter(DagRun.run_type == run_type) - run_state = request.args.get("run_state") - if run_state: - query = query.where(DagRun.state == run_state) + run_state = request.args.get("run_state") + if run_state: + query = query.filter(DagRun.state == run_state) - dag_runs = wwwutils.sorted_dag_runs( - query, ordering=dag.timetable.run_ordering, limit=num_runs, session=session - ) - encoded_runs = [wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs] - data = { - "groups": dag_to_grid(dag, dag_runs, session), - "dag_runs": encoded_runs, - "ordering": dag.timetable.run_ordering, - } + dag_runs = wwwutils.sorted_dag_runs(query, ordering=dag.timetable.run_ordering, limit=num_runs) + encoded_runs = [ + wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs + ] + data = { + "groups": dag_to_grid(dag, dag_runs, session), + "dag_runs": encoded_runs, + "ordering": dag.timetable.run_ordering, + } # avoid spaces to reduce payload size return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -3873,34 +3863,37 @@ def historical_metrics_data(self): end_date = _safe_parse_datetime(request.args.get("end_date")) with create_session() as session: # DagRuns - dag_runs_type = session.execute( - select(DagRun.run_type, func.count(DagRun.run_id)) - .where( + dag_runs_type = ( + session.query(DagRun.run_type, func.count(DagRun.run_id)) + .filter( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.run_type) - ).all() + .all() + ) - dag_run_states = session.execute( - select(DagRun.state, func.count(DagRun.run_id)) - .where( + dag_run_states = ( + session.query(DagRun.state, func.count(DagRun.run_id)) + .filter( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.state) - ).all() + .all() + ) # TaskInstances - task_instance_states = session.execute( - select(TaskInstance.state, func.count(TaskInstance.run_id)) + task_instance_states = ( + session.query(TaskInstance.state, func.count(TaskInstance.run_id)) .join(TaskInstance.dag_run) - .where( + .filter( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(TaskInstance.state) - ).all() + .all() + ) data = { "dag_run_types": { @@ -3934,32 +3927,28 @@ def next_run_datasets(self, dag_id): with create_session() as session: data = [ dict(info) - for info in session.execute( - select( - DatasetModel.id, - DatasetModel.uri, - func.max(DatasetEvent.timestamp).label("lastUpdate"), - ) - .join( - DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id - ) - .join( - DatasetDagRunQueue, - and_( - DatasetDagRunQueue.dataset_id == DatasetModel.id, - DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, - ), - isouter=True, - ) - .join( - DatasetEvent, - DatasetEvent.dataset_id == DatasetModel.id, - isouter=True, - ) - .where(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) - .group_by(DatasetModel.id, DatasetModel.uri) - .order_by(DatasetModel.uri) + for info in session.query( + DatasetModel.id, + DatasetModel.uri, + func.max(DatasetEvent.timestamp).label("lastUpdate"), ) + .join(DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id) + .join( + DatasetDagRunQueue, + and_( + DatasetDagRunQueue.dataset_id == DatasetModel.id, + DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, + ), + isouter=True, + ) + .join( + DatasetEvent, + DatasetEvent.dataset_id == DatasetModel.id, + isouter=True, + ) + .filter(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) + .group_by(DatasetModel.id, DatasetModel.uri) + .order_by(DatasetModel.uri) ] return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -4057,12 +4046,12 @@ def datasets_summary(self): if session.bind.dialect.name == "postgresql": order_by = (order_by[0].nulls_first(), *order_by[1:]) - count_query = select(func.count(DatasetModel.id)) + count_query = session.query(func.count(DatasetModel.id)) has_event_filters = bool(updated_before or updated_after) query = ( - select( + session.query( DatasetModel.id, DatasetModel.uri, func.max(DatasetEvent.timestamp).label("last_dataset_update"), @@ -4087,12 +4076,11 @@ def datasets_summary(self): if updated_before: filters.append(DatasetEvent.timestamp <= updated_before) - query = query.where(*filters).offset(offset).limit(limit) - count_query = count_query.where(*filters) + query = query.filter(*filters).offset(offset).limit(limit) + count_query = count_query.filter(*filters) - query = session.execute(query) datasets = [dict(dataset) for dataset in query] - data = {"datasets": datasets, "total_entries": session.scalar(count_query)} + data = {"datasets": datasets, "total_entries": count_query.scalar()} return ( htmlsafe_json_dumps(data, separators=(",", ":"), cls=utils_json.WebEncoder), @@ -4138,20 +4126,20 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): included_events_raw = conf.get("webserver", "audit_view_included_events", fallback=None) excluded_events_raw = conf.get("webserver", "audit_view_excluded_events", fallback=None) - query = select(Log).where(Log.dag_id == dag_id) + query = session.query(Log).filter(Log.dag_id == dag_id) if included_events_raw: included_events = {event.strip() for event in included_events_raw.split(",")} - query = query.where(Log.event.in_(included_events)) + query = query.filter(Log.event.in_(included_events)) elif excluded_events_raw: excluded_events = {event.strip() for event in excluded_events_raw.split(",")} - query = query.where(Log.event.notin_(excluded_events)) + query = query.filter(Log.event.notin_(excluded_events)) current_page = request.args.get("page", default=0, type=int) arg_sorting_key = request.args.get("sorting_key", "dttm") arg_sorting_direction = request.args.get("sorting_direction", default="desc") logs_per_page = PAGE_SIZE - audit_logs_count = session.scalar(select(func.count()).select_from(query)) + audit_logs_count = query.count() num_of_pages = int(math.ceil(audit_logs_count / float(logs_per_page))) start = current_page * logs_per_page @@ -4163,7 +4151,7 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): sort_column = sort_column.desc() query = query.order_by(sort_column) - dag_audit_logs = session.scalars(query.offset(start).limit(logs_per_page)).all() + dag_audit_logs = query.offset(start).limit(logs_per_page).all() return self.render_template( "airflow/dag_audit_log.html", dag=dag, @@ -4284,7 +4272,7 @@ def apply(self, query, func): if get_airflow_app().appbuilder.sm.has_all_dags_access(g.user): return query filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - return query.where(self.model.dag_id.in_(filter_dag_ids)) + return query.filter(self.model.dag_id.in_(filter_dag_ids)) class AirflowModelView(ModelView): @@ -4729,11 +4717,9 @@ def action_mulduplicate(self, connections, session: Session = NEW_SESSION): potential_connection_ids = [f"{base_conn_id}_copy{i}" for i in range(1, 11)] - query = session.scalars( - select(Connection.conn_id).where(Connection.conn_id.in_(potential_connection_ids)) - ) + query = session.query(Connection.conn_id).filter(Connection.conn_id.in_(potential_connection_ids)) - found_conn_id_set = {conn_id for conn_id in query} + found_conn_id_set = {conn_id for conn_id, in query} possible_conn_id_iter = ( connection_id @@ -5403,7 +5389,7 @@ def _set_dag_runs_to_active_state(self, drs: list[DagRun], state: str, session: """This routine only supports Running and Queued state.""" try: count = 0 - for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): + for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): count += 1 if state == State.RUNNING: dr.start_date = timezone.utcnow() @@ -5429,7 +5415,7 @@ def action_set_failed(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): + for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): count += 1 altered_tis += set_dag_run_state_to_failed( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5457,7 +5443,7 @@ def action_set_success(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): + for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): count += 1 altered_tis += set_dag_run_state_to_success( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5481,7 +5467,7 @@ def action_clear(self, drs: list[DagRun], session: Session = NEW_SESSION): count = 0 cleared_ti_count = 0 dag_to_tis: dict[DAG, list[TaskInstance]] = {} - for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): + for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): count += 1 dag = get_airflow_app().dag_bag.get_dag(dr.dag_id) tis_to_clear = dag_to_tis.setdefault(dag, []) @@ -5897,37 +5883,35 @@ def autocomplete(self, session: Session = NEW_SESSION): return flask.json.jsonify([]) # Provide suggestions of dag_ids and owners - dag_ids_query = select( + dag_ids_query = session.query( sqla.literal("dag").label("type"), DagModel.dag_id.label("name"), - ).where(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) + ).filter(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) owners_query = ( - select( + session.query( sqla.literal("owner").label("type"), DagModel.owners.label("name"), ) .distinct() - .where(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) + .filter(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) ) # Hide DAGs if not showing status: "all" status = flask_session.get(FILTER_STATUS_COOKIE) if status == "active": - dag_ids_query = dag_ids_query.where(~DagModel.is_paused) - owners_query = owners_query.where(~DagModel.is_paused) + dag_ids_query = dag_ids_query.filter(~DagModel.is_paused) + owners_query = owners_query.filter(~DagModel.is_paused) elif status == "paused": - dag_ids_query = dag_ids_query.where(DagModel.is_paused) - owners_query = owners_query.where(DagModel.is_paused) + dag_ids_query = dag_ids_query.filter(DagModel.is_paused) + owners_query = owners_query.filter(DagModel.is_paused) filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - dag_ids_query = dag_ids_query.where(DagModel.dag_id.in_(filter_dag_ids)) - owners_query = owners_query.where(DagModel.dag_id.in_(filter_dag_ids)) - payload = [ - row._asdict() - for row in session.execute(dag_ids_query.union(owners_query).order_by("name").limit(10)) - ] + dag_ids_query = dag_ids_query.filter(DagModel.dag_id.in_(filter_dag_ids)) + owners_query = owners_query.filter(DagModel.dag_id.in_(filter_dag_ids)) + + payload = [row._asdict() for row in dag_ids_query.union(owners_query).order_by("name").limit(10)] return flask.json.jsonify(payload) From e4c3ecf8ceaefa17525b495e4bcb5b2f41309603 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Tue, 4 Jul 2023 19:29:29 +0530 Subject: [PATCH 035/533] Disable default allowing the testing of connections in UI, API and CLI (#32052) * Disable allowing by default testing of connnections in UI Users can enable test connection functionaility in UI with caution by setting the `enable_test_connection` key to `True` in the `[webserver]` section of airflow.cfg or by setting the environment variable `AIRFLOW__WEBSERVER__ENABLE_TEST_CONNECTION` to `True`. --- .../endpoints/connection_endpoint.py | 10 ++++++++- airflow/cli/commands/connection_command.py | 7 +++++++ airflow/config_templates/config.yml | 19 +++++++++++++---- airflow/config_templates/default_airflow.cfg | 11 ++++++++++ airflow/www/extensions/init_jinja_globals.py | 1 + airflow/www/static/js/connection_form.js | 21 +++++++++++++++++++ .../www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + newsfragments/32052.significant.rst | 17 +++++++++++++++ .../endpoints/test_connection_endpoint.py | 16 ++++++++++++++ tests/cli/commands/test_connection_command.py | 14 ++++++++++++- tests/www/views/test_views.py | 1 + 12 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 newsfragments/32052.significant.rst diff --git a/airflow/api_connexion/endpoints/connection_endpoint.py b/airflow/api_connexion/endpoints/connection_endpoint.py index 737ee54d6c73..cfb185d44b4c 100644 --- a/airflow/api_connexion/endpoints/connection_endpoint.py +++ b/airflow/api_connexion/endpoints/connection_endpoint.py @@ -20,7 +20,7 @@ from http import HTTPStatus from connexion import NoContent -from flask import request +from flask import Response, request from marshmallow import ValidationError from sqlalchemy import func, select from sqlalchemy.orm import Session @@ -36,6 +36,7 @@ connection_test_schema, ) from airflow.api_connexion.types import APIResponse, UpdateMask +from airflow.configuration import conf from airflow.models import Connection from airflow.secrets.environment_variables import CONN_ENV_PREFIX from airflow.security import permissions @@ -180,6 +181,13 @@ def test_connection() -> APIResponse: env var, as some hook classes tries to find out the conn from their __init__ method & errors out if not found. It also deletes the conn id env variable after the test. """ + if conf.get("core", "test_connection", fallback="Disabled").lower().strip() != "enabled": + return Response( + "Testing connections is disabled in Airflow configuration. Contact your deployment admin to " + "enable it.", + 403, + ) + body = request.json transient_conn_id = get_random_string() conn_env_var = f"{CONN_ENV_PREFIX}{transient_conn_id.upper()}" diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py index 6d137bc89685..63888c0ec96e 100644 --- a/airflow/cli/commands/connection_command.py +++ b/airflow/cli/commands/connection_command.py @@ -30,6 +30,7 @@ from airflow.cli.simple_table import AirflowConsole from airflow.compat.functools import cache +from airflow.configuration import conf from airflow.exceptions import AirflowNotFoundException from airflow.hooks.base import BaseHook from airflow.models import Connection @@ -343,6 +344,12 @@ def _import_helper(file_path: str, overwrite: bool) -> None: def connections_test(args) -> None: """Test an Airflow connection.""" console = AirflowConsole() + if conf.get("core", "test_connection", fallback="Disabled").lower().strip() != "enabled": + console.print( + "[bold yellow]\nTesting connections is disabled in Airflow configuration. " + "Contact your deployment admin to enable it.\n" + ) + raise SystemExit(1) print(f"Retrieving connection: {args.conn_id!r}") try: diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 88aa055a3901..1ad8705a7047 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -450,6 +450,21 @@ core: type: string default: ~ example: 'http://localhost:8080' + test_connection: + description: | + The ability to allow testing connections across Airflow UI, API and CLI. + Supported options: Disabled, Enabled, Hidden. Default: Disabled + Disabled - Disables the test connection functionality and disables the Test Connection button in UI. + Enabled - Enables the test connection functionality and shows the Test Connection button in UI. + Hidden - Disables the test connection functionality and hides the Test Connection button in UI. + Before setting this to Enabled, make sure that you review the users who are able to add/edit + connections and ensure they are trusted. Connection testing can be done maliciously leading to + undesired and insecure outcomes. For more information on capabilities of users, see the documentation: + https://airflow.apache.org/docs/apache-airflow/stable/security/index.html#capabilities-of-authenticated-ui-users + version_added: 2.7.0 + type: string + example: ~ + default: "Disabled" database: description: ~ options: @@ -891,8 +906,6 @@ logging: type: boolean example: ~ default: "False" - - metrics: description: | StatsD (https://github.com/etsy/statsd) integration settings. @@ -1801,7 +1814,6 @@ webserver: type: string example: "sha256" default: "md5" - email: description: | Configuration email backend and whether to @@ -1859,7 +1871,6 @@ email: type: string example: "Airflow " default: ~ - smtp: description: | If you want airflow to send emails on retries, failure, and you want to use diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index c28155ac2c40..0cc99a3f4e5f 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -257,6 +257,17 @@ database_access_isolation = False # Example: internal_api_url = http://localhost:8080 # internal_api_url = +# The ability to allow testing connections across Airflow UI, API and CLI. +# Supported options: Disabled, Enabled, Hidden. Default: Disabled +# Disabled - Disables the test connection functionality and disables the Test Connection button in UI. +# Enabled - Enables the test connection functionality and shows the Test Connection button in UI. +# Hidden - Disables the test connection functionality and hides the Test Connection button in UI. +# Before setting this to Enabled, make sure that you review the users who are able to add/edit +# connections and ensure they are trusted. Connection testing can be done maliciously leading to +# undesired and insecure outcomes. For more information on capabilities of users, see the documentation: +# https://airflow.apache.org/docs/apache-airflow/stable/security/index.html#capabilities-of-authenticated-ui-users +test_connection = Disabled + [database] # Path to the ``alembic.ini`` file. You can either provide the file path relative # to the Airflow home directory or the absolute path if it is located elsewhere. diff --git a/airflow/www/extensions/init_jinja_globals.py b/airflow/www/extensions/init_jinja_globals.py index 2d5778af4b4f..0674a8e4a3ea 100644 --- a/airflow/www/extensions/init_jinja_globals.py +++ b/airflow/www/extensions/init_jinja_globals.py @@ -68,6 +68,7 @@ def prepare_jinja_globals(): "git_version": git_version, "k8s_or_k8scelery_executor": IS_K8S_OR_K8SCELERY_EXECUTOR, "rest_api_enabled": False, + "config_test_connection": conf.get("core", "test_connection", fallback="Disabled"), } backends = conf.get("api", "auth_backends") diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 41df55dc07bb..453be58411a9 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -23,6 +23,9 @@ /* global document, DOMParser, $, CodeMirror */ import { getMetaValue } from "./utils"; +const configTestConnection = getMetaValue("config_test_connection") + .toLowerCase() + .trim(); const restApiEnabled = getMetaValue("rest_api_enabled") === "True"; const connectionTestUrl = getMetaValue("test_url"); @@ -126,6 +129,24 @@ function applyFieldBehaviours(connection) { */ function handleTestConnection(connectionType, testableConnections) { const testButton = document.getElementById("test-connection"); + + if (configTestConnection === "hidden") { + // If test connection is hidden in config, hide button and return. + $(testButton).hide(); + return; + } + if (configTestConnection === "disabled") { + // If test connection is not enabled in config, disable button and display toolip + // alerting the user. + $(testButton) + .prop("disabled", true) + .attr( + "title", + "Testing connections is disabled in Airflow configuration. Contact your deployment admin to enable it." + ); + return; + } + const testConnEnabled = testableConnections.includes(connectionType); if (testConnEnabled) { diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 34c2f32dedb8..ac92b967f7e3 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -21,6 +21,7 @@ {% block head_css %} {{ super() }} + diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index e963b8ac0cb5..653b0dd3ce07 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -21,6 +21,7 @@ {% block head_css %} {{ super() }} + diff --git a/newsfragments/32052.significant.rst b/newsfragments/32052.significant.rst new file mode 100644 index 000000000000..d4ec9a934f9d --- /dev/null +++ b/newsfragments/32052.significant.rst @@ -0,0 +1,17 @@ +Disable default allowing the testing of connections in UI, API and CLI + +The test connection functionality is disabled by default across Airflow UI, +API and CLI. The availability of the functionality can be controlled by the +``test_connection`` flag in the ``core`` section of the Airflow +configuration (``airflow.cfg``). It can also be controlled by the +environment variable ``AIRFLOW__CORE__TEST_CONNECTION``. +The following values are accepted for this config param: +1. ``Disabled``: Disables the test connection functionality and +disables(greys out) the Test Connection button in the UI. +This is also the default value set in the Airflow configuration. +2. ``Enabled``: Enables the test connection functionality and +activates the Test Connection button in the UI. +3. ``Hidden``: Disables the test connection functionality and +hides the Test Connection button in UI. +For more information on capabilities of users, see the documentation: +https://airflow.apache.org/docs/apache-airflow/stable/security/index.html#capabilities-of-authenticated-ui-users diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py b/tests/api_connexion/endpoints/test_connection_endpoint.py index bce7150f8bda..de0b7aad973f 100644 --- a/tests/api_connexion/endpoints/test_connection_endpoint.py +++ b/tests/api_connexion/endpoints/test_connection_endpoint.py @@ -16,6 +16,9 @@ # under the License. from __future__ import annotations +import os +from unittest import mock + import pytest from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP @@ -603,6 +606,7 @@ def test_should_raises_401_unauthenticated(self): class TestConnection(TestConnectionEndpoint): + @mock.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) def test_should_respond_200(self): payload = {"connection_id": "test-connection-id", "conn_type": "sqlite"} response = self.client.post( @@ -614,6 +618,7 @@ def test_should_respond_200(self): "message": "Connection successfully tested", } + @mock.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) def test_post_should_respond_400_for_invalid_payload(self): payload = { "connection_id": "test-connection-id", @@ -635,3 +640,14 @@ def test_should_raises_401_unauthenticated(self): ) assert_401(response) + + def test_should_respond_403_by_default(self): + payload = {"connection_id": "test-connection-id", "conn_type": "sqlite"} + response = self.client.post( + "/api/v1/connections/test", json=payload, environ_overrides={"REMOTE_USER": "test"} + ) + assert response.status_code == 403 + assert response.text == ( + "Testing connections is disabled in Airflow configuration. " + "Contact your deployment admin to enable it." + ) diff --git a/tests/cli/commands/test_connection_command.py b/tests/cli/commands/test_connection_command.py index 42f6fc8cc204..12b3de6170c0 100644 --- a/tests/cli/commands/test_connection_command.py +++ b/tests/cli/commands/test_connection_command.py @@ -18,6 +18,7 @@ import io import json +import os import re import shlex import warnings @@ -938,6 +939,7 @@ class TestCliTestConnections: def setup_class(self): clear_db_connections() + @mock.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) @mock.patch("airflow.providers.http.hooks.http.HttpHook.test_connection") def test_cli_connections_test_success(self, mock_test_conn): """Check that successful connection test result is displayed properly.""" @@ -948,6 +950,7 @@ def test_cli_connections_test_success(self, mock_test_conn): assert "Connection success!" in stdout.getvalue() + @mock.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) @mock.patch("airflow.providers.http.hooks.http.HttpHook.test_connection") def test_cli_connections_test_fail(self, mock_test_conn): """Check that failed connection test result is displayed properly.""" @@ -958,9 +961,18 @@ def test_cli_connections_test_fail(self, mock_test_conn): assert "Connection failed!\nFailed.\n\n" in stdout.getvalue() + @mock.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) def test_cli_connections_test_missing_conn(self): """Check a connection test on a non-existent connection raises a "Connection not found" message.""" with redirect_stdout(io.StringIO()) as stdout, pytest.raises(SystemExit): connection_command.connections_test(self.parser.parse_args(["connections", "test", "missing"])) + assert "Connection not found.\n\n" in stdout.getvalue() - assert "Connection not found.\n\n" in stdout.getvalue() + def test_cli_connections_test_disabled_by_default(self): + """Check that test connection functionality is disabled by default.""" + with redirect_stdout(io.StringIO()) as stdout, pytest.raises(SystemExit): + connection_command.connections_test(self.parser.parse_args(["connections", "test", "missing"])) + assert ( + "Testing connections is disabled in Airflow configuration. Contact your deployment admin to " + "enable it.\n\n" + ) in stdout.getvalue() diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index e266b9b8c1a1..755df3b7aa7f 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -84,6 +84,7 @@ def test_redoc_should_render_template(capture_templates, admin_client): assert len(templates) == 1 assert templates[0].name == "airflow/redoc.html" assert templates[0].local_context == { + "config_test_connection": "Disabled", "openapi_spec_url": "/api/v1/openapi.yaml", "rest_api_enabled": True, "get_docs_url": get_docs_url, From d1625188ff20b4b6a3bf8f41792c4d16b79c8bf1 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Tue, 4 Jul 2023 16:25:19 +0200 Subject: [PATCH 036/533] Remove outdated comment from chart values in the workers KEDA conf section (#32300) Signed-off-by: Hussein Awala --- chart/values.schema.json | 2 +- chart/values.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 395506701bec..80e25b9b10a6 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1381,7 +1381,7 @@ "additionalProperties": false, "properties": { "enabled": { - "description": "Allow KEDA autoscaling. `Persistence.enabled` must be set to false to use KEDA.", + "description": "Allow KEDA autoscaling.", "type": "boolean", "default": false }, diff --git a/chart/values.yaml b/chart/values.yaml index b1879f0d4466..3beeb3765cfa 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -510,7 +510,6 @@ workers: annotations: {} # Allow KEDA autoscaling. - # Persistence.enabled must be set to false to use KEDA. keda: enabled: false namespaceLabels: {} From 575bf2f04089b7c99a8ee30637f1d88492ef4742 Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Tue, 4 Jul 2023 20:54:11 +0530 Subject: [PATCH 037/533] Bug fix GCSToS3Operator: avoid `ValueError` when `replace=False` with files already in S3 (#32322) --- .../amazon/aws/transfers/gcs_to_s3.py | 5 ++ .../amazon/aws/transfers/test_gcs_to_s3.py | 69 ++++++++++++++----- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 2213de2b60c1..d57de7e11efb 100644 --- a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -161,6 +161,11 @@ def execute(self, context: Context) -> list[str]: # and only keep those files which are present in # Google Cloud Storage and not in S3 bucket_name, prefix = S3Hook.parse_s3_url(self.dest_s3_key) + # if prefix is empty, do not add "/" at end since it would + # filter all the objects (return empty list) instead of empty + # prefix returning all the objects + if prefix: + prefix = prefix if prefix.endswith("/") else f"{prefix}/" # look for the bucket and the prefix to avoid look into # parent directories/keys existing_files = s3_hook.list_keys(bucket_name, prefix=prefix) diff --git a/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py b/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py index a7a0b2e4305a..5e64f167ba45 100644 --- a/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py +++ b/tests/providers/amazon/aws/transfers/test_gcs_to_s3.py @@ -33,6 +33,7 @@ S3_BUCKET = "s3://bucket/" MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv"] S3_ACL_POLICY = "private-read" +deprecated_call_match = "Usage of 'delimiter' is deprecated, please use 'match_glob' instead" def _create_test_bucket(): @@ -47,8 +48,6 @@ def _create_test_bucket(): @mock_s3 class TestGCSToS3Operator: - - # Test0: match_glob @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute__match_glob(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES @@ -73,7 +72,6 @@ def test_execute__match_glob(self, mock_hook): bucket_name=GCS_BUCKET, delimiter=None, match_glob=f"**/*{DELIMITER}", prefix=PREFIX ) - # Test1: incremental behaviour (just some files missing) @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_incremental(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES @@ -81,7 +79,7 @@ def test_execute_incremental(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -100,15 +98,17 @@ def test_execute_incremental(self, mock_hook): assert sorted(MOCK_FILES[1:]) == sorted(uploaded_files) assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) - # Test2: All the files are already in origin and destination without replace @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_without_replace(self, mock_hook): + """ + Tests scenario where all the files are already in origin and destination without replace + """ mock_hook.return_value.list.return_value = MOCK_FILES with NamedTemporaryFile() as f: gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -128,15 +128,53 @@ def test_execute_without_replace(self, mock_hook): assert [] == uploaded_files assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) - # Test3: There are no files in destination bucket + @pytest.mark.parametrize( + argnames="dest_s3_url", + argvalues=[f"{S3_BUCKET}/test/", f"{S3_BUCKET}/test"], + ) + @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") + def test_execute_without_replace_with_folder_structure(self, mock_hook, dest_s3_url): + mock_files_gcs = [f"test{idx}/{mock_file}" for idx, mock_file in enumerate(MOCK_FILES)] + mock_files_s3 = [f"test/test{idx}/{mock_file}" for idx, mock_file in enumerate(MOCK_FILES)] + mock_hook.return_value.list.return_value = mock_files_gcs + + hook, bucket = _create_test_bucket() + for mock_file_s3 in mock_files_s3: + bucket.put_object(Key=mock_file_s3, Body=b"testing") + + with NamedTemporaryFile() as f: + gcs_provide_file = mock_hook.return_value.provide_file + gcs_provide_file.return_value.__enter__.return_value.name = f.name + + with pytest.deprecated_call(match=deprecated_call_match): + operator = GCSToS3Operator( + task_id=TASK_ID, + bucket=GCS_BUCKET, + prefix=PREFIX, + delimiter=DELIMITER, + dest_aws_conn_id="aws_default", + dest_s3_key=dest_s3_url, + replace=False, + ) + + # we expect nothing to be uploaded + # and all the MOCK_FILES to be present at the S3 bucket + uploaded_files = operator.execute(None) + + assert [] == uploaded_files + assert sorted(mock_files_s3) == sorted(hook.list_keys("bucket", prefix="test/")) + @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute(self, mock_hook): + """ + Tests the scenario where there are no files in destination bucket + """ mock_hook.return_value.list.return_value = MOCK_FILES with NamedTemporaryFile() as f: gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -154,7 +192,6 @@ def test_execute(self, mock_hook): assert sorted(MOCK_FILES) == sorted(uploaded_files) assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) - # Test4: Destination and Origin are in sync but replace all files in destination @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_with_replace(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES @@ -162,7 +199,7 @@ def test_execute_with_replace(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -182,7 +219,6 @@ def test_execute_with_replace(self, mock_hook): assert sorted(MOCK_FILES) == sorted(uploaded_files) assert sorted(MOCK_FILES) == sorted(hook.list_keys("bucket", delimiter="/")) - # Test5: Incremental sync with replace @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_incremental_with_replace(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES @@ -190,7 +226,7 @@ def test_execute_incremental_with_replace(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -218,7 +254,7 @@ def test_execute_should_handle_with_default_dest_s3_extra_args(self, s3_mock_hoo s3_mock_hook.return_value = mock.Mock() s3_mock_hook.parse_s3_url.return_value = mock.Mock() - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -241,7 +277,7 @@ def test_execute_should_pass_dest_s3_extra_args_to_s3_hook(self, s3_mock_hook, m s3_mock_hook.return_value = mock.Mock() s3_mock_hook.parse_s3_url.return_value = mock.Mock() - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -259,7 +295,6 @@ def test_execute_should_pass_dest_s3_extra_args_to_s3_hook(self, s3_mock_hook, m aws_conn_id="aws_default", extra_args={"ContentLanguage": "value"}, verify=None ) - # Test6: s3_acl_policy parameter is set @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.load_file") def test_execute_with_s3_acl_policy(self, mock_load_file, mock_gcs_hook): @@ -268,7 +303,7 @@ def test_execute_with_s3_acl_policy(self, mock_load_file, mock_gcs_hook): gcs_provide_file = mock_gcs_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, @@ -293,7 +328,7 @@ def test_execute_without_keep_director_structure(self, mock_hook): gcs_provide_file = mock_hook.return_value.provide_file gcs_provide_file.return_value.__enter__.return_value.name = f.name - with pytest.deprecated_call(): + with pytest.deprecated_call(match=deprecated_call_match): operator = GCSToS3Operator( task_id=TASK_ID, bucket=GCS_BUCKET, From 836bdb29c0c455a5e9f84957947452e50c4d477e Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 4 Jul 2023 17:47:43 +0200 Subject: [PATCH 038/533] feat: Use 'use_ssl' parameter instead of 'ssl' + Use attributes of the patched function (#32305) --- tests/providers/apache/impala/hooks/test_impala.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/providers/apache/impala/hooks/test_impala.py b/tests/providers/apache/impala/hooks/test_impala.py index 829a543e0822..a3e1f119360d 100644 --- a/tests/providers/apache/impala/hooks/test_impala.py +++ b/tests/providers/apache/impala/hooks/test_impala.py @@ -35,17 +35,22 @@ def impala_hook_fixture() -> ImpalaHook: return hook -@patch("airflow.providers.apache.impala.hooks.impala.connect") +@patch("airflow.providers.apache.impala.hooks.impala.connect", autospec=True) def test_get_conn(mock_connect): hook = ImpalaHook() hook.get_connection = MagicMock( return_value=Connection( - login="login", password="password", host="host", port=21050, schema="test", extra={"ssl": True} + login="login", + password="password", + host="host", + port=21050, + schema="test", + extra={"use_ssl": True}, ) ) hook.get_conn() mock_connect.assert_called_once_with( - host="host", port=21050, user="login", password="password", database="test", ssl=True + host="host", port=21050, user="login", password="password", database="test", use_ssl=True ) From 65fad4affc24b33c4499ad0fbcdfff535fbae3bf Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Tue, 4 Jul 2023 17:48:41 +0200 Subject: [PATCH 039/533] Change default schema behaviour in SQLParser. (#32347) Signed-off-by: Jakub Dardzinski --- airflow/providers/openlineage/sqlparser.py | 3 +- airflow/providers/openlineage/utils/sql.py | 9 +++-- tests/providers/openlineage/utils/test_sql.py | 4 ++ .../openlineage/utils/test_sqlparser.py | 39 ++++++++++++------- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/airflow/providers/openlineage/sqlparser.py b/airflow/providers/openlineage/sqlparser.py index 657428549eaa..ed3e92e58b5a 100644 --- a/airflow/providers/openlineage/sqlparser.py +++ b/airflow/providers/openlineage/sqlparser.py @@ -103,7 +103,7 @@ def __init__(self, dialect: str | None = None, default_schema: str | None = None def parse(self, sql: list[str] | str) -> SqlMeta | None: """Parse a single or a list of SQL statements.""" - return parse(sql=sql, dialect=self.dialect, default_schema=self.default_schema) + return parse(sql=sql, dialect=self.dialect) def parse_table_schemas( self, @@ -126,6 +126,7 @@ def parse_table_schemas( return get_table_schemas( hook, namespace, + self.default_schema, database or database_info.database, self.create_information_schema_query(tables=inputs, **database_kwargs) if inputs else None, self.create_information_schema_query(tables=outputs, **database_kwargs) if outputs else None, diff --git a/airflow/providers/openlineage/utils/sql.py b/airflow/providers/openlineage/utils/sql.py index 317e46e44255..fe43a25bae67 100644 --- a/airflow/providers/openlineage/utils/sql.py +++ b/airflow/providers/openlineage/utils/sql.py @@ -63,12 +63,12 @@ class TableSchema: database: str | None fields: list[SchemaField] - def to_dataset(self, namespace: str, database: str | None = None) -> Dataset: + def to_dataset(self, namespace: str, database: str | None = None, schema: str | None = None) -> Dataset: # Prefix the table name with database and schema name using # the format: {database_name}.{table_schema}.{table_name}. name = ".".join( part - for part in [self.database if self.database else database, self.schema, self.table] + for part in [self.database or database, self.schema or schema, self.table] if part is not None ) return Dataset( @@ -81,6 +81,7 @@ def to_dataset(self, namespace: str, database: str | None = None) -> Dataset: def get_table_schemas( hook: BaseHook, namespace: str, + schema: str | None, database: str | None, in_query: str | None, out_query: str | None, @@ -97,12 +98,12 @@ def get_table_schemas( with closing(hook.get_conn()) as conn, closing(conn.cursor()) as cursor: if in_query: cursor.execute(in_query) - in_datasets = [x.to_dataset(namespace, database) for x in parse_query_result(cursor)] + in_datasets = [x.to_dataset(namespace, database, schema) for x in parse_query_result(cursor)] else: in_datasets = [] if out_query: cursor.execute(out_query) - out_datasets = [x.to_dataset(namespace, database) for x in parse_query_result(cursor)] + out_datasets = [x.to_dataset(namespace, database, schema) for x in parse_query_result(cursor)] else: out_datasets = [] return in_datasets, out_datasets diff --git a/tests/providers/openlineage/utils/test_sql.py b/tests/providers/openlineage/utils/test_sql.py index be929a1ad643..a82ab36bda26 100644 --- a/tests/providers/openlineage/utils/test_sql.py +++ b/tests/providers/openlineage/utils/test_sql.py @@ -77,6 +77,7 @@ def test_get_table_schemas(): hook=hook, namespace="bigquery", database=DB_NAME, + schema=DB_SCHEMA_NAME, in_query="fake_sql", out_query="another_fake_sql", ) @@ -139,6 +140,7 @@ def test_get_table_schemas_with_mixed_databases(): hook=hook, namespace="bigquery", database=DB_NAME, + schema=DB_SCHEMA_NAME, in_query="fake_sql", out_query="another_fake_sql", ) @@ -179,6 +181,7 @@ def test_get_table_schemas_with_mixed_schemas(): hook=hook, namespace="bigquery", database=DB_NAME, + schema=DB_SCHEMA_NAME, in_query="fake_sql", out_query="another_fake_sql", ) @@ -237,6 +240,7 @@ def test_get_table_schemas_with_other_database(): hook=hook, namespace="bigquery", database=DB_NAME, + schema=DB_SCHEMA_NAME, in_query="fake_sql", out_query="another_fake_sql", ) diff --git a/tests/providers/openlineage/utils/test_sqlparser.py b/tests/providers/openlineage/utils/test_sqlparser.py index 6f11a7ad9438..31611c7c11fe 100644 --- a/tests/providers/openlineage/utils/test_sqlparser.py +++ b/tests/providers/openlineage/utils/test_sqlparser.py @@ -19,6 +19,7 @@ from unittest import mock from unittest.mock import MagicMock +import pytest from openlineage.client.facet import SchemaDatasetFacet, SchemaField, SqlJobFacet from openlineage.client.run import Dataset from openlineage.common.sql import DbTableMeta @@ -155,19 +156,20 @@ def test_parse_table_schemas(self): database_info=db_info, ) + @pytest.mark.parametrize("parser_returns_schema", [True, False]) @mock.patch("airflow.providers.openlineage.sqlparser.SQLParser.parse") - def test_generate_openlineage_metadata_from_sql(self, mock_parse): - parser = SQLParser() + def test_generate_openlineage_metadata_from_sql(self, mock_parse, parser_returns_schema): + parser = SQLParser(default_schema="ANOTHER_SCHEMA") db_info = DatabaseInfo(scheme="myscheme", authority="host:port") hook = MagicMock() - rows = lambda name: [ - (DB_SCHEMA_NAME, name, "ID", 1, "int4"), - (DB_SCHEMA_NAME, name, "AMOUNT_OFF", 2, "int4"), - (DB_SCHEMA_NAME, name, "CUSTOMER_EMAIL", 3, "varchar"), - (DB_SCHEMA_NAME, name, "STARTS_ON", 4, "timestamp"), - (DB_SCHEMA_NAME, name, "ENDS_ON", 5, "timestamp"), + rows = lambda schema, table: [ + (schema, table, "ID", 1, "int4"), + (schema, table, "AMOUNT_OFF", 2, "int4"), + (schema, table, "CUSTOMER_EMAIL", 3, "varchar"), + (schema, table, "STARTS_ON", 4, "timestamp"), + (schema, table, "ENDS_ON", 5, "timestamp"), ] sql = """CREATE TABLE table_out ( @@ -182,13 +184,17 @@ def test_generate_openlineage_metadata_from_sql(self, mock_parse): """ hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [ - rows("TABLE_IN"), - rows("TABLE_OUT"), + rows(DB_SCHEMA_NAME if parser_returns_schema else None, "TABLE_IN"), + rows(DB_SCHEMA_NAME if parser_returns_schema else None, "TABLE_OUT"), ] mock_sql_meta = MagicMock() - mock_sql_meta.in_tables = [DbTableMeta("PUBLIC.TABLE_IN")] - mock_sql_meta.out_tables = [DbTableMeta("PUBLIC.TABLE_OUT")] + if parser_returns_schema: + mock_sql_meta.in_tables = [DbTableMeta("PUBLIC.TABLE_IN")] + mock_sql_meta.out_tables = [DbTableMeta("PUBLIC.TABLE_OUT")] + else: + mock_sql_meta.in_tables = [DbTableMeta("TABLE_IN")] + mock_sql_meta.out_tables = [DbTableMeta("TABLE_OUT")] mock_sql_meta.errors = [] mock_parse.return_value = mock_sql_meta @@ -201,15 +207,20 @@ def test_generate_openlineage_metadata_from_sql(self, mock_parse): ENDS_ON timestamp )""" + expected_schema = "PUBLIC" if parser_returns_schema else "ANOTHER_SCHEMA" expected = OperatorLineage( inputs=[ Dataset( - namespace="myscheme://host:port", name="PUBLIC.TABLE_IN", facets={"schema": SCHEMA_FACET} + namespace="myscheme://host:port", + name=f"{expected_schema}.TABLE_IN", + facets={"schema": SCHEMA_FACET}, ) ], outputs=[ Dataset( - namespace="myscheme://host:port", name="PUBLIC.TABLE_OUT", facets={"schema": SCHEMA_FACET} + namespace="myscheme://host:port", + name=f"{expected_schema}.TABLE_OUT", + facets={"schema": SCHEMA_FACET}, ) ], job_facets={"sql": SqlJobFacet(query=formatted_sql)}, From 1d6f6131df7e420b9e9dd1535ea7cd1a29e3c548 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 5 Jul 2023 03:54:28 +0800 Subject: [PATCH 040/533] refresh connection if an exception is caught in "AzureDataFactory" (#32323) --- .../microsoft/azure/hooks/data_factory.py | 43 +++--- .../microsoft/azure/triggers/data_factory.py | 126 ++++++++++++------ .../azure/hooks/test_azure_data_factory.py | 36 +++-- .../azure/triggers/test_azure_data_factory.py | 6 +- 4 files changed, 131 insertions(+), 80 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/data_factory.py b/airflow/providers/microsoft/azure/hooks/data_factory.py index b9e7feea0a13..590f2af92b36 100644 --- a/airflow/providers/microsoft/azure/hooks/data_factory.py +++ b/airflow/providers/microsoft/azure/hooks/data_factory.py @@ -36,6 +36,7 @@ from typing import Any, Callable, TypeVar, Union, cast from asgiref.sync import sync_to_async +from azure.core.exceptions import ServiceRequestError from azure.core.polling import LROPoller from azure.identity import ClientSecretCredential, DefaultAzureCredential from azure.identity.aio import ( @@ -214,6 +215,10 @@ def get_conn(self) -> DataFactoryManagementClient: return self._conn + def refresh_conn(self) -> DataFactoryManagementClient: + self._conn = None + return self.get_conn() + @provide_targeted_factory def get_factory( self, resource_group_name: str | None = None, factory_name: str | None = None, **config: Any @@ -812,6 +817,7 @@ def wait_for_pipeline_run_status( resource_group_name=resource_group_name, ) pipeline_run_status = self.get_pipeline_run_status(**pipeline_run_info) + executed_after_token_refresh = True start_time = time.monotonic() @@ -828,7 +834,14 @@ def wait_for_pipeline_run_status( # Wait to check the status of the pipeline run based on the ``check_interval`` configured. time.sleep(check_interval) - pipeline_run_status = self.get_pipeline_run_status(**pipeline_run_info) + try: + pipeline_run_status = self.get_pipeline_run_status(**pipeline_run_info) + executed_after_token_refresh = True + except ServiceRequestError: + if executed_after_token_refresh: + self.refresh_conn() + continue + raise return pipeline_run_status in expected_statuses @@ -1132,6 +1145,10 @@ async def get_async_conn(self) -> AsyncDataFactoryManagementClient: return self._async_conn + async def refresh_conn(self) -> AsyncDataFactoryManagementClient: + self._conn = None + return await self.get_async_conn() + @provide_targeted_factory_async async def get_pipeline_run( self, @@ -1149,11 +1166,8 @@ async def get_pipeline_run( :param config: Extra parameters for the ADF client. """ client = await self.get_async_conn() - try: - pipeline_run = await client.pipeline_runs.get(resource_group_name, factory_name, run_id) - return pipeline_run - except Exception as e: - raise AirflowException(e) + pipeline_run = await client.pipeline_runs.get(resource_group_name, factory_name, run_id) + return pipeline_run async def get_adf_pipeline_run_status( self, run_id: str, resource_group_name: str | None = None, factory_name: str | None = None @@ -1165,16 +1179,13 @@ async def get_adf_pipeline_run_status( :param resource_group_name: The resource group name. :param factory_name: The factory name. """ - try: - pipeline_run = await self.get_pipeline_run( - run_id=run_id, - factory_name=factory_name, - resource_group_name=resource_group_name, - ) - status: str = pipeline_run.status - return status - except Exception as e: - raise AirflowException(e) + pipeline_run = await self.get_pipeline_run( + run_id=run_id, + factory_name=factory_name, + resource_group_name=resource_group_name, + ) + status: str = pipeline_run.status + return status @provide_targeted_factory_async async def cancel_pipeline_run( diff --git a/airflow/providers/microsoft/azure/triggers/data_factory.py b/airflow/providers/microsoft/azure/triggers/data_factory.py index 40b95559405c..e3dd38ad6646 100644 --- a/airflow/providers/microsoft/azure/triggers/data_factory.py +++ b/airflow/providers/microsoft/azure/triggers/data_factory.py @@ -20,6 +20,8 @@ import time from typing import Any, AsyncIterator +from azure.core.exceptions import ServiceRequestError + from airflow.providers.microsoft.azure.hooks.data_factory import ( AzureDataFactoryAsyncHook, AzureDataFactoryPipelineRunStatus, @@ -68,24 +70,41 @@ def serialize(self) -> tuple[str, dict[str, Any]]: async def run(self) -> AsyncIterator[TriggerEvent]: """Make async connection to Azure Data Factory, polls for the pipeline run status.""" hook = AzureDataFactoryAsyncHook(azure_data_factory_conn_id=self.azure_data_factory_conn_id) + executed_after_token_refresh = False try: while True: - pipeline_status = await hook.get_adf_pipeline_run_status( - run_id=self.run_id, - resource_group_name=self.resource_group_name, - factory_name=self.factory_name, - ) - if pipeline_status == AzureDataFactoryPipelineRunStatus.FAILED: - yield TriggerEvent( - {"status": "error", "message": f"Pipeline run {self.run_id} has Failed."} + try: + pipeline_status = await hook.get_adf_pipeline_run_status( + run_id=self.run_id, + resource_group_name=self.resource_group_name, + factory_name=self.factory_name, ) - elif pipeline_status == AzureDataFactoryPipelineRunStatus.CANCELLED: - msg = f"Pipeline run {self.run_id} has been Cancelled." - yield TriggerEvent({"status": "error", "message": msg}) - elif pipeline_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED: - msg = f"Pipeline run {self.run_id} has been Succeeded." - yield TriggerEvent({"status": "success", "message": msg}) - await asyncio.sleep(self.poke_interval) + executed_after_token_refresh = False + if pipeline_status == AzureDataFactoryPipelineRunStatus.FAILED: + yield TriggerEvent( + {"status": "error", "message": f"Pipeline run {self.run_id} has Failed."} + ) + return + elif pipeline_status == AzureDataFactoryPipelineRunStatus.CANCELLED: + msg = f"Pipeline run {self.run_id} has been Cancelled." + yield TriggerEvent({"status": "error", "message": msg}) + return + elif pipeline_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED: + msg = f"Pipeline run {self.run_id} has been Succeeded." + yield TriggerEvent({"status": "success", "message": msg}) + return + await asyncio.sleep(self.poke_interval) + except ServiceRequestError: + # conn might expire during long running pipeline. + # If expcetion is caught, it tries to refresh connection once. + # If it still doesn't fix the issue, + # than the execute_after_token_refresh would still be False + # and an exception will be raised + if executed_after_token_refresh: + await hook.refresh_conn() + executed_after_token_refresh = False + continue + raise except Exception as e: yield TriggerEvent({"status": "error", "message": str(e)}) @@ -147,33 +166,49 @@ async def run(self) -> AsyncIterator[TriggerEvent]: resource_group_name=self.resource_group_name, factory_name=self.factory_name, ) + executed_after_token_refresh = True if self.wait_for_termination: while self.end_time > time.time(): - pipeline_status = await hook.get_adf_pipeline_run_status( - run_id=self.run_id, - resource_group_name=self.resource_group_name, - factory_name=self.factory_name, - ) - if pipeline_status in AzureDataFactoryPipelineRunStatus.FAILURE_STATES: - yield TriggerEvent( - { - "status": "error", - "message": f"The pipeline run {self.run_id} has {pipeline_status}.", - "run_id": self.run_id, - } + try: + pipeline_status = await hook.get_adf_pipeline_run_status( + run_id=self.run_id, + resource_group_name=self.resource_group_name, + factory_name=self.factory_name, ) - elif pipeline_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED: - yield TriggerEvent( - { - "status": "success", - "message": f"The pipeline run {self.run_id} has {pipeline_status}.", - "run_id": self.run_id, - } + executed_after_token_refresh = True + if pipeline_status in AzureDataFactoryPipelineRunStatus.FAILURE_STATES: + yield TriggerEvent( + { + "status": "error", + "message": f"The pipeline run {self.run_id} has {pipeline_status}.", + "run_id": self.run_id, + } + ) + return + elif pipeline_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED: + yield TriggerEvent( + { + "status": "success", + "message": f"The pipeline run {self.run_id} has {pipeline_status}.", + "run_id": self.run_id, + } + ) + return + self.log.info( + "Sleeping for %s. The pipeline state is %s.", self.check_interval, pipeline_status ) - self.log.info( - "Sleeping for %s. The pipeline state is %s.", self.check_interval, pipeline_status - ) - await asyncio.sleep(self.check_interval) + await asyncio.sleep(self.check_interval) + except ServiceRequestError: + # conn might expire during long running pipeline. + # If expcetion is caught, it tries to refresh connection once. + # If it still doesn't fix the issue, + # than the execute_after_token_refresh would still be False + # and an exception will be raised + if executed_after_token_refresh: + await hook.refresh_conn() + executed_after_token_refresh = False + continue + raise yield TriggerEvent( { @@ -192,10 +227,13 @@ async def run(self) -> AsyncIterator[TriggerEvent]: ) except Exception as e: if self.run_id: - await hook.cancel_pipeline_run( - run_id=self.run_id, - resource_group_name=self.resource_group_name, - factory_name=self.factory_name, - ) - self.log.info("Unexpected error %s caught. Cancel pipeline run %s", str(e), self.run_id) + try: + await hook.cancel_pipeline_run( + run_id=self.run_id, + resource_group_name=self.resource_group_name, + factory_name=self.factory_name, + ) + self.log.info("Unexpected error %s caught. Cancel pipeline run %s", str(e), self.run_id) + except Exception as err: + yield TriggerEvent({"status": "error", "message": str(err), "run_id": self.run_id}) yield TriggerEvent({"status": "error", "message": str(e), "run_id": self.run_id}) diff --git a/tests/providers/microsoft/azure/hooks/test_azure_data_factory.py b/tests/providers/microsoft/azure/hooks/test_azure_data_factory.py index c7b256e9cb28..57f7bc6178dc 100644 --- a/tests/providers/microsoft/azure/hooks/test_azure_data_factory.py +++ b/tests/providers/microsoft/azure/hooks/test_azure_data_factory.py @@ -720,6 +720,14 @@ def test_backcompat_prefix_both_prefers_short(mock_connect): mock_connect.return_value.factories.delete.assert_called_with("non-prefixed", "n/a") +def test_refresh_conn(hook): + """Test refresh_conn method _conn is reset and get_conn is called""" + with patch.object(hook, "get_conn") as mock_get_conn: + hook.refresh_conn() + assert not hook._conn + assert mock_get_conn.called + + class TestAzureDataFactoryAsyncHook: @pytest.mark.asyncio @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_async_conn") @@ -780,16 +788,6 @@ async def test_get_adf_pipeline_run_status_cancelled(self, mock_get_pipeline_run response = await hook.get_adf_pipeline_run_status(RUN_ID, RESOURCE_GROUP_NAME, DATAFACTORY_NAME) assert response == mock_status - @pytest.mark.asyncio - @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_async_conn") - @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_pipeline_run") - async def test_get_adf_pipeline_run_status_exception(self, mock_get_pipeline_run, mock_conn): - """Test get_adf_pipeline_run_status function with exception""" - mock_get_pipeline_run.side_effect = Exception("Test exception") - hook = AzureDataFactoryAsyncHook(AZURE_DATA_FACTORY_CONN_ID) - with pytest.raises(AirflowException): - await hook.get_adf_pipeline_run_status(RUN_ID, RESOURCE_GROUP_NAME, DATAFACTORY_NAME) - @pytest.mark.asyncio @mock.patch("azure.mgmt.datafactory.models._models_py3.PipelineRun") @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_connection") @@ -810,15 +808,6 @@ async def test_get_pipeline_run_exception_without_resource( with pytest.raises(AirflowException): await hook.get_pipeline_run(RUN_ID, None, DATAFACTORY_NAME) - @pytest.mark.asyncio - @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_async_conn") - async def test_get_pipeline_run_exception(self, mock_conn): - """Test get_pipeline_run function with exception""" - mock_conn.return_value.pipeline_runs.get.side_effect = Exception("Test exception") - hook = AzureDataFactoryAsyncHook(AZURE_DATA_FACTORY_CONN_ID) - with pytest.raises(AirflowException): - await hook.get_pipeline_run(RUN_ID, RESOURCE_GROUP_NAME, DATAFACTORY_NAME) - @pytest.mark.asyncio @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_connection") async def test_get_async_conn(self, mock_connection): @@ -958,3 +947,12 @@ def test_get_field_non_prefixed_extras(self): assert get_field(extras, "factory_name", strict=True) == DATAFACTORY_NAME with pytest.raises(KeyError): get_field(extras, "non-existent-field", strict=True) + + @pytest.mark.asyncio + @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_async_conn") + async def test_refresh_conn(self, mock_get_async_conn): + """Test refresh_conn method _conn is reset and get_async_conn is called""" + hook = AzureDataFactoryAsyncHook(AZURE_DATA_FACTORY_CONN_ID) + await hook.refresh_conn() + assert not hook._conn + assert mock_get_async_conn.called diff --git a/tests/providers/microsoft/azure/triggers/test_azure_data_factory.py b/tests/providers/microsoft/azure/triggers/test_azure_data_factory.py index 9df51bae71a5..fd1a20655404 100644 --- a/tests/providers/microsoft/azure/triggers/test_azure_data_factory.py +++ b/tests/providers/microsoft/azure/triggers/test_azure_data_factory.py @@ -163,10 +163,14 @@ async def test_adf_pipeline_run_status_sensors_trigger_cancelled( assert TriggerEvent({"status": "error", "message": mock_message}) == actual @pytest.mark.asyncio + @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.refresh_conn") @mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_adf_pipeline_run_status") - async def test_adf_pipeline_run_status_sensors_trigger_exception(self, mock_data_factory): + async def test_adf_pipeline_run_status_sensors_trigger_exception( + self, mock_data_factory, mock_refresh_token + ): """Test EMR container sensors with raise exception""" mock_data_factory.side_effect = Exception("Test exception") + mock_refresh_token.side_effect = Exception("Test exception") task = [i async for i in self.TRIGGER.run()] assert len(task) == 1 From 9ef9dc46f393a9e093b2d30928c30e72508b9394 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Tue, 4 Jul 2023 22:04:22 +0200 Subject: [PATCH 041/533] Adds support for custom query in workers KEDA trigger (#32308) Signed-off-by: Hussein Awala --- .../workers/worker-kedaautoscaler.yaml | 8 +--- chart/values.schema.json | 5 +++ chart/values.yaml | 9 ++++ tests/charts/airflow_core/test_worker.py | 45 +++++++++++++++++++ 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml index 036e172d8964..44e17e445b59 100644 --- a/chart/templates/workers/worker-kedaautoscaler.yaml +++ b/chart/templates/workers/worker-kedaautoscaler.yaml @@ -51,11 +51,5 @@ spec: metadata: targetQueryValue: "1" connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB - query: >- - SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) - FROM task_instance - WHERE (state='running' OR state='queued') - {{- if eq .Values.executor "CeleryKubernetesExecutor" }} - AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' - {{- end }} + query: {{ tpl .Values.workers.keda.query . | quote }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 80e25b9b10a6..56689532fb9a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1433,6 +1433,11 @@ } } } + }, + "query": { + "description": "Query to use for KEDA autoscaling. Must return a single integer.", + "type": "string", + "default": "SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') {{- if eq .Values.executor \"CeleryKubernetesExecutor\" }} AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' {{- end }}" } } }, diff --git a/chart/values.yaml b/chart/values.yaml index 3beeb3765cfa..3b07b17049dc 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -538,6 +538,15 @@ workers: # value: 100 # periodSeconds: 15 + # Query to use for KEDA autoscaling. Must return a single integer. + query: >- + SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) + FROM task_instance + WHERE (state='running' OR state='queued') + {{- if eq .Values.executor "CeleryKubernetesExecutor" }} + AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' + {{- end }} + persistence: # Enable persistent volumes enabled: true diff --git a/tests/charts/airflow_core/test_worker.py b/tests/charts/airflow_core/test_worker.py index a49b78fee433..f4e02024a0f0 100644 --- a/tests/charts/airflow_core/test_worker.py +++ b/tests/charts/airflow_core/test_worker.py @@ -640,6 +640,51 @@ def test_should_remove_replicas_field(self): assert "replicas" not in jmespath.search("spec", docs[0]) + @pytest.mark.parametrize( + "query, executor, expected_query", + [ + # default query with CeleryExecutor + ( + None, + "CeleryExecutor", + "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance" + " WHERE (state='running' OR state='queued')", + ), + # default query with CeleryKubernetesExecutor + ( + None, + "CeleryKubernetesExecutor", + "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance" + " WHERE (state='running' OR state='queued') AND queue != 'kubernetes'", + ), + # test custom static query + ( + "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance", + "CeleryKubernetesExecutor", + "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance", + ), + # test custom template query + ( + "SELECT ceil(COUNT(*)::decimal / {{ mul .Values.config.celery.worker_concurrency 2 }})" + " FROM task_instance", + "CeleryKubernetesExecutor", + "SELECT ceil(COUNT(*)::decimal / 32) FROM task_instance", + ), + ], + ) + def test_should_use_keda_query(self, query, executor, expected_query): + + docs = render_chart( + values={ + "executor": executor, + "workers": { + "keda": {"enabled": True, **({"query": query} if query else {})}, + }, + }, + show_only=["templates/workers/worker-kedaautoscaler.yaml"], + ) + assert expected_query == jmespath.search("spec.triggers[0].metadata.query", docs[0]) + class TestWorkerNetworkPolicy: def test_should_add_component_specific_labels(self): From c68a319f04d90bd91a116d80a1f0d2b06879571e Mon Sep 17 00:00:00 2001 From: Changhoon Oh <81631424+okayhooni@users.noreply.github.com> Date: Wed, 5 Jul 2023 05:08:02 +0900 Subject: [PATCH 042/533] Introducing AirflowClusterPolicySkipDag exception (#32013) --- airflow/exceptions.py | 9 +++++- airflow/models/dagbag.py | 5 +++- .../cluster-policies.rst | 4 +++ docs/apache-airflow/best-practices.rst | 28 +++++++++++++++++++ tests/cluster_policies/__init__.py | 9 ++++-- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 9cdb8c418a1b..8c65a1f66f97 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -169,8 +169,15 @@ class AirflowClusterPolicyViolation(AirflowException): """Raise when there is a violation of a Cluster Policy in DAG definition.""" +class AirflowClusterPolicySkipDag(AirflowException): + """Raise when skipping dag is needed in Cluster Policy.""" + + class AirflowClusterPolicyError(AirflowException): - """Raise when there is an error except AirflowClusterPolicyViolation in Cluster Policy.""" + """ + Raise when there is an error in Cluster Policy, + except AirflowClusterPolicyViolation and AirflowClusterPolicySkipDag. + """ class AirflowTimetableInvalid(AirflowException): diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index b422d4699e99..1b80cb01582a 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -38,6 +38,7 @@ from airflow.configuration import conf from airflow.exceptions import ( AirflowClusterPolicyError, + AirflowClusterPolicySkipDag, AirflowClusterPolicyViolation, AirflowDagCycleException, AirflowDagDuplicatedIdException, @@ -442,6 +443,8 @@ def _process_modules(self, filepath, mods, file_last_changed_on_disk): try: dag.validate() self.bag_dag(dag=dag, root_dag=dag) + except AirflowClusterPolicySkipDag: + pass except Exception as e: self.log.exception("Failed to bag_dag: %s", dag.fileloc) self.import_errors[dag.fileloc] = f"{type(e).__name__}: {e}" @@ -477,7 +480,7 @@ def _bag_dag(self, *, dag, root_dag, recursive): for task in dag.tasks: settings.task_policy(task) - except AirflowClusterPolicyViolation: + except (AirflowClusterPolicyViolation, AirflowClusterPolicySkipDag): raise except Exception as e: self.log.exception(e) diff --git a/docs/apache-airflow/administration-and-deployment/cluster-policies.rst b/docs/apache-airflow/administration-and-deployment/cluster-policies.rst index ddf05fde69c0..c664d0b5037c 100644 --- a/docs/apache-airflow/administration-and-deployment/cluster-policies.rst +++ b/docs/apache-airflow/administration-and-deployment/cluster-policies.rst @@ -43,6 +43,10 @@ There are three main types of cluster policy: The DAG and Task cluster policies can raise the :class:`~airflow.exceptions.AirflowClusterPolicyViolation` exception to indicate that the dag/task they were passed is not compliant and should not be loaded. +They can also raise the :class:`~airflow.exceptions.AirflowClusterPolicySkipDag` exception +when skipping that DAG is needed intentionally. Unlike :class:`~airflow.exceptions.AirflowClusterPolicyViolation`, +this exception is not displayed on the Airflow web UI (Internally, it's not recorded on ``import_error`` table on meta database.) + Any extra attributes set by a cluster policy take priority over those defined in your DAG file; for example, if you set an ``sla`` on your Task in the DAG file, and then your cluster policy also sets an ``sla``, the cluster policy's value will take precedence. diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index 2e09f9141ded..b7202ef5a097 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -394,6 +394,34 @@ It's important to note, that without ``watcher`` task, the whole DAG Run will ge If we want the ``watcher`` to monitor the state of all tasks, we need to make it dependent on all of them separately. Thanks to this, we can fail the DAG Run if any of the tasks fail. Note that the watcher task has a trigger rule set to ``"one_failed"``. On the other hand, without the ``teardown`` task, the ``watcher`` task will not be needed, because ``failing_task`` will propagate its ``failed`` state to downstream task ``passed_task`` and the whole DAG Run will also get the ``failed`` status. + +Using AirflowClusterPolicySkipDag exception in cluster policies to skip specific DAGs +------------------------------------------------------------------------------------- + +.. versionadded:: 2.7 + +Airflow DAGs can usually be deployed and updated with the specific branch of Git repository via ``git-sync``. +But, when you have to run multiple Airflow clusters for some operational reasons, it's very cumbersome to maintain multiple Git branches. +Especially, you have some difficulties when you need to synchronize two separate branches(like ``prod`` and ``beta``) periodically with proper branching strategy. + +- cherry-pick is too cumbersome to maintain Git repository. +- hard-reset is not recommended way for GitOps + +So, you can consider connecting multiple Airflow clusters with same Git branch (like ``main``), and maintaining those with different environment variables and different connection configurations with same ``connection_id``. +you can also raise :class:`~airflow.exceptions.AirflowClusterPolicySkipDag` exception on the cluster policy, to load specific DAGs to :class:`~airflow.models.dagbag.DagBag` on the specific Airflow deployment only, if needed. + +.. code-block:: python + + def dag_policy(dag: DAG): + """Skipping the DAG with `only_for_beta` tag.""" + + if "only_for_beta" in dag.tags: + raise AirflowClusterPolicySkipDag( + f"DAG {dag.dag_id} is not loaded on the production cluster, due to `only_for_beta` tag." + ) + +The example above, shows the ``dag_policy`` code snippet to skip the DAG depending on the tags it has. + .. _best_practices/reducing_dag_complexity: Reducing DAG complexity diff --git a/tests/cluster_policies/__init__.py b/tests/cluster_policies/__init__.py index 40c3b2140681..5aaf5701acc6 100644 --- a/tests/cluster_policies/__init__.py +++ b/tests/cluster_policies/__init__.py @@ -22,7 +22,7 @@ from typing import Callable from airflow.configuration import conf -from airflow.exceptions import AirflowClusterPolicyViolation +from airflow.exceptions import AirflowClusterPolicySkipDag, AirflowClusterPolicyViolation from airflow.models import DAG, TaskInstance from airflow.models.baseoperator import BaseOperator @@ -73,12 +73,17 @@ def example_task_policy(task: BaseOperator): # [START example_dag_cluster_policy] def dag_policy(dag: DAG): - """Ensure that DAG has at least one tag""" + """Ensure that DAG has at least one tag and skip the DAG with `only_for_beta` tag.""" if not dag.tags: raise AirflowClusterPolicyViolation( f"DAG {dag.dag_id} has no tags. At least one tag required. File path: {dag.fileloc}" ) + if "only_for_beta" in dag.tags: + raise AirflowClusterPolicySkipDag( + f"DAG {dag.dag_id} is not loaded on the production cluster, due to `only_for_beta` tag." + ) + # [END example_dag_cluster_policy] From 5623a21a1fc738ccb97ade4d4197b181bf1128d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Tue, 4 Jul 2023 13:28:15 -0700 Subject: [PATCH 043/533] deprecate arbitrary parameter passing to RDS hook (#32352) * deprecate arbitrary parameter passing to RDS hook * add url to PR --- airflow/providers/amazon/aws/operators/rds.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/rds.py b/airflow/providers/amazon/aws/operators/rds.py index 440d895afa15..9aef6701660e 100644 --- a/airflow/providers/amazon/aws/operators/rds.py +++ b/airflow/providers/amazon/aws/operators/rds.py @@ -18,12 +18,13 @@ from __future__ import annotations import json +import warnings from datetime import timedelta from typing import TYPE_CHECKING, Sequence from mypy_boto3_rds.type_defs import TagTypeDef -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.rds import RdsHook from airflow.providers.amazon.aws.triggers.rds import RdsDbInstanceTrigger @@ -42,6 +43,15 @@ class RdsBaseOperator(BaseOperator): ui_fgcolor = "#ffffff" def __init__(self, *args, aws_conn_id: str = "aws_conn_id", hook_params: dict | None = None, **kwargs): + if hook_params is not None: + warnings.warn( + "The parameter hook_params is deprecated and will be removed. " + "If you were using it, please get in touch either on airflow slack, " + "or by opening a github issue on the project. " + "You can mention https://github.com/apache/airflow/pull/32352", + AirflowProviderDeprecationWarning, + stacklevel=3, # 2 is in the operator's init, 3 is in the user code creating the operator + ) self.hook_params = hook_params or {} self.hook = RdsHook(aws_conn_id=aws_conn_id, **self.hook_params) super().__init__(*args, **kwargs) From 8a6766f0b95cd270b52117115de48fa328bac18c Mon Sep 17 00:00:00 2001 From: Francesco Caronte Date: Tue, 4 Jul 2023 22:43:33 +0200 Subject: [PATCH 044/533] Runtime class name (#31868) --- chart/files/pod-template-file.kubernetes-helm-yaml | 3 +++ chart/templates/workers/worker-deployment.yaml | 3 +++ chart/values.schema.json | 8 ++++++++ chart/values.yaml | 1 + tests/charts/airflow_core/test_worker.py | 10 ++++++++++ 5 files changed, 25 insertions(+) diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 1f3dbc909d39..70ac65c8e738 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.workers.priorityClassName }} priorityClassName: {{ .Values.workers.priorityClassName }} {{- end }} + {{- if .Values.workers.runtimeClassName }} + priorityClassName: {{ .Values.workers.runtimeClassName }} + {{- end }} {{- if or .Values.registry.secretName .Values.registry.connection }} imagePullSecrets: - name: {{ template "registry_secret" . }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index bacba8ff0306..75b2b1b60a4c 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -99,6 +99,9 @@ spec: {{- toYaml .Values.workers.podAnnotations | nindent 8 }} {{- end }} spec: + {{- if .Values.workers.runtimeClassName }} + runtimeClassName: {{ .Values.workers.runtimeClassName }} + {{- end }} {{- if .Values.workers.priorityClassName }} priorityClassName: {{ .Values.workers.priorityClassName }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 56689532fb9a..3f9b7356796e 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1625,6 +1625,14 @@ "type": "string" } }, + "runtimeClassName": { + "description": "Specify runtime for worker pods.", + "type": [ + "string", + "null" + ], + "default": null + }, "priorityClassName": { "description": "Specify priority for worker pods.", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index 3b07b17049dc..70d862b2fb0a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -617,6 +617,7 @@ workers: # Select certain nodes for airflow worker pods. nodeSelector: {} + runtimeClassName: ~ priorityClassName: ~ affinity: {} # default worker affinity is: diff --git a/tests/charts/airflow_core/test_worker.py b/tests/charts/airflow_core/test_worker.py index f4e02024a0f0..f3127b3bdf46 100644 --- a/tests/charts/airflow_core/test_worker.py +++ b/tests/charts/airflow_core/test_worker.py @@ -378,6 +378,16 @@ def test_should_create_default_affinity(self): docs[0], ) + def test_runtime_class_name_values_are_configurable(self): + docs = render_chart( + values={ + "workers": {"runtimeClassName": "nvidia"}, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert jmespath.search("spec.template.spec.runtimeClassName", docs[0]) == "nvidia" + def test_livenessprobe_values_are_configurable(self): docs = render_chart( values={ From 891c2e401928ecafea78f7c6c3b453663ef03dce Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Wed, 5 Jul 2023 02:19:47 +0530 Subject: [PATCH 045/533] Add Deferrable switch to SnowflakeSqlApiOperator (#31596) --- .../snowflake/hooks/snowflake_sql_api.py | 43 +++++-- .../snowflake/operators/snowflake.py | 46 ++++++- airflow/providers/snowflake/provider.yaml | 5 + .../providers/snowflake/triggers/__init__.py | 16 +++ .../snowflake/triggers/snowflake_trigger.py | 109 ++++++++++++++++ .../operators/snowflake.rst | 2 + .../snowflake/hooks/test_snowflake_sql_api.py | 56 +++++++- .../snowflake/operators/test_snowflake.py | 96 +++++++++++++- .../providers/snowflake/triggers/__init__.py | 16 +++ .../snowflake/triggers/test_snowflake.py | 120 ++++++++++++++++++ 10 files changed, 490 insertions(+), 19 deletions(-) create mode 100644 airflow/providers/snowflake/triggers/__init__.py create mode 100644 airflow/providers/snowflake/triggers/snowflake_trigger.py create mode 100644 tests/providers/snowflake/triggers/__init__.py create mode 100644 tests/providers/snowflake/triggers/test_snowflake.py diff --git a/airflow/providers/snowflake/hooks/snowflake_sql_api.py b/airflow/providers/snowflake/hooks/snowflake_sql_api.py index 0d808291ffa0..eec3c7349e1e 100644 --- a/airflow/providers/snowflake/hooks/snowflake_sql_api.py +++ b/airflow/providers/snowflake/hooks/snowflake_sql_api.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Any +import aiohttp import requests from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -59,7 +60,8 @@ class SnowflakeSqlApiHook(SnowflakeHook): :param session_parameters: You can set session-level parameters at the time you connect to Snowflake :param token_life_time: lifetime of the JWT Token in timedelta - :param token_renewal_delta: Renewal time of the JWT Token in timedelta + :param token_renewal_delta: Renewal time of the JWT Token in timedelta + :param deferrable: Run operator in the deferrable mode. """ LIFETIME = timedelta(minutes=59) # The tokens will have a 59 minute lifetime @@ -225,17 +227,7 @@ def check_query_output(self, query_ids: list[str]) -> None: f"Response: {e.response.content}, Status Code: {e.response.status_code}" ) - def get_sql_api_query_status(self, query_id: str) -> dict[str, str | list[str]]: - """ - Based on the query id async HTTP request is made to snowflake SQL API and return response. - - :param query_id: statement handle id for the individual statements. - """ - self.log.info("Retrieving status for query id %s", {query_id}) - header, params, url = self.get_request_url_header_params(query_id) - response = requests.get(url, params=params, headers=header) - status_code = response.status_code - resp = response.json() + def _process_response(self, status_code, resp): self.log.info("Snowflake SQL GET statements status API response: %s", resp) if status_code == 202: return {"status": "running", "message": "Query statements are still running"} @@ -254,3 +246,30 @@ def get_sql_api_query_status(self, query_id: str) -> dict[str, str | list[str]]: } else: return {"status": "error", "message": resp["message"]} + + def get_sql_api_query_status(self, query_id: str) -> dict[str, str | list[str]]: + """ + Based on the query id async HTTP request is made to snowflake SQL API and return response. + + :param query_id: statement handle id for the individual statements. + """ + self.log.info("Retrieving status for query id %s", query_id) + header, params, url = self.get_request_url_header_params(query_id) + response = requests.get(url, params=params, headers=header) + status_code = response.status_code + resp = response.json() + return self._process_response(status_code, resp) + + async def get_sql_api_query_status_async(self, query_id: str) -> dict[str, str | list[str]]: + """ + Based on the query id async HTTP request is made to snowflake SQL API and return response. + + :param query_id: statement handle id for the individual statements. + """ + self.log.info("Retrieving status for query id %s", query_id) + header, params, url = self.get_request_url_header_params(query_id) + async with aiohttp.ClientSession(headers=header) as session: + async with session.get(url, params=params) as response: + status_code = response.status + resp = await response.json() + return self._process_response(status_code, resp) diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index b56b14b0d5a4..db35fa000731 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -20,7 +20,7 @@ import time import warnings from datetime import timedelta -from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence, SupportsAbs +from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Sequence, SupportsAbs, cast from airflow import AirflowException from airflow.exceptions import AirflowProviderDeprecationWarning @@ -33,6 +33,7 @@ from airflow.providers.snowflake.hooks.snowflake_sql_api import ( SnowflakeSqlApiHook, ) +from airflow.providers.snowflake.triggers.snowflake_trigger import SnowflakeSqlApiTrigger if TYPE_CHECKING: from airflow.utils.context import Context @@ -430,6 +431,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator): :param bindings: (Optional) Values of bind variables in the SQL statement. When executing the statement, Snowflake replaces placeholders (? and :name) in the statement with these specified values. + :param deferrable: Run operator in the deferrable mode. """ # noqa LIFETIME = timedelta(minutes=59) # The tokens will have a 59 minutes lifetime @@ -450,6 +452,7 @@ def __init__( token_life_time: timedelta = LIFETIME, token_renewal_delta: timedelta = RENEWAL_DELTA, bindings: dict[str, Any] | None = None, + deferrable: bool = False, **kwargs: Any, ) -> None: self.snowflake_conn_id = snowflake_conn_id @@ -459,6 +462,7 @@ def __init__( self.token_renewal_delta = token_renewal_delta self.bindings = bindings self.execute_async = False + self.deferrable = deferrable if any([warehouse, database, role, schema, authenticator, session_parameters]): # pragma: no cover hook_params = kwargs.pop("hook_params", {}) # pragma: no cover kwargs["hook_params"] = { @@ -482,6 +486,7 @@ def execute(self, context: Context) -> None: snowflake_conn_id=self.snowflake_conn_id, token_life_time=self.token_life_time, token_renewal_delta=self.token_renewal_delta, + deferrable=self.deferrable, ) self.query_ids = self._hook.execute_query( self.sql, statement_count=self.statement_count, bindings=self.bindings # type: ignore[arg-type] @@ -491,10 +496,23 @@ def execute(self, context: Context) -> None: if self.do_xcom_push: context["ti"].xcom_push(key="query_ids", value=self.query_ids) - statement_status = self.poll_on_queries() - if statement_status["error"]: - raise AirflowException(statement_status["error"]) - self._hook.check_query_output(self.query_ids) + if self.deferrable: + self.defer( + timeout=self.execution_timeout, + trigger=SnowflakeSqlApiTrigger( + poll_interval=self.poll_interval, + query_ids=self.query_ids, + snowflake_conn_id=self.snowflake_conn_id, + token_life_time=self.token_life_time, + token_renewal_delta=self.token_renewal_delta, + ), + method_name="execute_complete", + ) + else: + statement_status = self.poll_on_queries() + if statement_status["error"]: + raise AirflowException(statement_status["error"]) + self._hook.check_query_output(self.query_ids) def poll_on_queries(self): """Poll on requested queries.""" @@ -517,3 +535,21 @@ def poll_on_queries(self): queries_in_progress.remove(query_id) time.sleep(self.poll_interval) return {"success": statement_success_status, "error": statement_error_status} + + def execute_complete(self, context: Context, event: dict[str, str | list[str]] | None = None) -> None: + """ + Callback for when the trigger fires - returns immediately. + Relies on trigger to throw an exception, otherwise it assumes execution was + successful. + """ + if event: + if "status" in event and event["status"] == "error": + msg = f"{event['status']}: {event['message']}" + raise AirflowException(msg) + elif "status" in event and event["status"] == "success": + hook = SnowflakeSqlApiHook(snowflake_conn_id=self.snowflake_conn_id) + query_ids = cast(List[str], event["statement_query_ids"]) + hook.check_query_output(query_ids) + self.log.info("%s completed successfully.", self.task_id) + else: + self.log.info("%s completed successfully.", self.task_id) diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 1e68fbddca55..2cea953ab428 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -100,3 +100,8 @@ transfers: connection-types: - hook-class-name: airflow.providers.snowflake.hooks.snowflake.SnowflakeHook connection-type: snowflake + +triggers: + - integration-name: Snowflake + python-modules: + - airflow.providers.snowflake.triggers.snowflake_trigger diff --git a/airflow/providers/snowflake/triggers/__init__.py b/airflow/providers/snowflake/triggers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/airflow/providers/snowflake/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/snowflake/triggers/snowflake_trigger.py b/airflow/providers/snowflake/triggers/snowflake_trigger.py new file mode 100644 index 000000000000..4f1e0cffb299 --- /dev/null +++ b/airflow/providers/snowflake/triggers/snowflake_trigger.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import asyncio +from datetime import timedelta +from typing import Any, AsyncIterator + +from airflow.providers.snowflake.hooks.snowflake_sql_api import SnowflakeSqlApiHook +from airflow.triggers.base import BaseTrigger, TriggerEvent + + +class SnowflakeSqlApiTrigger(BaseTrigger): + """ + Fetch the status for the query ids passed. + + :param poll_interval: polling period in seconds to check for the status + :param query_ids: List of Query ids to run and poll for the status + :param snowflake_conn_id: Reference to Snowflake connection id + :param token_life_time: lifetime of the JWT Token in timedelta + :param token_renewal_delta: Renewal time of the JWT Token in timedelta + """ + + def __init__( + self, + poll_interval: float, + query_ids: list[str], + snowflake_conn_id: str, + token_life_time: timedelta, + token_renewal_delta: timedelta, + ): + super().__init__() + self.poll_interval = poll_interval + self.query_ids = query_ids + self.snowflake_conn_id = snowflake_conn_id + self.token_life_time = token_life_time + self.token_renewal_delta = token_renewal_delta + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serializes SnowflakeSqlApiTrigger arguments and classpath.""" + return ( + "airflow.providers.snowflake.triggers.snowflake_trigger.SnowflakeSqlApiTrigger", + { + "poll_interval": self.poll_interval, + "query_ids": self.query_ids, + "snowflake_conn_id": self.snowflake_conn_id, + "token_life_time": self.token_life_time, + "token_renewal_delta": self.token_renewal_delta, + }, + ) + + async def run(self) -> AsyncIterator[TriggerEvent]: + """Wait for the query the snowflake query to complete.""" + SnowflakeSqlApiHook( + self.snowflake_conn_id, + self.token_life_time, + self.token_renewal_delta, + ) + try: + statement_query_ids: list[str] = [] + for query_id in self.query_ids: + while True: + statement_status = await self.get_query_status(query_id) + if statement_status["status"] not in ["running"]: + break + await asyncio.sleep(self.poll_interval) + if statement_status["status"] == "error": + yield TriggerEvent(statement_status) + return + if statement_status["status"] == "success": + statement_query_ids.extend(statement_status["statement_handles"]) + yield TriggerEvent( + { + "status": "success", + "statement_query_ids": statement_query_ids, + } + ) + except Exception as e: + yield TriggerEvent({"status": "error", "message": str(e)}) + + async def get_query_status(self, query_id: str) -> dict[str, Any]: + """ + Async function to check whether the query statement submitted via SQL API is still + running state and returns True if it is still running else + return False. + """ + hook = SnowflakeSqlApiHook( + self.snowflake_conn_id, + self.token_life_time, + self.token_renewal_delta, + ) + return await hook.get_sql_api_query_status_async(query_id) + + def _set_context(self, context): + pass diff --git a/docs/apache-airflow-providers-snowflake/operators/snowflake.rst b/docs/apache-airflow-providers-snowflake/operators/snowflake.rst index 1e80f3af2955..d3ffbec5a4cf 100644 --- a/docs/apache-airflow-providers-snowflake/operators/snowflake.rst +++ b/docs/apache-airflow-providers-snowflake/operators/snowflake.rst @@ -66,6 +66,8 @@ SnowflakeSqlApiOperator Use the :class:`SnowflakeSqlApiHook ` to execute SQL commands in a `Snowflake `__ database. +You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. +This will ensure that the task is deferred from the Airflow worker slot and polling for the task status happens on the trigger. Using the Operator ^^^^^^^^^^^^^^^^^^ diff --git a/tests/providers/snowflake/hooks/test_snowflake_sql_api.py b/tests/providers/snowflake/hooks/test_snowflake_sql_api.py index 61e88de864df..fd2da72c928e 100644 --- a/tests/providers/snowflake/hooks/test_snowflake_sql_api.py +++ b/tests/providers/snowflake/hooks/test_snowflake_sql_api.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Any from unittest import mock +from unittest.mock import AsyncMock import pytest import requests @@ -396,7 +397,6 @@ def test_get_private_key_should_support_private_auth_with_unencrypted_key( ), pytest.raises(TypeError, match="Password was given but private key is not encrypted."): SnowflakeSqlApiHook(snowflake_conn_id="test_conn").get_private_key() - @pytest.mark.asyncio @pytest.mark.parametrize( "status_code,response,expected_response", [ @@ -456,3 +456,57 @@ def json(self): mock_requests.get.return_value = MockResponse(status_code, response) hook = SnowflakeSqlApiHook(snowflake_conn_id="test_conn") assert hook.get_sql_api_query_status("uuid") == expected_response + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "status_code,response,expected_response", + [ + ( + 200, + { + "status": "success", + "message": "Statement executed successfully.", + "statementHandle": "uuid", + }, + { + "status": "success", + "message": "Statement executed successfully.", + "statement_handles": ["uuid"], + }, + ), + ( + 200, + { + "status": "success", + "message": "Statement executed successfully.", + "statementHandles": ["uuid", "uuid1"], + }, + { + "status": "success", + "message": "Statement executed successfully.", + "statement_handles": ["uuid", "uuid1"], + }, + ), + (202, {}, {"status": "running", "message": "Query statements are still running"}), + (422, {"status": "error", "message": "test"}, {"status": "error", "message": "test"}), + (404, {"status": "error", "message": "test"}, {"status": "error", "message": "test"}), + ], + ) + @mock.patch( + "airflow.providers.snowflake.hooks.snowflake_sql_api.SnowflakeSqlApiHook." + "get_request_url_header_params" + ) + @mock.patch("airflow.providers.snowflake.hooks.snowflake_sql_api.aiohttp.ClientSession.get") + async def test_get_sql_api_query_status_async( + self, mock_get, mock_geturl_header_params, status_code, response, expected_response + ): + """Test Async get_sql_api_query_status_async function by mocking the status, + response and expected response""" + req_id = uuid.uuid4() + params = {"requestId": str(req_id), "page": 2, "pageSize": 10} + mock_geturl_header_params.return_value = HEADERS, params, "/test/airflow/" + mock_get.return_value.__aenter__.return_value.status = status_code + mock_get.return_value.__aenter__.return_value.json = AsyncMock(return_value=response) + hook = SnowflakeSqlApiHook(snowflake_conn_id="test_conn") + response = await hook.get_sql_api_query_status_async("uuid") + assert response == expected_response diff --git a/tests/providers/snowflake/operators/test_snowflake.py b/tests/providers/snowflake/operators/test_snowflake.py index 8f32c6e62d8d..41cbfe671746 100644 --- a/tests/providers/snowflake/operators/test_snowflake.py +++ b/tests/providers/snowflake/operators/test_snowflake.py @@ -19,10 +19,13 @@ from unittest import mock +import pendulum import pytest -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, TaskDeferred from airflow.models.dag import DAG +from airflow.models.dagrun import DagRun +from airflow.models.taskinstance import TaskInstance from airflow.providers.snowflake.operators.snowflake import ( SnowflakeCheckOperator, SnowflakeIntervalCheckOperator, @@ -30,7 +33,9 @@ SnowflakeSqlApiOperator, SnowflakeValueCheckOperator, ) +from airflow.providers.snowflake.triggers.snowflake_trigger import SnowflakeSqlApiTrigger from airflow.utils import timezone +from airflow.utils.types import DagRunType DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() @@ -88,6 +93,34 @@ def test_get_db_hook( mock_get_db_hook.assert_called_once() +def create_context(task, dag=None): + if dag is None: + dag = DAG(dag_id="dag") + tzinfo = pendulum.timezone("UTC") + execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=execution_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), + ) + + task_instance = TaskInstance(task=task) + task_instance.dag_run = dag_run + task_instance.xcom_push = mock.Mock() + return { + "dag": dag, + "ts": execution_date.isoformat(), + "task": task, + "ti": task_instance, + "task_instance": task_instance, + "run_id": dag_run.run_id, + "dag_run": dag_run, + "execution_date": execution_date, + "data_interval_end": execution_date, + "logical_date": execution_date, + } + + class TestSnowflakeSqlApiOperator: @pytest.fixture def mock_execute_query(self): @@ -142,3 +175,64 @@ def test_snowflake_sql_api_to_fails_when_one_query_fails( mock_get_sql_api_query_status.side_effect = [{"status": "error"}, {"status": "success"}] with pytest.raises(AirflowException): operator.execute(context=None) + + @pytest.mark.parametrize("mock_sql, statement_count", [(SQL_MULTIPLE_STMTS, 4), (SINGLE_STMT, 1)]) + @mock.patch("airflow.providers.snowflake.hooks.snowflake_sql_api.SnowflakeSqlApiHook.execute_query") + def test_snowflake_sql_api_execute_operator_async(self, mock_db_hook, mock_sql, statement_count): + """ + Asserts that a task is deferred and an SnowflakeSqlApiTrigger will be fired + when the SnowflakeSqlApiOperator is executed. + """ + operator = SnowflakeSqlApiOperator( + task_id=TASK_ID, + snowflake_conn_id=CONN_ID, + sql=mock_sql, + statement_count=statement_count, + deferrable=True, + ) + + with pytest.raises(TaskDeferred) as exc: + operator.execute(create_context(operator)) + + assert isinstance( + exc.value.trigger, SnowflakeSqlApiTrigger + ), "Trigger is not a SnowflakeSqlApiTrigger" + + def test_snowflake_sql_api_execute_complete_failure(self): + """Test SnowflakeSqlApiOperator raise AirflowException of error event""" + + operator = SnowflakeSqlApiOperator( + task_id=TASK_ID, + snowflake_conn_id=CONN_ID, + sql=SQL_MULTIPLE_STMTS, + statement_count=4, + deferrable=True, + ) + with pytest.raises(AirflowException): + operator.execute_complete( + context=None, + event={"status": "error", "message": "Test failure message", "type": "FAILED_WITH_ERROR"}, + ) + + @pytest.mark.parametrize( + "mock_event", + [ + None, + ({"status": "success", "statement_query_ids": ["uuid", "uuid"]}), + ], + ) + @mock.patch("airflow.providers.snowflake.hooks.snowflake_sql_api.SnowflakeSqlApiHook.check_query_output") + def test_snowflake_sql_api_execute_complete(self, mock_conn, mock_event): + """Tests execute_complete assert with successful message""" + + operator = SnowflakeSqlApiOperator( + task_id=TASK_ID, + snowflake_conn_id=CONN_ID, + sql=SQL_MULTIPLE_STMTS, + statement_count=4, + deferrable=True, + ) + + with mock.patch.object(operator.log, "info") as mock_log_info: + operator.execute_complete(context=None, event=mock_event) + mock_log_info.assert_called_with("%s completed successfully.", TASK_ID) diff --git a/tests/providers/snowflake/triggers/__init__.py b/tests/providers/snowflake/triggers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/providers/snowflake/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/snowflake/triggers/test_snowflake.py b/tests/providers/snowflake/triggers/test_snowflake.py new file mode 100644 index 000000000000..9fc14591625e --- /dev/null +++ b/tests/providers/snowflake/triggers/test_snowflake.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import asyncio +from datetime import timedelta +from unittest import mock + +import pytest + +from airflow.providers.snowflake.triggers.snowflake_trigger import SnowflakeSqlApiTrigger +from airflow.triggers.base import TriggerEvent + +TASK_ID = "snowflake_check" +POLL_INTERVAL = 1.0 +LIFETIME = timedelta(minutes=59) +RENEWAL_DELTA = timedelta(minutes=54) +MODULE = "airflow.providers.snowflake" + + +class TestSnowflakeSqlApiTrigger: + TRIGGER = SnowflakeSqlApiTrigger( + poll_interval=POLL_INTERVAL, + query_ids=["uuid"], + snowflake_conn_id="test_conn", + token_life_time=LIFETIME, + token_renewal_delta=RENEWAL_DELTA, + ) + + def test_snowflake_sql_trigger_serialization(self): + """ + Asserts that the SnowflakeSqlApiTrigger correctly serializes its arguments + and classpath. + """ + classpath, kwargs = self.TRIGGER.serialize() + assert classpath == "airflow.providers.snowflake.triggers.snowflake_trigger.SnowflakeSqlApiTrigger" + assert kwargs == { + "poll_interval": POLL_INTERVAL, + "query_ids": ["uuid"], + "snowflake_conn_id": "test_conn", + "token_life_time": LIFETIME, + "token_renewal_delta": RENEWAL_DELTA, + } + + @pytest.mark.asyncio + @mock.patch(f"{MODULE}.triggers.snowflake_trigger.SnowflakeSqlApiTrigger.get_query_status") + @mock.patch(f"{MODULE}.hooks.snowflake_sql_api.SnowflakeSqlApiHook.get_sql_api_query_status_async") + async def test_snowflake_sql_trigger_running( + self, mock_get_sql_api_query_status_async, mock_get_query_status + ): + """Tests that the SnowflakeSqlApiTrigger in running by mocking get_query_status to true""" + mock_get_query_status.return_value = {"status": "running"} + + task = asyncio.create_task(self.TRIGGER.run().__anext__()) + await asyncio.sleep(0.5) + + # TriggerEvent was not returned + assert task.done() is False + asyncio.get_event_loop().stop() + + @pytest.mark.asyncio + @mock.patch(f"{MODULE}.triggers.snowflake_trigger.SnowflakeSqlApiTrigger.get_query_status") + @mock.patch(f"{MODULE}.hooks.snowflake_sql_api.SnowflakeSqlApiHook.get_sql_api_query_status_async") + async def test_snowflake_sql_trigger_completed( + self, mock_get_sql_api_query_status_async, mock_get_query_status + ): + """ + Test SnowflakeSqlApiTrigger run method with success status and mock the get_sql_api_query_status + result and get_query_status to False. + """ + mock_get_query_status.return_value = {"status": "success", "statement_handles": ["uuid", "uuid1"]} + statement_query_ids = ["uuid", "uuid1"] + mock_get_sql_api_query_status_async.return_value = { + "message": "Statement executed successfully.", + "status": "success", + "statement_handles": statement_query_ids, + } + + generator = self.TRIGGER.run() + actual = await generator.asend(None) + assert TriggerEvent({"status": "success", "statement_query_ids": statement_query_ids}) == actual + + @pytest.mark.asyncio + @mock.patch(f"{MODULE}.hooks.snowflake_sql_api.SnowflakeSqlApiHook.get_sql_api_query_status_async") + async def test_snowflake_sql_trigger_failure_status(self, mock_get_sql_api_query_status_async): + """Test SnowflakeSqlApiTrigger task is executed and triggered with failure status.""" + mock_response = { + "status": "error", + "message": "An error occurred when executing the statement. Check " + "the error code and error message for details", + } + mock_get_sql_api_query_status_async.return_value = mock_response + + generator = self.TRIGGER.run() + actual = await generator.asend(None) + assert TriggerEvent(mock_response) == actual + + @pytest.mark.asyncio + @mock.patch(f"{MODULE}.hooks.snowflake_sql_api.SnowflakeSqlApiHook.get_sql_api_query_status_async") + async def test_snowflake_sql_trigger_exception(self, mock_get_sql_api_query_status_async): + """Tests the SnowflakeSqlApiTrigger does not fire if there is an exception.""" + mock_get_sql_api_query_status_async.side_effect = Exception("Test exception") + + task = [i async for i in self.TRIGGER.run()] + assert len(task) == 1 + assert TriggerEvent({"status": "error", "message": "Test exception"}) in task From 56eae9c9a4b784e7b239335560c31ed30f57e0a1 Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Tue, 4 Jul 2023 22:52:58 +0200 Subject: [PATCH 046/533] Deprecate `runtime_parameters` in favor of options in `hook_params` (#32345) for PostgresOperator. Signed-off-by: Jakub Dardzinski --- airflow/providers/postgres/hooks/postgres.py | 9 +++++- .../providers/postgres/operators/postgres.py | 32 ++++++------------- .../postgres_operator_howto_guide.rst | 9 +++--- .../providers/postgres/hooks/test_postgres.py | 14 ++++++++ .../postgres/operators/test_postgres.py | 1 + .../providers/postgres/example_postgres.py | 2 +- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index 0a7aa0732a43..2e76029028da 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -59,6 +59,9 @@ class PostgresHook(DbApiHook): :param postgres_conn_id: The :ref:`postgres conn id ` reference to a specific postgres database. + :param options: Optional. Specifies command-line options to send to the server + at connection start. For example, setting this to ``-c search_path=myschema`` + sets the session's value of the ``search_path`` to ``myschema``. """ conn_name_attr = "postgres_conn_id" @@ -67,7 +70,7 @@ class PostgresHook(DbApiHook): hook_name = "Postgres" supports_autocommit = True - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args, options: str | None = None, **kwargs) -> None: if "schema" in kwargs: warnings.warn( 'The "schema" arg has been renamed to "database" as it contained the database name.' @@ -80,6 +83,7 @@ def __init__(self, *args, **kwargs) -> None: self.connection: Connection | None = kwargs.pop("connection", None) self.conn: connection = None self.database: str | None = kwargs.pop("database", None) + self.options = options @property def schema(self): @@ -131,6 +135,9 @@ def get_conn(self) -> connection: if raw_cursor: conn_args["cursor_factory"] = self._get_cursor(raw_cursor) + if self.options: + conn_args["options"] = self.options + for arg_name, arg_val in conn.extra_dejson.items(): if arg_name not in [ "iam", diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py index e70ac720278f..ef92b51dda75 100644 --- a/airflow/providers/postgres/operators/postgres.py +++ b/airflow/providers/postgres/operators/postgres.py @@ -20,8 +20,6 @@ import warnings from typing import Mapping, Sequence -from psycopg2.sql import SQL, Identifier - from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator @@ -41,6 +39,7 @@ class PostgresOperator(SQLExecuteQueryOperator): :param database: name of database which overwrite defined one in connection :param runtime_parameters: a mapping of runtime params added to the final sql being executed. For example, you could set the schema via `{"search_path": "CUSTOM_SCHEMA"}`. + Deprecated - use `hook_params={'options': '-c '}` instead. """ template_fields: Sequence[str] = ("sql",) @@ -61,26 +60,15 @@ def __init__( kwargs["hook_params"] = {"schema": database, **hook_params} if runtime_parameters: - sql = kwargs.pop("sql") - parameters = kwargs.pop("parameters", {}) - - final_sql = [] - sql_param = {} - for param in runtime_parameters: - set_param_sql = f"SET {{}} TO %({param})s;" - dynamic_sql = SQL(set_param_sql).format(Identifier(f"{param}")) - final_sql.append(dynamic_sql) - for param, val in runtime_parameters.items(): - sql_param.update({f"{param}": f"{val}"}) - if parameters: - sql_param.update(parameters) - if isinstance(sql, str): - final_sql.append(SQL(sql)) - else: - final_sql.extend(list(map(SQL, sql))) - - kwargs["sql"] = final_sql - kwargs["parameters"] = sql_param + warnings.warn( + """`runtime_parameters` is deprecated. + Please use `hook_params={'options': '-c }`.""", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + hook_params = kwargs.pop("hook_params", {}) + options = " ".join(f"-c {param}={val}" for param, val in runtime_parameters.items()) + kwargs["hook_params"] = {"options": options, **hook_params} super().__init__(conn_id=postgres_conn_id, **kwargs) warnings.warn( diff --git a/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst b/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst index 7fb4150d8aec..962bbf87de77 100644 --- a/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst +++ b/docs/apache-airflow-providers-postgres/operators/postgres_operator_howto_guide.rst @@ -159,8 +159,9 @@ class. Passing Server Configuration Parameters into PostgresOperator ------------------------------------------------------------- -PostgresOperator provides the optional ``runtime_parameters`` attribute which makes it possible to set -the `server configuration parameter values `_ for the SQL request during runtime. +PostgresOperator provides ``hook_params`` attribute that allows you to pass add parameters to PostgresHook. +You can pass ``options`` argument this way so that you specify `command-line options `_ +sent to the server at connection start. .. exampleinclude:: /../../tests/system/providers/postgres/example_postgres.py :language: python @@ -186,5 +187,5 @@ In this how-to guide we explored the Apache Airflow PostgreOperator. Let's quick It is best practice to create subdirectory called ``sql`` in your ``dags`` directory where you can store your sql files. This will make your code more elegant and more maintainable. And finally, we looked at the different ways you can dynamically pass parameters into our PostgresOperator -tasks using ``parameters`` or ``params`` attribute and how you can control the server configuration parameters by passing -the ``runtime_parameters`` attribute. +tasks using ``parameters`` or ``params`` attribute and how you can control the session parameters by passing +options in the ``hook_params`` attribute. diff --git a/tests/providers/postgres/hooks/test_postgres.py b/tests/providers/postgres/hooks/test_postgres.py index 804ee9da9a00..a6fa619634c4 100644 --- a/tests/providers/postgres/hooks/test_postgres.py +++ b/tests/providers/postgres/hooks/test_postgres.py @@ -101,6 +101,20 @@ def test_get_conn_from_connection_with_database(self, mock_connect): user="login-conn", password="password-conn", host="host", dbname="database-override", port=None ) + @mock.patch("airflow.providers.postgres.hooks.postgres.psycopg2.connect") + def test_get_conn_from_connection_with_options(self, mock_connect): + conn = Connection(login="login-conn", password="password-conn", host="host", schema="database") + hook = PostgresHook(connection=conn, options="-c statement_timeout=3000ms") + hook.get_conn() + mock_connect.assert_called_once_with( + user="login-conn", + password="password-conn", + host="host", + dbname="database", + port=None, + options="-c statement_timeout=3000ms", + ) + @mock.patch("airflow.providers.postgres.hooks.postgres.psycopg2.connect") @mock.patch("airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook") @pytest.mark.parametrize("aws_conn_id", [NOTSET, None, "mock_aws_conn"]) diff --git a/tests/providers/postgres/operators/test_postgres.py b/tests/providers/postgres/operators/test_postgres.py index 4b615917ea28..b58abf3497dc 100644 --- a/tests/providers/postgres/operators/test_postgres.py +++ b/tests/providers/postgres/operators/test_postgres.py @@ -112,3 +112,4 @@ def test_runtime_parameter_setting(self): runtime_parameters={"statement_timeout": "3000ms"}, ) op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + assert op.get_db_hook().get_first("SHOW statement_timeout;")[0] == "3s" diff --git a/tests/system/providers/postgres/example_postgres.py b/tests/system/providers/postgres/example_postgres.py index ceab5271958a..fd0a80783c9f 100644 --- a/tests/system/providers/postgres/example_postgres.py +++ b/tests/system/providers/postgres/example_postgres.py @@ -73,7 +73,7 @@ task_id="get_birth_date", sql="SELECT * FROM pet WHERE birth_date BETWEEN SYMMETRIC %(begin_date)s AND %(end_date)s", parameters={"begin_date": "2020-01-01", "end_date": "2020-12-31"}, - runtime_parameters={"statement_timeout": "3000ms"}, + hook_params={"options": "-c statement_timeout=3000ms"}, ) # [END postgres_operator_howto_guide_get_birth_date] From c2b9fcc108eae4373350fd0559d6c166853bec40 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Tue, 4 Jul 2023 13:56:01 -0700 Subject: [PATCH 047/533] Add ferruzzi as committer (#32353) --- dev/breeze/src/airflow_breeze/global_constants.py | 1 + docs/apache-airflow/project.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 510996a4b591..0a66f7d98ced 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -225,6 +225,7 @@ def get_default_platform_machine() -> str: "ephraimbuddy", "feluelle", "feng-tao", + "ferruzzi", "houqp", "hussein-awala", "jedcunningham", diff --git a/docs/apache-airflow/project.rst b/docs/apache-airflow/project.rst index 75bc4b2e7fc3..3516a6b8c093 100644 --- a/docs/apache-airflow/project.rst +++ b/docs/apache-airflow/project.rst @@ -51,6 +51,7 @@ Committers - Dan Davydov (@aoen) - Daniel Imberman (@dimberman) - Daniel Standish (@dstandish) +- Dennis Ferruzzi (@ferruzzi) - Elad Kalif (@eladkal) - Ephraim Anierobi (@ephraimbuddy) - Felix Uellendall (@feluelle) From bc3b2d16d3563d5b9bccd283db3f9e290d1d823d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 4 Jul 2023 22:56:24 +0200 Subject: [PATCH 048/533] Add kerberos dependency to Impala Provider (#32304) --- airflow/providers/apache/impala/provider.yaml | 5 ++++ .../apache/impala/hooks/test_impala.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index b08374bc8223..4d02db35c9df 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -44,3 +44,8 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.impala.hooks.impala.ImpalaHook connection-type: impala + +additional-extras: + - name: kerberos + dependencies: + - kerberos>=1.3.0 diff --git a/tests/providers/apache/impala/hooks/test_impala.py b/tests/providers/apache/impala/hooks/test_impala.py index a3e1f119360d..97f1822eef37 100644 --- a/tests/providers/apache/impala/hooks/test_impala.py +++ b/tests/providers/apache/impala/hooks/test_impala.py @@ -54,6 +54,31 @@ def test_get_conn(mock_connect): ) +@patch("airflow.providers.apache.impala.hooks.impala.connect", autospec=True) +def test_get_conn_kerberos(mock_connect): + hook = ImpalaHook() + hook.get_connection = MagicMock( + return_value=Connection( + login="login", + password="password", + host="host", + port=21050, + schema="test", + extra={"auth_mechanism": "GSSAPI", "use_ssl": True}, + ) + ) + hook.get_conn() + mock_connect.assert_called_once_with( + host="host", + port=21050, + user="login", + password="password", + database="test", + use_ssl=True, + auth_mechanism="GSSAPI", + ) + + @patch("airflow.providers.common.sql.hooks.sql.DbApiHook.insert_rows") def test_insert_rows(mock_insert_rows, impala_hook_fixture): table = "table" From ab2c861dd8a96f22b0fda692368ce9b103175322 Mon Sep 17 00:00:00 2001 From: Vijayasarathi Balasubramanian Date: Tue, 4 Jul 2023 16:57:49 -0400 Subject: [PATCH 049/533] Clarify Listener API behavior (#32269) * Fixes: #31180 - Plugin for listeners - on_dag_run_running hook ignored Documentation update for Plugin for listeners - on_dag_run_running hook ignored Co-authored-by: Tzu-ping Chung --- docs/apache-airflow/administration-and-deployment/listeners.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/administration-and-deployment/listeners.rst b/docs/apache-airflow/administration-and-deployment/listeners.rst index c34689c53fa4..2847eb514d0f 100644 --- a/docs/apache-airflow/administration-and-deployment/listeners.rst +++ b/docs/apache-airflow/administration-and-deployment/listeners.rst @@ -54,7 +54,7 @@ Airflow defines the specification as `hookspec ` -Listener API is meant to be called across all DAGs and all operators. You can't listen to events generated by specific DAGs. For that behavior, try methods like ``on_success_callback`` and ``pre_execute``. These provide callbacks for particular DAG authors or operator creators. +Listener API is meant to be called across all DAGs and all operators. You can't listen to events generated by specific DAGs. For that behavior, try methods like ``on_success_callback`` and ``pre_execute``. These provide callbacks for particular DAG authors or operator creators. The logs and ``print()`` calls will be handled as part of the listeners. |experimental| From f8593503cbe252c2f4dc5ff48a3f292c9e13baad Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 5 Jul 2023 13:55:26 +0800 Subject: [PATCH 050/533] Add default_deferrable config (#31712) --- airflow/config_templates/config.yml | 7 +++++ airflow/config_templates/default_airflow.cfg | 3 ++ airflow/operators/trigger_dagrun.py | 6 ++-- .../providers/amazon/aws/operators/athena.py | 3 +- .../providers/amazon/aws/operators/batch.py | 5 ++-- airflow/providers/amazon/aws/operators/ecs.py | 19 ++++-------- airflow/providers/amazon/aws/operators/eks.py | 11 ++++--- airflow/providers/amazon/aws/operators/emr.py | 9 +++--- .../providers/amazon/aws/operators/glue.py | 3 +- .../amazon/aws/operators/glue_crawler.py | 3 +- airflow/providers/amazon/aws/operators/rds.py | 5 ++-- .../amazon/aws/operators/redshift_cluster.py | 12 ++++---- .../amazon/aws/operators/sagemaker.py | 11 +++---- airflow/providers/amazon/aws/sensors/batch.py | 3 +- airflow/providers/amazon/aws/sensors/ec2.py | 3 +- airflow/providers/amazon/aws/sensors/emr.py | 7 +++-- airflow/providers/amazon/aws/sensors/s3.py | 7 +++-- .../providers/apache/livy/operators/livy.py | 4 +-- .../cncf/kubernetes/operators/pod.py | 3 +- .../databricks/operators/databricks.py | 5 ++-- airflow/providers/dbt/cloud/operators/dbt.py | 3 +- airflow/providers/dbt/cloud/sensors/dbt.py | 3 +- .../google/cloud/operators/bigquery.py | 12 ++++---- .../google/cloud/operators/bigquery_dts.py | 3 +- .../google/cloud/operators/cloud_build.py | 3 +- .../google/cloud/operators/cloud_composer.py | 7 +++-- .../google/cloud/operators/cloud_sql.py | 3 +- .../google/cloud/operators/dataflow.py | 6 ++-- .../google/cloud/operators/dataproc.py | 17 ++++++----- .../cloud/operators/kubernetes_engine.py | 6 ++-- .../google/cloud/operators/mlengine.py | 8 ++--- .../google/cloud/sensors/bigquery.py | 5 ++-- airflow/providers/google/cloud/sensors/gcs.py | 12 ++++---- .../providers/google/cloud/sensors/pubsub.py | 4 +-- .../google/cloud/transfers/bigquery_to_gcs.py | 3 +- .../google/cloud/transfers/gcs_to_bigquery.py | 5 ++-- .../microsoft/azure/operators/data_factory.py | 3 +- .../microsoft/azure/sensors/data_factory.py | 3 +- .../providers/microsoft/azure/sensors/wasb.py | 5 ++-- .../authoring-and-scheduling/deferring.rst | 30 +++++++++++++++++++ tests/models/test_baseoperator.py | 1 - 41 files changed, 160 insertions(+), 111 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 1ad8705a7047..d588a7f26e3b 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1305,6 +1305,13 @@ operators: type: string example: ~ default: "airflow" + default_deferrable: + description: | + The default value of attribute "deferrable" in operators and sensors. + version_added: ~ + type: boolean + example: ~ + default: "false" default_cpus: description: ~ version_added: ~ diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 0cc99a3f4e5f..ae6bdec08524 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -703,6 +703,9 @@ password = # The default owner assigned to each new operator, unless # provided explicitly or passed via ``default_args`` default_owner = airflow + +# The default value of attribute "deferrable" in operators and sensors. +default_deferrable = false default_cpus = 1 default_ram = 512 default_disk = 512 diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py index 0165bb470ed9..548ef9189418 100644 --- a/airflow/operators/trigger_dagrun.py +++ b/airflow/operators/trigger_dagrun.py @@ -25,6 +25,7 @@ from sqlalchemy.orm.exc import NoResultFound from airflow.api.common.trigger_dag import trigger_dag +from airflow.configuration import conf from airflow.exceptions import AirflowException, DagNotFound, DagRunAlreadyExists from airflow.models.baseoperator import BaseOperator, BaseOperatorLink from airflow.models.dag import DagModel @@ -113,7 +114,7 @@ def __init__( poke_interval: int = 60, allowed_states: list | None = None, failed_states: list | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -135,7 +136,6 @@ def __init__( self.execution_date = execution_date def execute(self, context: Context): - if isinstance(self.execution_date, datetime.datetime): parsed_execution_date = self.execution_date elif isinstance(self.execution_date, str): @@ -187,7 +187,6 @@ def execute(self, context: Context): ti.xcom_push(key=XCOM_RUN_ID, value=dag_run.run_id) if self.wait_for_completion: - # Kick off the deferral process if self._defer: self.defer( @@ -219,7 +218,6 @@ def execute(self, context: Context): @provide_session def execute_complete(self, context: Context, session: Session, event: tuple[str, dict[str, Any]]): - # This execution date is parsed from the return trigger event provided_execution_date = event[1]["execution_dates"][0] try: diff --git a/airflow/providers/amazon/aws/operators/athena.py b/airflow/providers/amazon/aws/operators/athena.py index 0467fe6d11ae..6dd1432ea480 100644 --- a/airflow/providers/amazon/aws/operators/athena.py +++ b/airflow/providers/amazon/aws/operators/athena.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Any, Sequence from airflow import AirflowException +from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.athena import AthenaHook from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger @@ -74,7 +75,7 @@ def __init__( sleep_time: int = 30, max_polling_attempts: int | None = None, log_query: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index b9b3322c4910..9dd954d05cd8 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -29,6 +29,7 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook @@ -154,7 +155,7 @@ def __init__( region_name: str | None = None, tags: dict | None = None, wait_for_completion: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: int = 30, awslogs_enabled: bool = False, awslogs_fetch_interval: timedelta = timedelta(seconds=30), @@ -437,7 +438,7 @@ def __init__( max_retries: int | None = None, aws_conn_id: str | None = None, region_name: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if "status_retries" in kwargs: diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 91533cfa6211..e5833bf4c3d5 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -26,20 +26,14 @@ import boto3 +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator, XCom from airflow.providers.amazon.aws.exceptions import EcsOperatorError, EcsTaskFailToStart from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook -from airflow.providers.amazon.aws.hooks.ecs import ( - EcsClusterStates, - EcsHook, - should_retry_eni, -) +from airflow.providers.amazon.aws.hooks.ecs import EcsClusterStates, EcsHook, should_retry_eni from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook -from airflow.providers.amazon.aws.triggers.ecs import ( - ClusterWaiterTrigger, - TaskDoneTrigger, -) +from airflow.providers.amazon.aws.triggers.ecs import ClusterWaiterTrigger, TaskDoneTrigger from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher from airflow.utils.helpers import prune_dict from airflow.utils.session import provide_session @@ -118,7 +112,7 @@ def __init__( wait_for_completion: bool = True, waiter_delay: int = 15, waiter_max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -201,7 +195,7 @@ def __init__( wait_for_completion: bool = True, waiter_delay: int = 15, waiter_max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -482,7 +476,7 @@ def __init__( wait_for_completion: bool = True, waiter_delay: int = 6, waiter_max_attempts: int = 100, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -727,7 +721,6 @@ def _check_success_task(self) -> None: raise AirflowException(response) for task in response["tasks"]: - if task.get("stopCode", "") == "TaskFailedToStart": # Reset task arn here otherwise the retry run will not start # a new task but keep polling the old dead one diff --git a/airflow/providers/amazon/aws/operators/eks.py b/airflow/providers/amazon/aws/operators/eks.py index bea422398726..56e9269f88d7 100644 --- a/airflow/providers/amazon/aws/operators/eks.py +++ b/airflow/providers/amazon/aws/operators/eks.py @@ -25,6 +25,7 @@ from botocore.exceptions import ClientError, WaiterError +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.eks import EksHook @@ -83,7 +84,6 @@ def _create_compute( log = logging.getLogger(__name__) eks_hook = EksHook(aws_conn_id=aws_conn_id, region_name=region) if compute == "nodegroup" and nodegroup_name: - # this is to satisfy mypy subnets = subnets or [] create_nodegroup_kwargs = create_nodegroup_kwargs or {} @@ -107,7 +107,6 @@ def _create_compute( status_args=["nodegroup.status"], ) elif compute == "fargate" and fargate_profile_name: - # this is to satisfy mypy create_fargate_profile_kwargs = create_fargate_profile_kwargs or {} fargate_selectors = fargate_selectors or [] @@ -366,7 +365,7 @@ def __init__( region: str | None = None, waiter_delay: int = 30, waiter_max_attempts: int = 80, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: self.nodegroup_subnets = nodegroup_subnets @@ -489,7 +488,7 @@ def __init__( region: str | None = None, waiter_delay: int = 10, waiter_max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: self.cluster_name = cluster_name @@ -690,7 +689,7 @@ def __init__( region: str | None = None, waiter_delay: int = 30, waiter_max_attempts: int = 40, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: self.cluster_name = cluster_name @@ -780,7 +779,7 @@ def __init__( region: str | None = None, waiter_delay: int = 30, waiter_max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/operators/emr.py b/airflow/providers/amazon/aws/operators/emr.py index f9eacdb79d19..8330a586e442 100644 --- a/airflow/providers/amazon/aws/operators/emr.py +++ b/airflow/providers/amazon/aws/operators/emr.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Any, Sequence from uuid import uuid4 +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook @@ -96,7 +97,7 @@ def __init__( waiter_delay: int | None = None, waiter_max_attempts: int | None = None, execution_role_arn: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if not exactly_one(job_flow_id is None, job_flow_name is None): @@ -510,7 +511,7 @@ def __init__( max_tries: int | None = None, tags: dict | None = None, max_polling_attempts: int | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -695,7 +696,7 @@ def __init__( waiter_delay: int | None | ArgNotSet = NOTSET, waiter_countdown: int | None = None, waiter_check_interval_seconds: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ): if waiter_max_attempts is NOTSET: @@ -900,7 +901,7 @@ def __init__( aws_conn_id: str = "aws_default", waiter_delay: int = 60, waiter_max_attempts: int = 20, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/operators/glue.py b/airflow/providers/amazon/aws/operators/glue.py index 060ac358a40c..265d057de51a 100644 --- a/airflow/providers/amazon/aws/operators/glue.py +++ b/airflow/providers/amazon/aws/operators/glue.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Sequence from airflow import AirflowException +from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.glue import GlueJobHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -98,7 +99,7 @@ def __init__( create_job_kwargs: dict | None = None, run_job_kwargs: dict | None = None, wait_for_completion: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), verbose: bool = False, update_config: bool = False, job_poll_interval: int | float = 6, diff --git a/airflow/providers/amazon/aws/operators/glue_crawler.py b/airflow/providers/amazon/aws/operators/glue_crawler.py index a7efb9f5c0cd..71e2607039c3 100644 --- a/airflow/providers/amazon/aws/operators/glue_crawler.py +++ b/airflow/providers/amazon/aws/operators/glue_crawler.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Sequence from airflow import AirflowException +from airflow.configuration import conf from airflow.providers.amazon.aws.triggers.glue_crawler import GlueCrawlerCompleteTrigger if TYPE_CHECKING: @@ -61,7 +62,7 @@ def __init__( region_name: str | None = None, poll_interval: int = 5, wait_for_completion: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/operators/rds.py b/airflow/providers/amazon/aws/operators/rds.py index 9aef6701660e..c58961db2e8f 100644 --- a/airflow/providers/amazon/aws/operators/rds.py +++ b/airflow/providers/amazon/aws/operators/rds.py @@ -24,6 +24,7 @@ from mypy_boto3_rds.type_defs import TagTypeDef +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.rds import RdsHook @@ -554,7 +555,7 @@ def __init__( rds_kwargs: dict | None = None, aws_conn_id: str = "aws_default", wait_for_completion: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), waiter_delay: int = 30, waiter_max_attempts: int = 60, **kwargs, @@ -645,7 +646,7 @@ def __init__( rds_kwargs: dict | None = None, aws_conn_id: str = "aws_default", wait_for_completion: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), waiter_delay: int = 30, waiter_max_attempts: int = 60, **kwargs, diff --git a/airflow/providers/amazon/aws/operators/redshift_cluster.py b/airflow/providers/amazon/aws/operators/redshift_cluster.py index 905c34ff3ac5..cde4a32226e9 100644 --- a/airflow/providers/amazon/aws/operators/redshift_cluster.py +++ b/airflow/providers/amazon/aws/operators/redshift_cluster.py @@ -20,6 +20,7 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftHook @@ -148,7 +149,7 @@ def __init__( wait_for_completion: bool = False, max_attempt: int = 5, poll_interval: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -327,7 +328,7 @@ def __init__( poll_interval: int = 15, max_attempt: int = 20, aws_conn_id: str = "aws_default", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -470,7 +471,7 @@ def __init__( cluster_identifier: str, aws_conn_id: str = "aws_default", wait_for_completion: bool = False, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: int = 10, max_attempts: int = 10, **kwargs, @@ -560,7 +561,7 @@ def __init__( *, cluster_identifier: str, aws_conn_id: str = "aws_default", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: int = 10, max_attempts: int = 15, **kwargs, @@ -647,7 +648,7 @@ def __init__( wait_for_completion: bool = True, aws_conn_id: str = "aws_default", poll_interval: int = 30, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), max_attempts: int = 30, **kwargs, ): @@ -668,7 +669,6 @@ def __init__( self.max_attempts = max_attempts def execute(self, context: Context): - while self._attempts >= 1: try: self.redshift_hook.delete_cluster( diff --git a/airflow/providers/amazon/aws/operators/sagemaker.py b/airflow/providers/amazon/aws/operators/sagemaker.py index 4dac7df00790..ac1b7a73d2de 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker.py +++ b/airflow/providers/amazon/aws/operators/sagemaker.py @@ -25,6 +25,7 @@ from botocore.exceptions import ClientError +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook @@ -198,7 +199,7 @@ def __init__( max_attempts: int | None = None, max_ingestion_time: int | None = None, action_if_job_exists: str = "timestamp", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config=config, aws_conn_id=aws_conn_id, **kwargs) @@ -392,7 +393,7 @@ def __init__( check_interval: int = CHECK_INTERVAL_SECOND, max_ingestion_time: int | None = None, operation: str = "create", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config=config, aws_conn_id=aws_conn_id, **kwargs) @@ -551,7 +552,7 @@ def __init__( max_ingestion_time: int | None = None, check_if_job_exists: bool = True, action_if_job_exists: str = "timestamp", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config=config, aws_conn_id=aws_conn_id, **kwargs) @@ -700,7 +701,7 @@ def __init__( wait_for_completion: bool = True, check_interval: int = CHECK_INTERVAL_SECOND, max_ingestion_time: int | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config=config, aws_conn_id=aws_conn_id, **kwargs) @@ -862,7 +863,7 @@ def __init__( max_ingestion_time: int | None = None, check_if_job_exists: bool = True, action_if_job_exists: str = "timestamp", - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config=config, aws_conn_id=aws_conn_id, **kwargs) diff --git a/airflow/providers/amazon/aws/sensors/batch.py b/airflow/providers/amazon/aws/sensors/batch.py index 2033d1e86bee..32da5b4cf252 100644 --- a/airflow/providers/amazon/aws/sensors/batch.py +++ b/airflow/providers/amazon/aws/sensors/batch.py @@ -22,6 +22,7 @@ from deprecated import deprecated +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook from airflow.providers.amazon.aws.triggers.batch import BatchSensorTrigger @@ -58,7 +59,7 @@ def __init__( job_id: str, aws_conn_id: str = "aws_default", region_name: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poke_interval: float = 5, max_retries: int = 5, **kwargs, diff --git a/airflow/providers/amazon/aws/sensors/ec2.py b/airflow/providers/amazon/aws/sensors/ec2.py index c5d77610319e..2b7b63f7e6c7 100644 --- a/airflow/providers/amazon/aws/sensors/ec2.py +++ b/airflow/providers/amazon/aws/sensors/ec2.py @@ -20,6 +20,7 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.ec2 import EC2Hook from airflow.providers.amazon.aws.triggers.ec2 import EC2StateSensorTrigger @@ -55,7 +56,7 @@ def __init__( instance_id: str, aws_conn_id: str = "aws_default", region_name: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if target_state not in self.valid_states: diff --git a/airflow/providers/amazon/aws/sensors/emr.py b/airflow/providers/amazon/aws/sensors/emr.py index 2f44caab0619..9953dfa78260 100644 --- a/airflow/providers/amazon/aws/sensors/emr.py +++ b/airflow/providers/amazon/aws/sensors/emr.py @@ -23,6 +23,7 @@ from deprecated import deprecated +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook from airflow.providers.amazon.aws.links.emr import EmrClusterLink, EmrLogsLink, get_log_uri @@ -271,7 +272,7 @@ def __init__( max_retries: int | None = None, aws_conn_id: str = "aws_default", poll_interval: int = 10, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -425,7 +426,7 @@ def __init__( target_states: Iterable[str] | None = None, failed_states: Iterable[str] | None = None, max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -549,7 +550,7 @@ def __init__( target_states: Iterable[str] | None = None, failed_states: Iterable[str] | None = None, max_attempts: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) diff --git a/airflow/providers/amazon/aws/sensors/s3.py b/airflow/providers/amazon/aws/sensors/s3.py index 7192585afd58..4d15cdc2125d 100644 --- a/airflow/providers/amazon/aws/sensors/s3.py +++ b/airflow/providers/amazon/aws/sensors/s3.py @@ -26,6 +26,8 @@ from deprecated import deprecated +from airflow.configuration import conf + if TYPE_CHECKING: from airflow.utils.context import Context @@ -87,7 +89,7 @@ def __init__( check_fn: Callable[..., bool] | None = None, aws_conn_id: str = "aws_default", verify: str | bool | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -238,10 +240,9 @@ def __init__( min_objects: int = 1, previous_objects: set[str] | None = None, allow_delete: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: - super().__init__(**kwargs) self.bucket_name = bucket_name diff --git a/airflow/providers/apache/livy/operators/livy.py b/airflow/providers/apache/livy/operators/livy.py index fa4f35734371..f5e519315f4b 100644 --- a/airflow/providers/apache/livy/operators/livy.py +++ b/airflow/providers/apache/livy/operators/livy.py @@ -20,6 +20,7 @@ from time import sleep from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.apache.livy.hooks.livy import BatchState, LivyHook @@ -88,10 +89,9 @@ def __init__( extra_options: dict[str, Any] | None = None, extra_headers: dict[str, Any] | None = None, retry_args: dict[str, Any] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: - super().__init__(**kwargs) self.spark_params = { diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 696611c6c2e9..e3ac7708e74c 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -33,6 +33,7 @@ from slugify import slugify from urllib3.exceptions import HTTPError +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException from airflow.kubernetes import pod_generator from airflow.kubernetes.pod_generator import PodGenerator @@ -305,7 +306,7 @@ def __init__( configmaps: list[str] | None = None, skip_on_exit_code: int | Container[int] | None = None, base_container_name: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 2, log_pod_spec_on_failure: bool = True, on_finish_action: str = "delete_pod", diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index ab93d8f49b7c..fb27f0c01a11 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -24,6 +24,7 @@ from logging import Logger from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator, BaseOperatorLink, XCom from airflow.providers.databricks.hooks.databricks import DatabricksHook, RunState @@ -315,7 +316,7 @@ def __init__( access_control_list: list[dict[str, str]] | None = None, wait_for_termination: bool = True, git_source: dict[str, str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: """Creates a new ``DatabricksSubmitRunOperator``.""" @@ -605,7 +606,7 @@ def __init__( databricks_retry_args: dict[Any, Any] | None = None, do_xcom_push: bool = True, wait_for_termination: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: """Creates a new ``DatabricksRunNowOperator``.""" diff --git a/airflow/providers/dbt/cloud/operators/dbt.py b/airflow/providers/dbt/cloud/operators/dbt.py index f316c47f3db4..c977539afb73 100644 --- a/airflow/providers/dbt/cloud/operators/dbt.py +++ b/airflow/providers/dbt/cloud/operators/dbt.py @@ -22,6 +22,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import BaseOperator, BaseOperatorLink, XCom from airflow.providers.dbt.cloud.hooks.dbt import ( @@ -99,7 +100,7 @@ def __init__( timeout: int = 60 * 60 * 24 * 7, check_interval: int = 60, additional_run_config: dict[str, Any] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/dbt/cloud/sensors/dbt.py b/airflow/providers/dbt/cloud/sensors/dbt.py index 5838f6d6247d..3b5ae549a35a 100644 --- a/airflow/providers/dbt/cloud/sensors/dbt.py +++ b/airflow/providers/dbt/cloud/sensors/dbt.py @@ -20,6 +20,7 @@ import warnings from typing import TYPE_CHECKING, Any +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook, DbtCloudJobRunException, DbtCloudJobRunStatus from airflow.providers.dbt.cloud.triggers.dbt import DbtCloudRunJobTrigger @@ -50,7 +51,7 @@ def __init__( dbt_cloud_conn_id: str = DbtCloudHook.default_conn_name, run_id: int, account_id: int | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: if deferrable: diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 98929b6e6dec..970e7813ed9c 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -29,6 +29,7 @@ from google.cloud.bigquery import DEFAULT_RETRY, CopyJob, ExtractJob, LoadJob, QueryJob from google.cloud.bigquery.table import RowIterator +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException from airflow.models import BaseOperator, BaseOperatorLink from airflow.models.xcom import XCom @@ -200,7 +201,7 @@ def __init__( location: str | None = None, impersonation_chain: str | Sequence[str] | None = None, labels: dict | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 4.0, **kwargs, ) -> None: @@ -320,7 +321,7 @@ def __init__( location: str | None = None, impersonation_chain: str | Sequence[str] | None = None, labels: dict | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 4.0, **kwargs, ) -> None: @@ -460,7 +461,7 @@ def __init__( location: str | None = None, impersonation_chain: str | Sequence[str] | None = None, labels: dict | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 4.0, **kwargs, ) -> None: @@ -854,7 +855,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", location: str | None = None, impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 4.0, as_dict: bool = False, use_legacy_sql: bool = True, @@ -1876,7 +1877,6 @@ def __init__( exists_ok: bool | None = None, **kwargs, ) -> None: - self.dataset_id = dataset_id self.project_id = project_id self.location = location @@ -2623,7 +2623,7 @@ def __init__( cancel_on_kill: bool = True, result_retry: Retry = DEFAULT_RETRY, result_timeout: float | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: float = 4.0, **kwargs, ) -> None: diff --git a/airflow/providers/google/cloud/operators/bigquery_dts.py b/airflow/providers/google/cloud/operators/bigquery_dts.py index d9e013afa68c..e10618bc39bd 100644 --- a/airflow/providers/google/cloud/operators/bigquery_dts.py +++ b/airflow/providers/google/cloud/operators/bigquery_dts.py @@ -32,6 +32,7 @@ ) from airflow import AirflowException +from airflow.configuration import conf from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook, get_object_id from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator @@ -279,7 +280,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id="google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/google/cloud/operators/cloud_build.py b/airflow/providers/google/cloud/operators/cloud_build.py index 4242f561c100..14fed55a3af6 100644 --- a/airflow/providers/google/cloud/operators/cloud_build.py +++ b/airflow/providers/google/cloud/operators/cloud_build.py @@ -28,6 +28,7 @@ from google.api_core.retry import Retry from google.cloud.devtools.cloudbuild_v1.types import Build, BuildTrigger, RepoSource +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildHook from airflow.providers.google.cloud.links.cloud_build import ( @@ -176,7 +177,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, poll_interval: float = 4.0, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), location: str = "global", **kwargs, ) -> None: diff --git a/airflow/providers/google/cloud/operators/cloud_composer.py b/airflow/providers/google/cloud/operators/cloud_composer.py index d04a1606fc84..c9b52d855915 100644 --- a/airflow/providers/google/cloud/operators/cloud_composer.py +++ b/airflow/providers/google/cloud/operators/cloud_composer.py @@ -27,6 +27,7 @@ from google.protobuf.field_mask_pb2 import FieldMask from airflow import AirflowException +from airflow.configuration import conf from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerHook from airflow.providers.google.cloud.links.base import BaseGoogleLink from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator @@ -135,7 +136,7 @@ def __init__( retry: Retry | _MethodDefault = DEFAULT, timeout: float | None = None, metadata: Sequence[tuple[str, str]] = (), - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), pooling_period_seconds: int = 30, **kwargs, ) -> None: @@ -264,7 +265,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), pooling_period_seconds: int = 30, **kwargs, ) -> None: @@ -509,7 +510,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), pooling_period_seconds: int = 30, **kwargs, ) -> None: diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 5c77cbd86c94..b1144663c706 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -22,6 +22,7 @@ from googleapiclient.errors import HttpError +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.models import Connection @@ -955,7 +956,7 @@ def __init__( api_version: str = "v1beta4", validate_body: bool = True, impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poke_interval: int = 10, **kwargs, ) -> None: diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py index 5ae1115a34ae..a5e958821485 100644 --- a/airflow/providers/google/cloud/operators/dataflow.py +++ b/airflow/providers/google/cloud/operators/dataflow.py @@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Any, Sequence from airflow import AirflowException +from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType from airflow.providers.google.cloud.hooks.dataflow import ( @@ -419,7 +420,6 @@ def set_current_job_id(job_id): variables=pipeline_options, ) while is_running and self.check_if_running == CheckJobRunning.WaitForRun: - is_running = self.dataflow_hook.is_job_dataflow_running( name=self.job_name, variables=pipeline_options, @@ -611,7 +611,7 @@ def __init__( cancel_timeout: int | None = 10 * 60, wait_until_finished: bool | None = None, append_job_name: bool = True, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -801,7 +801,7 @@ def __init__( cancel_timeout: int | None = 10 * 60, wait_until_finished: bool | None = None, impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), append_job_name: bool = True, *args, **kwargs, diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index db7d785347a8..d14a495bc0cb 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -36,6 +36,7 @@ from google.protobuf.duration_pb2 import Duration from google.protobuf.field_mask_pb2 import FieldMask +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.dataproc import DataprocHook, DataProcJobBuilder from airflow.providers.google.cloud.hooks.gcs import GCSHook @@ -484,7 +485,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ) -> None: @@ -849,7 +850,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ): @@ -981,7 +982,7 @@ def __init__( job_error_states: set[str] | None = None, impersonation_chain: str | Sequence[str] | None = None, asynchronous: bool = False, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ) -> None: @@ -1731,7 +1732,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ) -> None: @@ -1859,7 +1860,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ) -> None: @@ -1979,7 +1980,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, asynchronous: bool = False, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, cancel_on_kill: bool = True, wait_timeout: int | None = None, @@ -2139,7 +2140,7 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 10, **kwargs, ): @@ -2270,7 +2271,7 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, result_retry: Retry | _MethodDefault = DEFAULT, asynchronous: bool = False, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), polling_interval_seconds: int = 5, **kwargs, ): diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index bf14828d8702..086a7d99b7e6 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -26,6 +26,7 @@ from google.cloud.container_v1.types import Cluster from kubernetes.client.models import V1Pod +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction @@ -34,6 +35,7 @@ except ImportError: # preserve backward compatibility for older versions of cncf.kubernetes provider from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator + from airflow.providers.google.cloud.hooks.kubernetes_engine import GKEHook, GKEPodHook from airflow.providers.google.cloud.links.kubernetes_engine import ( KubernetesEngineClusterLink, @@ -108,7 +110,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", api_version: str = "v2", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval: int = 10, **kwargs, ) -> None: @@ -255,7 +257,7 @@ def __init__( api_version: str = "v2", impersonation_chain: str | Sequence[str] | None = None, poll_interval: int = 10, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/airflow/providers/google/cloud/operators/mlengine.py index b776d20dff42..20d9d1988623 100644 --- a/airflow/providers/google/cloud/operators/mlengine.py +++ b/airflow/providers/google/cloud/operators/mlengine.py @@ -27,6 +27,7 @@ from googleapiclient.errors import HttpError +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.mlengine import MLEngineHook from airflow.providers.google.cloud.links.mlengine import ( @@ -722,7 +723,6 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, **kwargs, ) -> None: - super().__init__(**kwargs) self._project_id = project_id self._model_name = model_name @@ -804,7 +804,6 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, **kwargs, ) -> None: - super().__init__(**kwargs) self._project_id = project_id self._model_name = model_name @@ -883,7 +882,6 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, **kwargs, ) -> None: - super().__init__(**kwargs) self._project_id = project_id self._model_name = model_name @@ -961,7 +959,6 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, **kwargs, ) -> None: - super().__init__(**kwargs) self._project_id = project_id self._model_name = model_name @@ -1098,7 +1095,7 @@ def __init__( labels: dict[str, str] | None = None, impersonation_chain: str | Sequence[str] | None = None, hyperparameters: dict | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs, ) -> None: @@ -1370,7 +1367,6 @@ def __init__( raise AirflowException("Google Cloud project id is required.") def execute(self, context: Context): - hook = MLEngineHook( gcp_conn_id=self._gcp_conn_id, impersonation_chain=self._impersonation_chain, diff --git a/airflow/providers/google/cloud/sensors/bigquery.py b/airflow/providers/google/cloud/sensors/bigquery.py index db109bf2c1e8..e4e1819ef134 100644 --- a/airflow/providers/google/cloud/sensors/bigquery.py +++ b/airflow/providers/google/cloud/sensors/bigquery.py @@ -22,6 +22,7 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.cloud.triggers.bigquery import ( @@ -71,7 +72,7 @@ def __init__( table_id: str, gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: if deferrable and "poke_interval" not in kwargs: @@ -184,7 +185,7 @@ def __init__( partition_id: str, gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: if deferrable and "poke_interval" not in kwargs: diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/airflow/providers/google/cloud/sensors/gcs.py index 08fd37022dfe..7048789601e4 100644 --- a/airflow/providers/google/cloud/sensors/gcs.py +++ b/airflow/providers/google/cloud/sensors/gcs.py @@ -27,6 +27,7 @@ from google.api_core.retry import Retry from google.cloud.storage.retry import DEFAULT_RETRY +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.gcs import GCSHook from airflow.providers.google.cloud.triggers.gcs import ( @@ -76,10 +77,9 @@ def __init__( google_cloud_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, retry: Retry = DEFAULT_RETRY, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: - super().__init__(**kwargs) self.bucket = bucket self.object = object @@ -208,10 +208,9 @@ def __init__( ts_func: Callable = ts_function, google_cloud_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: - super().__init__(**kwargs) self.bucket = bucket self.object = object @@ -298,7 +297,7 @@ def __init__( prefix: str, google_cloud_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -412,10 +411,9 @@ def __init__( allow_delete: bool = True, google_cloud_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: - super().__init__(**kwargs) self.bucket = bucket diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/airflow/providers/google/cloud/sensors/pubsub.py index 2e03b3669d3d..db9f39b19b0b 100644 --- a/airflow/providers/google/cloud/sensors/pubsub.py +++ b/airflow/providers/google/cloud/sensors/pubsub.py @@ -23,6 +23,7 @@ from google.cloud.pubsub_v1.types import ReceivedMessage +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.google.cloud.hooks.pubsub import PubSubHook from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger @@ -103,10 +104,9 @@ def __init__( messages_callback: Callable[[list[ReceivedMessage], Context], Any] | None = None, impersonation_chain: str | Sequence[str] | None = None, poke_interval: float = 10.0, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: - super().__init__(**kwargs) self.gcp_conn_id = gcp_conn_id self.project_id = project_id diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py index 7ec62db9bfbd..8836e3ee354e 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py @@ -25,6 +25,7 @@ from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob from airflow import AirflowException +from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink @@ -114,7 +115,7 @@ def __init__( job_id: str | None = None, force_rerun: bool = False, reattach_states: set[str] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 88b6d0932370..da462eae74df 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -36,6 +36,7 @@ from google.cloud.bigquery.table import EncryptionConfiguration, Table, TableReference from airflow import AirflowException +from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob from airflow.providers.google.cloud.hooks.gcs import GCSHook @@ -218,7 +219,7 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, labels=None, description=None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), result_retry: Retry = DEFAULT_RETRY, result_timeout: float | None = None, cancel_on_kill: bool = True, @@ -228,7 +229,6 @@ def __init__( project_id: str | None = None, **kwargs, ) -> None: - super().__init__(**kwargs) self.hook: BigQueryHook | None = None self.configuration: dict[str, Any] = {} @@ -718,7 +718,6 @@ def _validate_src_fmt_configs( def _cleanse_time_partitioning( self, destination_dataset_table: str | None, time_partitioning_in: dict | None ) -> dict: # if it is a partitioned table ($ is in the table name) add partition load option - if time_partitioning_in is None: time_partitioning_in = {} diff --git a/airflow/providers/microsoft/azure/operators/data_factory.py b/airflow/providers/microsoft/azure/operators/data_factory.py index 8906c02ae12c..a2b2c528bf9b 100644 --- a/airflow/providers/microsoft/azure/operators/data_factory.py +++ b/airflow/providers/microsoft/azure/operators/data_factory.py @@ -20,6 +20,7 @@ import warnings from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.models import BaseOperator, BaseOperatorLink, XCom @@ -140,7 +141,7 @@ def __init__( parameters: dict[str, Any] | None = None, timeout: int = 60 * 60 * 24 * 7, check_interval: int = 60, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/airflow/providers/microsoft/azure/sensors/data_factory.py index 70ae1f69dc3f..b4ebedce698a 100644 --- a/airflow/providers/microsoft/azure/sensors/data_factory.py +++ b/airflow/providers/microsoft/azure/sensors/data_factory.py @@ -20,6 +20,7 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.microsoft.azure.hooks.data_factory import ( AzureDataFactoryHook, @@ -60,7 +61,7 @@ def __init__( azure_data_factory_conn_id: str = AzureDataFactoryHook.default_conn_name, resource_group_name: str | None = None, factory_name: str | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py index 4e2ec2d502c1..0c227f2ea37b 100644 --- a/airflow/providers/microsoft/azure/sensors/wasb.py +++ b/airflow/providers/microsoft/azure/sensors/wasb.py @@ -21,6 +21,7 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger @@ -53,7 +54,7 @@ def __init__( wasb_conn_id: str = "wasb_default", check_options: dict | None = None, public_read: bool = False, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -151,7 +152,7 @@ def __init__( prefix: str, wasb_conn_id: str = "wasb_default", check_options: dict | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/docs/apache-airflow/authoring-and-scheduling/deferring.rst b/docs/apache-airflow/authoring-and-scheduling/deferring.rst index b1e6c6be983d..81526737e8d1 100644 --- a/docs/apache-airflow/authoring-and-scheduling/deferring.rst +++ b/docs/apache-airflow/authoring-and-scheduling/deferring.rst @@ -56,6 +56,36 @@ Writing a deferrable operator takes a bit more work. There are some main points * You can defer multiple times, and you can defer before/after your Operator does significant work, or only defer if certain conditions are met (e.g. a system does not have an immediate answer). Deferral is entirely under your control. * Any Operator can defer; no special marking on its class is needed, and it's not limited to Sensors. * In order for any changes to a Trigger to be reflected, the *triggerer* needs to be restarted whenever the Trigger is modified. +* If you want add an operator or sensor that supports both deferrable and non-deferrable modes. It's suggested to add ``deferable: bool = conf.getboolean("operators", "default_deferrable", fallback=False)`` to the ``__init__`` method of the operator and use it to decide whether to run the operator in deferrable mode. You'll be able to configure the default value of ``deferrable`` of all the operators and sensors that supports switch between deferrable and non-deferrable mode through ``default_deferrable`` in the ``operator`` section. Here's an example of a sensor that supports both modes.:: + + import time + from datetime import timedelta + + from airflow.sensors.base import BaseSensorOperator + from airflow.triggers.temporal import TimeDeltaTrigger + + + class WaitOneHourSensor(BaseSensorOperator): + def __init__( + self, + deferable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + **kwargs + ): + super().__init__(**kwargs) + self.deferrable = deferable + + def execute(self, context): + if deferrable: + self.defer( + trigger=TimeDeltaTrigger(timedelta(hours=1)), + method_name="execute_complete" + ) + else: + time.sleep(3600) + + def execute_complete(self, context, event=None): + # We have no more work to do here. Mark as complete. + return Triggering Deferral diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index 2b118176721d..c49e0bb03479 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -919,7 +919,6 @@ def test_render_template_fields_logging( caplog, monkeypatch, task, context, expected_exception, expected_rendering, expected_log, not_expected_log ): """Verify if operator attributes are correctly templated.""" - # Trigger templating and verify results def _do_render(): task.render_template_fields(context=context) From 1240dcc167c4b47331db81deff61fc688df118c2 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Tue, 4 Jul 2023 23:10:26 -0700 Subject: [PATCH 051/533] D205 Support - Providers: GRPC to Oracle (inclusive) (#32357) --- .../hashicorp/_internal_client/vault_client.py | 15 +++++++-------- airflow/providers/hashicorp/secrets/vault.py | 5 +++-- airflow/providers/http/sensors/http.py | 3 +-- airflow/providers/imap/hooks/imap.py | 4 ++-- airflow/providers/jenkins/hooks/jenkins.py | 6 ++++-- .../jenkins/operators/jenkins_job_trigger.py | 4 ++-- airflow/providers/microsoft/azure/hooks/adx.py | 5 +++-- .../microsoft/azure/hooks/base_azure.py | 6 ++++-- .../providers/microsoft/azure/hooks/cosmos.py | 5 +---- .../providers/microsoft/azure/hooks/fileshare.py | 2 ++ airflow/providers/microsoft/azure/hooks/wasb.py | 2 ++ .../microsoft/azure/log/wasb_task_handler.py | 13 ++++++------- .../providers/microsoft/azure/operators/asb.py | 10 ++-------- .../microsoft/azure/operators/data_factory.py | 4 ++-- .../microsoft/azure/secrets/key_vault.py | 1 + .../microsoft/azure/sensors/data_factory.py | 4 ++-- .../providers/microsoft/azure/sensors/wasb.py | 8 ++++---- .../microsoft/azure/transfers/local_to_adls.py | 1 + .../azure/transfers/oracle_to_azure_data_lake.py | 4 +--- .../providers/microsoft/azure/triggers/wasb.py | 8 +++++--- airflow/providers/microsoft/azure/utils.py | 2 ++ airflow/providers/microsoft/mssql/hooks/mssql.py | 16 +++++++++------- airflow/providers/microsoft/psrp/hooks/psrp.py | 5 +++-- airflow/providers/mongo/hooks/mongo.py | 1 + .../providers/mysql/transfers/presto_to_mysql.py | 7 ++++--- .../providers/mysql/transfers/trino_to_mysql.py | 7 ++++--- airflow/providers/neo4j/hooks/neo4j.py | 8 ++------ airflow/providers/openlineage/extractors/bash.py | 6 ++++-- .../providers/openlineage/extractors/python.py | 2 ++ airflow/providers/openlineage/plugins/adapter.py | 5 +---- .../providers/openlineage/plugins/listener.py | 5 +---- airflow/providers/openlineage/plugins/macros.py | 11 ++++++----- .../providers/openlineage/plugins/openlineage.py | 7 +++++-- airflow/providers/openlineage/sqlparser.py | 3 ++- airflow/providers/openlineage/utils/utils.py | 5 ++++- airflow/providers/opsgenie/hooks/opsgenie.py | 1 + airflow/providers/opsgenie/operators/opsgenie.py | 13 +++++++++---- 37 files changed, 115 insertions(+), 99 deletions(-) diff --git a/airflow/providers/hashicorp/_internal_client/vault_client.py b/airflow/providers/hashicorp/_internal_client/vault_client.py index 3ec4384829be..aea8bfb01d76 100644 --- a/airflow/providers/hashicorp/_internal_client/vault_client.py +++ b/airflow/providers/hashicorp/_internal_client/vault_client.py @@ -48,10 +48,12 @@ class _VaultClient(LoggingMixin): """ - Retrieves Authenticated client from Hashicorp Vault. This is purely internal class promoting - authentication code reuse between the Hook and the SecretBackend, it should not be used directly in - Airflow DAGs. Use VaultBackend for backend integration and Hook in case you want to communicate - with VaultHook using standard Airflow Connection definition. + Retrieves Authenticated client from Hashicorp Vault. + + This is purely internal class promoting authentication code reuse between the Hook and the + SecretBackend, it should not be used directly in Airflow DAGs. Use VaultBackend for backend + integration and Hook in case you want to communicate with VaultHook using standard Airflow + Connection definition. :param url: Base URL for the Vault instance being addressed. :param auth_type: Authentication Type for Vault. Default is ``token``. Available values are in @@ -172,12 +174,9 @@ def __init__( @property def client(self): """ - Authentication to Vault can expire. This wrapper function checks that - it is still authenticated to Vault, and invalidates the cache if this - is not the case. + Checks that it is still authenticated to Vault and invalidates the cache if this is not the case. :return: Vault Client - """ if not self._client.is_authenticated(): # Invalidate the cache: diff --git a/airflow/providers/hashicorp/secrets/vault.py b/airflow/providers/hashicorp/secrets/vault.py index 661707d9b853..dd93aeb6c848 100644 --- a/airflow/providers/hashicorp/secrets/vault.py +++ b/airflow/providers/hashicorp/secrets/vault.py @@ -209,8 +209,9 @@ def get_conn_uri(self, conn_id: str) -> str | None: def get_connection(self, conn_id: str) -> Connection | None: """ - Get connection from Vault as secret. Prioritize conn_uri if exists, - if not fall back to normal Connection creation. + Get connection from Vault as secret. + + Prioritize conn_uri if exists, if not fall back to normal Connection creation. :return: A Connection object constructed from Vault data """ diff --git a/airflow/providers/http/sensors/http.py b/airflow/providers/http/sensors/http.py index f691fd0a6922..302ea98ef2a4 100644 --- a/airflow/providers/http/sensors/http.py +++ b/airflow/providers/http/sensors/http.py @@ -29,8 +29,7 @@ class HttpSensor(BaseSensorOperator): """ - Executes a HTTP GET statement and returns False on failure caused by - 404 Not Found or `response_check` returning False. + Execute HTTP GET statement; return False on failure 404 Not Found or `response_check` returning False. HTTP Error codes other than 404 (like 403) or Connection Refused Error would raise an exception and fail the sensor itself directly (no more poking). diff --git a/airflow/providers/imap/hooks/imap.py b/airflow/providers/imap/hooks/imap.py index 7368eedfdb07..4a00e6965a64 100644 --- a/airflow/providers/imap/hooks/imap.py +++ b/airflow/providers/imap/hooks/imap.py @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. """ -This module provides everything to be able to search in mails for a specific attachment -and also to download it. +This module provides everything to search mail for a specific attachment and download it. + It uses the imaplib library that is already integrated in python 3. """ from __future__ import annotations diff --git a/airflow/providers/jenkins/hooks/jenkins.py b/airflow/providers/jenkins/hooks/jenkins.py index e9024b6a32e4..3a91fd151bcd 100644 --- a/airflow/providers/jenkins/hooks/jenkins.py +++ b/airflow/providers/jenkins/hooks/jenkins.py @@ -27,8 +27,10 @@ def _ensure_prefixes(conn_type): """ - Remove when provider min airflow version >= 2.5.0 since this is handled by - provider manager from that version. + Deprecated. + + Remove when provider min airflow version >= 2.5.0 since + this is handled by provider manager from that version. """ def dec(func): diff --git a/airflow/providers/jenkins/operators/jenkins_job_trigger.py b/airflow/providers/jenkins/operators/jenkins_job_trigger.py index 5b00007748bb..006c199f5957 100644 --- a/airflow/providers/jenkins/operators/jenkins_job_trigger.py +++ b/airflow/providers/jenkins/operators/jenkins_job_trigger.py @@ -21,7 +21,7 @@ import json import socket import time -from typing import Any, Iterable, List, Mapping, Sequence, Union +from typing import Any, Iterable, Mapping, Sequence, Union from urllib.error import HTTPError, URLError import jenkins @@ -33,7 +33,7 @@ from airflow.providers.jenkins.hooks.jenkins import JenkinsHook JenkinsRequest = Mapping[str, Any] -ParamType = Union[str, dict, List, None] +ParamType = Union[str, dict, list, None] def jenkins_request_with_headers(jenkins_server: Jenkins, req: Request) -> JenkinsRequest | None: diff --git a/airflow/providers/microsoft/azure/hooks/adx.py b/airflow/providers/microsoft/azure/hooks/adx.py index 96518b7e0ad4..e3cda1d08f06 100644 --- a/airflow/providers/microsoft/azure/hooks/adx.py +++ b/airflow/providers/microsoft/azure/hooks/adx.py @@ -186,8 +186,9 @@ def get_required_param(name: str) -> str: def run_query(self, query: str, database: str, options: dict | None = None) -> KustoResponseDataSetV2: """ - Run KQL query using provided configuration, and return - `azure.kusto.data.response.KustoResponseDataSet` instance. + Run KQL query using provided configuration, and return KustoResponseDataSet instance. + + See: `azure.kusto.data.response.KustoResponseDataSet` If query is unsuccessful AirflowException is raised. :param query: KQL query to run diff --git a/airflow/providers/microsoft/azure/hooks/base_azure.py b/airflow/providers/microsoft/azure/hooks/base_azure.py index 4190a2bf03ee..2a4d250e1ca5 100644 --- a/airflow/providers/microsoft/azure/hooks/base_azure.py +++ b/airflow/providers/microsoft/azure/hooks/base_azure.py @@ -27,8 +27,10 @@ class AzureBaseHook(BaseHook): """ - This hook acts as a base hook for azure services. It offers several authentication mechanisms to - authenticate the client library used for upstream azure hooks. + This hook acts as a base hook for azure services. + + It offers several authentication mechanisms to authenticate + the client library used for upstream azure hooks. :param sdk_client: The SDKClient to use. :param conn_id: The :ref:`Azure connection id` diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/airflow/providers/microsoft/azure/hooks/cosmos.py index 238abc4c70f6..45b1b0dab2dc 100644 --- a/airflow/providers/microsoft/azure/hooks/cosmos.py +++ b/airflow/providers/microsoft/azure/hooks/cosmos.py @@ -237,10 +237,7 @@ def delete_collection(self, collection_name: str, database_name: str | None = No ) def upsert_document(self, document, database_name=None, collection_name=None, document_id=None): - """ - Inserts a new document (or updates an existing one) into an existing - collection in the CosmosDB database. - """ + """Insert or update a document into an existing collection in the CosmosDB database.""" # Assign unique ID if one isn't provided if document_id is None: document_id = str(uuid.uuid4()) diff --git a/airflow/providers/microsoft/azure/hooks/fileshare.py b/airflow/providers/microsoft/azure/hooks/fileshare.py index 8067907706fe..0244b7ce727e 100644 --- a/airflow/providers/microsoft/azure/hooks/fileshare.py +++ b/airflow/providers/microsoft/azure/hooks/fileshare.py @@ -28,6 +28,8 @@ def _ensure_prefixes(conn_type): """ + Deprecated. + Remove when provider min airflow version >= 2.5.0 since this is handled by provider manager from that version. """ diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index ad9c4754c525..794da8dd8575 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -53,6 +53,8 @@ def _ensure_prefixes(conn_type): """ + Deprecated. + Remove when provider min airflow version >= 2.5.0 since this is handled by provider manager from that version. """ diff --git a/airflow/providers/microsoft/azure/log/wasb_task_handler.py b/airflow/providers/microsoft/azure/log/wasb_task_handler.py index 51c0c4dc182b..96c87219ca13 100644 --- a/airflow/providers/microsoft/azure/log/wasb_task_handler.py +++ b/airflow/providers/microsoft/azure/log/wasb_task_handler.py @@ -45,9 +45,9 @@ def get_default_delete_local_copy(): class WasbTaskHandler(FileTaskHandler, LoggingMixin): """ - WasbTaskHandler is a python log handler that handles and reads - task instance logs. It extends airflow FileTaskHandler and - uploads to and reads from Wasb remote storage. + WasbTaskHandler is a python log handler that handles and reads task instance logs. + + It extends airflow FileTaskHandler and uploads to and reads from Wasb remote storage. """ trigger_should_wrap = True @@ -171,6 +171,7 @@ def _read( ) -> tuple[str, dict[str, bool]]: """ Read logs of given task instance and try_number from Wasb remote storage. + If failed, read the log from task instance host machine. todo: when min airflow version >= 2.6, remove this method @@ -207,8 +208,7 @@ def wasb_log_exists(self, remote_log_location: str) -> bool: def wasb_read(self, remote_log_location: str, return_error: bool = False): """ - Returns the log found at the remote_log_location. Returns '' if no - logs are found or there is an error. + Return the log found at the remote_log_location. Returns '' if no logs are found or there is an error. :param remote_log_location: the log's location in remote storage :param return_error: if True, returns a string error message if an @@ -226,8 +226,7 @@ def wasb_read(self, remote_log_location: str, return_error: bool = False): def wasb_write(self, log: str, remote_log_location: str, append: bool = True) -> bool: """ - Writes the log to the remote_log_location. Fails silently if no hook - was created. + Writes the log to the remote_log_location. Fails silently if no hook was created. :param log: the log to write to the remote_log_location :param remote_log_location: the log's location in remote storage diff --git a/airflow/providers/microsoft/azure/operators/asb.py b/airflow/providers/microsoft/azure/operators/asb.py index 333fc82d2b4d..d9a460b77d23 100644 --- a/airflow/providers/microsoft/azure/operators/asb.py +++ b/airflow/providers/microsoft/azure/operators/asb.py @@ -155,10 +155,7 @@ def __init__( self.max_wait_time = max_wait_time def execute(self, context: Context) -> None: - """ - Receive Message in specific queue in Service Bus namespace, - by connecting to Service Bus client. - """ + """Receive Message in specific queue in Service Bus namespace by connecting to Service Bus client.""" # Create the hook hook = MessageHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id) @@ -527,10 +524,7 @@ def __init__( self.azure_service_bus_conn_id = azure_service_bus_conn_id def execute(self, context: Context) -> None: - """ - Receive Message in specific queue in Service Bus namespace, - by connecting to Service Bus client. - """ + """Receive Message in specific queue in Service Bus namespace by connecting to Service Bus client.""" # Create the hook hook = MessageHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id) diff --git a/airflow/providers/microsoft/azure/operators/data_factory.py b/airflow/providers/microsoft/azure/operators/data_factory.py index a2b2c528bf9b..1823212473f6 100644 --- a/airflow/providers/microsoft/azure/operators/data_factory.py +++ b/airflow/providers/microsoft/azure/operators/data_factory.py @@ -233,8 +233,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, str]) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event["status"] == "error": diff --git a/airflow/providers/microsoft/azure/secrets/key_vault.py b/airflow/providers/microsoft/azure/secrets/key_vault.py index df696b05821b..177da5d23532 100644 --- a/airflow/providers/microsoft/azure/secrets/key_vault.py +++ b/airflow/providers/microsoft/azure/secrets/key_vault.py @@ -162,6 +162,7 @@ def get_config(self, key: str) -> str | None: def build_path(path_prefix: str, secret_id: str, sep: str = "-") -> str: """ Given a path_prefix and secret_id, build a valid secret name for the Azure Key Vault Backend. + Also replaces underscore in the path with dashes to support easy switching between environment variables, so ``connection_default`` becomes ``connection-default``. diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/airflow/providers/microsoft/azure/sensors/data_factory.py index b4ebedce698a..f0651494bfac 100644 --- a/airflow/providers/microsoft/azure/sensors/data_factory.py +++ b/airflow/providers/microsoft/azure/sensors/data_factory.py @@ -113,8 +113,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, str]) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event["status"] == "error": diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py index 0c227f2ea37b..18ac2331e018 100644 --- a/airflow/providers/microsoft/azure/sensors/wasb.py +++ b/airflow/providers/microsoft/azure/sensors/wasb.py @@ -97,8 +97,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, str]) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event["status"] == "error": @@ -193,8 +193,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, str]) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event["status"] == "error": diff --git a/airflow/providers/microsoft/azure/transfers/local_to_adls.py b/airflow/providers/microsoft/azure/transfers/local_to_adls.py index f39f837c9820..7eee5009afcc 100644 --- a/airflow/providers/microsoft/azure/transfers/local_to_adls.py +++ b/airflow/providers/microsoft/azure/transfers/local_to_adls.py @@ -102,6 +102,7 @@ def execute(self, context: Context) -> None: class LocalToAzureDataLakeStorageOperator(LocalFilesystemToADLSOperator): """ This class is deprecated. + Please use `airflow.providers.microsoft.azure.transfers.local_to_adls.LocalFilesystemToADLSOperator`. """ diff --git a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py index 93e8a7b14d3f..f891e080a490 100644 --- a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py +++ b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py @@ -32,9 +32,7 @@ class OracleToAzureDataLakeOperator(BaseOperator): """ - Moves data from Oracle to Azure Data Lake. The operator runs the query against - Oracle and stores the file locally before loading it into Azure Data Lake. - + Runs the query against Oracle and stores the file locally before loading it into Azure Data Lake. :param filename: file name to be used by the csv file. :param azure_data_lake_conn_id: destination azure data lake connection. diff --git a/airflow/providers/microsoft/azure/triggers/wasb.py b/airflow/providers/microsoft/azure/triggers/wasb.py index 0bc75f065f3a..944c7ddae1d3 100644 --- a/airflow/providers/microsoft/azure/triggers/wasb.py +++ b/airflow/providers/microsoft/azure/triggers/wasb.py @@ -25,8 +25,9 @@ class WasbBlobSensorTrigger(BaseTrigger): """ - WasbBlobSensorTrigger is fired as deferred class with params to run the task in - trigger worker to check for existence of the given blob in the provided container. + Checks for existence of the given blob in the provided container. + + WasbBlobSensorTrigger is fired as deferred class with params to run the task in trigger worker. :param container_name: name of the container in which the blob should be searched for :param blob_name: name of the blob to check existence for @@ -90,8 +91,9 @@ async def run(self) -> AsyncIterator[TriggerEvent]: class WasbPrefixSensorTrigger(BaseTrigger): """ + Checks for the existence of a blob with the given prefix in the provided container. + WasbPrefixSensorTrigger is fired as a deferred class with params to run the task in trigger. - It checks for the existence of a blob with the given prefix in the provided container. :param container_name: name of the container in which the blob should be searched for :param prefix: prefix of the blob to check existence for diff --git a/airflow/providers/microsoft/azure/utils.py b/airflow/providers/microsoft/azure/utils.py index 8c0110046905..e4161a1b12bb 100644 --- a/airflow/providers/microsoft/azure/utils.py +++ b/airflow/providers/microsoft/azure/utils.py @@ -23,6 +23,8 @@ def _ensure_prefixes(conn_type): """ + Deprecated. + Remove when provider min airflow version >= 2.5.0 since this is handled by provider manager from that version. """ diff --git a/airflow/providers/microsoft/mssql/hooks/mssql.py b/airflow/providers/microsoft/mssql/hooks/mssql.py index 01e1c9e47b4a..7b21e77c7fab 100644 --- a/airflow/providers/microsoft/mssql/hooks/mssql.py +++ b/airflow/providers/microsoft/mssql/hooks/mssql.py @@ -26,7 +26,14 @@ class MsSqlHook(DbApiHook): - """Interact with Microsoft SQL Server.""" + """ + Interact with Microsoft SQL Server. + + :param args: passed to DBApiHook + :param sqlalchemy_scheme: Scheme sqlalchemy connection. Default is ``mssql+pymssql`` Only used for + ``get_sqlalchemy_engine`` and ``get_sqlalchemy_connection`` methods. + :param kwargs: passed to DbApiHook + """ conn_name_attr = "mssql_conn_id" default_conn_name = "mssql_default" @@ -41,12 +48,6 @@ def __init__( sqlalchemy_scheme: str | None = None, **kwargs, ) -> None: - """ - :param args: passed to DBApiHook - :param sqlalchemy_scheme: Scheme sqlalchemy connection. Default is ``mssql+pymssql`` Only used for - ``get_sqlalchemy_engine`` and ``get_sqlalchemy_connection`` methods. - :param kwargs: passed to DbApiHook - """ super().__init__(*args, **kwargs) self.schema = kwargs.pop("schema", None) self._sqlalchemy_scheme = sqlalchemy_scheme @@ -55,6 +56,7 @@ def __init__( def connection_extra_lower(self) -> dict: """ ``connection.extra_dejson`` but where keys are converted to lower case. + This is used internally for case-insensitive access of mssql params. """ conn = self.get_connection(self.mssql_conn_id) # type: ignore[attr-defined] diff --git a/airflow/providers/microsoft/psrp/hooks/psrp.py b/airflow/providers/microsoft/psrp/hooks/psrp.py index fe7f4c271fa7..06e3eb93a7fa 100644 --- a/airflow/providers/microsoft/psrp/hooks/psrp.py +++ b/airflow/providers/microsoft/psrp/hooks/psrp.py @@ -158,8 +158,9 @@ def apply_extra(d, keys): @contextmanager def invoke(self) -> Generator[PowerShell, None, None]: """ - Context manager that yields a PowerShell object to which commands can be - added. Upon exit, the commands will be invoked. + Yields a PowerShell object to which commands can be added. + + Upon exit, the commands will be invoked. """ logger = copy(self.log) logger.setLevel(self._logging_level) diff --git a/airflow/providers/mongo/hooks/mongo.py b/airflow/providers/mongo/hooks/mongo.py index b49a9365afcc..cfa34e0e7510 100644 --- a/airflow/providers/mongo/hooks/mongo.py +++ b/airflow/providers/mongo/hooks/mongo.py @@ -98,6 +98,7 @@ def get_conn(self) -> MongoClient: def _create_uri(self) -> str: """ Create URI string from the given credentials. + :return: URI string. """ srv = self.extras.pop("srv", False) diff --git a/airflow/providers/mysql/transfers/presto_to_mysql.py b/airflow/providers/mysql/transfers/presto_to_mysql.py index b38e6b8654d6..0d80b58bff8c 100644 --- a/airflow/providers/mysql/transfers/presto_to_mysql.py +++ b/airflow/providers/mysql/transfers/presto_to_mysql.py @@ -29,9 +29,10 @@ class PrestoToMySqlOperator(BaseOperator): """ - Moves data from Presto to MySQL, note that for now the data is loaded - into memory before being pushed to MySQL, so this operator should - be used for smallish amount of data. + Moves data from Presto to MySQL. + + Note that for now the data is loaded into memory before being pushed + to MySQL, so this operator should be used for smallish amount of data. :param sql: SQL query to execute against Presto. (templated) :param mysql_table: target MySQL table, use dot notation to target a diff --git a/airflow/providers/mysql/transfers/trino_to_mysql.py b/airflow/providers/mysql/transfers/trino_to_mysql.py index 8ff5ed0446b5..e96aaafd5bf0 100644 --- a/airflow/providers/mysql/transfers/trino_to_mysql.py +++ b/airflow/providers/mysql/transfers/trino_to_mysql.py @@ -29,9 +29,10 @@ class TrinoToMySqlOperator(BaseOperator): """ - Moves data from Trino to MySQL, note that for now the data is loaded - into memory before being pushed to MySQL, so this operator should - be used for smallish amount of data. + Moves data from Trino to MySQL. + + Note that for now the data is loaded into memory before being pushed + to MySQL, so this operator should be used for smallish amount of data. :param sql: SQL query to execute against Trino. (templated) :param mysql_table: target MySQL table, use dot notation to target a diff --git a/airflow/providers/neo4j/hooks/neo4j.py b/airflow/providers/neo4j/hooks/neo4j.py index ed1df254a87a..137b1e00c032 100644 --- a/airflow/providers/neo4j/hooks/neo4j.py +++ b/airflow/providers/neo4j/hooks/neo4j.py @@ -48,10 +48,7 @@ def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: self.client: Driver | None = None def get_conn(self) -> Driver: - """ - Function that initiates a new Neo4j connection - with username, password and database schema. - """ + """Function that initiates a new Neo4j connection with username, password and database schema.""" if self.client is not None: return self.client @@ -112,8 +109,7 @@ def get_uri(self, conn: Connection) -> str: def run(self, query) -> list[Any]: """ - Function to create a neo4j session - and execute the query in the session. + Function to create a neo4j session and execute the query in the session. :param query: Neo4j query :return: Result diff --git a/airflow/providers/openlineage/extractors/bash.py b/airflow/providers/openlineage/extractors/bash.py index 5a2bc92f65bf..9d7c40b114c1 100644 --- a/airflow/providers/openlineage/extractors/bash.py +++ b/airflow/providers/openlineage/extractors/bash.py @@ -32,9 +32,11 @@ class BashExtractor(BaseExtractor): """ + Extract executed bash command and put it into SourceCodeJobFacet. + This extractor provides visibility on what bash task does by extracting - executed bash command and putting it into SourceCodeJobFacet. It does not extract - datasets. + executed bash command and putting it into SourceCodeJobFacet. It does + not extract datasets. :meta private: """ diff --git a/airflow/providers/openlineage/extractors/python.py b/airflow/providers/openlineage/extractors/python.py index 0c5720ac8672..50d84014fdee 100644 --- a/airflow/providers/openlineage/extractors/python.py +++ b/airflow/providers/openlineage/extractors/python.py @@ -35,6 +35,8 @@ class PythonExtractor(BaseExtractor): """ + Extract executed source code and put it into SourceCodeJobFacet. + This extractor provides visibility on what particular task does by extracting executed source code and putting it into SourceCodeJobFacet. It does not extract datasets yet. diff --git a/airflow/providers/openlineage/plugins/adapter.py b/airflow/providers/openlineage/plugins/adapter.py index aecd0a65314f..0e530e5c53fe 100644 --- a/airflow/providers/openlineage/plugins/adapter.py +++ b/airflow/providers/openlineage/plugins/adapter.py @@ -58,10 +58,7 @@ class OpenLineageAdapter(LoggingMixin): - """ - Adapter for translating Airflow metadata to OpenLineage events, - instead of directly creating them from Airflow code. - """ + """Translate Airflow metadata to OpenLineage events instead of creating them from Airflow code.""" def __init__(self, client: OpenLineageClient | None = None, secrets_masker: SecretsMasker | None = None): super().__init__() diff --git a/airflow/providers/openlineage/plugins/listener.py b/airflow/providers/openlineage/plugins/listener.py index 9b57bf919fb6..99394863f514 100644 --- a/airflow/providers/openlineage/plugins/listener.py +++ b/airflow/providers/openlineage/plugins/listener.py @@ -38,10 +38,7 @@ class OpenLineageListener: - """ - OpenLineage listener - Sends events on task instance and dag run starts, completes and failures. - """ + """OpenLineage listener sends events on task instance and dag run starts, completes and failures.""" def __init__(self): self.log = logging.getLogger(__name__) diff --git a/airflow/providers/openlineage/plugins/macros.py b/airflow/providers/openlineage/plugins/macros.py index 19cbb59f299e..61af81a1eb71 100644 --- a/airflow/providers/openlineage/plugins/macros.py +++ b/airflow/providers/openlineage/plugins/macros.py @@ -30,9 +30,9 @@ def lineage_run_id(task_instance: TaskInstance): """ - Macro function which returns the generated run id for a given task. This - can be used to forward the run id from a task to a child run so the job - hierarchy is preserved. + Macro function which returns the generated run id for a given task. + + This can be used to forward the run id from a task to a child run so the job hierarchy is preserved. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -45,8 +45,9 @@ def lineage_run_id(task_instance: TaskInstance): def lineage_parent_id(run_id: str, task_instance: TaskInstance): """ - Macro function which returns the generated job and run id for a given task. This - can be used to forward the ids from a task to a child run so the job + Macro function which returns the generated job and run id for a given task. + + This can be used to forward the ids from a task to a child run so the job hierarchy is preserved. Child run can create ParentRunFacet from those ids. .. seealso:: diff --git a/airflow/providers/openlineage/plugins/openlineage.py b/airflow/providers/openlineage/plugins/openlineage.py index 9aade083b482..2ec080114753 100644 --- a/airflow/providers/openlineage/plugins/openlineage.py +++ b/airflow/providers/openlineage/plugins/openlineage.py @@ -31,8 +31,11 @@ def _is_disabled() -> bool: class OpenLineageProviderPlugin(AirflowPlugin): - """OpenLineage Plugin provides listener that emits OL events on DAG start, complete and failure - and TaskInstances start, complete and failure. + """ + Listener that emits numerous Events. + + OpenLineage Plugin provides listener that emits OL events on DAG start, + complete and failure and TaskInstances start, complete and failure. """ name = "OpenLineageProviderPlugin" diff --git a/airflow/providers/openlineage/sqlparser.py b/airflow/providers/openlineage/sqlparser.py index ed3e92e58b5a..8e85d3706c52 100644 --- a/airflow/providers/openlineage/sqlparser.py +++ b/airflow/providers/openlineage/sqlparser.py @@ -207,7 +207,8 @@ def normalize_sql(cls, sql: list[str] | str) -> str: @classmethod def split_sql_string(cls, sql: list[str] | str) -> list[str]: """ - Split SQL string into list of statements + Split SQL string into list of statements. + Tries to use `DbApiHook.split_sql_string` if available. Otherwise, uses the same logic. """ diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py index 84ad41e23701..9d1cab8eca87 100644 --- a/airflow/providers/openlineage/utils/utils.py +++ b/airflow/providers/openlineage/utils/utils.py @@ -101,6 +101,7 @@ def url_to_https(url) -> str | None: def redacted_connection_uri(conn: Connection, filtered_params=None, filtered_prefixes=None): """ Return the connection URI for the given Connection. + This method additionally filters URI by removing query parameters that are known to carry sensitive data like username, password, access key. """ @@ -324,7 +325,9 @@ def get_airflow_run_facet( class OpenLineageRedactor(SecretsMasker): - """This class redacts sensitive data similar to SecretsMasker in Airflow logs. + """ + This class redacts sensitive data similar to SecretsMasker in Airflow logs. + The difference is that our default max recursion depth is way higher - due to the structure of OL events we need more depth. Additionally, we allow data structures to specify data that needs not to be diff --git a/airflow/providers/opsgenie/hooks/opsgenie.py b/airflow/providers/opsgenie/hooks/opsgenie.py index 7c28b2a76e1e..239f67a7e212 100644 --- a/airflow/providers/opsgenie/hooks/opsgenie.py +++ b/airflow/providers/opsgenie/hooks/opsgenie.py @@ -33,6 +33,7 @@ class OpsgenieAlertHook(BaseHook): """ This hook allows you to post alerts to Opsgenie. + Accepts a connection that has an Opsgenie API key as the connection's password. This hook sets the domain to conn_id.host, and if not set will default to ``https://api.opsgenie.com``. diff --git a/airflow/providers/opsgenie/operators/opsgenie.py b/airflow/providers/opsgenie/operators/opsgenie.py index a4777a8eb72f..d12db2087356 100644 --- a/airflow/providers/opsgenie/operators/opsgenie.py +++ b/airflow/providers/opsgenie/operators/opsgenie.py @@ -29,6 +29,7 @@ class OpsgenieCreateAlertOperator(BaseOperator): """ This operator allows you to post alerts to Opsgenie. + Accepts a connection that has an Opsgenie API key as the connection's password. This operator sets the domain to conn_id.host, and if not set will default to ``https://api.opsgenie.com``. @@ -101,8 +102,9 @@ def __init__( def _build_opsgenie_payload(self) -> dict[str, Any]: """ - Construct the Opsgenie JSON payload. All relevant parameters are combined here - to a valid Opsgenie JSON payload. + Construct the Opsgenie JSON payload. + + All relevant parameters are combined here to a valid Opsgenie JSON payload. :return: Opsgenie payload (dict) to send """ @@ -137,6 +139,7 @@ def execute(self, context: Context) -> None: class OpsgenieCloseAlertOperator(BaseOperator): """ This operator allows you to close alerts to Opsgenie. + Accepts a connection that has an Opsgenie API key as the connection's password. This operator sets the domain to conn_id.host, and if not set will default to ``https://api.opsgenie.com``. @@ -183,8 +186,9 @@ def __init__( def _build_opsgenie_close_alert_payload(self) -> dict[str, Any]: """ - Construct the Opsgenie JSON payload. All relevant parameters are combined here - to a valid Opsgenie JSON payload. + Construct the Opsgenie JSON payload. + + All relevant parameters are combined here to a valid Opsgenie JSON payload. :return: Opsgenie close alert payload (dict) to send """ @@ -214,6 +218,7 @@ def execute(self, context: Context) -> None: class OpsgenieDeleteAlertOperator(BaseOperator): """ This operator allows you to delete alerts in Opsgenie. + Accepts a connection that has an Opsgenie API key as the connection's password. This operator sets the domain to conn_id.host, and if not set will default to ``https://api.opsgenie.com``. From cb4927a01887e2413c45d8d9cb63e74aa994ee74 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:12:51 +0300 Subject: [PATCH 052/533] Prepare docs for July 2023 wave of Providers (#32298) * Prepare docs for July 2023 wave of Providers * Add common.sql and more changes to Amazon * Add amazon and microsoft.azure --- .../airbyte/.latest-doc-only-change.txt | 2 +- airflow/providers/alibaba/CHANGELOG.rst | 13 +++++ airflow/providers/alibaba/__init__.py | 2 +- airflow/providers/alibaba/provider.yaml | 1 + airflow/providers/amazon/CHANGELOG.rst | 50 +++++++++++++++++++ airflow/providers/amazon/__init__.py | 2 +- airflow/providers/amazon/provider.yaml | 1 + .../apache/beam/.latest-doc-only-change.txt | 2 +- .../cassandra/.latest-doc-only-change.txt | 2 +- .../apache/drill/.latest-doc-only-change.txt | 2 +- .../apache/druid/.latest-doc-only-change.txt | 2 +- .../apache/hdfs/.latest-doc-only-change.txt | 2 +- .../apache/hive/.latest-doc-only-change.txt | 2 +- airflow/providers/apache/kafka/CHANGELOG.rst | 13 +++++ airflow/providers/apache/kafka/__init__.py | 2 +- airflow/providers/apache/kafka/provider.yaml | 1 + .../apache/kylin/.latest-doc-only-change.txt | 2 +- airflow/providers/apache/livy/CHANGELOG.rst | 13 +++++ airflow/providers/apache/livy/__init__.py | 2 +- airflow/providers/apache/livy/provider.yaml | 1 + .../apache/pig/.latest-doc-only-change.txt | 2 +- .../apache/pinot/.latest-doc-only-change.txt | 2 +- .../apache/spark/.latest-doc-only-change.txt | 2 +- .../apache/sqoop/.latest-doc-only-change.txt | 2 +- airflow/providers/apprise/CHANGELOG.rst | 5 ++ .../arangodb/.latest-doc-only-change.txt | 2 +- .../asana/.latest-doc-only-change.txt | 2 +- .../celery/.latest-doc-only-change.txt | 2 +- .../cloudant/.latest-doc-only-change.txt | 2 +- .../providers/cncf/kubernetes/CHANGELOG.rst | 20 ++++++++ airflow/providers/cncf/kubernetes/__init__.py | 2 +- .../providers/cncf/kubernetes/provider.yaml | 1 + airflow/providers/common/sql/CHANGELOG.rst | 13 +++++ airflow/providers/common/sql/__init__.py | 2 +- airflow/providers/common/sql/provider.yaml | 1 + airflow/providers/databricks/CHANGELOG.rst | 13 +++++ airflow/providers/databricks/__init__.py | 2 +- airflow/providers/databricks/provider.yaml | 1 + .../datadog/.latest-doc-only-change.txt | 2 +- .../dbt/cloud/.latest-doc-only-change.txt | 2 +- .../dingding/.latest-doc-only-change.txt | 2 +- .../discord/.latest-doc-only-change.txt | 2 +- .../docker/.latest-doc-only-change.txt | 2 +- airflow/providers/elasticsearch/CHANGELOG.rst | 8 +++ airflow/providers/elasticsearch/__init__.py | 2 +- .../exasol/.latest-doc-only-change.txt | 2 +- .../facebook/.latest-doc-only-change.txt | 2 +- .../providers/ftp/.latest-doc-only-change.txt | 2 +- .../github/.latest-doc-only-change.txt | 2 +- airflow/providers/google/CHANGELOG.rst | 32 ++++++++++++ airflow/providers/google/__init__.py | 2 +- airflow/providers/google/provider.yaml | 1 + .../grpc/.latest-doc-only-change.txt | 2 +- airflow/providers/hashicorp/CHANGELOG.rst | 12 +++++ airflow/providers/hashicorp/__init__.py | 2 +- airflow/providers/hashicorp/provider.yaml | 1 + .../http/.latest-doc-only-change.txt | 2 +- .../imap/.latest-doc-only-change.txt | 2 +- .../influxdb/.latest-doc-only-change.txt | 2 +- .../jdbc/.latest-doc-only-change.txt | 2 +- .../jenkins/.latest-doc-only-change.txt | 2 +- .../providers/microsoft/azure/CHANGELOG.rst | 27 ++++++++++ airflow/providers/microsoft/azure/__init__.py | 2 +- .../providers/microsoft/azure/provider.yaml | 1 + .../mssql/.latest-doc-only-change.txt | 2 +- .../psrp/.latest-doc-only-change.txt | 2 +- .../winrm/.latest-doc-only-change.txt | 2 +- .../mongo/.latest-doc-only-change.txt | 2 +- .../mysql/.latest-doc-only-change.txt | 2 +- .../neo4j/.latest-doc-only-change.txt | 2 +- .../odbc/.latest-doc-only-change.txt | 2 +- .../openfaas/.latest-doc-only-change.txt | 2 +- .../opsgenie/.latest-doc-only-change.txt | 2 +- .../oracle/.latest-doc-only-change.txt | 2 +- .../pagerduty/.latest-doc-only-change.txt | 2 +- .../papermill/.latest-doc-only-change.txt | 2 +- .../plexus/.latest-doc-only-change.txt | 2 +- .../postgres/.latest-doc-only-change.txt | 2 +- .../presto/.latest-doc-only-change.txt | 2 +- .../qubole/.latest-doc-only-change.txt | 2 +- .../redis/.latest-doc-only-change.txt | 2 +- .../salesforce/.latest-doc-only-change.txt | 2 +- .../samba/.latest-doc-only-change.txt | 2 +- .../segment/.latest-doc-only-change.txt | 2 +- .../sendgrid/.latest-doc-only-change.txt | 2 +- .../sftp/.latest-doc-only-change.txt | 2 +- .../singularity/.latest-doc-only-change.txt | 2 +- .../slack/.latest-doc-only-change.txt | 2 +- .../snowflake/.latest-doc-only-change.txt | 2 +- .../sqlite/.latest-doc-only-change.txt | 2 +- .../providers/ssh/.latest-doc-only-change.txt | 2 +- .../tableau/.latest-doc-only-change.txt | 2 +- .../tabular/.latest-doc-only-change.txt | 2 +- .../telegram/.latest-doc-only-change.txt | 2 +- airflow/providers/trino/CHANGELOG.rst | 12 +++++ airflow/providers/trino/__init__.py | 2 +- airflow/providers/trino/provider.yaml | 1 + airflow/providers/vertica/CHANGELOG.rst | 12 +++++ airflow/providers/vertica/__init__.py | 2 +- airflow/providers/vertica/provider.yaml | 1 + .../zendesk/.latest-doc-only-change.txt | 2 +- .../prepare_provider_packages.py | 7 ++- .../commits.rst | 14 ++++++ .../index.rst | 2 +- .../commits.rst | 43 ++++++++++++++++ .../apache-airflow-providers-amazon/index.rst | 8 +-- .../commits.rst | 14 ++++++ .../index.rst | 2 +- .../commits.rst | 14 ++++++ .../index.rst | 6 +-- .../commits.rst | 23 ++++++++- .../commits.rst | 17 +++++++ .../index.rst | 2 +- .../commits.rst | 22 ++++++-- .../index.rst | 30 ++++++++++- .../commits.rst | 14 ++++++ .../index.rst | 6 +-- .../commits.rst | 14 ++++++ .../commits.rst | 24 +++++++++ .../apache-airflow-providers-google/index.rst | 6 +-- .../commits.rst | 13 +++++ .../index.rst | 6 +-- .../commits.rst | 20 ++++++++ .../index.rst | 6 +-- .../commits.rst | 13 +++++ docs/apache-airflow-providers-trino/index.rst | 6 +-- .../commits.rst | 13 +++++ .../index.rst | 6 +-- 128 files changed, 647 insertions(+), 109 deletions(-) diff --git a/airflow/providers/airbyte/.latest-doc-only-change.txt b/airflow/providers/airbyte/.latest-doc-only-change.txt index ff7136e07d74..370cc38c596d 100644 --- a/airflow/providers/airbyte/.latest-doc-only-change.txt +++ b/airflow/providers/airbyte/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +0bc689ee6d4b6967d7ae99a202031aac14d181a2 diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/airflow/providers/alibaba/CHANGELOG.rst index be0b33e07a97..af267238d0f8 100644 --- a/airflow/providers/alibaba/CHANGELOG.rst +++ b/airflow/providers/alibaba/CHANGELOG.rst @@ -24,6 +24,19 @@ Changelog for ``apache-airflow-providers-alibaba`` -------------------------------------------------- +2.5.0 +..... + +Features +~~~~~~~~ + +* ``Add Alibaba Cloud AnalyticDB Spark Support (#31787)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Airbyte and Alibaba (#32214)`` + * ``Improve provider documentation and README structure (#32125)`` + 2.4.1 ..... diff --git a/airflow/providers/alibaba/__init__.py b/airflow/providers/alibaba/__init__.py index 2442a7218247..1d3407921c34 100644 --- a/airflow/providers/alibaba/__init__.py +++ b/airflow/providers/alibaba/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "2.4.1" +__version__ = "2.5.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index 660dccdcd648..367209552523 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 2.5.0 - 2.4.1 - 2.4.0 - 2.3.0 diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index ea6abb7c61ac..1b3b94fefc23 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -24,6 +24,56 @@ Changelog for ``apache-airflow-providers-amazon`` ------------------------------------------------- +8.3.0 +..... + +Features +~~~~~~~~ + +* ``Add 'ChimeWebhookHook' (#31939)`` +* ``Add 'ChimeNotifier' (#32222)`` +* ``Add deferrable mode to S3KeysUnchangedSensor (#31940)`` +* ``Add deferrable mode to 'RdsCreateDbInstanceOperator' and 'RdsDeleteDbInstanceOperator' (#32171)`` +* ``Add deferrable mode for 'AthenaOperator' (#32186)`` +* ``Add a deferrable mode to 'BatchCreateComputeEnvironmentOperator' (#32036)`` +* ``Add deferrable mode in EMR operator and sensor (#32029)`` +* ``add async wait method to the "with logging" aws utils (#32055)`` +* ``Add custom waiters to EMR Serverless (#30463)`` +* ``Add an option to 'GlueJobOperator' to stop the job run when the TI is killed (#32155)`` +* ``deferrable mode for 'SageMakerTuningOperator' and 'SageMakerEndpointOperator' (#32112)`` +* ``EKS Create/Delete Nodegroup Deferrable mode (#32165)`` +* ``Deferrable mode for ECS operators (#31881)`` +* ``feature: AWS - GlueJobOperator - job_poll_interval (#32147)`` +* ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` + +Bug Fixes +~~~~~~~~~ + +* ``bugfix: break down run+wait method in ECS operator (#32104)`` +* ``Handle 'UnboundLocalError' while parsing invalid 's3_url' (#32120)`` +* ``Fix 'LambdaInvokeFunctionOperator' payload parameter type (#32259)`` + +Misc +~~~~ + +* ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` +* ``aws waiter util: log status info with error level on waiter error (#32247)`` +* ``rewrite method used in ecs to fetch less logs (#31786)`` +* ``Refactor Eks Create Cluster Operator code (#31960)`` +* ``Use a waiter in 'AthenaHook' (#31942)`` +* ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Revert "add deferrable mode for 'AthenaOperator' (#32110)" (#32172)`` + * ``add deferrable mode for 'AthenaOperator' (#32110)`` + * ``D205 Support - Auto-fixes and Stragglers (#32212)`` + * ``D205 Support - Providers: Amazon/AWS (#32224)`` + * ``Improve provider documentation and README structure (#32125)`` + * ``Minor name change for the util wait method. (#32152)`` + * ``Clean up string concatenation (#32129)`` + * ``cleanup Amazon CHANGELOG.rst (#32031)`` + 8.2.0 ..... diff --git a/airflow/providers/amazon/__init__.py b/airflow/providers/amazon/__init__.py index cf6ef41b5ebc..204607dbae69 100644 --- a/airflow/providers/amazon/__init__.py +++ b/airflow/providers/amazon/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "8.2.0" +__version__ = "8.3.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 575a2b5cab1b..390a5866368a 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 8.3.0 - 8.2.0 - 8.1.0 - 8.0.0 diff --git a/airflow/providers/apache/beam/.latest-doc-only-change.txt b/airflow/providers/apache/beam/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/apache/beam/.latest-doc-only-change.txt +++ b/airflow/providers/apache/beam/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt +++ b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/drill/.latest-doc-only-change.txt b/airflow/providers/apache/drill/.latest-doc-only-change.txt index 8399fdc43bc0..33a07c855e0f 100644 --- a/airflow/providers/apache/drill/.latest-doc-only-change.txt +++ b/airflow/providers/apache/drill/.latest-doc-only-change.txt @@ -1 +1 @@ -428a43995390b3623a51aa7bac7e21da69a8db22 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/apache/druid/.latest-doc-only-change.txt b/airflow/providers/apache/druid/.latest-doc-only-change.txt index e7c3c940c9c7..4d0c8bcde51e 100644 --- a/airflow/providers/apache/druid/.latest-doc-only-change.txt +++ b/airflow/providers/apache/druid/.latest-doc-only-change.txt @@ -1 +1 @@ -602abe8394fafe7de54df7e73af56de848cdf617 +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt +++ b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/hive/.latest-doc-only-change.txt b/airflow/providers/apache/hive/.latest-doc-only-change.txt index e7c3c940c9c7..4d0c8bcde51e 100644 --- a/airflow/providers/apache/hive/.latest-doc-only-change.txt +++ b/airflow/providers/apache/hive/.latest-doc-only-change.txt @@ -1 +1 @@ -602abe8394fafe7de54df7e73af56de848cdf617 +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/kafka/CHANGELOG.rst b/airflow/providers/apache/kafka/CHANGELOG.rst index 3bde0c11ded1..95898617e6c7 100644 --- a/airflow/providers/apache/kafka/CHANGELOG.rst +++ b/airflow/providers/apache/kafka/CHANGELOG.rst @@ -23,6 +23,19 @@ Changelog for ``apache-airflow-providers-apache-kafka`` ------------------------------------------------------- +1.1.2 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Break AwaitMessageTrigger execution when finding a message with the desired format (#31803)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` + 1.1.1 ..... diff --git a/airflow/providers/apache/kafka/__init__.py b/airflow/providers/apache/kafka/__init__.py index 45b378cab2fd..d8fb72160e58 100644 --- a/airflow/providers/apache/kafka/__init__.py +++ b/airflow/providers/apache/kafka/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "1.1.1" +__version__ = "1.1.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/apache/kafka/provider.yaml b/airflow/providers/apache/kafka/provider.yaml index c78c5952f078..7617440e82a1 100644 --- a/airflow/providers/apache/kafka/provider.yaml +++ b/airflow/providers/apache/kafka/provider.yaml @@ -23,6 +23,7 @@ suspended: false description: | `Apache Kafka `__ versions: + - 1.1.2 - 1.1.1 - 1.1.0 - 1.0.0 diff --git a/airflow/providers/apache/kylin/.latest-doc-only-change.txt b/airflow/providers/apache/kylin/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/apache/kylin/.latest-doc-only-change.txt +++ b/airflow/providers/apache/kylin/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/apache/livy/CHANGELOG.rst b/airflow/providers/apache/livy/CHANGELOG.rst index 40cf76463937..93dc82bbbfe8 100644 --- a/airflow/providers/apache/livy/CHANGELOG.rst +++ b/airflow/providers/apache/livy/CHANGELOG.rst @@ -24,6 +24,19 @@ Changelog for ``apache-airflow-providers-apache-livy`` ------------------------------------------------------ +3.5.2 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Fix 'LivyHook' TypeError exception on 'session_id' log format (#32051)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` + 3.5.1 ..... diff --git a/airflow/providers/apache/livy/__init__.py b/airflow/providers/apache/livy/__init__.py index 5ce66cbc351b..fe23053440bc 100644 --- a/airflow/providers/apache/livy/__init__.py +++ b/airflow/providers/apache/livy/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "3.5.1" +__version__ = "3.5.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index 7f06bf3093b9..1a22652df86e 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 3.5.2 - 3.5.1 - 3.5.0 - 3.4.0 diff --git a/airflow/providers/apache/pig/.latest-doc-only-change.txt b/airflow/providers/apache/pig/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/apache/pig/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pig/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/apache/pinot/.latest-doc-only-change.txt b/airflow/providers/apache/pinot/.latest-doc-only-change.txt index d2f4035cfe09..4d0c8bcde51e 100644 --- a/airflow/providers/apache/pinot/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pinot/.latest-doc-only-change.txt @@ -1 +1 @@ -487b174073c01e03ae64760405a8d88f6a488ca6 +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/spark/.latest-doc-only-change.txt b/airflow/providers/apache/spark/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/apache/spark/.latest-doc-only-change.txt +++ b/airflow/providers/apache/spark/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/apache/sqoop/.latest-doc-only-change.txt b/airflow/providers/apache/sqoop/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/apache/sqoop/.latest-doc-only-change.txt +++ b/airflow/providers/apache/sqoop/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/apprise/CHANGELOG.rst b/airflow/providers/apprise/CHANGELOG.rst index 0b1586bbfafd..9b5d755cbec9 100644 --- a/airflow/providers/apprise/CHANGELOG.rst +++ b/airflow/providers/apprise/CHANGELOG.rst @@ -20,3 +20,8 @@ Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. + +1.0.0 +..... + +Initial version of the provider. diff --git a/airflow/providers/arangodb/.latest-doc-only-change.txt b/airflow/providers/arangodb/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/arangodb/.latest-doc-only-change.txt +++ b/airflow/providers/arangodb/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/asana/.latest-doc-only-change.txt b/airflow/providers/asana/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/asana/.latest-doc-only-change.txt +++ b/airflow/providers/asana/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/celery/.latest-doc-only-change.txt b/airflow/providers/celery/.latest-doc-only-change.txt index ff7136e07d74..4d0c8bcde51e 100644 --- a/airflow/providers/celery/.latest-doc-only-change.txt +++ b/airflow/providers/celery/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +8c37b74a208a808d905c1b86d081d69d7a1aa900 diff --git a/airflow/providers/cloudant/.latest-doc-only-change.txt b/airflow/providers/cloudant/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/cloudant/.latest-doc-only-change.txt +++ b/airflow/providers/cloudant/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index bae95c0297dd..f90802cbe402 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -24,6 +24,26 @@ Changelog for ``apache-airflow-providers-cncf-kubernetes`` ---------------------------------------------------------- +7.2.0 +..... + +Features +~~~~~~~~ + +* ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` + +Bug Fixes +~~~~~~~~~ + +* ``Fix KubernetesPodOperator validate xcom json and add retries (#32113)`` +* ``Fix 'KubernetesPodTrigger' waiting strategy (#31348)`` +* ``fix spark-kubernetes-operator compatibality (#31798)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` + 7.1.0 ..... diff --git a/airflow/providers/cncf/kubernetes/__init__.py b/airflow/providers/cncf/kubernetes/__init__.py index 3e3bd8ce3117..3235c02fd586 100644 --- a/airflow/providers/cncf/kubernetes/__init__.py +++ b/airflow/providers/cncf/kubernetes/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "7.1.0" +__version__ = "7.2.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 7375b7b35276..f65527ded795 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 7.2.0 - 7.1.0 - 7.0.0 - 6.1.0 diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst index 73400cd5c627..5710d8e39dda 100644 --- a/airflow/providers/common/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -24,6 +24,19 @@ Changelog for ``apache-airflow-providers-common-sql`` ----------------------------------------------------- +1.6.0 +..... + +Features +~~~~~~~~ + +* ``openlineage, common.sql: provide OL SQL parser as internal OpenLineage provider API (#31398)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` + 1.5.2 ..... diff --git a/airflow/providers/common/sql/__init__.py b/airflow/providers/common/sql/__init__.py index 58ca3b0bc20f..6180b06a7bca 100644 --- a/airflow/providers/common/sql/__init__.py +++ b/airflow/providers/common/sql/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "1.5.2" +__version__ = "1.6.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index dc2630336055..e1064b37f51f 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 1.6.0 - 1.5.2 - 1.5.1 - 1.5.0 diff --git a/airflow/providers/databricks/CHANGELOG.rst b/airflow/providers/databricks/CHANGELOG.rst index de5a5df4cc7c..21bf6efebcf9 100644 --- a/airflow/providers/databricks/CHANGELOG.rst +++ b/airflow/providers/databricks/CHANGELOG.rst @@ -24,6 +24,19 @@ Changelog for ``apache-airflow-providers-databricks`` ----------------------------------------------------- +4.3.1 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Modify 'template_fields' of 'DatabricksSqlOperator' to support parent class fields (#32253)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` + * ``Improve provider documentation and README structure (#32125)`` + 4.3.0 ..... diff --git a/airflow/providers/databricks/__init__.py b/airflow/providers/databricks/__init__.py index 78d3cd8c42cf..704af2e7674b 100644 --- a/airflow/providers/databricks/__init__.py +++ b/airflow/providers/databricks/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.3.0" +__version__ = "4.3.1" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 9aad0384eea0..87ca83c99e47 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.3.1 - 4.3.0 - 4.2.0 - 4.1.0 diff --git a/airflow/providers/datadog/.latest-doc-only-change.txt b/airflow/providers/datadog/.latest-doc-only-change.txt index ff7136e07d74..4ca04d3bd48b 100644 --- a/airflow/providers/datadog/.latest-doc-only-change.txt +++ b/airflow/providers/datadog/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/dbt/cloud/.latest-doc-only-change.txt b/airflow/providers/dbt/cloud/.latest-doc-only-change.txt index ab24993f5713..4ca04d3bd48b 100644 --- a/airflow/providers/dbt/cloud/.latest-doc-only-change.txt +++ b/airflow/providers/dbt/cloud/.latest-doc-only-change.txt @@ -1 +1 @@ -8b6b0848a3cacf9999477d6af4d2a87463f03026 +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/dingding/.latest-doc-only-change.txt b/airflow/providers/dingding/.latest-doc-only-change.txt index ff7136e07d74..4ca04d3bd48b 100644 --- a/airflow/providers/dingding/.latest-doc-only-change.txt +++ b/airflow/providers/dingding/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/discord/.latest-doc-only-change.txt b/airflow/providers/discord/.latest-doc-only-change.txt index ff7136e07d74..4ca04d3bd48b 100644 --- a/airflow/providers/discord/.latest-doc-only-change.txt +++ b/airflow/providers/discord/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/docker/.latest-doc-only-change.txt b/airflow/providers/docker/.latest-doc-only-change.txt index 9ab50d6e01d3..4ca04d3bd48b 100644 --- a/airflow/providers/docker/.latest-doc-only-change.txt +++ b/airflow/providers/docker/.latest-doc-only-change.txt @@ -1 +1 @@ -6933022e94acf139b2dea9a589bb8b25c62a5d20 +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index bb489325a6e4..c104ed62ece0 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -27,9 +27,17 @@ Changelog for ``apache-airflow-providers-elasticsearch`` 5.0.0 ..... +Breaking changes +~~~~~~~~~~~~~~~~ + .. note:: Deprecate non-official elasticsearch libraries. Only the official elasticsearch library was used +* ``Deprecate the 2 non-official elasticsearch libraries (#31920)`` + +.. Review and move the new changes to one of the sections above: + * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` + * ``Improve provider documentation and README structure (#32125)`` 4.5.1 ..... diff --git a/airflow/providers/elasticsearch/__init__.py b/airflow/providers/elasticsearch/__init__.py index f7905972eeea..b5638d95b1cb 100644 --- a/airflow/providers/elasticsearch/__init__.py +++ b/airflow/providers/elasticsearch/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.5.1" +__version__ = "5.0.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/exasol/.latest-doc-only-change.txt b/airflow/providers/exasol/.latest-doc-only-change.txt index e7c3c940c9c7..33a07c855e0f 100644 --- a/airflow/providers/exasol/.latest-doc-only-change.txt +++ b/airflow/providers/exasol/.latest-doc-only-change.txt @@ -1 +1 @@ -602abe8394fafe7de54df7e73af56de848cdf617 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/facebook/.latest-doc-only-change.txt b/airflow/providers/facebook/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/facebook/.latest-doc-only-change.txt +++ b/airflow/providers/facebook/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/ftp/.latest-doc-only-change.txt b/airflow/providers/ftp/.latest-doc-only-change.txt index ff7136e07d74..4ca04d3bd48b 100644 --- a/airflow/providers/ftp/.latest-doc-only-change.txt +++ b/airflow/providers/ftp/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/airflow/providers/github/.latest-doc-only-change.txt index ff7136e07d74..4ca04d3bd48b 100644 --- a/airflow/providers/github/.latest-doc-only-change.txt +++ b/airflow/providers/github/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +d1aa509bbd1941ceb3fe31789efeebbddd58d32f diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 7c39d83e3c8d..630467f4b9e5 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -23,6 +23,38 @@ Changelog for ``apache-airflow-providers-google`` ------------------------------------------------- +10.3.0 +...... + +Features +~~~~~~~~ + +* ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` +* ``Add deferrable mode to CloudSQLExportInstanceOperator (#30852)`` +* ``Adding 'src_fmt_configs' to the list of template fields. (#32097)`` + +Bug Fixes +~~~~~~~~~ + +* ``[Issue-32069] Fix name format in the batch requests (#32070)`` +* ``Fix 'BigQueryInsertJobOperator' error handling in deferrable mode (#32034)`` +* ``Fix 'BIGQUERY_JOB_DETAILS_LINK_FMT' in 'BigQueryConsoleLink' (#31953)`` +* ``Make the deferrable version of DataprocCreateBatchOperator handle a batch_id that already exists (#32216)`` + + +Misc +~~~~ + +* ``Switch Google Ads API version from v13 to v14 (#32028)`` +* ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` +* ``Refactor GKECreateClusterOperator's body validation (#31923)`` +* ``Optimize deferrable mode execution for 'BigQueryValueCheckOperator' (#31872)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Improve provider documentation and README structure (#32125)`` + * ``Google provider docstring improvements (#31731)`` + 10.2.0 ...... diff --git a/airflow/providers/google/__init__.py b/airflow/providers/google/__init__.py index 245255b39e61..0bf8186b4a75 100644 --- a/airflow/providers/google/__init__.py +++ b/airflow/providers/google/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "10.2.0" +__version__ = "10.3.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index f22ff420821d..02168a12c953 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -30,6 +30,7 @@ description: | suspended: false versions: + - 10.3.0 - 10.2.0 - 10.1.1 - 10.1.0 diff --git a/airflow/providers/grpc/.latest-doc-only-change.txt b/airflow/providers/grpc/.latest-doc-only-change.txt index 86f00ad558af..33a07c855e0f 100644 --- a/airflow/providers/grpc/.latest-doc-only-change.txt +++ b/airflow/providers/grpc/.latest-doc-only-change.txt @@ -1 +1 @@ -25bdbc8e6768712bad6043618242eec9c6632618 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/hashicorp/CHANGELOG.rst b/airflow/providers/hashicorp/CHANGELOG.rst index 88ca2da0256d..0b314faafb74 100644 --- a/airflow/providers/hashicorp/CHANGELOG.rst +++ b/airflow/providers/hashicorp/CHANGELOG.rst @@ -24,6 +24,18 @@ Changelog for ``apache-airflow-providers-hashicorp`` ---------------------------------------------------- +3.4.2 +..... + +Misc +~~~~ + +* ``Add default retry to hvac client requests (#31073)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Improve provider documentation and README structure (#32125)`` + 3.4.1 ..... diff --git a/airflow/providers/hashicorp/__init__.py b/airflow/providers/hashicorp/__init__.py index f58108903de8..d9d0ceaadc35 100644 --- a/airflow/providers/hashicorp/__init__.py +++ b/airflow/providers/hashicorp/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "3.4.1" +__version__ = "3.4.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index 2faa1009a245..46399874d9c5 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 3.4.2 - 3.4.1 - 3.4.0 - 3.3.1 diff --git a/airflow/providers/http/.latest-doc-only-change.txt b/airflow/providers/http/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/http/.latest-doc-only-change.txt +++ b/airflow/providers/http/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/imap/.latest-doc-only-change.txt b/airflow/providers/imap/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/imap/.latest-doc-only-change.txt +++ b/airflow/providers/imap/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/influxdb/.latest-doc-only-change.txt b/airflow/providers/influxdb/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/influxdb/.latest-doc-only-change.txt +++ b/airflow/providers/influxdb/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/jdbc/.latest-doc-only-change.txt b/airflow/providers/jdbc/.latest-doc-only-change.txt index ab24993f5713..33a07c855e0f 100644 --- a/airflow/providers/jdbc/.latest-doc-only-change.txt +++ b/airflow/providers/jdbc/.latest-doc-only-change.txt @@ -1 +1 @@ -8b6b0848a3cacf9999477d6af4d2a87463f03026 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/jenkins/.latest-doc-only-change.txt b/airflow/providers/jenkins/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/jenkins/.latest-doc-only-change.txt +++ b/airflow/providers/jenkins/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index 87424de9b624..223aea264997 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -24,6 +24,33 @@ Changelog for ``apache-airflow-providers-microsoft-azure`` ---------------------------------------------------------- +6.2.0 +..... + +Features +~~~~~~~~ + +* ``Adds connection test for ADLS Gen2 (#32126)`` +* ``Add option to pass extra configs to ClientSecretCredential (#31783)`` +* ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` + +Bug Fixes +~~~~~~~~~ + +* ``Cancel pipeline if unexpected exception caught (#32238)`` +* ``Fix where account url is build if not provided using login (account name) (#32082)`` + +Misc +~~~~ + +* ``Doc changes: Added Transfers section in Azure provider docs (#32241)`` +* ``Adds Sensor section in the Azure providers docs (#32299)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Improve provider documentation and README structure (#32125)`` + * ``invalid args fix (#32326)`` + 6.1.2 ..... diff --git a/airflow/providers/microsoft/azure/__init__.py b/airflow/providers/microsoft/azure/__init__.py index 6a11fab18c2f..40f553f79fb7 100644 --- a/airflow/providers/microsoft/azure/__init__.py +++ b/airflow/providers/microsoft/azure/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "6.1.2" +__version__ = "6.2.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index c255da931522..a1070177bc11 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -22,6 +22,7 @@ description: | `Microsoft Azure `__ suspended: false versions: + - 6.2.0 - 6.1.2 - 6.1.1 - 6.1.0 diff --git a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt index e7c3c940c9c7..33a07c855e0f 100644 --- a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt @@ -1 +1 @@ -602abe8394fafe7de54df7e73af56de848cdf617 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt b/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/mongo/.latest-doc-only-change.txt b/airflow/providers/mongo/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/mongo/.latest-doc-only-change.txt +++ b/airflow/providers/mongo/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/mysql/.latest-doc-only-change.txt b/airflow/providers/mysql/.latest-doc-only-change.txt index 13020f96f448..33a07c855e0f 100644 --- a/airflow/providers/mysql/.latest-doc-only-change.txt +++ b/airflow/providers/mysql/.latest-doc-only-change.txt @@ -1 +1 @@ -df00436569bb6fb79ce8c0b7ca71dddf02b854ef +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/neo4j/.latest-doc-only-change.txt b/airflow/providers/neo4j/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/neo4j/.latest-doc-only-change.txt +++ b/airflow/providers/neo4j/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/odbc/.latest-doc-only-change.txt b/airflow/providers/odbc/.latest-doc-only-change.txt index e7e8156d80b9..33a07c855e0f 100644 --- a/airflow/providers/odbc/.latest-doc-only-change.txt +++ b/airflow/providers/odbc/.latest-doc-only-change.txt @@ -1 +1 @@ -b916b7507921129dc48d6add1bdc4b923b60c9b9 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/openfaas/.latest-doc-only-change.txt b/airflow/providers/openfaas/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/openfaas/.latest-doc-only-change.txt +++ b/airflow/providers/openfaas/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/opsgenie/.latest-doc-only-change.txt b/airflow/providers/opsgenie/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/opsgenie/.latest-doc-only-change.txt +++ b/airflow/providers/opsgenie/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/oracle/.latest-doc-only-change.txt b/airflow/providers/oracle/.latest-doc-only-change.txt index ab24993f5713..a780a5d07a3c 100644 --- a/airflow/providers/oracle/.latest-doc-only-change.txt +++ b/airflow/providers/oracle/.latest-doc-only-change.txt @@ -1 +1 @@ -8b6b0848a3cacf9999477d6af4d2a87463f03026 +2794c4172e7b7655b813236fe47222cec46f57e4 diff --git a/airflow/providers/pagerduty/.latest-doc-only-change.txt b/airflow/providers/pagerduty/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/pagerduty/.latest-doc-only-change.txt +++ b/airflow/providers/pagerduty/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/papermill/.latest-doc-only-change.txt b/airflow/providers/papermill/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/papermill/.latest-doc-only-change.txt +++ b/airflow/providers/papermill/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/plexus/.latest-doc-only-change.txt b/airflow/providers/plexus/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/plexus/.latest-doc-only-change.txt +++ b/airflow/providers/plexus/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/postgres/.latest-doc-only-change.txt b/airflow/providers/postgres/.latest-doc-only-change.txt index 570fad6daee2..33a07c855e0f 100644 --- a/airflow/providers/postgres/.latest-doc-only-change.txt +++ b/airflow/providers/postgres/.latest-doc-only-change.txt @@ -1 +1 @@ -97496ba2b41063fa24393c58c5c648a0cdb5a7f8 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/presto/.latest-doc-only-change.txt b/airflow/providers/presto/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/presto/.latest-doc-only-change.txt +++ b/airflow/providers/presto/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/qubole/.latest-doc-only-change.txt b/airflow/providers/qubole/.latest-doc-only-change.txt index ab24993f5713..33a07c855e0f 100644 --- a/airflow/providers/qubole/.latest-doc-only-change.txt +++ b/airflow/providers/qubole/.latest-doc-only-change.txt @@ -1 +1 @@ -8b6b0848a3cacf9999477d6af4d2a87463f03026 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/redis/.latest-doc-only-change.txt b/airflow/providers/redis/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/redis/.latest-doc-only-change.txt +++ b/airflow/providers/redis/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/salesforce/.latest-doc-only-change.txt b/airflow/providers/salesforce/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/salesforce/.latest-doc-only-change.txt +++ b/airflow/providers/salesforce/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/samba/.latest-doc-only-change.txt b/airflow/providers/samba/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/samba/.latest-doc-only-change.txt +++ b/airflow/providers/samba/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/segment/.latest-doc-only-change.txt b/airflow/providers/segment/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/segment/.latest-doc-only-change.txt +++ b/airflow/providers/segment/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/sendgrid/.latest-doc-only-change.txt b/airflow/providers/sendgrid/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/sendgrid/.latest-doc-only-change.txt +++ b/airflow/providers/sendgrid/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/sftp/.latest-doc-only-change.txt b/airflow/providers/sftp/.latest-doc-only-change.txt index 28124098645c..33a07c855e0f 100644 --- a/airflow/providers/sftp/.latest-doc-only-change.txt +++ b/airflow/providers/sftp/.latest-doc-only-change.txt @@ -1 +1 @@ -6c3a67d4fccafe4ab6cd9ec8c7bacf2677f17038 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/singularity/.latest-doc-only-change.txt b/airflow/providers/singularity/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/singularity/.latest-doc-only-change.txt +++ b/airflow/providers/singularity/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/slack/.latest-doc-only-change.txt b/airflow/providers/slack/.latest-doc-only-change.txt index d947e4964a38..33a07c855e0f 100644 --- a/airflow/providers/slack/.latest-doc-only-change.txt +++ b/airflow/providers/slack/.latest-doc-only-change.txt @@ -1 +1 @@ -b5b1fae2dfe92751d6aaaace00009ce29625095b +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/snowflake/.latest-doc-only-change.txt b/airflow/providers/snowflake/.latest-doc-only-change.txt index 5bda81758205..33a07c855e0f 100644 --- a/airflow/providers/snowflake/.latest-doc-only-change.txt +++ b/airflow/providers/snowflake/.latest-doc-only-change.txt @@ -1 +1 @@ -814e471d137aad68bd64a21d20736e7b88403f97 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/sqlite/.latest-doc-only-change.txt b/airflow/providers/sqlite/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/sqlite/.latest-doc-only-change.txt +++ b/airflow/providers/sqlite/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/ssh/.latest-doc-only-change.txt b/airflow/providers/ssh/.latest-doc-only-change.txt index 570fad6daee2..33a07c855e0f 100644 --- a/airflow/providers/ssh/.latest-doc-only-change.txt +++ b/airflow/providers/ssh/.latest-doc-only-change.txt @@ -1 +1 @@ -97496ba2b41063fa24393c58c5c648a0cdb5a7f8 +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/tableau/.latest-doc-only-change.txt b/airflow/providers/tableau/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/tableau/.latest-doc-only-change.txt +++ b/airflow/providers/tableau/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/tabular/.latest-doc-only-change.txt b/airflow/providers/tabular/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/tabular/.latest-doc-only-change.txt +++ b/airflow/providers/tabular/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/telegram/.latest-doc-only-change.txt b/airflow/providers/telegram/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/telegram/.latest-doc-only-change.txt +++ b/airflow/providers/telegram/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/trino/CHANGELOG.rst b/airflow/providers/trino/CHANGELOG.rst index 7e5b05538ebc..da3fd0695c17 100644 --- a/airflow/providers/trino/CHANGELOG.rst +++ b/airflow/providers/trino/CHANGELOG.rst @@ -24,6 +24,18 @@ Changelog for ``apache-airflow-providers-trino`` ------------------------------------------------ +5.2.0 +..... + +Features +~~~~~~~~ + +* ``Trino Hook: Add ability to read JWT from file (#31950)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Improve provider documentation and README structure (#32125)`` + 5.1.1 ..... diff --git a/airflow/providers/trino/__init__.py b/airflow/providers/trino/__init__.py index 37cb07d1a46e..7ecb43088fc6 100644 --- a/airflow/providers/trino/__init__.py +++ b/airflow/providers/trino/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "5.1.1" +__version__ = "5.2.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index 6c4c61a63ffe..97fe20574b11 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 5.2.0 - 5.1.1 - 5.1.0 - 5.0.0 diff --git a/airflow/providers/vertica/CHANGELOG.rst b/airflow/providers/vertica/CHANGELOG.rst index db9a5335b85c..36e06b2cf5a6 100644 --- a/airflow/providers/vertica/CHANGELOG.rst +++ b/airflow/providers/vertica/CHANGELOG.rst @@ -24,6 +24,18 @@ Changelog for ``apache-airflow-providers-vertica`` -------------------------------------------------- +3.5.0 +..... + +Features +~~~~~~~~ + +* ``Add various Vertica connection parameters (#32089)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Improve provider documentation and README structure (#32125)`` + 3.4.1 ..... diff --git a/airflow/providers/vertica/__init__.py b/airflow/providers/vertica/__init__.py index feba3e282653..94565d1472c8 100644 --- a/airflow/providers/vertica/__init__.py +++ b/airflow/providers/vertica/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "3.4.1" +__version__ = "3.5.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 90498fa1a1cd..d9f30e37b014 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 3.5.0 - 3.4.1 - 3.4.0 - 3.3.1 diff --git a/airflow/providers/zendesk/.latest-doc-only-change.txt b/airflow/providers/zendesk/.latest-doc-only-change.txt index ff7136e07d74..33a07c855e0f 100644 --- a/airflow/providers/zendesk/.latest-doc-only-change.txt +++ b/airflow/providers/zendesk/.latest-doc-only-change.txt @@ -1 +1 @@ -06acf40a4337759797f666d5bb27a5a393b74fed +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 8bf6e24852dc..cde0918381cf 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -2014,7 +2014,12 @@ def generate_new_changelog(package_id, provider_details, changelog_path, changes template_name="UPDATE_CHANGELOG", context=context, extension=".rst" ) else: - classified_changes = get_changes_classified(changes[0]) + if changes: + classified_changes = get_changes_classified(changes[0]) + else: + # change log exist but without version 1.0.0 entry + classified_changes = None + context = { "version": latest_version, "version_header": "." * len(latest_version), diff --git a/docs/apache-airflow-providers-alibaba/commits.rst b/docs/apache-airflow-providers-alibaba/commits.rst index f01e83594ba2..bb9559028c2a 100644 --- a/docs/apache-airflow-providers-alibaba/commits.rst +++ b/docs/apache-airflow-providers-alibaba/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-28 ``D205 Support - Providers: Airbyte and Alibaba (#32214)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`f05beb7219 `_ 2023-06-27 ``Add Alibaba Cloud AnalyticDB Spark Support (#31787)`` +================================================================================================= =========== ================================================================ + 2.4.1 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` `9fa75aaf7a `_ 2023-05-29 ``Remove Python 3.7 support (#30963)`` diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/docs/apache-airflow-providers-alibaba/index.rst index 2b468623ff99..2c8f671b549a 100644 --- a/docs/apache-airflow-providers-alibaba/index.rst +++ b/docs/apache-airflow-providers-alibaba/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-alibaba Alibaba Cloud integration (including `Alibaba Cloud `__). -Release: 2.4.1 +Release: 2.5.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index 94b8c2bbfa60..fdceb0af6634 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -28,6 +28,48 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-04 ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` +`5c72befcfd `_ 2023-07-03 ``Fix 'LambdaInvokeFunctionOperator' payload parameter type (#32259)`` +`d6e254db68 `_ 2023-06-30 ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` +`dd937e51fe `_ 2023-06-30 ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` +`e781aef1a7 `_ 2023-06-29 ``aws waiter util: log status info with error level on waiter error (#32247)`` +`a46b942f48 `_ 2023-06-29 ``Add deferrable mode to S3KeysUnchangedSensor (#31940)`` +`b1b69af88f `_ 2023-06-28 ``Add deferrable mode to 'RdsCreateDbInstanceOperator' and 'RdsDeleteDbInstanceOperator' (#32171)`` +`d029e043f4 `_ 2023-06-28 ``Add 'ChimeNotifier' (#32222)`` +`1d60332cf8 `_ 2023-06-28 ``Add an option to 'GlueJobOperator' to stop the job run when the TI is killed (#32155)`` +`bd2f156bc8 `_ 2023-06-28 ``D205 Support - Providers: Amazon/AWS (#32224)`` +`25526a112f `_ 2023-06-28 ``Add deferrable mode for 'AthenaOperator' (#32186)`` +`1fb2831239 `_ 2023-06-28 ``Add a deferrable mode to 'BatchCreateComputeEnvironmentOperator' (#32036)`` +`14785bc84c `_ 2023-06-27 ``deferrable mode for 'SageMakerTuningOperator' and 'SageMakerEndpointOperator' (#32112)`` +`2794c4172e `_ 2023-06-27 ``D205 Support - Auto-fixes and Stragglers (#32212)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`06b5a1e850 `_ 2023-06-27 ``Add deferrable mode in EMR operator and sensor (#32029)`` +`3a85d4e7e8 `_ 2023-06-27 ``Revert "add deferrable mode for 'AthenaOperator' (#32110)" (#32172)`` +`256438c3d6 `_ 2023-06-27 ``add deferrable mode for 'AthenaOperator' (#32110)`` +`e0f4de347f `_ 2023-06-27 ``EKS Create/Delete Nodegroup Deferrable mode (#32165)`` +`10df7436f3 `_ 2023-06-27 ``Add 'ChimeWebhookHook' (#31939)`` +`e4eb19866e `_ 2023-06-26 ``rewrite method used in ecs to fetch less logs (#31786)`` +`2f96127b9e `_ 2023-06-26 ``Minor name change for the util wait method. (#32152)`` +`cc87ae578e `_ 2023-06-26 ``feature: AWS - GlueJobOperator - job_poll_interval (#32147)`` +`e4468ed380 `_ 2023-06-26 ``Clean up string concatenation (#32129)`` +`8057659e96 `_ 2023-06-25 ``Handle 'UnboundLocalError' while parsing invalid 's3_url' (#32120)`` +`d49fa999a9 `_ 2023-06-23 ``bugfix: break down run+wait method in ECS operator (#32104)`` +`72d09a677f `_ 2023-06-23 ``Use a waiter in 'AthenaHook' (#31942)`` +`5c887988b0 `_ 2023-06-23 ``Refactor Eks Create Cluster Operator code (#31960)`` +`415e076761 `_ 2023-06-23 ``Deferrable mode for ECS operators (#31881)`` +`479719297f `_ 2023-06-22 ``add async wait method to the "with logging" aws utils (#32055)`` +`afe293d934 `_ 2023-06-21 ``cleanup Amazon CHANGELOG.rst (#32031)`` +`743bf5a0ae `_ 2023-06-20 ``Add custom waiters to EMR Serverless (#30463)`` +================================================================================================= =========== ============================================================================================================ + 8.2.0 ..... @@ -36,6 +78,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== =============================================================================================================================== Commit Committed Subject ================================================================================================= =========== =============================================================================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `f0b91ac6a7 `_ 2023-06-20 ``Add 'deferrable' param in 'EmrContainerSensor' (#30945)`` `e01ff4749c `_ 2023-06-19 ``Add realtime container execution logs for BatchOperator (#31837)`` diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index 85d5542f8c00..b018d27cb0a7 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -82,7 +82,7 @@ Package apache-airflow-providers-amazon Amazon integration (including `Amazon Web Services (AWS) `__). -Release: 8.2.0 +Release: 8.3.0 Provider package ---------------- @@ -107,6 +107,7 @@ PIP package Version required ======================================= ================== ``apache-airflow`` ``>=2.4.0`` ``apache-airflow-providers-common-sql`` ``>=1.3.1`` +``apache-airflow-providers-http`` ``boto3`` ``>=1.24.0`` ``asgiref`` ``watchtower`` ``~=2.0.1`` @@ -144,6 +145,7 @@ Dependent package `apache-airflow-providers-google `_ ``google`` `apache-airflow-providers-http `_ ``http`` `apache-airflow-providers-imap `_ ``imap`` +`apache-airflow-providers-microsoft-azure `_ ``microsoft.azure`` `apache-airflow-providers-mongo `_ ``mongo`` `apache-airflow-providers-salesforce `_ ``salesforce`` `apache-airflow-providers-ssh `_ ``ssh`` @@ -155,5 +157,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-amazon 8.2.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-amazon 8.2.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 8.3.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 8.3.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apache-kafka/commits.rst b/docs/apache-airflow-providers-apache-kafka/commits.rst index e15017bf58ee..94450c54b479 100644 --- a/docs/apache-airflow-providers-apache-kafka/commits.rst +++ b/docs/apache-airflow-providers-apache-kafka/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-30 ``Break AwaitMessageTrigger execution when finding a message with the desired format (#31803)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== =============================================================================================== + 1.1.1 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ==================================================================== Commit Committed Subject ================================================================================================= =========== ==================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `547f5846bf `_ 2023-06-20 ``Add note about dropping Python 3.7 for kafka and impala (#32017)`` `dc5bf3fd02 `_ 2023-06-02 ``Add discoverability for triggers in provider.yaml (#31576)`` `a473facf6c `_ 2023-06-01 ``Add D400 pydocstyle check - Apache providers only (#31424)`` diff --git a/docs/apache-airflow-providers-apache-kafka/index.rst b/docs/apache-airflow-providers-apache-kafka/index.rst index 2c56ce00460f..d3d7a8f66b34 100644 --- a/docs/apache-airflow-providers-apache-kafka/index.rst +++ b/docs/apache-airflow-providers-apache-kafka/index.rst @@ -82,7 +82,7 @@ Package apache-airflow-providers-apache-kafka `Apache Kafka `__ -Release: 1.1.1 +Release: 1.1.2 Provider package ---------------- diff --git a/docs/apache-airflow-providers-apache-livy/commits.rst b/docs/apache-airflow-providers-apache-livy/commits.rst index 3580de1bbf5b..f3685d6d9363 100644 --- a/docs/apache-airflow-providers-apache-livy/commits.rst +++ b/docs/apache-airflow-providers-apache-livy/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`58784ade88 `_ 2023-06-27 ``Fix 'LivyHook' TypeError exception on 'session_id' log format (#32051)`` +================================================================================================= =========== ========================================================================== + 3.5.1 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== =========================================================================== Commit Committed Subject ================================================================================================= =========== =========================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `13890788ae `_ 2023-06-07 ``Apache provider docstring improvements (#31730)`` `361aa77d5d `_ 2023-06-05 ``Optimize deferred mode execution (#31685)`` diff --git a/docs/apache-airflow-providers-apache-livy/index.rst b/docs/apache-airflow-providers-apache-livy/index.rst index 4db63c3ae908..6b7704c37023 100644 --- a/docs/apache-airflow-providers-apache-livy/index.rst +++ b/docs/apache-airflow-providers-apache-livy/index.rst @@ -75,7 +75,7 @@ Package apache-airflow-providers-apache-livy `Apache Livy `__ -Release: 3.5.1 +Release: 3.5.2 Provider package ---------------- @@ -129,5 +129,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-livy 3.5.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-livy 3.5.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-livy 3.5.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-livy 3.5.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apprise/commits.rst b/docs/apache-airflow-providers-apprise/commits.rst index 9514a7d2d7be..af71b748389e 100644 --- a/docs/apache-airflow-providers-apprise/commits.rst +++ b/docs/apache-airflow-providers-apprise/commits.rst @@ -1,3 +1,4 @@ + .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -15,7 +16,27 @@ specific language governing permissions and limitations under the License. + Package apache-airflow-providers-apprise ----------------------------------------- +------------------------------------------------------ `Apprise `__ + + +This is detailed commit list of changes for versions provider package: ``apprise``. +For high-level changelog, see :doc:`package information including changelog `. + + + +1.0.0 +..... + +Latest change: 2023-06-28 + +================================================================================================= =========== =================================================================== +Commit Committed Subject +================================================================================================= =========== =================================================================== +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`f4c4b77486 `_ 2023-06-21 ``Add Apprise Provider (#31533)`` +================================================================================================= =========== =================================================================== diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst index d0be3b36022e..070badf99a5a 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst @@ -28,6 +28,22 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-01 ``Fix KubernetesPodOperator validate xcom json and add retries (#32113)`` +`dd937e51fe `_ 2023-06-30 ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` +`8f5de83ee6 `_ 2023-06-29 ``Fix 'KubernetesPodTrigger' waiting strategy (#31348)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`6693bdd72d `_ 2023-06-27 ``fix spark-kubernetes-operator compatibality (#31798)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ========================================================================= + 7.1.0 ..... @@ -36,6 +52,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================================================================= Commit Committed Subject ================================================================================================= =========== ============================================================================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `66299338eb `_ 2023-06-18 ``add a return when the event is yielded in a loop to stop the execution (#31985)`` `07ea574fed `_ 2023-06-07 ``Add D400 pydocstyle check (#31742)`` diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index f511ea5ecb58..d087d1d75759 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-cncf-kubernetes `Kubernetes `__ -Release: 7.1.0 +Release: 7.2.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-common-sql/commits.rst b/docs/apache-airflow-providers-common-sql/commits.rst index 385dfa45462e..d09b7c398104 100644 --- a/docs/apache-airflow-providers-common-sql/commits.rst +++ b/docs/apache-airflow-providers-common-sql/commits.rst @@ -28,18 +28,32 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-29 ``openlineage, common.sql: provide OL SQL parser as internal OpenLineage provider API (#31398)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================================================= + 1.5.2 ..... -Latest change: 2023-06-05 +Latest change: 2023-06-20 -================================================================================================= =========== ================================================== +================================================================================================= =========== ============================================================= Commit Committed Subject -================================================================================================= =========== ================================================== +================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` `9fa75aaf7a `_ 2023-05-29 ``Remove Python 3.7 support (#30963)`` -================================================================================================= =========== ================================================== +================================================================================================= =========== ============================================================= 1.5.1 ..... diff --git a/docs/apache-airflow-providers-common-sql/index.rst b/docs/apache-airflow-providers-common-sql/index.rst index 027c9a436d3c..0e68b27f8957 100644 --- a/docs/apache-airflow-providers-common-sql/index.rst +++ b/docs/apache-airflow-providers-common-sql/index.rst @@ -76,7 +76,7 @@ Package apache-airflow-providers-common-sql `Common SQL Provider `__ -Release: 1.5.2 +Release: 1.6.0 Provider package ---------------- @@ -102,3 +102,31 @@ PIP package Version required ``apache-airflow`` ``>=2.4.0`` ``sqlparse`` ``>=0.4.2`` ================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-common-sql[openlineage] + + +============================================================================================================== =============== +Dependent package Extra +============================================================================================================== =============== +`apache-airflow-providers-openlineage `_ ``openlineage`` +============================================================================================================== =============== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-common-sql 1.6.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-sql 1.6.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-databricks/commits.rst b/docs/apache-airflow-providers-databricks/commits.rst index bfc05853edbd..b0df4f06c724 100644 --- a/docs/apache-airflow-providers-databricks/commits.rst +++ b/docs/apache-airflow-providers-databricks/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-29 ``Modify 'template_fields' of 'DatabricksSqlOperator' to support parent class fields (#32253)`` +`d1aa509bbd `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== =============================================================================================== + 4.3.0 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== =========================================================================================== Commit Committed Subject ================================================================================================= =========== =========================================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `69bc90b824 `_ 2023-06-19 ``Fix type annotation (#31888)`` `66299338eb `_ 2023-06-18 ``add a return when the event is yielded in a loop to stop the execution (#31985)`` diff --git a/docs/apache-airflow-providers-databricks/index.rst b/docs/apache-airflow-providers-databricks/index.rst index c04b6d656c46..3d9e3672ec69 100644 --- a/docs/apache-airflow-providers-databricks/index.rst +++ b/docs/apache-airflow-providers-databricks/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-databricks `Databricks `__ -Release: 4.3.0 +Release: 4.3.1 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-databricks 4.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-databricks 4.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 4.3.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 4.3.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-elasticsearch/commits.rst b/docs/apache-airflow-providers-elasticsearch/commits.rst index 48b8cdbeefb8..774d5ea55f50 100644 --- a/docs/apache-airflow-providers-elasticsearch/commits.rst +++ b/docs/apache-airflow-providers-elasticsearch/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`b28c90354f `_ 2023-06-24 ``Deprecate the 2 non-official elasticsearch libraries (#31920)`` +================================================================================================= =========== ======================================================================= + 4.5.1 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` diff --git a/docs/apache-airflow-providers-google/commits.rst b/docs/apache-airflow-providers-google/commits.rst index 22631f117d14..d69b4d009e78 100644 --- a/docs/apache-airflow-providers-google/commits.rst +++ b/docs/apache-airflow-providers-google/commits.rst @@ -35,6 +35,29 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-30 ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` +`dd937e51fe `_ 2023-06-30 ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` +`c0eaa9b25d `_ 2023-06-29 ``Add deferrable mode to CloudSQLExportInstanceOperator (#30852)`` +`f3f69bf1e0 `_ 2023-06-29 ``Refactor GKECreateClusterOperator's body validation (#31923)`` +`7d2ec76c72 `_ 2023-06-29 ``Make the deferrable version of DataprocCreateBatchOperator handle a batch_id that already exists (#32216)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`b156db3a70 `_ 2023-06-23 ``Adding 'src_fmt_configs' to the list of template fields. (#32097)`` +`59d64d8f2e `_ 2023-06-22 ``[Issue-32069] Fix name format in the batch requests (#32070)`` +`fd116cc196 `_ 2023-06-22 ``Google provider docstring improvements (#31731)`` +`8f41584777 `_ 2023-06-21 ``Optimize deferrable mode execution for 'BigQueryValueCheckOperator' (#31872)`` +`fe7a1aa096 `_ 2023-06-21 ``Switch Google Ads API version from v13 to v14 (#32028)`` +`43fa157e51 `_ 2023-06-20 ``Fix 'BigQueryInsertJobOperator' error handling in deferrable mode (#32034)`` +`2a79fb74fd `_ 2023-06-20 ``Fix 'BIGQUERY_JOB_DETAILS_LINK_FMT' in 'BigQueryConsoleLink' (#31953)`` +================================================================================================= =========== ============================================================================================================= + 10.2.0 ...... @@ -43,6 +66,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ======================================================================================================== Commit Committed Subject ================================================================================================= =========== ======================================================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `f2ebc292fe `_ 2023-06-19 ``Provide missing project id and creds for TabularDataset (#31991)`` `66299338eb `_ 2023-06-18 ``add a return when the event is yielded in a loop to stop the execution (#31985)`` diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 5e322f9d0054..6721dfc0399e 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -88,7 +88,7 @@ Google services including: - `Google Workspace `__ (formerly Google Suite) -Release: 10.2.0 +Release: 10.3.0 Provider package ---------------- @@ -206,5 +206,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-google 10.2.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-google 10.2.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.3.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.3.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-hashicorp/commits.rst b/docs/apache-airflow-providers-hashicorp/commits.rst index e6bfc9a669d4..8dff6dbb5bc4 100644 --- a/docs/apache-airflow-providers-hashicorp/commits.rst +++ b/docs/apache-airflow-providers-hashicorp/commits.rst @@ -28,6 +28,18 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`d136457686 `_ 2023-06-20 ``Add default retry to hvac client requests (#31073)`` +================================================================================================= =========== ================================================================ + 3.4.1 ..... @@ -36,6 +48,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` `41ea700cbd `_ 2023-05-30 ``Fix hashicorp some client authentication methods (#31593)`` diff --git a/docs/apache-airflow-providers-hashicorp/index.rst b/docs/apache-airflow-providers-hashicorp/index.rst index a1b4007398a7..36e122c58df5 100644 --- a/docs/apache-airflow-providers-hashicorp/index.rst +++ b/docs/apache-airflow-providers-hashicorp/index.rst @@ -69,7 +69,7 @@ Package apache-airflow-providers-hashicorp Hashicorp including `Hashicorp Vault `__ -Release: 3.4.1 +Release: 3.4.2 Provider package ---------------- @@ -121,5 +121,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-hashicorp 3.4.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-hashicorp 3.4.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-hashicorp 3.4.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-hashicorp 3.4.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-microsoft-azure/commits.rst b/docs/apache-airflow-providers-microsoft-azure/commits.rst index a812135ab721..04df0a44ec97 100644 --- a/docs/apache-airflow-providers-microsoft-azure/commits.rst +++ b/docs/apache-airflow-providers-microsoft-azure/commits.rst @@ -28,6 +28,25 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-04 ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` +`985a6cd2a8 `_ 2023-07-03 ``invalid args fix (#32326)`` +`c03d7b7c33 `_ 2023-07-03 ``Adds Sensor section in the Azure providers docs (#32299)`` +`2ce51ac5d3 `_ 2023-06-29 ``Cancel pipeline if unexpected exception caught (#32238)`` +`57b7ba16a3 `_ 2023-06-29 ``Doc changes: Added Transfers section in Azure provider docs (#32241)`` +`46ee1c2c8d `_ 2023-06-28 ``Fix where account url is build if not provided using login (account name) (#32082)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`dc9b516494 `_ 2023-06-26 ``Adds connection test for ADLS Gen2 (#32126)`` +`c508b8e531 `_ 2023-06-21 ``Add option to pass extra configs to ClientSecretCredential (#31783)`` +================================================================================================= =========== ====================================================================================== + 6.1.2 ..... @@ -36,6 +55,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ====================================================================== Commit Committed Subject ================================================================================================= =========== ====================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `4266a545b0 `_ 2023-06-20 ``Microsoft provider docstring improvements (#31708)`` `94128303e1 `_ 2023-06-08 ``Removed unused variables in AzureBlobStorageToGCSOperator (#31765)`` diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index a10c5017314a..c2ee6d69df49 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -81,7 +81,7 @@ Package apache-airflow-providers-microsoft-azure `Microsoft Azure `__ -Release: 6.1.2 +Release: 6.2.0 Provider package ---------------- @@ -151,5 +151,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-microsoft-azure 6.1.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-microsoft-azure 6.1.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 6.2.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 6.2.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-trino/commits.rst b/docs/apache-airflow-providers-trino/commits.rst index 83c13556851c..3ddb3cd8ef9c 100644 --- a/docs/apache-airflow-providers-trino/commits.rst +++ b/docs/apache-airflow-providers-trino/commits.rst @@ -28,6 +28,18 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +`371833e076 `_ 2023-06-24 ``Trino Hook: Add ability to read JWT from file (#31950)`` +================================================================================================= =========== ================================================================ + 5.1.1 ..... @@ -36,6 +48,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` ================================================================================================= =========== ============================================================= diff --git a/docs/apache-airflow-providers-trino/index.rst b/docs/apache-airflow-providers-trino/index.rst index ebd3b2e0cd81..ea550a64b0ff 100644 --- a/docs/apache-airflow-providers-trino/index.rst +++ b/docs/apache-airflow-providers-trino/index.rst @@ -78,7 +78,7 @@ Package apache-airflow-providers-trino `Trino `__ -Release: 5.1.1 +Release: 5.2.0 Provider package ---------------- @@ -133,5 +133,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-trino 5.1.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-trino 5.1.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-trino 5.2.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-trino 5.2.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-vertica/commits.rst b/docs/apache-airflow-providers-vertica/commits.rst index 35f09c68892c..489949baab86 100644 --- a/docs/apache-airflow-providers-vertica/commits.rst +++ b/docs/apache-airflow-providers-vertica/commits.rst @@ -28,6 +28,18 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-06-30 ``Add various Vertica connection parameters (#32089)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================ + 3.4.1 ..... @@ -36,6 +48,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` ================================================================================================= =========== ============================================================= diff --git a/docs/apache-airflow-providers-vertica/index.rst b/docs/apache-airflow-providers-vertica/index.rst index ae7c2457b811..78aa37666e11 100644 --- a/docs/apache-airflow-providers-vertica/index.rst +++ b/docs/apache-airflow-providers-vertica/index.rst @@ -68,7 +68,7 @@ Package apache-airflow-providers-vertica `Vertica `__ -Release: 3.4.1 +Release: 3.5.0 Provider package ---------------- @@ -121,5 +121,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-vertica 3.4.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-vertica 3.4.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-vertica 3.5.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-vertica 3.5.0 wheel package `_ (`asc `__, `sha512 `__) From 43f3e57bf162293b92154f16a8ce33e6922fbf4e Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 5 Jul 2023 13:10:32 +0530 Subject: [PATCH 053/533] Update index.rst doc to correct grammar (#32315) * Update index.rst doc to correct grammar * Address @uranusjr's comments --- docs/apache-airflow/index.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index feaf2d388b50..adc5f47eff59 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -98,14 +98,22 @@ Rich scheduling and execution semantics enable you to easily define complex pipe intervals. Backfilling allows you to (re-)run pipelines on historical data after making changes to your logic. And the ability to rerun partial pipelines after resolving an error helps maximize efficiency. -Airflow's user interface provides both in-depth views of pipelines and individual tasks, and an overview of -pipelines over time. From the interface, you can inspect logs and manage tasks, for example retrying a task in +Airflow's user interface provides: + + 1. In-depth views of two things: + + i. Pipelines + ii. Tasks + + 2. Overview of your pipelines over time + +From the interface, you can inspect logs and manage tasks, for example retrying a task in case of failure. The open-source nature of Airflow ensures you work on components developed, tested, and used by many other `companies `_ around the world. In the active `community `_ you can find plenty of helpful resources in the form of -blogs posts, articles, conferences, books, and more. You can connect with other peers via several channels +blog posts, articles, conferences, books, and more. You can connect with other peers via several channels such as `Slack `_ and mailing lists. Airflow as a Platform is highly customizable. By utilizing :doc:`public-airflow-interface` you can extend @@ -115,7 +123,7 @@ Why not Airflow™? ================= Airflow™ was built for finite batch workflows. While the CLI and REST API do allow triggering workflows, -Airflow was not built for infinitely-running event-based workflows. Airflow is not a streaming solution. +Airflow was not built for infinitely running event-based workflows. Airflow is not a streaming solution. However, a streaming system such as Apache Kafka is often seen working together with Apache Airflow. Kafka can be used for ingestion and processing in real-time, event data is written to a storage location, and Airflow periodically starts a workflow processing a batch of data. From a8e4b8aee602e8c672ab879b7392a65b5c2bb34e Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:00:51 +0530 Subject: [PATCH 054/533] Fix try_number shown in the `task?task_id=&dag_id=&execution_date=` (#32361) * try_number fix * change test name --- airflow/www/views.py | 3 ++- tests/www/views/test_views_tasks.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/airflow/www/views.py b/airflow/www/views.py index 3f0965da3889..d7059156f575 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -1801,7 +1801,8 @@ def task(self, session: Session = NEW_SESSION): with warnings.catch_warnings(): warnings.simplefilter("ignore", RemovedInAirflow3Warning) all_ti_attrs = ( - (name, getattr(ti, name)) + # fetching the value of _try_number to be shown under name try_number in UI + (name, getattr(ti, "_try_number" if name == "try_number" else name)) for name in dir(ti) if not name.startswith("_") and name not in ti_attrs_to_skip ) diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 8ed0ba825a3e..21c573fe5049 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -323,6 +323,15 @@ def test_views_get(admin_client, url, contents): check_content_in_response(content, resp) +def test_rendered_task_view(admin_client): + url = f"task?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + resp = admin_client.get(url, follow_redirects=True) + resp_html = resp.data.decode("utf-8") + assert resp.status_code == 200 + assert "_try_number" not in resp_html + assert "try_number" in resp_html + + def test_rendered_k8s(admin_client): url = f"rendered-k8s?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" with unittest.mock.patch.object(settings, "IS_K8S_OR_K8SCELERY_EXECUTOR", True): From 7722b6f226e9db3a89b01b89db5fdb7a1ab2256f Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Wed, 5 Jul 2023 14:08:29 +0530 Subject: [PATCH 055/533] Ensure dynamic tasks inside dynamic task group only marks the (#32354) corresponding EmptyOperator in downstream as success. --- airflow/models/dagrun.py | 4 ++-- tests/models/test_dagrun.py | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 8f3b3a33018a..b0b56f3c6c5a 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -1338,7 +1338,7 @@ def schedule_tis( and not ti.task.on_success_callback and not ti.task.outlets ): - dummy_ti_ids.append(ti.task_id) + dummy_ti_ids.append((ti.task_id, ti.map_index)) else: schedulable_ti_ids.append((ti.task_id, ti.map_index)) @@ -1369,7 +1369,7 @@ def schedule_tis( .where( TI.dag_id == self.dag_id, TI.run_id == self.run_id, - TI.task_id.in_(dummy_ti_ids_chunk), + tuple_in_condition((TI.task_id, TI.map_index), dummy_ti_ids_chunk), ) .values( state=TaskInstanceState.SUCCESS, diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 5d7db547d31c..c90ec1aef328 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -1811,6 +1811,52 @@ def tg(x): ] +def test_mapped_task_group_empty_operator(dag_maker, session): + """ + Test that dynamic task inside a dynamic task group only marks + the corresponding downstream EmptyOperator as success. + """ + + literal = [1, 2, 3] + + with dag_maker(session=session) as dag: + + @task_group + def tg(x): + @task + def t1(x): + return x + + t2 = EmptyOperator(task_id="t2") + + @task + def t3(x): + return x + + t1(x) >> t2 >> t3(x) + + tg.expand(x=literal) + + dr = dag_maker.create_dagrun() + + t2_task = dag.get_task("tg.t2") + t2_0 = dr.get_task_instance(task_id="tg.t2", map_index=0) + t2_0.refresh_from_task(t2_task) + assert t2_0.state is None + + t2_1 = dr.get_task_instance(task_id="tg.t2", map_index=1) + t2_1.refresh_from_task(t2_task) + assert t2_1.state is None + + dr.schedule_tis([t2_0]) + + t2_0 = dr.get_task_instance(task_id="tg.t2", map_index=0) + assert t2_0.state == TaskInstanceState.SUCCESS + + t2_1 = dr.get_task_instance(task_id="tg.t2", map_index=1) + assert t2_1.state is None + + def test_ti_scheduling_mapped_zero_length(dag_maker, session): with dag_maker(session=session): task = BaseOperator(task_id="task_1") From 61f33304d587b3b0a48a876d3bfedab82e42bacc Mon Sep 17 00:00:00 2001 From: Phani Kumar <94376113+phanikumv@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:26:36 +0530 Subject: [PATCH 056/533] Refactor Sqlalchemy queries to 2.0 style (Part 3) (#32350) --------- Co-authored-by: Ephraim Anierobi --- airflow/utils/db.py | 38 ++-- airflow/www/utils.py | 11 +- airflow/www/views.py | 461 ++++++++++++++++++++++--------------------- 3 files changed, 266 insertions(+), 244 deletions(-) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index a76f0d4f675d..46ddcbfb3453 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -92,7 +92,7 @@ def _format_airflow_moved_table_name(source_table, version, category): @provide_session def merge_conn(conn, session: Session = NEW_SESSION): """Add new Connection.""" - if not session.query(conn.__class__).filter_by(conn_id=conn.conn_id).first(): + if not session.scalar(select(conn.__class__).filter_by(conn_id=conn.conn_id).limit(1)): session.add(conn) session.commit() @@ -959,7 +959,9 @@ def check_conn_id_duplicates(session: Session) -> Iterable[str]: dups = [] try: - dups = session.query(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1).all() + dups = session.execute( + select(Connection.conn_id).group_by(Connection.conn_id).having(func.count() > 1) + ).all() except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -984,12 +986,11 @@ def check_username_duplicates(session: Session) -> Iterable[str]: for model in [User, RegisterUser]: dups = [] try: - dups = ( - session.query(model.username) # type: ignore[attr-defined] + dups = session.execute( + select(model.username) # type: ignore[attr-defined] .group_by(model.username) # type: ignore[attr-defined] .having(func.count() > 1) - .all() - ) + ).all() except (exc.OperationalError, exc.ProgrammingError): # fallback if tables hasn't been created yet session.rollback() @@ -1058,13 +1059,13 @@ def check_task_fail_for_duplicates(session): """ minimal_table_obj = table(table_name, *[column(x) for x in uniqueness]) try: - subquery = ( - session.query(minimal_table_obj, func.count().label("dupe_count")) + subquery = session.execute( + select(minimal_table_obj, func.count().label("dupe_count")) .group_by(*[text(x) for x in uniqueness]) .having(func.count() > text("1")) .subquery() ) - dupe_count = session.query(func.sum(subquery.c.dupe_count)).scalar() + dupe_count = session.scalar(select(func.sum(subquery.c.dupe_count))) if not dupe_count: # there are no duplicates; nothing to do. return @@ -1101,7 +1102,7 @@ def check_conn_type_null(session: Session) -> Iterable[str]: n_nulls = [] try: - n_nulls = session.query(Connection.conn_id).filter(Connection.conn_type.is_(None)).all() + n_nulls = session.scalars(select(Connection.conn_id).where(Connection.conn_type.is_(None))).all() except (exc.OperationalError, exc.ProgrammingError, exc.InternalError): # fallback if tables hasn't been created yet session.rollback() @@ -1143,7 +1144,7 @@ def check_run_id_null(session: Session) -> Iterable[str]: dagrun_table.c.run_id.is_(None), dagrun_table.c.execution_date.is_(None), ) - invalid_dagrun_count = session.query(func.count(dagrun_table.c.id)).filter(invalid_dagrun_filter).scalar() + invalid_dagrun_count = session.scalar(select(func.count(dagrun_table.c.id)).where(invalid_dagrun_filter)) if invalid_dagrun_count > 0: dagrun_dangling_table_name = _format_airflow_moved_table_name(dagrun_table.name, "2.2", "dangling") if dagrun_dangling_table_name in inspect(session.get_bind()).get_table_names(): @@ -1240,7 +1241,7 @@ def _move_dangling_data_to_new_table( pk_cols = source_table.primary_key.columns delete = source_table.delete().where( - tuple_(*pk_cols).in_(session.query(*target_table.primary_key.columns).subquery()) + tuple_(*pk_cols).in_(session.select(*target_table.primary_key.columns).subquery()) ) else: delete = source_table.delete().where( @@ -1262,10 +1263,11 @@ def _dangling_against_dag_run(session, source_table, dag_run): source_table.c.dag_id == dag_run.c.dag_id, source_table.c.execution_date == dag_run.c.execution_date, ) + return ( - session.query(*[c.label(c.name) for c in source_table.c]) + select(*[c.label(c.name) for c in source_table.c]) .join(dag_run, source_to_dag_run_join_cond, isouter=True) - .filter(dag_run.c.dag_id.is_(None)) + .where(dag_run.c.dag_id.is_(None)) ) @@ -1304,10 +1306,10 @@ def _dangling_against_task_instance(session, source_table, dag_run, task_instanc ) return ( - session.query(*[c.label(c.name) for c in source_table.c]) + select(*[c.label(c.name) for c in source_table.c]) .join(dag_run, dr_join_cond, isouter=True) .join(task_instance, ti_join_cond, isouter=True) - .filter(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) + .where(or_(task_instance.c.dag_id.is_(None), dag_run.c.dag_id.is_(None))) ) @@ -1331,9 +1333,9 @@ def _move_duplicate_data_to_new_table( """ bind = session.get_bind() dialect_name = bind.dialect.name + query = ( - session.query(source_table) - .with_entities(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) + select(*[getattr(source_table.c, x.name).label(str(x.name)) for x in source_table.columns]) .select_from(source_table) .join(subquery, and_(*[getattr(source_table.c, x) == getattr(subquery.c, x) for x in uniqueness])) ) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 76914dd9cdcd..b31f9326d988 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -39,6 +39,7 @@ from pygments.lexer import Lexer from sqlalchemy import delete, func, types from sqlalchemy.ext.associationproxy import AssociationProxy +from sqlalchemy.sql import Select from airflow.exceptions import RemovedInAirflow3Warning from airflow.models import errors @@ -53,7 +54,6 @@ from airflow.www.widgets import AirflowDateTimePickerWidget if TYPE_CHECKING: - from sqlalchemy.orm.query import Query from sqlalchemy.orm.session import Session from sqlalchemy.sql.operators import ColumnOperators @@ -518,18 +518,21 @@ def _get_run_ordering_expr(name: str) -> ColumnOperators: return expr.desc() -def sorted_dag_runs(query: Query, *, ordering: Sequence[str], limit: int) -> Sequence[DagRun]: +def sorted_dag_runs( + query: Select, *, ordering: Sequence[str], limit: int, session: Session +) -> Sequence[DagRun]: """Produce DAG runs sorted by specified columns. - :param query: An ORM query object against *DagRun*. + :param query: An ORM select object against *DagRun*. :param ordering: Column names to sort the runs. should generally come from a timetable's ``run_ordering``. :param limit: Number of runs to limit to. + :param session: SQLAlchemy ORM session object :return: A list of DagRun objects ordered by the specified columns. The list contains only the *last* objects, but in *ascending* order. """ ordering_exprs = (_get_run_ordering_expr(name) for name in ordering) - runs = query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit).all() + runs = session.scalars(query.order_by(*ordering_exprs, DagRun.id.desc()).limit(limit)).all() runs.reverse() return runs diff --git a/airflow/www/views.py b/airflow/www/views.py index d7059156f575..b304dd0ddf02 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -69,7 +69,7 @@ from pendulum.parsing.exceptions import ParserError from pygments import highlight, lexers from pygments.formatters import HtmlFormatter -from sqlalchemy import Date, and_, case, desc, func, inspect, or_, union_all +from sqlalchemy import Date, and_, case, desc, func, inspect, or_, select, union_all from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload from wtforms import SelectField, validators @@ -95,7 +95,7 @@ from airflow.jobs.job import Job from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner -from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, XCom, errors +from airflow.models import Connection, DagModel, DagTag, Log, SlaMiss, TaskFail, Trigger, XCom, errors from airflow.models.abstractoperator import AbstractOperator from airflow.models.dag import DAG, get_dataset_triggered_next_run_info from airflow.models.dagcode import DagCode @@ -231,16 +231,15 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): # loaded and the actual requested run would be excluded by the limit(). Once # the user has changed base date to be anything else we want to use that instead. query_date = base_date - if date_time < base_date and date_time + datetime.timedelta(seconds=1) >= base_date: + if date_time < base_date <= date_time + datetime.timedelta(seconds=1): query_date = date_time - drs = ( - session.query(DagRun) - .filter(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) + drs = session.scalars( + select(DagRun) + .where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) .order_by(desc(DagRun.execution_date)) .limit(num_runs) - .all() - ) + ).all() dr_choices = [] dr_state = None for dr in drs: @@ -291,8 +290,8 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): Create a nested dict representation of the DAG's TaskGroup and its children used to construct the Graph and Grid views. """ - query = ( - session.query( + query = session.execute( + select( TaskInstance.task_id, TaskInstance.run_id, TaskInstance.state, @@ -303,7 +302,7 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): func.max(TaskInstance.end_date).label("end_date"), ) .join(TaskInstance.task_instance_note, isouter=True) - .filter( + .where( TaskInstance.dag_id == dag.dag_id, TaskInstance.run_id.in_([dag_run.run_id for dag_run in dag_runs]), ) @@ -426,11 +425,9 @@ def get_summary(dag_run: DagRun): } def get_mapped_group_summaries(): - mapped_ti_query = ( - session.query( - TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index - ) - .filter( + mapped_ti_query = session.execute( + select(TaskInstance.task_id, TaskInstance.state, TaskInstance.run_id, TaskInstance.map_index) + .where( TaskInstance.dag_id == dag.dag_id, TaskInstance.task_id.in_(child["id"] for child in children), TaskInstance.run_id.in_(r.run_id for r in dag_runs), @@ -738,21 +735,20 @@ def index(self): with create_session() as session: # read orm_dags from the db - dags_query = session.query(DagModel).filter(~DagModel.is_subdag, DagModel.is_active) + dags_query = select(DagModel).where(~DagModel.is_subdag, DagModel.is_active) if arg_search_query: escaped_arg_search_query = arg_search_query.replace("_", r"\_") - dags_query = dags_query.filter( + dags_query = dags_query.where( DagModel.dag_id.ilike("%" + escaped_arg_search_query + "%", escape="\\") | DagModel.owners.ilike("%" + escaped_arg_search_query + "%", escape="\\") ) if arg_tags_filter: - dags_query = dags_query.filter(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) + dags_query = dags_query.where(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) - dags_query = dags_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - - filtered_dag_count = dags_query.count() + dags_query = dags_query.where(DagModel.dag_id.in_(filter_dag_ids)) + filtered_dag_count = session.scalar(select(func.count()).select_from(dags_query)) if filtered_dag_count == 0 and len(arg_tags_filter): flash( "No matching DAG tags found.", @@ -762,28 +758,28 @@ def index(self): return redirect(url_for("Airflow.index")) all_dags = dags_query - active_dags = dags_query.filter(~DagModel.is_paused) - paused_dags = dags_query.filter(DagModel.is_paused) + active_dags = dags_query.where(~DagModel.is_paused) + paused_dags = dags_query.where(DagModel.is_paused) # find DAGs which have a RUNNING DagRun - running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).filter( + running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).where( DagRun.state == State.RUNNING ) # find DAGs for which the latest DagRun is FAILED subq_all = ( - session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) .group_by(DagRun.dag_id) .subquery() ) subq_failed = ( - session.query(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) - .filter(DagRun.state == State.FAILED) + select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) + .where(DagRun.state == State.FAILED) .group_by(DagRun.dag_id) .subquery() ) subq_join = ( - session.query(subq_all.c.dag_id, subq_all.c.start_date) + select(subq_all.c.dag_id, subq_all.c.start_date) .join( subq_failed, and_( @@ -796,16 +792,18 @@ def index(self): failed_dags = dags_query.join(subq_join, DagModel.dag_id == subq_join.c.dag_id) is_paused_count = dict( - all_dags.with_entities(DagModel.is_paused, func.count(DagModel.dag_id)).group_by( - DagModel.is_paused - ) + session.execute( + select(DagModel.is_paused, func.count(DagModel.dag_id)) + .group_by(DagModel.is_paused) + .select_from(all_dags) + ).all() ) status_count_active = is_paused_count.get(False, 0) status_count_paused = is_paused_count.get(True, 0) - status_count_running = running_dags.count() - status_count_failed = failed_dags.count() + status_count_running = session.scalar(select(func.count()).select_from(running_dags)) + status_count_failed = session.scalar(select(func.count()).select_from(failed_dags)) all_dags_count = status_count_active + status_count_paused if arg_status_filter == "active": @@ -826,7 +824,7 @@ def index(self): if arg_sorting_key == "last_dagrun": dag_run_subquery = ( - session.query( + select( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) @@ -854,7 +852,13 @@ def index(self): else: current_dags = current_dags.order_by(null_case, sort_column) - dags = current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page).all() + dags = ( + session.scalars( + current_dags.options(joinedload(DagModel.tags)).offset(start).limit(dags_per_page) + ) + .unique() + .all() + ) user_permissions = g.user.perms can_create_dag_run = ( permissions.ACTION_CAN_CREATE, @@ -874,7 +878,7 @@ def index(self): dag.can_trigger = dag.can_edit and can_create_dag_run dag.can_delete = get_airflow_app().appbuilder.sm.can_delete_dag(dag.dag_id, g.user) - dagtags = session.query(func.distinct(DagTag.name)).order_by(DagTag.name).all() + dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() tags = [ {"name": name, "selected": bool(arg_tags_filter and name in arg_tags_filter)} for name, in dagtags @@ -882,14 +886,15 @@ def index(self): owner_links_dict = DagOwnerAttributes.get_all(session) - import_errors = session.query(errors.ImportError).order_by(errors.ImportError.id) + import_errors = select(errors.ImportError).order_by(errors.ImportError.id) if (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG) not in user_permissions: # if the user doesn't have access to all DAGs, only display errors from visible DAGs import_errors = import_errors.join( DagModel, DagModel.fileloc == errors.ImportError.filename - ).filter(DagModel.dag_id.in_(filter_dag_ids)) + ).where(DagModel.dag_id.in_(filter_dag_ids)) + import_errors = session.scalars(import_errors) for import_error in import_errors: flash( f"Broken DAG: [{import_error.filename}] {import_error.stacktrace}", @@ -933,10 +938,12 @@ def _iter_parsed_moved_data_table_names(): permissions.RESOURCE_ADMIN_MENU, ) in user_permissions and conf.getboolean("webserver", "warn_deployment_exposure"): robots_file_access_count = ( - session.query(Log) - .filter(Log.event == "robots") - .filter(Log.dttm > (utcnow() - datetime.timedelta(days=7))) - .count() + select(Log) + .where(Log.event == "robots") + .where(Log.dttm > (utcnow() - datetime.timedelta(days=7))) + ) + robots_file_access_count = session.scalar( + select(func.count()).select_from(robots_file_access_count) ) if robots_file_access_count > 0: flash( @@ -1038,9 +1045,11 @@ def next_run_datasets_summary(self, session: Session = NEW_SESSION): dataset_triggered_dag_ids = [ dag.dag_id for dag in ( - session.query(DagModel.dag_id) - .filter(DagModel.dag_id.in_(filter_dag_ids)) - .filter(DagModel.schedule_interval == "Dataset") + session.scalars( + select(DagModel.dag_id) + .where(DagModel.dag_id.in_(filter_dag_ids)) + .where(DagModel.schedule_interval == "Dataset") + ) ) ] @@ -1071,10 +1080,10 @@ def dag_stats(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify({}) - dag_state_stats = ( - session.query(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) + dag_state_stats = session.execute( + select(DagRun.dag_id, DagRun.state, sqla.func.count(DagRun.state)) .group_by(DagRun.dag_id, DagRun.state) - .filter(DagRun.dag_id.in_(filter_dag_ids)) + .where(DagRun.dag_id.in_(filter_dag_ids)) ) dag_state_data = {(dag_id, state): count for dag_id, state, count in dag_state_stats} @@ -1112,17 +1121,17 @@ def task_stats(self, session: Session = NEW_SESSION): filter_dag_ids = allowed_dag_ids running_dag_run_query_result = ( - session.query(DagRun.dag_id, DagRun.run_id) + select(DagRun.dag_id, DagRun.run_id) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .filter(DagRun.state == State.RUNNING, DagModel.is_active) + .where(DagRun.state == State.RUNNING, DagModel.is_active) ) - running_dag_run_query_result = running_dag_run_query_result.filter(DagRun.dag_id.in_(filter_dag_ids)) + running_dag_run_query_result = running_dag_run_query_result.where(DagRun.dag_id.in_(filter_dag_ids)) running_dag_run_query_result = running_dag_run_query_result.subquery("running_dag_run") # Select all task_instances from active dag_runs. - running_task_instance_query_result = session.query( + running_task_instance_query_result = select( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(True).label("is_dag_running"), @@ -1136,19 +1145,19 @@ def task_stats(self, session: Session = NEW_SESSION): if conf.getboolean("webserver", "SHOW_RECENT_STATS_FOR_COMPLETED_RUNS", fallback=True): last_dag_run = ( - session.query(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) + select(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .filter(DagRun.state != State.RUNNING, DagModel.is_active) + .where(DagRun.state != State.RUNNING, DagModel.is_active) .group_by(DagRun.dag_id) ) - last_dag_run = last_dag_run.filter(DagRun.dag_id.in_(filter_dag_ids)) + last_dag_run = last_dag_run.where(DagRun.dag_id.in_(filter_dag_ids)) last_dag_run = last_dag_run.subquery("last_dag_run") # Select all task_instances from active dag_runs. # If no dag_run is active, return task instances from most recent dag_run. last_task_instance_query_result = ( - session.query( + select( TaskInstance.dag_id.label("dag_id"), TaskInstance.state.label("state"), sqla.literal(False).label("is_dag_running"), @@ -1169,8 +1178,8 @@ def task_stats(self, session: Session = NEW_SESSION): else: final_task_instance_query_result = running_task_instance_query_result.subquery("final_ti") - qry = ( - session.query( + qry = session.execute( + select( final_task_instance_query_result.c.dag_id, final_task_instance_query_result.c.state, final_task_instance_query_result.c.is_dag_running, @@ -1186,7 +1195,6 @@ def task_stats(self, session: Session = NEW_SESSION): final_task_instance_query_result.c.is_dag_running.desc(), ) ) - data = get_task_stats_from_query(qry) payload: dict[str, list[dict[str, Any]]] = collections.defaultdict(list) for dag_id in filter_dag_ids: @@ -1219,29 +1227,31 @@ def last_dagruns(self, session: Session = NEW_SESSION): return flask.json.jsonify({}) last_runs_subquery = ( - session.query( + select( DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("max_execution_date"), ) .group_by(DagRun.dag_id) - .filter(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. + .where(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. .subquery("last_runs") ) - query = session.query( - DagRun.dag_id, - DagRun.start_date, - DagRun.end_date, - DagRun.state, - DagRun.execution_date, - DagRun.data_interval_start, - DagRun.data_interval_end, - ).join( - last_runs_subquery, - and_( - last_runs_subquery.c.dag_id == DagRun.dag_id, - last_runs_subquery.c.max_execution_date == DagRun.execution_date, - ), + query = session.execute( + select( + DagRun.dag_id, + DagRun.start_date, + DagRun.end_date, + DagRun.state, + DagRun.execution_date, + DagRun.data_interval_start, + DagRun.data_interval_end, + ).join( + last_runs_subquery, + and_( + last_runs_subquery.c.dag_id == DagRun.dag_id, + last_runs_subquery.c.max_execution_date == DagRun.execution_date, + ), + ) ) resp = { @@ -1340,19 +1350,18 @@ def dag_details(self, dag_id, session: Session = NEW_SESSION): title = "DAG Details" root = request.args.get("root", "") - states = ( - session.query(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) - .filter(TaskInstance.dag_id == dag_id) + states = session.execute( + select(TaskInstance.state, sqla.func.count(TaskInstance.dag_id)) + .where(TaskInstance.dag_id == dag_id) .group_by(TaskInstance.state) - .all() - ) + ).all() active_runs = models.DagRun.find(dag_id=dag_id, state=DagRunState.RUNNING, external_trigger=False) - tags = session.query(models.DagTag).filter(models.DagTag.dag_id == dag_id).all() + tags = session.scalars(select(models.DagTag).where(models.DagTag.dag_id == dag_id)).all() # TODO: convert this to a relationship - owner_links = session.query(DagOwnerAttributes).filter_by(dag_id=dag_id).all() + owner_links = session.execute(select(DagOwnerAttributes).filter_by(dag_id=dag_id)).all() attrs_to_avoid = [ "schedule_datasets", @@ -1613,18 +1622,17 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): "metadata": {"end_of_log": True}, } - ti = ( - session.query(models.TaskInstance) - .filter( + ti = session.scalar( + select(models.TaskInstance) + .where( TaskInstance.task_id == task_id, TaskInstance.dag_id == dag_id, TaskInstance.execution_date == execution_date, TaskInstance.map_index == map_index, ) .join(TaskInstance.dag_run) - .options(joinedload("trigger")) - .options(joinedload("trigger.triggerer_job")) - .first() + .options(joinedload(TaskInstance.trigger).joinedload(Trigger.triggerer_job)) + .limit(1) ) if ti is None: @@ -1682,10 +1690,10 @@ def log(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) dag_model = DagModel.get_dagmodel(dag_id) - ti = ( - session.query(models.TaskInstance) + ti = session.scalar( + select(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .first() + .limit(1) ) num_logs = 0 @@ -1726,10 +1734,10 @@ def redirect_to_external_log(self, session: Session = NEW_SESSION): map_index = request.args.get("map_index", -1, type=int) try_number = request.args.get("try_number", 1) - ti = ( - session.query(models.TaskInstance) + ti = session.scalar( + select(models.TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) - .first() + .limit(1) ) if not ti: @@ -1771,8 +1779,8 @@ def task(self, session: Session = NEW_SESSION): task = copy.copy(dag.get_task(task_id)) task.resolve_template_files() - ti: TaskInstance | None = ( - session.query(TaskInstance) + ti: TaskInstance | None = session.scalar( + select(TaskInstance) .options( # HACK: Eager-load relationships. This is needed because # multiple properties mis-use provide_session() that destroys @@ -1781,7 +1789,6 @@ def task(self, session: Session = NEW_SESSION): joinedload(TaskInstance.trigger, innerjoin=False), ) .filter_by(execution_date=dttm, dag_id=dag_id, task_id=task_id, map_index=map_index) - .one_or_none() ) if ti is None: ti_attrs: list[tuple[str, Any]] | None = None @@ -1899,17 +1906,19 @@ def xcom(self, session: Session = NEW_SESSION): form = DateTimeForm(data={"execution_date": dttm}) root = request.args.get("root", "") dag = DagModel.get_dagmodel(dag_id) - ti = session.query(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).first() + ti = session.scalar(select(TaskInstance).filter_by(dag_id=dag_id, task_id=task_id).limit(1)) if not ti: flash(f"Task [{dag_id}.{task_id}] doesn't seem to exist at the moment", "error") return redirect(url_for("Airflow.index")) - xcom_query = session.query(XCom.key, XCom.value).filter( - XCom.dag_id == dag_id, - XCom.task_id == task_id, - XCom.execution_date == dttm, - XCom.map_index == map_index, + xcom_query = session.execute( + select(XCom.key, XCom.value).where( + XCom.dag_id == dag_id, + XCom.task_id == task_id, + XCom.execution_date == dttm, + XCom.map_index == map_index, + ) ) attributes = [(k, v) for k, v in xcom_query if not k.startswith("_")] @@ -1979,7 +1988,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): request_execution_date = request.values.get("execution_date", default=timezone.utcnow().isoformat()) is_dag_run_conf_overrides_params = conf.getboolean("core", "dag_run_conf_overrides_params") dag = get_airflow_app().dag_bag.get_dag(dag_id) - dag_orm: DagModel = session.query(DagModel).filter(DagModel.dag_id == dag_id).first() + dag_orm: DagModel = session.scalar(select(DagModel).where(DagModel.dag_id == dag_id).limit(1)) # Prepare form fields with param struct details to render a proper form with schema information form_fields = {} @@ -2017,11 +2026,9 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): flash(f"Cannot create dagruns because the dag {dag_id} has import errors", "error") return redirect(origin) - recent_runs = ( - session.query( - DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date) - ) - .filter( + recent_runs = session.execute( + select(DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date)) + .where( DagRun.dag_id == dag_id, DagRun.run_type == DagRunType.MANUAL, DagRun.conf.isnot(None), @@ -2294,15 +2301,17 @@ def clear(self, *, session: Session = NEW_SESSION): # Lock the related dag runs to prevent from possible dead lock. # https://github.com/apache/airflow/pull/26658 - dag_runs_query = session.query(DagRun.id).filter(DagRun.dag_id == dag_id).with_for_update() + dag_runs_query = session.scalars( + select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() + ) if start_date is None and end_date is None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date == start_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) else: if start_date is not None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date >= start_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date >= start_date) if end_date is not None: - dag_runs_query = dag_runs_query.filter(DagRun.execution_date <= end_date) + dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) locked_dag_run_ids = dag_runs_query.all() elif task_id: @@ -2391,10 +2400,10 @@ def blocked(self, session: Session = NEW_SESSION): if not filter_dag_ids: return flask.json.jsonify([]) - dags = ( - session.query(DagRun.dag_id, sqla.func.count(DagRun.id)) - .filter(DagRun.state == DagRunState.RUNNING) - .filter(DagRun.dag_id.in_(filter_dag_ids)) + dags = session.execute( + select(DagRun.dag_id, sqla.func.count(DagRun.id)) + .where(DagRun.state == DagRunState.RUNNING) + .where(DagRun.dag_id.in_(filter_dag_ids)) .group_by(DagRun.dag_id) ) @@ -2475,9 +2484,11 @@ def _mark_dagrun_state_as_queued( # Identify tasks that will be queued up to run when confirmed all_task_ids = [task.task_id for task in dag.tasks] - existing_tis = session.query(TaskInstance.task_id).filter( - TaskInstance.dag_id == dag.dag_id, - TaskInstance.run_id == dag_run_id, + existing_tis = session.execute( + select(TaskInstance.task_id).where( + TaskInstance.dag_id == dag.dag_id, + TaskInstance.run_id == dag_run_id, + ) ) completed_tis_ids = [task_id for task_id, in existing_tis] @@ -2959,19 +2970,18 @@ def _convert_to_date(session, column): if root: dag = dag.partial_subset(task_ids_or_regex=root, include_downstream=False, include_upstream=True) - dag_states = ( - session.query( - (_convert_to_date(session, DagRun.execution_date)).label("date"), + dag_states = session.execute( + select( + _convert_to_date(session, DagRun.execution_date).label("date"), DagRun.state, func.max(DagRun.data_interval_start).label("data_interval_start"), func.max(DagRun.data_interval_end).label("data_interval_end"), func.count("*").label("count"), ) - .filter(DagRun.dag_id == dag.dag_id) + .where(DagRun.dag_id == dag.dag_id) .group_by(_convert_to_date(session, DagRun.execution_date), DagRun.state) .order_by(_convert_to_date(session, DagRun.execution_date).asc()) - .all() - ) + ).all() data_dag_states = [ { @@ -3242,16 +3252,17 @@ def duration(self, dag_id: str, session: Session = NEW_SESSION): else: min_date = timezone.utc_epoch() ti_fails = ( - session.query(TaskFail) + select(TaskFail) .join(TaskFail.dag_run) - .filter( + .where( TaskFail.dag_id == dag.dag_id, DagRun.execution_date >= min_date, DagRun.execution_date <= base_date, ) ) if dag.partial: - ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = session.scalars(ti_fails) fails_totals: dict[tuple[str, str, str], int] = defaultdict(int) for failed_task_instance in ti_fails: dict_key = ( @@ -3592,21 +3603,22 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): form = DateTimeWithNumRunsWithDagRunsForm(data=dt_nr_dr_data) form.execution_date.choices = dt_nr_dr_data["dr_choices"] - tis = ( - session.query(TaskInstance) - .filter( + tis = session.scalars( + select(TaskInstance) + .where( TaskInstance.dag_id == dag_id, TaskInstance.run_id == dag_run_id, TaskInstance.start_date.is_not(None), TaskInstance.state.is_not(None), ) .order_by(TaskInstance.start_date) - ) + ).all() - ti_fails = session.query(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) + ti_fails = select(TaskFail).filter_by(run_id=dag_run_id, dag_id=dag_id) if dag.partial: - ti_fails = ti_fails.filter(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = ti_fails.where(TaskFail.task_id.in_([t.task_id for t in dag.tasks])) + ti_fails = session.scalars(ti_fails) tasks = [] for ti in tis: if not dag.has_task(ti.task_id): @@ -3712,12 +3724,13 @@ def extra_links(self, *, session: Session = NEW_SESSION): if link_name is None: return {"url": None, "error": "Link name not passed"}, 400 - ti = ( - session.query(TaskInstance) + ti = session.scalar( + select(TaskInstance) .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) .options(joinedload(TaskInstance.dag_run)) - .first() + .limit(1) ) + if not ti: return {"url": None, "error": "Task Instances not found"}, 404 try: @@ -3825,27 +3838,25 @@ def grid_data(self): base_date = dag.get_latest_execution_date() or timezone.utcnow() with create_session() as session: - query = session.query(DagRun).filter( - DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date - ) + query = select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date) - run_type = request.args.get("run_type") - if run_type: - query = query.filter(DagRun.run_type == run_type) + run_type = request.args.get("run_type") + if run_type: + query = query.where(DagRun.run_type == run_type) - run_state = request.args.get("run_state") - if run_state: - query = query.filter(DagRun.state == run_state) + run_state = request.args.get("run_state") + if run_state: + query = query.where(DagRun.state == run_state) - dag_runs = wwwutils.sorted_dag_runs(query, ordering=dag.timetable.run_ordering, limit=num_runs) - encoded_runs = [ - wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs - ] - data = { - "groups": dag_to_grid(dag, dag_runs, session), - "dag_runs": encoded_runs, - "ordering": dag.timetable.run_ordering, - } + dag_runs = wwwutils.sorted_dag_runs( + query, ordering=dag.timetable.run_ordering, limit=num_runs, session=session + ) + encoded_runs = [wwwutils.encode_dag_run(dr, json_encoder=utils_json.WebEncoder) for dr in dag_runs] + data = { + "groups": dag_to_grid(dag, dag_runs, session), + "dag_runs": encoded_runs, + "ordering": dag.timetable.run_ordering, + } # avoid spaces to reduce payload size return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -3864,37 +3875,34 @@ def historical_metrics_data(self): end_date = _safe_parse_datetime(request.args.get("end_date")) with create_session() as session: # DagRuns - dag_runs_type = ( - session.query(DagRun.run_type, func.count(DagRun.run_id)) - .filter( + dag_runs_type = session.execute( + select(DagRun.run_type, func.count(DagRun.run_id)) + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.run_type) - .all() - ) + ).all() - dag_run_states = ( - session.query(DagRun.state, func.count(DagRun.run_id)) - .filter( + dag_run_states = session.execute( + select(DagRun.state, func.count(DagRun.run_id)) + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(DagRun.state) - .all() - ) + ).all() # TaskInstances - task_instance_states = ( - session.query(TaskInstance.state, func.count(TaskInstance.run_id)) + task_instance_states = session.execute( + select(TaskInstance.state, func.count(TaskInstance.run_id)) .join(TaskInstance.dag_run) - .filter( + .where( DagRun.start_date >= start_date, or_(DagRun.end_date.is_(None), DagRun.end_date <= end_date), ) .group_by(TaskInstance.state) - .all() - ) + ).all() data = { "dag_run_types": { @@ -3928,28 +3936,32 @@ def next_run_datasets(self, dag_id): with create_session() as session: data = [ dict(info) - for info in session.query( - DatasetModel.id, - DatasetModel.uri, - func.max(DatasetEvent.timestamp).label("lastUpdate"), - ) - .join(DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id) - .join( - DatasetDagRunQueue, - and_( - DatasetDagRunQueue.dataset_id == DatasetModel.id, - DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, - ), - isouter=True, - ) - .join( - DatasetEvent, - DatasetEvent.dataset_id == DatasetModel.id, - isouter=True, + for info in session.execute( + select( + DatasetModel.id, + DatasetModel.uri, + func.max(DatasetEvent.timestamp).label("lastUpdate"), + ) + .join( + DagScheduleDatasetReference, DagScheduleDatasetReference.dataset_id == DatasetModel.id + ) + .join( + DatasetDagRunQueue, + and_( + DatasetDagRunQueue.dataset_id == DatasetModel.id, + DatasetDagRunQueue.target_dag_id == DagScheduleDatasetReference.dag_id, + ), + isouter=True, + ) + .join( + DatasetEvent, + DatasetEvent.dataset_id == DatasetModel.id, + isouter=True, + ) + .where(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) + .group_by(DatasetModel.id, DatasetModel.uri) + .order_by(DatasetModel.uri) ) - .filter(DagScheduleDatasetReference.dag_id == dag_id, ~DatasetModel.is_orphaned) - .group_by(DatasetModel.id, DatasetModel.uri) - .order_by(DatasetModel.uri) ] return ( htmlsafe_json_dumps(data, separators=(",", ":"), dumps=flask.json.dumps), @@ -4047,12 +4059,12 @@ def datasets_summary(self): if session.bind.dialect.name == "postgresql": order_by = (order_by[0].nulls_first(), *order_by[1:]) - count_query = session.query(func.count(DatasetModel.id)) + count_query = select(func.count(DatasetModel.id)) has_event_filters = bool(updated_before or updated_after) query = ( - session.query( + select( DatasetModel.id, DatasetModel.uri, func.max(DatasetEvent.timestamp).label("last_dataset_update"), @@ -4077,11 +4089,12 @@ def datasets_summary(self): if updated_before: filters.append(DatasetEvent.timestamp <= updated_before) - query = query.filter(*filters).offset(offset).limit(limit) - count_query = count_query.filter(*filters) + query = query.where(*filters).offset(offset).limit(limit) + count_query = count_query.where(*filters) + query = session.execute(query) datasets = [dict(dataset) for dataset in query] - data = {"datasets": datasets, "total_entries": count_query.scalar()} + data = {"datasets": datasets, "total_entries": session.scalar(count_query)} return ( htmlsafe_json_dumps(data, separators=(",", ":"), cls=utils_json.WebEncoder), @@ -4127,20 +4140,20 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): included_events_raw = conf.get("webserver", "audit_view_included_events", fallback=None) excluded_events_raw = conf.get("webserver", "audit_view_excluded_events", fallback=None) - query = session.query(Log).filter(Log.dag_id == dag_id) + query = select(Log).where(Log.dag_id == dag_id) if included_events_raw: included_events = {event.strip() for event in included_events_raw.split(",")} - query = query.filter(Log.event.in_(included_events)) + query = query.where(Log.event.in_(included_events)) elif excluded_events_raw: excluded_events = {event.strip() for event in excluded_events_raw.split(",")} - query = query.filter(Log.event.notin_(excluded_events)) + query = query.where(Log.event.notin_(excluded_events)) current_page = request.args.get("page", default=0, type=int) arg_sorting_key = request.args.get("sorting_key", "dttm") arg_sorting_direction = request.args.get("sorting_direction", default="desc") logs_per_page = PAGE_SIZE - audit_logs_count = query.count() + audit_logs_count = session.scalar(select(func.count()).select_from(query)) num_of_pages = int(math.ceil(audit_logs_count / float(logs_per_page))) start = current_page * logs_per_page @@ -4152,7 +4165,7 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): sort_column = sort_column.desc() query = query.order_by(sort_column) - dag_audit_logs = query.offset(start).limit(logs_per_page).all() + dag_audit_logs = session.scalars(query.offset(start).limit(logs_per_page)).all() return self.render_template( "airflow/dag_audit_log.html", dag=dag, @@ -4273,7 +4286,7 @@ def apply(self, query, func): if get_airflow_app().appbuilder.sm.has_all_dags_access(g.user): return query filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - return query.filter(self.model.dag_id.in_(filter_dag_ids)) + return query.where(self.model.dag_id.in_(filter_dag_ids)) class AirflowModelView(ModelView): @@ -4718,9 +4731,11 @@ def action_mulduplicate(self, connections, session: Session = NEW_SESSION): potential_connection_ids = [f"{base_conn_id}_copy{i}" for i in range(1, 11)] - query = session.query(Connection.conn_id).filter(Connection.conn_id.in_(potential_connection_ids)) + query = session.scalars( + select(Connection.conn_id).where(Connection.conn_id.in_(potential_connection_ids)) + ) - found_conn_id_set = {conn_id for conn_id, in query} + found_conn_id_set = {conn_id for conn_id in query} possible_conn_id_iter = ( connection_id @@ -5390,7 +5405,7 @@ def _set_dag_runs_to_active_state(self, drs: list[DagRun], state: str, session: """This routine only supports Running and Queued state.""" try: count = 0 - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 if state == State.RUNNING: dr.start_date = timezone.utcnow() @@ -5416,7 +5431,7 @@ def action_set_failed(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 altered_tis += set_dag_run_state_to_failed( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5444,7 +5459,7 @@ def action_set_success(self, drs: list[DagRun], session: Session = NEW_SESSION): try: count = 0 altered_tis = [] - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 altered_tis += set_dag_run_state_to_success( dag=get_airflow_app().dag_bag.get_dag(dr.dag_id), @@ -5468,7 +5483,7 @@ def action_clear(self, drs: list[DagRun], session: Session = NEW_SESSION): count = 0 cleared_ti_count = 0 dag_to_tis: dict[DAG, list[TaskInstance]] = {} - for dr in session.query(DagRun).filter(DagRun.id.in_(dagrun.id for dagrun in drs)): + for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 dag = get_airflow_app().dag_bag.get_dag(dr.dag_id) tis_to_clear = dag_to_tis.setdefault(dag, []) @@ -5884,35 +5899,37 @@ def autocomplete(self, session: Session = NEW_SESSION): return flask.json.jsonify([]) # Provide suggestions of dag_ids and owners - dag_ids_query = session.query( + dag_ids_query = select( sqla.literal("dag").label("type"), DagModel.dag_id.label("name"), - ).filter(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) + ).where(~DagModel.is_subdag, DagModel.is_active, DagModel.dag_id.ilike(f"%{query}%")) owners_query = ( - session.query( + select( sqla.literal("owner").label("type"), DagModel.owners.label("name"), ) .distinct() - .filter(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) + .where(~DagModel.is_subdag, DagModel.is_active, DagModel.owners.ilike(f"%{query}%")) ) # Hide DAGs if not showing status: "all" status = flask_session.get(FILTER_STATUS_COOKIE) if status == "active": - dag_ids_query = dag_ids_query.filter(~DagModel.is_paused) - owners_query = owners_query.filter(~DagModel.is_paused) + dag_ids_query = dag_ids_query.where(~DagModel.is_paused) + owners_query = owners_query.where(~DagModel.is_paused) elif status == "paused": - dag_ids_query = dag_ids_query.filter(DagModel.is_paused) - owners_query = owners_query.filter(DagModel.is_paused) + dag_ids_query = dag_ids_query.where(DagModel.is_paused) + owners_query = owners_query.where(DagModel.is_paused) filter_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) - dag_ids_query = dag_ids_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - owners_query = owners_query.filter(DagModel.dag_id.in_(filter_dag_ids)) - - payload = [row._asdict() for row in dag_ids_query.union(owners_query).order_by("name").limit(10)] + dag_ids_query = dag_ids_query.where(DagModel.dag_id.in_(filter_dag_ids)) + owners_query = owners_query.where(DagModel.dag_id.in_(filter_dag_ids)) + payload = [ + row._asdict() + for row in session.execute(dag_ids_query.union(owners_query).order_by("name").limit(10)) + ] return flask.json.jsonify(payload) From e53320d62030a53c6ffe896434bcf0fc85803f31 Mon Sep 17 00:00:00 2001 From: Sai Pragna Etikyala Date: Wed, 5 Jul 2023 02:22:30 -0700 Subject: [PATCH 057/533] Update python.rst with multi-Scheduler LocalExecutor info (#32310) Co-authored-by: Pankaj Koti --- docs/apache-airflow/core-concepts/executor/local.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/apache-airflow/core-concepts/executor/local.rst b/docs/apache-airflow/core-concepts/executor/local.rst index 256c6125538c..94a3f512112a 100644 --- a/docs/apache-airflow/core-concepts/executor/local.rst +++ b/docs/apache-airflow/core-concepts/executor/local.rst @@ -47,3 +47,11 @@ Arguably, :class:`~airflow.executors.sequential_executor.SequentialExecutor` cou parallelism of just 1 worker, i.e. ``self.parallelism = 1``. This option could lead to the unification of the executor implementations, running locally, into just one :class:`~airflow.executors.local_executor.LocalExecutor` with multiple modes. + +.. note:: + + When multiple Schedulers are configured with ``executor = LocalExecutor`` in the ``[core]`` section of your ``airflow.cfg``, each Scheduler will run a LocalExecutor. This means tasks would be processed in a distributed fashion across the machines running the Schedulers. + + One consideration should be taken into account: + + - Restarting a Scheduler: If a Scheduler is restarted, it may take some time for other Schedulers to recognize the orphaned tasks and restart or fail them. From c8a3c112a7bae345d37bb8b90d68c8d6ff2ef8fc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 5 Jul 2023 13:28:16 +0200 Subject: [PATCH 058/533] Retry transactions on occasional deadlocks for rendered fields (#32341) This is a follow-up on #18616 where we introduced retries on the occassional deadlocks when rendered task fields have been deleted by parallel threads (this is not a real deadlock, it's because MySQL locks too many things when queries are executed and will deadlock when one of those queries wait too much). Adding retry - while not perfect - should allow to handle the problem and significantly decrease the likelihood of such deadlocks. We can probably think about different approach for rendered fields, but for now retrying is - I think - acceptable short-term fix. Fixes: #32294 Fixes: #29687 --- airflow/models/renderedtifields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 8cea81715f37..958683447004 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -175,6 +175,7 @@ def get_k8s_pod_yaml(cls, ti: TaskInstance, session: Session = NEW_SESSION) -> d return result.k8s_pod_yaml if result else None @provide_session + @retry_db_transaction def write(self, session: Session = None): """Write instance to database. From 41954fe194afcd316153ae18f9e7448af4a7f368 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 5 Jul 2023 17:14:46 +0530 Subject: [PATCH 059/533] Improving LOCAL_VIRTUALENV.rst for ease of local venv setup (#32372) Co-authored-by: Amogh Desai --- LOCAL_VIRTUALENV.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 987a0df4bfa5..1f541270ec8e 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -58,12 +58,18 @@ Homebrew for macOS to install required software packages: Refer to the `Dockerfile.ci `__ for a comprehensive list of required packages. +.. note:: + + - MySql 2.2.0 needs pkgconf to be a pre requisite, refer `here `_ to install pkgconf + - MacOs with ARM architectures require graphviz for venv setup, refer `here `_ to install graphviz + Extra Packages -------------- .. note:: Only ``pip`` installation is currently officially supported. + Make sure you have the latest pip installed, reference `version `_ While there are some successes with using other tools like `poetry `_ or `pip-tools `_, they do not share the same workflow as From ee4a838d49461b3b053a9cbe660dbff06a17fff5 Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Wed, 5 Jul 2023 15:36:10 +0200 Subject: [PATCH 060/533] Pass SQLAlchemy engine to construct information schema query. (#32371) Signed-off-by: Jakub Dardzinski --- airflow/providers/common/sql/hooks/sql.py | 2 +- airflow/providers/common/sql/operators/sql.py | 6 +++++- airflow/providers/openlineage/sqlparser.py | 20 +++++++++++++++++-- .../common/sql/operators/test_sql_execute.py | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 4ff6e62e2497..3f2f964a0bb6 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -538,7 +538,7 @@ def get_openlineage_database_dialect(self, connection) -> str: """ return "generic" - def get_openlineage_default_schema(self) -> str: + def get_openlineage_default_schema(self) -> str | None: """ Returns default schema specific to database. diff --git a/airflow/providers/common/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py index 5f129e0817e6..53b499f50631 100644 --- a/airflow/providers/common/sql/operators/sql.py +++ b/airflow/providers/common/sql/operators/sql.py @@ -320,7 +320,11 @@ def get_openlineage_facets_on_start(self) -> OperatorLineage | None: return None operator_lineage = sql_parser.generate_openlineage_metadata_from_sql( - sql=self.sql, hook=hook, database_info=database_info, database=self.database + sql=self.sql, + hook=hook, + database_info=database_info, + database=self.database, + sqlalchemy_engine=hook.get_sqlalchemy_engine(), ) return operator_lineage diff --git a/airflow/providers/openlineage/sqlparser.py b/airflow/providers/openlineage/sqlparser.py index 8e85d3706c52..fdf36c94d37f 100644 --- a/airflow/providers/openlineage/sqlparser.py +++ b/airflow/providers/openlineage/sqlparser.py @@ -33,6 +33,8 @@ from openlineage.common.sql import DbTableMeta, SqlMeta, parse if TYPE_CHECKING: + from sqlalchemy.engine import Engine + from airflow.hooks.base import BaseHook DEFAULT_NAMESPACE = "default" @@ -113,6 +115,7 @@ def parse_table_schemas( database_info: DatabaseInfo, namespace: str = DEFAULT_NAMESPACE, database: str | None = None, + sqlalchemy_engine: Engine | None = None, ) -> tuple[list[Dataset], ...]: """Parse schemas for input and output tables.""" database_kwargs: GetTableSchemasParams = { @@ -128,8 +131,16 @@ def parse_table_schemas( namespace, self.default_schema, database or database_info.database, - self.create_information_schema_query(tables=inputs, **database_kwargs) if inputs else None, - self.create_information_schema_query(tables=outputs, **database_kwargs) if outputs else None, + self.create_information_schema_query( + tables=inputs, sqlalchemy_engine=sqlalchemy_engine, **database_kwargs + ) + if inputs + else None, + self.create_information_schema_query( + tables=outputs, sqlalchemy_engine=sqlalchemy_engine, **database_kwargs + ) + if outputs + else None, ) def generate_openlineage_metadata_from_sql( @@ -138,6 +149,7 @@ def generate_openlineage_metadata_from_sql( hook: BaseHook, database_info: DatabaseInfo, database: str | None = None, + sqlalchemy_engine: Engine | None = None, ) -> OperatorLineage: """Parses SQL statement(s) and generates OpenLineage metadata. @@ -152,6 +164,7 @@ def generate_openlineage_metadata_from_sql( :param hook: Airflow Hook used to connect to the database :param database_info: database specific information :param database: when passed it takes precedence over parsed database name + :param sqlalchemy_engine: when passed, engine's dialect is used to compile SQL queries """ job_facets: dict[str, BaseFacet] = {"sql": SqlJobFacet(query=self.normalize_sql(sql))} parse_result = self.parse(self.split_sql_string(sql)) @@ -182,6 +195,7 @@ def generate_openlineage_metadata_from_sql( namespace=namespace, database=database, database_info=database_info, + sqlalchemy_engine=sqlalchemy_engine, ) return OperatorLineage( @@ -236,6 +250,7 @@ def create_information_schema_query( information_schema_table, is_uppercase_names, database: str | None = None, + sqlalchemy_engine: Engine | None = None, ) -> str: """Creates SELECT statement to query information schema table.""" tables_hierarchy = cls._get_tables_hierarchy( @@ -249,6 +264,7 @@ def create_information_schema_query( information_schema_table_name=information_schema_table, tables_hierarchy=tables_hierarchy, uppercase_names=is_uppercase_names, + sqlalchemy_engine=sqlalchemy_engine, ) @staticmethod diff --git a/tests/providers/common/sql/operators/test_sql_execute.py b/tests/providers/common/sql/operators/test_sql_execute.py index 5bc2a932f878..48f59e0ffea4 100644 --- a/tests/providers/common/sql/operators/test_sql_execute.py +++ b/tests/providers/common/sql/operators/test_sql_execute.py @@ -316,7 +316,7 @@ def get_db_hook(self): (DB_SCHEMA_NAME, "popular_orders_day_of_week", "orders_placed", 3, "int4"), ] dbapi_hook.get_connection.return_value = Connection( - conn_id="sql_default", conn_type="postgres", host="host", port=1234 + conn_id="sql_default", conn_type="postgresql", host="host", port=1234 ) dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [rows, []] From 3878fe6fab3ccc1461932b456c48996f2763139f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 5 Jul 2023 16:40:00 +0200 Subject: [PATCH 061/533] Remove spurious headers for provider changelogs (#32373) --- airflow/providers/airbyte/CHANGELOG.rst | 8 +++++--- airflow/providers/alibaba/CHANGELOG.rst | 8 +++++--- airflow/providers/amazon/CHANGELOG.rst | 8 +++++--- airflow/providers/apache/beam/CHANGELOG.rst | 8 +++++--- airflow/providers/apache/cassandra/CHANGELOG.rst | 8 +++++--- airflow/providers/apache/drill/CHANGELOG.rst | 8 +++++--- airflow/providers/apache/druid/CHANGELOG.rst | 9 ++++++--- airflow/providers/apache/flink/CHANGELOG.rst | 8 +++++--- airflow/providers/apache/hdfs/CHANGELOG.rst | 9 ++++++--- airflow/providers/apache/hive/CHANGELOG.rst | 9 ++++++--- airflow/providers/apache/impala/CHANGELOG.rst | 9 ++++++--- airflow/providers/apache/kafka/CHANGELOG.rst | 10 +++++++--- airflow/providers/apache/kylin/CHANGELOG.rst | 11 ++++++++--- airflow/providers/apache/livy/CHANGELOG.rst | 10 +++++++--- airflow/providers/apache/pig/CHANGELOG.rst | 11 ++++++++--- airflow/providers/apache/pinot/CHANGELOG.rst | 11 ++++++++--- airflow/providers/apache/spark/CHANGELOG.rst | 11 ++++++++--- airflow/providers/apache/sqoop/CHANGELOG.rst | 10 +++++++--- airflow/providers/apprise/CHANGELOG.rst | 8 +++++++- airflow/providers/arangodb/CHANGELOG.rst | 9 ++++++--- airflow/providers/asana/CHANGELOG.rst | 9 ++++++--- airflow/providers/atlassian/jira/CHANGELOG.rst | 9 ++++++--- airflow/providers/celery/CHANGELOG.rst | 9 ++++++--- airflow/providers/cloudant/CHANGELOG.rst | 9 ++++++--- airflow/providers/cncf/kubernetes/CHANGELOG.rst | 9 ++++++--- airflow/providers/common/sql/CHANGELOG.rst | 5 ++--- airflow/providers/databricks/CHANGELOG.rst | 9 ++++++--- airflow/providers/datadog/CHANGELOG.rst | 9 ++++++--- airflow/providers/dbt/cloud/CHANGELOG.rst | 10 +++++++--- airflow/providers/dingding/CHANGELOG.rst | 9 ++++++--- airflow/providers/discord/CHANGELOG.rst | 9 ++++++--- airflow/providers/docker/CHANGELOG.rst | 9 ++++++--- airflow/providers/elasticsearch/CHANGELOG.rst | 9 ++++++--- airflow/providers/exasol/CHANGELOG.rst | 9 ++++++--- airflow/providers/facebook/CHANGELOG.rst | 9 ++++++--- airflow/providers/ftp/CHANGELOG.rst | 9 ++++++--- airflow/providers/github/CHANGELOG.rst | 10 +++++++--- airflow/providers/google/CHANGELOG.rst | 10 +++++++--- airflow/providers/grpc/CHANGELOG.rst | 9 ++++++--- airflow/providers/hashicorp/CHANGELOG.rst | 9 ++++++--- airflow/providers/http/CHANGELOG.rst | 9 ++++++--- airflow/providers/imap/CHANGELOG.rst | 8 +++++--- airflow/providers/influxdb/CHANGELOG.rst | 9 ++++++--- airflow/providers/jdbc/CHANGELOG.rst | 8 +++++--- airflow/providers/jenkins/CHANGELOG.rst | 9 ++++++--- airflow/providers/microsoft/azure/CHANGELOG.rst | 9 ++++++--- airflow/providers/microsoft/mssql/CHANGELOG.rst | 9 ++++++--- airflow/providers/microsoft/psrp/CHANGELOG.rst | 9 ++++++--- airflow/providers/microsoft/winrm/CHANGELOG.rst | 9 ++++++--- airflow/providers/mongo/CHANGELOG.rst | 9 ++++++--- airflow/providers/mysql/CHANGELOG.rst | 9 ++++++--- airflow/providers/neo4j/CHANGELOG.rst | 10 +++++++--- airflow/providers/odbc/CHANGELOG.rst | 9 ++++++--- airflow/providers/openfaas/CHANGELOG.rst | 9 ++++++--- airflow/providers/openlineage/CHANGELOG.rst | 9 ++++++--- airflow/providers/opsgenie/CHANGELOG.rst | 11 +++++++---- airflow/providers/oracle/CHANGELOG.rst | 9 ++++++--- airflow/providers/pagerduty/CHANGELOG.rst | 9 ++++++--- airflow/providers/papermill/CHANGELOG.rst | 9 ++++++--- airflow/providers/plexus/CHANGELOG.rst | 9 ++++++--- airflow/providers/postgres/CHANGELOG.rst | 9 ++++++--- airflow/providers/presto/CHANGELOG.rst | 9 ++++++--- airflow/providers/qubole/CHANGELOG.rst | 9 ++++++--- airflow/providers/redis/CHANGELOG.rst | 9 ++++++--- airflow/providers/salesforce/CHANGELOG.rst | 9 ++++++--- airflow/providers/samba/CHANGELOG.rst | 9 ++++++--- airflow/providers/segment/CHANGELOG.rst | 9 ++++++--- airflow/providers/sendgrid/CHANGELOG.rst | 9 ++++++--- airflow/providers/sftp/CHANGELOG.rst | 9 ++++++--- airflow/providers/singularity/CHANGELOG.rst | 9 ++++++--- airflow/providers/slack/CHANGELOG.rst | 9 ++++++--- airflow/providers/smtp/CHANGELOG.rst | 9 ++++++--- airflow/providers/snowflake/CHANGELOG.rst | 9 ++++++--- airflow/providers/sqlite/CHANGELOG.rst | 9 ++++++--- airflow/providers/ssh/CHANGELOG.rst | 9 ++++++--- airflow/providers/tableau/CHANGELOG.rst | 9 ++++++--- airflow/providers/tabular/CHANGELOG.rst | 9 ++++++--- airflow/providers/telegram/CHANGELOG.rst | 9 ++++++--- airflow/providers/trino/CHANGELOG.rst | 9 ++++++--- airflow/providers/vertica/CHANGELOG.rst | 10 +++++++--- airflow/providers/yandex/CHANGELOG.rst | 9 ++++++--- airflow/providers/zendesk/CHANGELOG.rst | 9 ++++++--- .../PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 | 3 --- dev/provider_packages/prepare_provider_packages.py | 7 ++++--- docs/apache-airflow-providers-airbyte/changelog.rst | 3 +-- docs/apache-airflow-providers-alibaba/changelog.rst | 3 +-- docs/apache-airflow-providers-amazon/changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../apache-airflow-providers-apache-pig/changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-apprise/changelog.rst | 3 +-- docs/apache-airflow-providers-arangodb/changelog.rst | 3 +-- docs/apache-airflow-providers-asana/changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-celery/changelog.rst | 3 +-- docs/apache-airflow-providers-cloudant/changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../apache-airflow-providers-common-sql/changelog.rst | 3 +-- .../apache-airflow-providers-databricks/changelog.rst | 3 +-- docs/apache-airflow-providers-datadog/changelog.rst | 3 +-- docs/apache-airflow-providers-dbt-cloud/changelog.rst | 3 +-- docs/apache-airflow-providers-dingding/changelog.rst | 3 +-- docs/apache-airflow-providers-discord/changelog.rst | 3 +-- docs/apache-airflow-providers-docker/changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-exasol/changelog.rst | 3 +-- docs/apache-airflow-providers-facebook/changelog.rst | 3 +-- docs/apache-airflow-providers-ftp/changelog.rst | 3 +-- docs/apache-airflow-providers-github/changelog.rst | 3 +-- docs/apache-airflow-providers-google/changelog.rst | 3 +-- docs/apache-airflow-providers-grpc/changelog.rst | 3 +-- docs/apache-airflow-providers-hashicorp/changelog.rst | 3 +-- docs/apache-airflow-providers-http/changelog.rst | 3 +-- docs/apache-airflow-providers-imap/changelog.rst | 3 +-- docs/apache-airflow-providers-influxdb/changelog.rst | 3 +-- docs/apache-airflow-providers-jdbc/changelog.rst | 3 +-- docs/apache-airflow-providers-jenkins/changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-mongo/changelog.rst | 3 +-- docs/apache-airflow-providers-mysql/changelog.rst | 3 +-- docs/apache-airflow-providers-neo4j/changelog.rst | 3 +-- docs/apache-airflow-providers-odbc/changelog.rst | 3 +-- docs/apache-airflow-providers-openfaas/changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-opsgenie/changelog.rst | 3 +-- docs/apache-airflow-providers-oracle/changelog.rst | 3 +-- docs/apache-airflow-providers-pagerduty/changelog.rst | 3 +-- docs/apache-airflow-providers-papermill/changelog.rst | 3 +-- docs/apache-airflow-providers-plexus/changelog.rst | 3 +-- docs/apache-airflow-providers-postgres/changelog.rst | 3 +-- docs/apache-airflow-providers-presto/changelog.rst | 3 +-- docs/apache-airflow-providers-qubole/changelog.rst | 3 +-- docs/apache-airflow-providers-redis/changelog.rst | 3 +-- .../apache-airflow-providers-salesforce/changelog.rst | 3 +-- docs/apache-airflow-providers-samba/changelog.rst | 3 +-- docs/apache-airflow-providers-segment/changelog.rst | 3 +-- docs/apache-airflow-providers-sendgrid/changelog.rst | 3 +-- docs/apache-airflow-providers-sftp/changelog.rst | 3 +-- .../changelog.rst | 3 +-- docs/apache-airflow-providers-slack/changelog.rst | 3 +-- docs/apache-airflow-providers-smtp/changelog.rst | 3 +-- docs/apache-airflow-providers-snowflake/changelog.rst | 3 +-- docs/apache-airflow-providers-sqlite/changelog.rst | 3 +-- docs/apache-airflow-providers-ssh/changelog.rst | 3 +-- docs/apache-airflow-providers-tableau/changelog.rst | 3 +-- docs/apache-airflow-providers-tabular/changelog.rst | 3 +-- docs/apache-airflow-providers-telegram/changelog.rst | 3 +-- docs/apache-airflow-providers-trino/changelog.rst | 3 +-- docs/apache-airflow-providers-vertica/changelog.rst | 3 +-- docs/apache-airflow-providers-zendesk/changelog.rst | 3 +-- 165 files changed, 582 insertions(+), 413 deletions(-) diff --git a/airflow/providers/airbyte/CHANGELOG.rst b/airflow/providers/airbyte/CHANGELOG.rst index 6d8f34d5cd49..00a9cece44ac 100644 --- a/airflow/providers/airbyte/CHANGELOG.rst +++ b/airflow/providers/airbyte/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-airbyte`` --------------------------------------------------- +``apache-airflow-providers-airbyte`` + +Changelog +--------- 3.3.1 diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/airflow/providers/alibaba/CHANGELOG.rst index af267238d0f8..b55e5aa6ea61 100644 --- a/airflow/providers/alibaba/CHANGELOG.rst +++ b/airflow/providers/alibaba/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-alibaba`` --------------------------------------------------- +``apache-airflow-providers-alibaba`` + +Changelog +--------- 2.5.0 ..... diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index 1b3b94fefc23..8af159f61104 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-amazon`` -------------------------------------------------- +``apache-airflow-providers-amazon`` + +Changelog +--------- 8.3.0 ..... diff --git a/airflow/providers/apache/beam/CHANGELOG.rst b/airflow/providers/apache/beam/CHANGELOG.rst index 495fba99e910..2176fc18c5e3 100644 --- a/airflow/providers/apache/beam/CHANGELOG.rst +++ b/airflow/providers/apache/beam/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-beam`` ------------------------------------------------------- +``apache-airflow-providers-apache-beam`` + +Changelog +--------- 5.1.1 ..... diff --git a/airflow/providers/apache/cassandra/CHANGELOG.rst b/airflow/providers/apache/cassandra/CHANGELOG.rst index 00bfcb83f816..10895c8d1507 100644 --- a/airflow/providers/apache/cassandra/CHANGELOG.rst +++ b/airflow/providers/apache/cassandra/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-cassandra`` ------------------------------------------------------------ +``apache-airflow-providers-apache-cassandra`` + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/apache/drill/CHANGELOG.rst b/airflow/providers/apache/drill/CHANGELOG.rst index 60df655ec462..37b6b3a5fdd0 100644 --- a/airflow/providers/apache/drill/CHANGELOG.rst +++ b/airflow/providers/apache/drill/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-drill`` -------------------------------------------------------- +``apache-airflow-providers-apache-drill`` + +Changelog +--------- 2.4.1 diff --git a/airflow/providers/apache/druid/CHANGELOG.rst b/airflow/providers/apache/druid/CHANGELOG.rst index 9a95a041ef71..8e7b0f67ee36 100644 --- a/airflow/providers/apache/druid/CHANGELOG.rst +++ b/airflow/providers/apache/druid/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-druid`` -------------------------------------------------------- +``apache-airflow-providers-apache-druid`` + + +Changelog +--------- 3.4.1 ..... diff --git a/airflow/providers/apache/flink/CHANGELOG.rst b/airflow/providers/apache/flink/CHANGELOG.rst index 40af6d54d452..25e177350b5c 100644 --- a/airflow/providers/apache/flink/CHANGELOG.rst +++ b/airflow/providers/apache/flink/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-flink`` -------------------------------------------------------- +``apache-airflow-providers-apache-flink`` + +Changelog +--------- 1.1.1 ..... diff --git a/airflow/providers/apache/hdfs/CHANGELOG.rst b/airflow/providers/apache/hdfs/CHANGELOG.rst index 7a8000b202a5..37de3a05cc49 100644 --- a/airflow/providers/apache/hdfs/CHANGELOG.rst +++ b/airflow/providers/apache/hdfs/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-hdfs`` ------------------------------------------------------- +``apache-airflow-providers-apache-hdfs`` + + +Changelog +--------- 4.1.0 ----- diff --git a/airflow/providers/apache/hive/CHANGELOG.rst b/airflow/providers/apache/hive/CHANGELOG.rst index 87c44ca909aa..a57689b1d857 100644 --- a/airflow/providers/apache/hive/CHANGELOG.rst +++ b/airflow/providers/apache/hive/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-hive`` ------------------------------------------------------- +``apache-airflow-providers-apache-hive`` + + +Changelog +--------- 6.1.1 diff --git a/airflow/providers/apache/impala/CHANGELOG.rst b/airflow/providers/apache/impala/CHANGELOG.rst index 95e6fca155f0..3dcd84dba83d 100644 --- a/airflow/providers/apache/impala/CHANGELOG.rst +++ b/airflow/providers/apache/impala/CHANGELOG.rst @@ -16,12 +16,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-impala`` --------------------------------------------------------- +``apache-airflow-providers-apache-impala`` + + +Changelog +--------- 1.1.1 ..... diff --git a/airflow/providers/apache/kafka/CHANGELOG.rst b/airflow/providers/apache/kafka/CHANGELOG.rst index 95898617e6c7..5c77dc4b349c 100644 --- a/airflow/providers/apache/kafka/CHANGELOG.rst +++ b/airflow/providers/apache/kafka/CHANGELOG.rst @@ -16,12 +16,16 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-kafka`` -------------------------------------------------------- + +``apache-airflow-providers-apache-kafka`` + + +Changelog +--------- 1.1.2 ..... diff --git a/airflow/providers/apache/kylin/CHANGELOG.rst b/airflow/providers/apache/kylin/CHANGELOG.rst index a40ddec0324c..48b03a0d9b15 100644 --- a/airflow/providers/apache/kylin/CHANGELOG.rst +++ b/airflow/providers/apache/kylin/CHANGELOG.rst @@ -17,12 +17,17 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-kylin`` -------------------------------------------------------- + +``apache-airflow-providers-apache-kylin`` + + + +Changelog +--------- 3.2.1 diff --git a/airflow/providers/apache/livy/CHANGELOG.rst b/airflow/providers/apache/livy/CHANGELOG.rst index 93dc82bbbfe8..36fb6f5cb330 100644 --- a/airflow/providers/apache/livy/CHANGELOG.rst +++ b/airflow/providers/apache/livy/CHANGELOG.rst @@ -17,12 +17,16 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-livy`` ------------------------------------------------------- + +``apache-airflow-providers-apache-livy`` + + +Changelog +--------- 3.5.2 ..... diff --git a/airflow/providers/apache/pig/CHANGELOG.rst b/airflow/providers/apache/pig/CHANGELOG.rst index 72ba0f4a3911..7279d51d7f6a 100644 --- a/airflow/providers/apache/pig/CHANGELOG.rst +++ b/airflow/providers/apache/pig/CHANGELOG.rst @@ -17,12 +17,17 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-pig`` ------------------------------------------------------ + +``apache-airflow-providers-apache-pig`` + + + +Changelog +--------- 4.1.1 ..... diff --git a/airflow/providers/apache/pinot/CHANGELOG.rst b/airflow/providers/apache/pinot/CHANGELOG.rst index eb4942319956..62fe3e9fa007 100644 --- a/airflow/providers/apache/pinot/CHANGELOG.rst +++ b/airflow/providers/apache/pinot/CHANGELOG.rst @@ -17,12 +17,17 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-pinot`` -------------------------------------------------------- + +``apache-airflow-providers-apache-pinot`` + + + +Changelog +--------- 4.1.1 diff --git a/airflow/providers/apache/spark/CHANGELOG.rst b/airflow/providers/apache/spark/CHANGELOG.rst index 2ec87940533d..8129b380a320 100644 --- a/airflow/providers/apache/spark/CHANGELOG.rst +++ b/airflow/providers/apache/spark/CHANGELOG.rst @@ -17,12 +17,17 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-spark`` -------------------------------------------------------- + +``apache-airflow-providers-apache-spark`` + + + +Changelog +--------- 4.1.1 ..... diff --git a/airflow/providers/apache/sqoop/CHANGELOG.rst b/airflow/providers/apache/sqoop/CHANGELOG.rst index a8a5dca3cd5f..d824b7dd1d9f 100644 --- a/airflow/providers/apache/sqoop/CHANGELOG.rst +++ b/airflow/providers/apache/sqoop/CHANGELOG.rst @@ -17,12 +17,16 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-apache-sqoop`` -------------------------------------------------------- + +``apache-airflow-providers-apache-sqoop`` + + +Changelog +--------- 3.2.1 diff --git a/airflow/providers/apprise/CHANGELOG.rst b/airflow/providers/apprise/CHANGELOG.rst index 9b5d755cbec9..bf8999d3a481 100644 --- a/airflow/providers/apprise/CHANGELOG.rst +++ b/airflow/providers/apprise/CHANGELOG.rst @@ -17,10 +17,16 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. +``apache-airflow-providers-apprise`` + + +Changelog +--------- + 1.0.0 ..... diff --git a/airflow/providers/arangodb/CHANGELOG.rst b/airflow/providers/arangodb/CHANGELOG.rst index e72e8963d432..d51c1f705eda 100644 --- a/airflow/providers/arangodb/CHANGELOG.rst +++ b/airflow/providers/arangodb/CHANGELOG.rst @@ -18,12 +18,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-arangodb`` ---------------------------------------------------- +``apache-airflow-providers-arangodb`` + + +Changelog +--------- 2.2.1 ..... diff --git a/airflow/providers/asana/CHANGELOG.rst b/airflow/providers/asana/CHANGELOG.rst index 0e8de8a3a558..1a811682b7e4 100644 --- a/airflow/providers/asana/CHANGELOG.rst +++ b/airflow/providers/asana/CHANGELOG.rst @@ -16,12 +16,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-asana`` ------------------------------------------------- +``apache-airflow-providers-asana`` + + +Changelog +--------- 2.2.1 ..... diff --git a/airflow/providers/atlassian/jira/CHANGELOG.rst b/airflow/providers/atlassian/jira/CHANGELOG.rst index 3358ac28cb85..f1a09c1b6768 100644 --- a/airflow/providers/atlassian/jira/CHANGELOG.rst +++ b/airflow/providers/atlassian/jira/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-atlassian-jira`` ---------------------------------------------------------- +``apache-airflow-providers-jira`` + + +Changelog +--------- 2.1.1 ..... diff --git a/airflow/providers/celery/CHANGELOG.rst b/airflow/providers/celery/CHANGELOG.rst index 1f3b37ac2d55..85ccbc5fc4be 100644 --- a/airflow/providers/celery/CHANGELOG.rst +++ b/airflow/providers/celery/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-celery`` -------------------------------------------------- +``apache-airflow-providers-celery`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/cloudant/CHANGELOG.rst b/airflow/providers/cloudant/CHANGELOG.rst index 93fea1598dbf..f1d30bb60620 100644 --- a/airflow/providers/cloudant/CHANGELOG.rst +++ b/airflow/providers/cloudant/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-cloudant`` ---------------------------------------------------- +``apache-airflow-providers-cloudant`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index f90802cbe402..c089b470adda 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-cncf-kubernetes`` ----------------------------------------------------------- +``apache-airflow-providers-cncf-kubernetes`` + + +Changelog +--------- 7.2.0 ..... diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst index 5710d8e39dda..566c2e6b4266 100644 --- a/airflow/providers/common/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -16,13 +16,12 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-common-sql`` ------------------------------------------------------ +------ 1.6.0 ..... diff --git a/airflow/providers/databricks/CHANGELOG.rst b/airflow/providers/databricks/CHANGELOG.rst index 21bf6efebcf9..b21db94e25af 100644 --- a/airflow/providers/databricks/CHANGELOG.rst +++ b/airflow/providers/databricks/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-databricks`` ------------------------------------------------------ +``apache-airflow-providers-databricks`` + + +Changelog +--------- 4.3.1 ..... diff --git a/airflow/providers/datadog/CHANGELOG.rst b/airflow/providers/datadog/CHANGELOG.rst index d7e966661e17..59ebe25fdf23 100644 --- a/airflow/providers/datadog/CHANGELOG.rst +++ b/airflow/providers/datadog/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-datadog`` --------------------------------------------------- +``apache-airflow-providers-datadog`` + + +Changelog +--------- 3.3.1 ..... diff --git a/airflow/providers/dbt/cloud/CHANGELOG.rst b/airflow/providers/dbt/cloud/CHANGELOG.rst index e87ecaf4b8ee..7d6b8c1e4c01 100644 --- a/airflow/providers/dbt/cloud/CHANGELOG.rst +++ b/airflow/providers/dbt/cloud/CHANGELOG.rst @@ -17,12 +17,16 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-dbt-cloud`` ----------------------------------------------------- + +``apache-airflow-providers-dbt-cloud`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/dingding/CHANGELOG.rst b/airflow/providers/dingding/CHANGELOG.rst index 78620c282a9c..06c1954a0e6c 100644 --- a/airflow/providers/dingding/CHANGELOG.rst +++ b/airflow/providers/dingding/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-dingding`` ---------------------------------------------------- +``apache-airflow-providers-dingding`` + + +Changelog +--------- 3.2.1 diff --git a/airflow/providers/discord/CHANGELOG.rst b/airflow/providers/discord/CHANGELOG.rst index 23b9b970dc0f..80f50187507e 100644 --- a/airflow/providers/discord/CHANGELOG.rst +++ b/airflow/providers/discord/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-discord`` --------------------------------------------------- +``apache-airflow-providers-discord`` + + +Changelog +--------- 3.3.0 ..... diff --git a/airflow/providers/docker/CHANGELOG.rst b/airflow/providers/docker/CHANGELOG.rst index 8d52fb8577a1..58f996d69459 100644 --- a/airflow/providers/docker/CHANGELOG.rst +++ b/airflow/providers/docker/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-docker`` -------------------------------------------------- +``apache-airflow-providers-docker`` + + +Changelog +--------- 3.7.1 ..... diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index c104ed62ece0..b2838c1590aa 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-elasticsearch`` --------------------------------------------------------- +``apache-airflow-providers-elasticsearch`` + + +Changelog +--------- 5.0.0 ..... diff --git a/airflow/providers/exasol/CHANGELOG.rst b/airflow/providers/exasol/CHANGELOG.rst index a6bbf1659568..c0309305ca8a 100644 --- a/airflow/providers/exasol/CHANGELOG.rst +++ b/airflow/providers/exasol/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-exasol`` -------------------------------------------------- +``apache-airflow-providers-exasol`` + + +Changelog +--------- 4.2.1 ..... diff --git a/airflow/providers/facebook/CHANGELOG.rst b/airflow/providers/facebook/CHANGELOG.rst index 4aca578d134c..0e24be9cb1b2 100644 --- a/airflow/providers/facebook/CHANGELOG.rst +++ b/airflow/providers/facebook/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-facebook`` ---------------------------------------------------- +``apache-airflow-providers-facebook`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/ftp/CHANGELOG.rst b/airflow/providers/ftp/CHANGELOG.rst index d1377ea1152b..9316062acb1d 100644 --- a/airflow/providers/ftp/CHANGELOG.rst +++ b/airflow/providers/ftp/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-ftp`` ----------------------------------------------- +``apache-airflow-providers-ftp`` + + +Changelog +--------- 3.4.2 ..... diff --git a/airflow/providers/github/CHANGELOG.rst b/airflow/providers/github/CHANGELOG.rst index 6e0fcca472e6..b9da973f1796 100644 --- a/airflow/providers/github/CHANGELOG.rst +++ b/airflow/providers/github/CHANGELOG.rst @@ -18,12 +18,16 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-github`` -------------------------------------------------- + +``apache-airflow-providers-github`` + + +Changelog +--------- 2.3.1 ..... diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 630467f4b9e5..a418a8d7b176 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -16,12 +16,16 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-google`` -------------------------------------------------- + +``apache-airflow-providers-google`` + + +Changelog +--------- 10.3.0 ...... diff --git a/airflow/providers/grpc/CHANGELOG.rst b/airflow/providers/grpc/CHANGELOG.rst index b65a9b0a7587..a752fbdaeb52 100644 --- a/airflow/providers/grpc/CHANGELOG.rst +++ b/airflow/providers/grpc/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-grpc`` ------------------------------------------------ +``apache-airflow-providers-grpc`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/hashicorp/CHANGELOG.rst b/airflow/providers/hashicorp/CHANGELOG.rst index 0b314faafb74..4f22ad96d0ab 100644 --- a/airflow/providers/hashicorp/CHANGELOG.rst +++ b/airflow/providers/hashicorp/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-hashicorp`` ----------------------------------------------------- +``apache-airflow-providers-hashicorp`` + + +Changelog +--------- 3.4.2 ..... diff --git a/airflow/providers/http/CHANGELOG.rst b/airflow/providers/http/CHANGELOG.rst index bab59fd533ba..d8d88f5f8088 100644 --- a/airflow/providers/http/CHANGELOG.rst +++ b/airflow/providers/http/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-http`` ------------------------------------------------ +``apache-airflow-providers-http`` + + +Changelog +--------- 4.4.2 ..... diff --git a/airflow/providers/imap/CHANGELOG.rst b/airflow/providers/imap/CHANGELOG.rst index e5ae86a416d1..24cf83e5b8f5 100644 --- a/airflow/providers/imap/CHANGELOG.rst +++ b/airflow/providers/imap/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-imap`` ------------------------------------------------ +``apache-airflow-providers-imap`` + +Changelog +--------- 3.2.2 ..... diff --git a/airflow/providers/influxdb/CHANGELOG.rst b/airflow/providers/influxdb/CHANGELOG.rst index 081456722f9a..432ff7fc7fd8 100644 --- a/airflow/providers/influxdb/CHANGELOG.rst +++ b/airflow/providers/influxdb/CHANGELOG.rst @@ -18,12 +18,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-influxdb`` ---------------------------------------------------- +``apache-airflow-providers-influxdb`` + + +Changelog +--------- 2.2.1 ..... diff --git a/airflow/providers/jdbc/CHANGELOG.rst b/airflow/providers/jdbc/CHANGELOG.rst index a8a079568ba8..c6c41c8d58bb 100644 --- a/airflow/providers/jdbc/CHANGELOG.rst +++ b/airflow/providers/jdbc/CHANGELOG.rst @@ -17,12 +17,14 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-jdbc`` ------------------------------------------------ +``apache-airflow-providers-jdbc`` + +Changelog +--------- 4.0.0 ..... diff --git a/airflow/providers/jenkins/CHANGELOG.rst b/airflow/providers/jenkins/CHANGELOG.rst index 1f9328f892aa..0e396f9269c0 100644 --- a/airflow/providers/jenkins/CHANGELOG.rst +++ b/airflow/providers/jenkins/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-jenkins`` --------------------------------------------------- +``apache-airflow-providers-jenkins`` + + +Changelog +--------- 3.3.1 ..... diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index 223aea264997..d82b4c128941 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-microsoft-azure`` ----------------------------------------------------------- +``apache-airflow-providers-microsoft-azure`` + + +Changelog +--------- 6.2.0 ..... diff --git a/airflow/providers/microsoft/mssql/CHANGELOG.rst b/airflow/providers/microsoft/mssql/CHANGELOG.rst index e161e6fa33e9..adf40ee6cbe7 100644 --- a/airflow/providers/microsoft/mssql/CHANGELOG.rst +++ b/airflow/providers/microsoft/mssql/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-microsoft-mssql`` ----------------------------------------------------------- +``apache-airflow-providers-microsoft-mssql`` + + +Changelog +--------- 3.4.1 ..... diff --git a/airflow/providers/microsoft/psrp/CHANGELOG.rst b/airflow/providers/microsoft/psrp/CHANGELOG.rst index f4fcf01bd10f..cad9578e332c 100644 --- a/airflow/providers/microsoft/psrp/CHANGELOG.rst +++ b/airflow/providers/microsoft/psrp/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-microsoft-psrp`` ---------------------------------------------------------- +``apache-airflow-providers-microsoft-psrp`` + + +Changelog +--------- 2.3.1 ..... diff --git a/airflow/providers/microsoft/winrm/CHANGELOG.rst b/airflow/providers/microsoft/winrm/CHANGELOG.rst index e30d84633268..19fef6082df0 100644 --- a/airflow/providers/microsoft/winrm/CHANGELOG.rst +++ b/airflow/providers/microsoft/winrm/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-microsoft-winrm`` ----------------------------------------------------------- +``apache-airflow-providers-microsoft-winrm`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/mongo/CHANGELOG.rst b/airflow/providers/mongo/CHANGELOG.rst index 2b07ba58d548..730b5286464a 100644 --- a/airflow/providers/mongo/CHANGELOG.rst +++ b/airflow/providers/mongo/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-mongo`` ------------------------------------------------- +``apache-airflow-providers-mongo`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/mysql/CHANGELOG.rst b/airflow/providers/mysql/CHANGELOG.rst index ec482a62ec96..c88a865f7113 100644 --- a/airflow/providers/mysql/CHANGELOG.rst +++ b/airflow/providers/mysql/CHANGELOG.rst @@ -16,12 +16,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-mysql`` ------------------------------------------------- +``apache-airflow-providers-mysql`` + + +Changelog +--------- 5.1.1 ..... diff --git a/airflow/providers/neo4j/CHANGELOG.rst b/airflow/providers/neo4j/CHANGELOG.rst index bf1ac7044607..57bbc05e1378 100644 --- a/airflow/providers/neo4j/CHANGELOG.rst +++ b/airflow/providers/neo4j/CHANGELOG.rst @@ -18,12 +18,16 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-neo4j`` ------------------------------------------------- + +``apache-airflow-providers-neo4j`` + + +Changelog +--------- 3.3.1 ..... diff --git a/airflow/providers/odbc/CHANGELOG.rst b/airflow/providers/odbc/CHANGELOG.rst index 9e2495ef292b..ae5cfcd35bb9 100644 --- a/airflow/providers/odbc/CHANGELOG.rst +++ b/airflow/providers/odbc/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-odbc`` ------------------------------------------------ +``apache-airflow-providers-odbc`` + + +Changelog +--------- 4.0.0 ..... diff --git a/airflow/providers/openfaas/CHANGELOG.rst b/airflow/providers/openfaas/CHANGELOG.rst index 5213b63000c8..ce7ee1fa4856 100644 --- a/airflow/providers/openfaas/CHANGELOG.rst +++ b/airflow/providers/openfaas/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-openfaas`` ---------------------------------------------------- +``apache-airflow-providers-openfaas`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/openlineage/CHANGELOG.rst b/airflow/providers/openlineage/CHANGELOG.rst index c54c6ebc0580..a8494511a54f 100644 --- a/airflow/providers/openlineage/CHANGELOG.rst +++ b/airflow/providers/openlineage/CHANGELOG.rst @@ -16,12 +16,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-openlineage`` ------------------------------------------------------- +``apache-airflow-providers-openlineage`` + + +Changelog +--------- 1.0.0 ..... diff --git a/airflow/providers/opsgenie/CHANGELOG.rst b/airflow/providers/opsgenie/CHANGELOG.rst index 69fd9383045e..3e29953b9a39 100644 --- a/airflow/providers/opsgenie/CHANGELOG.rst +++ b/airflow/providers/opsgenie/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-opsgenie`` ---------------------------------------------------- +``apache-airflow-providers-opsgenie`` + + +Changelog +--------- 5.1.1 ..... @@ -69,7 +72,7 @@ Misc Breaking changes ~~~~~~~~~~~~~~~~ -Remove 'OpsgenieAlertOperator' also removed hooks.opsgenie_alert path +Remove ``OpsgenieAlertOperator`` also removed hooks.opsgenie_alert path * ``Remove deprecated code from Opsgenie provider (#27252)`` diff --git a/airflow/providers/oracle/CHANGELOG.rst b/airflow/providers/oracle/CHANGELOG.rst index bc7e86d76c0b..c351ba422391 100644 --- a/airflow/providers/oracle/CHANGELOG.rst +++ b/airflow/providers/oracle/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-oracle`` -------------------------------------------------- +``apache-airflow-providers-oracle`` + + +Changelog +--------- 3.7.1 ..... diff --git a/airflow/providers/pagerduty/CHANGELOG.rst b/airflow/providers/pagerduty/CHANGELOG.rst index cddb9e34ee3d..27bdf89a56d6 100644 --- a/airflow/providers/pagerduty/CHANGELOG.rst +++ b/airflow/providers/pagerduty/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-pagerduty`` ----------------------------------------------------- +``apache-airflow-providers-pagerduty`` + + +Changelog +--------- 3.3.0 ..... diff --git a/airflow/providers/papermill/CHANGELOG.rst b/airflow/providers/papermill/CHANGELOG.rst index 933b456a49f4..1805d8aa6ded 100644 --- a/airflow/providers/papermill/CHANGELOG.rst +++ b/airflow/providers/papermill/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-papermill`` ----------------------------------------------------- +``apache-airflow-providers-papermill`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/plexus/CHANGELOG.rst b/airflow/providers/plexus/CHANGELOG.rst index d9ed577d220c..9d53bf994e09 100644 --- a/airflow/providers/plexus/CHANGELOG.rst +++ b/airflow/providers/plexus/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-plexus`` -------------------------------------------------- +``apache-airflow-providers-plexus`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/postgres/CHANGELOG.rst b/airflow/providers/postgres/CHANGELOG.rst index 46f391a0fad4..c86c013a64d6 100644 --- a/airflow/providers/postgres/CHANGELOG.rst +++ b/airflow/providers/postgres/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-postgres`` ---------------------------------------------------- +``apache-airflow-providers-postgres`` + + +Changelog +--------- 5.5.1 ..... diff --git a/airflow/providers/presto/CHANGELOG.rst b/airflow/providers/presto/CHANGELOG.rst index d35e42c51847..108bd30fda13 100644 --- a/airflow/providers/presto/CHANGELOG.rst +++ b/airflow/providers/presto/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-presto`` -------------------------------------------------- +``apache-airflow-providers-presto`` + + +Changelog +--------- 5.1.1 ..... diff --git a/airflow/providers/qubole/CHANGELOG.rst b/airflow/providers/qubole/CHANGELOG.rst index c8c7c76aa9dc..80dfff0f1699 100644 --- a/airflow/providers/qubole/CHANGELOG.rst +++ b/airflow/providers/qubole/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-qubole`` -------------------------------------------------- +``apache-airflow-providers-qubole`` + + +Changelog +--------- 3.4.1 ..... diff --git a/airflow/providers/redis/CHANGELOG.rst b/airflow/providers/redis/CHANGELOG.rst index 87b538ec7c07..2aab210268c8 100644 --- a/airflow/providers/redis/CHANGELOG.rst +++ b/airflow/providers/redis/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-redis`` ------------------------------------------------- +``apache-airflow-providers-redis`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/salesforce/CHANGELOG.rst b/airflow/providers/salesforce/CHANGELOG.rst index 78505ad7c870..319f82801a79 100644 --- a/airflow/providers/salesforce/CHANGELOG.rst +++ b/airflow/providers/salesforce/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-salesforce`` ------------------------------------------------------ +``apache-airflow-providers-salesforce`` + + +Changelog +--------- 5.4.1 ..... diff --git a/airflow/providers/samba/CHANGELOG.rst b/airflow/providers/samba/CHANGELOG.rst index 190b1f73f77b..28803623525c 100644 --- a/airflow/providers/samba/CHANGELOG.rst +++ b/airflow/providers/samba/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-samba`` ------------------------------------------------- +``apache-airflow-providers-samba`` + + +Changelog +--------- 4.2.1 ..... diff --git a/airflow/providers/segment/CHANGELOG.rst b/airflow/providers/segment/CHANGELOG.rst index 059ae0472def..b1856f183402 100644 --- a/airflow/providers/segment/CHANGELOG.rst +++ b/airflow/providers/segment/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-segment`` --------------------------------------------------- +``apache-airflow-providers-segment`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/sendgrid/CHANGELOG.rst b/airflow/providers/sendgrid/CHANGELOG.rst index 7e677678f5db..d49504984e8a 100644 --- a/airflow/providers/sendgrid/CHANGELOG.rst +++ b/airflow/providers/sendgrid/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-sendgrid`` ---------------------------------------------------- +``apache-airflow-providers-sendgrid`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/sftp/CHANGELOG.rst b/airflow/providers/sftp/CHANGELOG.rst index dcea4aad3171..951358a53b76 100644 --- a/airflow/providers/sftp/CHANGELOG.rst +++ b/airflow/providers/sftp/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-sftp`` ------------------------------------------------ +``apache-airflow-providers-sftp`` + + +Changelog +--------- 4.3.1 ..... diff --git a/airflow/providers/singularity/CHANGELOG.rst b/airflow/providers/singularity/CHANGELOG.rst index 1ad3b4bc1d54..0a543fa9f849 100644 --- a/airflow/providers/singularity/CHANGELOG.rst +++ b/airflow/providers/singularity/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-singularity`` ------------------------------------------------------- +``apache-airflow-providers-singularity`` + + +Changelog +--------- 3.2.1 ..... diff --git a/airflow/providers/slack/CHANGELOG.rst b/airflow/providers/slack/CHANGELOG.rst index ca1dc3895dfe..e43bbabb679a 100644 --- a/airflow/providers/slack/CHANGELOG.rst +++ b/airflow/providers/slack/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-slack`` ------------------------------------------------- +``apache-airflow-providers-slack`` + + +Changelog +--------- 7.3.1 ..... diff --git a/airflow/providers/smtp/CHANGELOG.rst b/airflow/providers/smtp/CHANGELOG.rst index 103e9a73c349..6e0a955e56b9 100644 --- a/airflow/providers/smtp/CHANGELOG.rst +++ b/airflow/providers/smtp/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-smtp`` ------------------------------------------------ +``apache-airflow-providers-smtp`` + + +Changelog +--------- 1.2.0 ..... diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/airflow/providers/snowflake/CHANGELOG.rst index ee9fca09d8e4..2fa9008d8a9a 100644 --- a/airflow/providers/snowflake/CHANGELOG.rst +++ b/airflow/providers/snowflake/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-snowflake`` ----------------------------------------------------- +``apache-airflow-providers-snowflake`` + + +Changelog +--------- 4.2.0 ..... diff --git a/airflow/providers/sqlite/CHANGELOG.rst b/airflow/providers/sqlite/CHANGELOG.rst index f84ff50a8555..10b34065369e 100644 --- a/airflow/providers/sqlite/CHANGELOG.rst +++ b/airflow/providers/sqlite/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-sqlite`` -------------------------------------------------- +``apache-airflow-providers-sqlite`` + + +Changelog +--------- 3.4.2 ..... diff --git a/airflow/providers/ssh/CHANGELOG.rst b/airflow/providers/ssh/CHANGELOG.rst index 9accc14048aa..b434c6997c14 100644 --- a/airflow/providers/ssh/CHANGELOG.rst +++ b/airflow/providers/ssh/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-ssh`` ----------------------------------------------- +``apache-airflow-providers-ssh`` + + +Changelog +--------- 3.7.1 ..... diff --git a/airflow/providers/tableau/CHANGELOG.rst b/airflow/providers/tableau/CHANGELOG.rst index 41af7710de9b..1cd1289890cc 100644 --- a/airflow/providers/tableau/CHANGELOG.rst +++ b/airflow/providers/tableau/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-tableau`` --------------------------------------------------- +``apache-airflow-providers-tableau`` + + +Changelog +--------- 4.2.1 ..... diff --git a/airflow/providers/tabular/CHANGELOG.rst b/airflow/providers/tabular/CHANGELOG.rst index 9db36926d24e..5ebb94af1b44 100644 --- a/airflow/providers/tabular/CHANGELOG.rst +++ b/airflow/providers/tabular/CHANGELOG.rst @@ -16,12 +16,15 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-tabular`` --------------------------------------------------- +``apache-airflow-providers-tabular`` + + +Changelog +--------- 1.2.1 ..... diff --git a/airflow/providers/telegram/CHANGELOG.rst b/airflow/providers/telegram/CHANGELOG.rst index dcd21d242ff1..00ee3cc07409 100644 --- a/airflow/providers/telegram/CHANGELOG.rst +++ b/airflow/providers/telegram/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-telegram`` ---------------------------------------------------- +``apache-airflow-providers-telegram`` + + +Changelog +--------- 4.1.1 ..... diff --git a/airflow/providers/trino/CHANGELOG.rst b/airflow/providers/trino/CHANGELOG.rst index da3fd0695c17..3b6ee23a3054 100644 --- a/airflow/providers/trino/CHANGELOG.rst +++ b/airflow/providers/trino/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-trino`` ------------------------------------------------- +``apache-airflow-providers-trino`` + + +Changelog +--------- 5.2.0 ..... diff --git a/airflow/providers/vertica/CHANGELOG.rst b/airflow/providers/vertica/CHANGELOG.rst index 36e06b2cf5a6..7793a3dedeb2 100644 --- a/airflow/providers/vertica/CHANGELOG.rst +++ b/airflow/providers/vertica/CHANGELOG.rst @@ -17,12 +17,16 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-vertica`` --------------------------------------------------- +``apache-airflow-providers-vertica`` + + + +Changelog +--------- 3.5.0 ..... diff --git a/airflow/providers/yandex/CHANGELOG.rst b/airflow/providers/yandex/CHANGELOG.rst index 8eee1ab6c99e..03b7ee0f9c1e 100644 --- a/airflow/providers/yandex/CHANGELOG.rst +++ b/airflow/providers/yandex/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-yandex`` -------------------------------------------------- +``apache-airflow-providers-yandex`` + + +Changelog +--------- .. note:: This release dropped support for Python 3.7 diff --git a/airflow/providers/zendesk/CHANGELOG.rst b/airflow/providers/zendesk/CHANGELOG.rst index 3c9f67d9d042..09dc18fc77c0 100644 --- a/airflow/providers/zendesk/CHANGELOG.rst +++ b/airflow/providers/zendesk/CHANGELOG.rst @@ -17,12 +17,15 @@ .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there are some breaking changes + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. -Changelog for ``apache-airflow-providers-zendex`` -------------------------------------------------- +``apache-airflow-providers-zendesk`` + + +Changelog +--------- 4.3.1 ..... diff --git a/dev/provider_packages/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 b/dev/provider_packages/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 index 81d0e42a23ab..3ace4d579405 100644 --- a/dev/provider_packages/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 +++ b/dev/provider_packages/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 @@ -40,7 +40,4 @@ specific language governing permissions and limitations under the License. -Package ``{{ PACKAGE_PIP_NAME }}`` -{{ '-' * (12 + PACKAGE_PIP_NAME | length) }} - .. include:: {{ CHANGELOG_RELATIVE_PATH }}/CHANGELOG.rst diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index cde0918381cf..82f3de998880 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -80,13 +80,14 @@ under the License. .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog for ..." header when there + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes and you want to add an explanation to the users on how they are supposed to deal with them. The changelog is updated and maintained semi-automatically by release manager. +``{{ package_name }}`` -Changelog for ``{{ package_name }}`` -{{ '-' * (18 + package_name | length) }} +Changelog +--------- 1.0.0 ..... diff --git a/docs/apache-airflow-providers-airbyte/changelog.rst b/docs/apache-airflow-providers-airbyte/changelog.rst index df72f14d050e..57b48befaed9 100644 --- a/docs/apache-airflow-providers-airbyte/changelog.rst +++ b/docs/apache-airflow-providers-airbyte/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-airbyte`` --------------------------------------------- + .. include:: ../../airflow/providers/airbyte/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-alibaba/changelog.rst b/docs/apache-airflow-providers-alibaba/changelog.rst index c032cb7d6a48..c4368e7ddbca 100644 --- a/docs/apache-airflow-providers-alibaba/changelog.rst +++ b/docs/apache-airflow-providers-alibaba/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-alibaba`` --------------------------------------------- + .. include:: ../../airflow/providers/alibaba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-amazon/changelog.rst b/docs/apache-airflow-providers-amazon/changelog.rst index aed399da661e..ed6e5a92f2f7 100644 --- a/docs/apache-airflow-providers-amazon/changelog.rst +++ b/docs/apache-airflow-providers-amazon/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-amazon`` -------------------------------------------- + .. include:: ../../airflow/providers/amazon/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-beam/changelog.rst b/docs/apache-airflow-providers-apache-beam/changelog.rst index f8bf09df1c3a..aa8651cc9f41 100644 --- a/docs/apache-airflow-providers-apache-beam/changelog.rst +++ b/docs/apache-airflow-providers-apache-beam/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-beam`` ------------------------------------------------- + .. include:: ../../airflow/providers/apache/beam/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-cassandra/changelog.rst b/docs/apache-airflow-providers-apache-cassandra/changelog.rst index ff78c1158e3e..c792cbe570c7 100644 --- a/docs/apache-airflow-providers-apache-cassandra/changelog.rst +++ b/docs/apache-airflow-providers-apache-cassandra/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-cassandra`` ------------------------------------------------------ + .. include:: ../../airflow/providers/apache/cassandra/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-drill/changelog.rst b/docs/apache-airflow-providers-apache-drill/changelog.rst index d1cdd2f3040d..38e2ce1ea797 100644 --- a/docs/apache-airflow-providers-apache-drill/changelog.rst +++ b/docs/apache-airflow-providers-apache-drill/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-drill`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/drill/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-druid/changelog.rst b/docs/apache-airflow-providers-apache-druid/changelog.rst index a77d36598a8e..3e7dad97ff5f 100644 --- a/docs/apache-airflow-providers-apache-druid/changelog.rst +++ b/docs/apache-airflow-providers-apache-druid/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-druid`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/druid/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-flink/changelog.rst b/docs/apache-airflow-providers-apache-flink/changelog.rst index 5738f55e53d2..8b3fc75fa18d 100644 --- a/docs/apache-airflow-providers-apache-flink/changelog.rst +++ b/docs/apache-airflow-providers-apache-flink/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-flink`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/flink/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hdfs/changelog.rst b/docs/apache-airflow-providers-apache-hdfs/changelog.rst index 85395a91c919..234612271e3f 100644 --- a/docs/apache-airflow-providers-apache-hdfs/changelog.rst +++ b/docs/apache-airflow-providers-apache-hdfs/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-hdfs`` ------------------------------------------------- + .. include:: ../../airflow/providers/apache/hdfs/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hive/changelog.rst b/docs/apache-airflow-providers-apache-hive/changelog.rst index d93dba54e397..d8ad824aff3d 100644 --- a/docs/apache-airflow-providers-apache-hive/changelog.rst +++ b/docs/apache-airflow-providers-apache-hive/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-hive`` ------------------------------------------------- + .. include:: ../../airflow/providers/apache/hive/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-impala/changelog.rst b/docs/apache-airflow-providers-apache-impala/changelog.rst index 5f504622be8a..0f07c87feef3 100644 --- a/docs/apache-airflow-providers-apache-impala/changelog.rst +++ b/docs/apache-airflow-providers-apache-impala/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-impala`` --------------------------------------------------- + .. include:: ../../airflow/providers/apache/impala/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kafka/changelog.rst b/docs/apache-airflow-providers-apache-kafka/changelog.rst index b2b294b36cc4..b1dafd854b3e 100644 --- a/docs/apache-airflow-providers-apache-kafka/changelog.rst +++ b/docs/apache-airflow-providers-apache-kafka/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-kafka`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/kafka/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kylin/changelog.rst b/docs/apache-airflow-providers-apache-kylin/changelog.rst index 30e11d3bde01..b67c0f97e573 100644 --- a/docs/apache-airflow-providers-apache-kylin/changelog.rst +++ b/docs/apache-airflow-providers-apache-kylin/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-kylin`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/kylin/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-livy/changelog.rst b/docs/apache-airflow-providers-apache-livy/changelog.rst index 1e0af37c44c6..05f40d97d28b 100644 --- a/docs/apache-airflow-providers-apache-livy/changelog.rst +++ b/docs/apache-airflow-providers-apache-livy/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-livy`` ------------------------------------------------- + .. include:: ../../airflow/providers/apache/livy/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-pig/changelog.rst b/docs/apache-airflow-providers-apache-pig/changelog.rst index 355f67835bf7..f1cc7943293d 100644 --- a/docs/apache-airflow-providers-apache-pig/changelog.rst +++ b/docs/apache-airflow-providers-apache-pig/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-pig`` ------------------------------------------------ + .. include:: ../../airflow/providers/apache/pig/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-pinot/changelog.rst b/docs/apache-airflow-providers-apache-pinot/changelog.rst index 4e510c766ef0..55e366743c38 100644 --- a/docs/apache-airflow-providers-apache-pinot/changelog.rst +++ b/docs/apache-airflow-providers-apache-pinot/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-pinot`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/pinot/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-spark/changelog.rst b/docs/apache-airflow-providers-apache-spark/changelog.rst index 9817d195020d..1a8c7b1c3ea1 100644 --- a/docs/apache-airflow-providers-apache-spark/changelog.rst +++ b/docs/apache-airflow-providers-apache-spark/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-spark`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/spark/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-sqoop/changelog.rst b/docs/apache-airflow-providers-apache-sqoop/changelog.rst index b17af28da0e9..3e7b24a33909 100644 --- a/docs/apache-airflow-providers-apache-sqoop/changelog.rst +++ b/docs/apache-airflow-providers-apache-sqoop/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apache-sqoop`` -------------------------------------------------- + .. include:: ../../airflow/providers/apache/sqoop/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apprise/changelog.rst b/docs/apache-airflow-providers-apprise/changelog.rst index 3dc9d66f9f63..75d15ffd0fb0 100644 --- a/docs/apache-airflow-providers-apprise/changelog.rst +++ b/docs/apache-airflow-providers-apprise/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-apprise`` --------------------------------------------- + .. include:: ../../airflow/providers/apprise/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-arangodb/changelog.rst b/docs/apache-airflow-providers-arangodb/changelog.rst index 53d58b062323..bb63458926a9 100644 --- a/docs/apache-airflow-providers-arangodb/changelog.rst +++ b/docs/apache-airflow-providers-arangodb/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-arangodb`` ---------------------------------------------- + .. include:: ../../airflow/providers/arangodb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-asana/changelog.rst b/docs/apache-airflow-providers-asana/changelog.rst index f27de03b77d9..eba1b4ee7c6d 100644 --- a/docs/apache-airflow-providers-asana/changelog.rst +++ b/docs/apache-airflow-providers-asana/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-asana`` ------------------------------------------- + .. include:: ../../airflow/providers/asana/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-atlassian-jira/changelog.rst b/docs/apache-airflow-providers-atlassian-jira/changelog.rst index f5b77baa5dc9..30962a6d1e0c 100644 --- a/docs/apache-airflow-providers-atlassian-jira/changelog.rst +++ b/docs/apache-airflow-providers-atlassian-jira/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-atlassian-jira`` ---------------------------------------------------- + .. include:: ../../airflow/providers/atlassian/jira/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-celery/changelog.rst b/docs/apache-airflow-providers-celery/changelog.rst index 3a3395be9862..301ee64f5426 100644 --- a/docs/apache-airflow-providers-celery/changelog.rst +++ b/docs/apache-airflow-providers-celery/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-celery`` -------------------------------------------- + .. include:: ../../airflow/providers/celery/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cloudant/changelog.rst b/docs/apache-airflow-providers-cloudant/changelog.rst index d92c198cfced..14c1a3715a4a 100644 --- a/docs/apache-airflow-providers-cloudant/changelog.rst +++ b/docs/apache-airflow-providers-cloudant/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-cloudant`` ---------------------------------------------- + .. include:: ../../airflow/providers/cloudant/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst index 284f10248847..758cd4c9abc7 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-cncf-kubernetes`` ----------------------------------------------------- + .. include:: ../../airflow/providers/cncf/kubernetes/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-common-sql/changelog.rst b/docs/apache-airflow-providers-common-sql/changelog.rst index 2b8d6ae7e899..be8dae1c039b 100644 --- a/docs/apache-airflow-providers-common-sql/changelog.rst +++ b/docs/apache-airflow-providers-common-sql/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-common-sql`` ------------------------------------------------ + .. include:: ../../airflow/providers/common/sql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-databricks/changelog.rst b/docs/apache-airflow-providers-databricks/changelog.rst index 9a262c2567c0..6c5cde733297 100644 --- a/docs/apache-airflow-providers-databricks/changelog.rst +++ b/docs/apache-airflow-providers-databricks/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-databricks`` ------------------------------------------------ + .. include:: ../../airflow/providers/databricks/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-datadog/changelog.rst b/docs/apache-airflow-providers-datadog/changelog.rst index 99ff68850f81..125edeffee29 100644 --- a/docs/apache-airflow-providers-datadog/changelog.rst +++ b/docs/apache-airflow-providers-datadog/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-datadog`` --------------------------------------------- + .. include:: ../../airflow/providers/datadog/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dbt-cloud/changelog.rst b/docs/apache-airflow-providers-dbt-cloud/changelog.rst index 7d262eb397b0..ab244b93a10e 100644 --- a/docs/apache-airflow-providers-dbt-cloud/changelog.rst +++ b/docs/apache-airflow-providers-dbt-cloud/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-dbt-cloud`` ----------------------------------------------- + .. include:: ../../airflow/providers/dbt/cloud/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dingding/changelog.rst b/docs/apache-airflow-providers-dingding/changelog.rst index 272bd9d017e0..6303afaa103e 100644 --- a/docs/apache-airflow-providers-dingding/changelog.rst +++ b/docs/apache-airflow-providers-dingding/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-dingding`` ---------------------------------------------- + .. include:: ../../airflow/providers/dingding/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-discord/changelog.rst b/docs/apache-airflow-providers-discord/changelog.rst index d26072461d1d..c9cfb5499faf 100644 --- a/docs/apache-airflow-providers-discord/changelog.rst +++ b/docs/apache-airflow-providers-discord/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-discord`` --------------------------------------------- + .. include:: ../../airflow/providers/discord/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-docker/changelog.rst b/docs/apache-airflow-providers-docker/changelog.rst index d92635e586b2..606d636484a0 100644 --- a/docs/apache-airflow-providers-docker/changelog.rst +++ b/docs/apache-airflow-providers-docker/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-docker`` -------------------------------------------- + .. include:: ../../airflow/providers/docker/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-elasticsearch/changelog.rst b/docs/apache-airflow-providers-elasticsearch/changelog.rst index 287030242def..78aa32de23aa 100644 --- a/docs/apache-airflow-providers-elasticsearch/changelog.rst +++ b/docs/apache-airflow-providers-elasticsearch/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-elasticsearch`` --------------------------------------------------- + .. include:: ../../airflow/providers/elasticsearch/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-exasol/changelog.rst b/docs/apache-airflow-providers-exasol/changelog.rst index 4dc893b56615..947bf8688e2f 100644 --- a/docs/apache-airflow-providers-exasol/changelog.rst +++ b/docs/apache-airflow-providers-exasol/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-exasol`` -------------------------------------------- + .. include:: ../../airflow/providers/exasol/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-facebook/changelog.rst b/docs/apache-airflow-providers-facebook/changelog.rst index d58f217b6488..62145f4fa244 100644 --- a/docs/apache-airflow-providers-facebook/changelog.rst +++ b/docs/apache-airflow-providers-facebook/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-facebook`` ---------------------------------------------- + .. include:: ../../airflow/providers/facebook/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-ftp/changelog.rst b/docs/apache-airflow-providers-ftp/changelog.rst index 6b7a72cefbe5..6dd0771a976b 100644 --- a/docs/apache-airflow-providers-ftp/changelog.rst +++ b/docs/apache-airflow-providers-ftp/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-ftp`` ----------------------------------------- + .. include:: ../../airflow/providers/ftp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-github/changelog.rst b/docs/apache-airflow-providers-github/changelog.rst index 964dfec0fd24..eb1c4bfc5e00 100644 --- a/docs/apache-airflow-providers-github/changelog.rst +++ b/docs/apache-airflow-providers-github/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-github`` -------------------------------------------- + .. include:: ../../airflow/providers/github/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-google/changelog.rst b/docs/apache-airflow-providers-google/changelog.rst index abda3f330d61..f3c708e00f11 100644 --- a/docs/apache-airflow-providers-google/changelog.rst +++ b/docs/apache-airflow-providers-google/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-google`` -------------------------------------------- + .. include:: ../../airflow/providers/google/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-grpc/changelog.rst b/docs/apache-airflow-providers-grpc/changelog.rst index 051d23b8a818..55e970f047ca 100644 --- a/docs/apache-airflow-providers-grpc/changelog.rst +++ b/docs/apache-airflow-providers-grpc/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-grpc`` ------------------------------------------ + .. include:: ../../airflow/providers/grpc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-hashicorp/changelog.rst b/docs/apache-airflow-providers-hashicorp/changelog.rst index 7ba4a83c0aa9..10da504065ad 100644 --- a/docs/apache-airflow-providers-hashicorp/changelog.rst +++ b/docs/apache-airflow-providers-hashicorp/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-hashicorp`` ----------------------------------------------- + .. include:: ../../airflow/providers/hashicorp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-http/changelog.rst b/docs/apache-airflow-providers-http/changelog.rst index 30efed25e3a3..ae63287e513b 100644 --- a/docs/apache-airflow-providers-http/changelog.rst +++ b/docs/apache-airflow-providers-http/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-http`` ------------------------------------------ + .. include:: ../../airflow/providers/http/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-imap/changelog.rst b/docs/apache-airflow-providers-imap/changelog.rst index b635271658f9..cd51a7872892 100644 --- a/docs/apache-airflow-providers-imap/changelog.rst +++ b/docs/apache-airflow-providers-imap/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-imap`` ------------------------------------------ + .. include:: ../../airflow/providers/imap/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-influxdb/changelog.rst b/docs/apache-airflow-providers-influxdb/changelog.rst index 2c32bdae5a4a..1cd3fad0746b 100644 --- a/docs/apache-airflow-providers-influxdb/changelog.rst +++ b/docs/apache-airflow-providers-influxdb/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-influxdb`` ---------------------------------------------- + .. include:: ../../airflow/providers/influxdb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-jdbc/changelog.rst b/docs/apache-airflow-providers-jdbc/changelog.rst index e8ff084db25d..00fe21958583 100644 --- a/docs/apache-airflow-providers-jdbc/changelog.rst +++ b/docs/apache-airflow-providers-jdbc/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-jdbc`` ------------------------------------------ + .. include:: ../../airflow/providers/jdbc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-jenkins/changelog.rst b/docs/apache-airflow-providers-jenkins/changelog.rst index c8b1b0600a78..fab80a472c16 100644 --- a/docs/apache-airflow-providers-jenkins/changelog.rst +++ b/docs/apache-airflow-providers-jenkins/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-jenkins`` --------------------------------------------- + .. include:: ../../airflow/providers/jenkins/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/changelog.rst b/docs/apache-airflow-providers-microsoft-azure/changelog.rst index 5bf8a1b215de..11ea0a86b489 100644 --- a/docs/apache-airflow-providers-microsoft-azure/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-azure/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-microsoft-azure`` ----------------------------------------------------- + .. include:: ../../airflow/providers/microsoft/azure/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-mssql/changelog.rst b/docs/apache-airflow-providers-microsoft-mssql/changelog.rst index 739c6b8b62a7..7d66f519f89b 100644 --- a/docs/apache-airflow-providers-microsoft-mssql/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-mssql/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-microsoft-mssql`` ----------------------------------------------------- + .. include:: ../../airflow/providers/microsoft/mssql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-psrp/changelog.rst b/docs/apache-airflow-providers-microsoft-psrp/changelog.rst index 07d8e032d52a..6a6ba838519a 100644 --- a/docs/apache-airflow-providers-microsoft-psrp/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-psrp/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-microsoft-psrp`` ---------------------------------------------------- + .. include:: ../../airflow/providers/microsoft/psrp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-winrm/changelog.rst b/docs/apache-airflow-providers-microsoft-winrm/changelog.rst index 201bbe2df89b..23787e7a7a9d 100644 --- a/docs/apache-airflow-providers-microsoft-winrm/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-winrm/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-microsoft-winrm`` ----------------------------------------------------- + .. include:: ../../airflow/providers/microsoft/winrm/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-mongo/changelog.rst b/docs/apache-airflow-providers-mongo/changelog.rst index 97e1e6c55d5b..2df335624949 100644 --- a/docs/apache-airflow-providers-mongo/changelog.rst +++ b/docs/apache-airflow-providers-mongo/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-mongo`` ------------------------------------------- + .. include:: ../../airflow/providers/mongo/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-mysql/changelog.rst b/docs/apache-airflow-providers-mysql/changelog.rst index 7221fd9fc662..e98c8b8a24ed 100644 --- a/docs/apache-airflow-providers-mysql/changelog.rst +++ b/docs/apache-airflow-providers-mysql/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-mysql`` ------------------------------------------- + .. include:: ../../airflow/providers/mysql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-neo4j/changelog.rst b/docs/apache-airflow-providers-neo4j/changelog.rst index dd14f25ba628..15a9791980c1 100644 --- a/docs/apache-airflow-providers-neo4j/changelog.rst +++ b/docs/apache-airflow-providers-neo4j/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-neo4j`` ------------------------------------------- + .. include:: ../../airflow/providers/neo4j/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-odbc/changelog.rst b/docs/apache-airflow-providers-odbc/changelog.rst index 58166fe379ab..22cc2b961fcf 100644 --- a/docs/apache-airflow-providers-odbc/changelog.rst +++ b/docs/apache-airflow-providers-odbc/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-odbc`` ------------------------------------------ + .. include:: ../../airflow/providers/odbc/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openfaas/changelog.rst b/docs/apache-airflow-providers-openfaas/changelog.rst index 9a2ff34f4895..0881442ad18b 100644 --- a/docs/apache-airflow-providers-openfaas/changelog.rst +++ b/docs/apache-airflow-providers-openfaas/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-openfaas`` ---------------------------------------------- + .. include:: ../../airflow/providers/openfaas/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-openlineage/changelog.rst b/docs/apache-airflow-providers-openlineage/changelog.rst index 8d5ab5278a60..7034f1b9e965 100644 --- a/docs/apache-airflow-providers-openlineage/changelog.rst +++ b/docs/apache-airflow-providers-openlineage/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-openlineage`` ------------------------------------------------- + .. include:: ../../airflow/providers/openlineage/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-opsgenie/changelog.rst b/docs/apache-airflow-providers-opsgenie/changelog.rst index 58a008331051..08f9e0ba2322 100644 --- a/docs/apache-airflow-providers-opsgenie/changelog.rst +++ b/docs/apache-airflow-providers-opsgenie/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-opsgenie`` ---------------------------------------------- + .. include:: ../../airflow/providers/opsgenie/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-oracle/changelog.rst b/docs/apache-airflow-providers-oracle/changelog.rst index ac5231a1d13b..6b05cb4fbf32 100644 --- a/docs/apache-airflow-providers-oracle/changelog.rst +++ b/docs/apache-airflow-providers-oracle/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-oracle`` -------------------------------------------- + .. include:: ../../airflow/providers/oracle/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-pagerduty/changelog.rst b/docs/apache-airflow-providers-pagerduty/changelog.rst index c463d381ad52..186bdeb52bac 100644 --- a/docs/apache-airflow-providers-pagerduty/changelog.rst +++ b/docs/apache-airflow-providers-pagerduty/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-pagerduty`` ----------------------------------------------- + .. include:: ../../airflow/providers/pagerduty/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-papermill/changelog.rst b/docs/apache-airflow-providers-papermill/changelog.rst index 4340d544f223..fc7fb2c86ec5 100644 --- a/docs/apache-airflow-providers-papermill/changelog.rst +++ b/docs/apache-airflow-providers-papermill/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-papermill`` ----------------------------------------------- + .. include:: ../../airflow/providers/papermill/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-plexus/changelog.rst b/docs/apache-airflow-providers-plexus/changelog.rst index 95c7f42b8d42..c9ccba5e642f 100644 --- a/docs/apache-airflow-providers-plexus/changelog.rst +++ b/docs/apache-airflow-providers-plexus/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-plexus`` -------------------------------------------- + .. include:: ../../airflow/providers/plexus/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-postgres/changelog.rst b/docs/apache-airflow-providers-postgres/changelog.rst index 6bbf23290870..4beb42c22798 100644 --- a/docs/apache-airflow-providers-postgres/changelog.rst +++ b/docs/apache-airflow-providers-postgres/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-postgres`` ---------------------------------------------- + .. include:: ../../airflow/providers/postgres/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-presto/changelog.rst b/docs/apache-airflow-providers-presto/changelog.rst index d1226474e2d5..0fb911447b88 100644 --- a/docs/apache-airflow-providers-presto/changelog.rst +++ b/docs/apache-airflow-providers-presto/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-presto`` -------------------------------------------- + .. include:: ../../airflow/providers/presto/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-qubole/changelog.rst b/docs/apache-airflow-providers-qubole/changelog.rst index 3566395943fe..5004b7cb1f2f 100644 --- a/docs/apache-airflow-providers-qubole/changelog.rst +++ b/docs/apache-airflow-providers-qubole/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-qubole`` -------------------------------------------- + .. include:: ../../airflow/providers/qubole/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-redis/changelog.rst b/docs/apache-airflow-providers-redis/changelog.rst index 8643b0621406..14e39687382f 100644 --- a/docs/apache-airflow-providers-redis/changelog.rst +++ b/docs/apache-airflow-providers-redis/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-redis`` ------------------------------------------- + .. include:: ../../airflow/providers/redis/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-salesforce/changelog.rst b/docs/apache-airflow-providers-salesforce/changelog.rst index a57d8371dfd8..aaa24b57c4a4 100644 --- a/docs/apache-airflow-providers-salesforce/changelog.rst +++ b/docs/apache-airflow-providers-salesforce/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-salesforce`` ------------------------------------------------ + .. include:: ../../airflow/providers/salesforce/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-samba/changelog.rst b/docs/apache-airflow-providers-samba/changelog.rst index c3a59099cfde..2d36cae155ee 100644 --- a/docs/apache-airflow-providers-samba/changelog.rst +++ b/docs/apache-airflow-providers-samba/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-samba`` ------------------------------------------- + .. include:: ../../airflow/providers/samba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-segment/changelog.rst b/docs/apache-airflow-providers-segment/changelog.rst index 43e6324969ad..baca90b57523 100644 --- a/docs/apache-airflow-providers-segment/changelog.rst +++ b/docs/apache-airflow-providers-segment/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-segment`` --------------------------------------------- + .. include:: ../../airflow/providers/segment/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sendgrid/changelog.rst b/docs/apache-airflow-providers-sendgrid/changelog.rst index 1963562bba92..ddb64b4ee8cd 100644 --- a/docs/apache-airflow-providers-sendgrid/changelog.rst +++ b/docs/apache-airflow-providers-sendgrid/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-sendgrid`` ---------------------------------------------- + .. include:: ../../airflow/providers/sendgrid/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sftp/changelog.rst b/docs/apache-airflow-providers-sftp/changelog.rst index 3cab8a64516d..3f0ce545ca8c 100644 --- a/docs/apache-airflow-providers-sftp/changelog.rst +++ b/docs/apache-airflow-providers-sftp/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-sftp`` ------------------------------------------ + .. include:: ../../airflow/providers/sftp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-singularity/changelog.rst b/docs/apache-airflow-providers-singularity/changelog.rst index 803e1201ab69..b8fd7c81138c 100644 --- a/docs/apache-airflow-providers-singularity/changelog.rst +++ b/docs/apache-airflow-providers-singularity/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-singularity`` ------------------------------------------------- + .. include:: ../../airflow/providers/singularity/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-slack/changelog.rst b/docs/apache-airflow-providers-slack/changelog.rst index cb130d2233cb..aa09de3fb0a5 100644 --- a/docs/apache-airflow-providers-slack/changelog.rst +++ b/docs/apache-airflow-providers-slack/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-slack`` ------------------------------------------- + .. include:: ../../airflow/providers/slack/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-smtp/changelog.rst b/docs/apache-airflow-providers-smtp/changelog.rst index e67bd0dcd652..d602eb0610a2 100644 --- a/docs/apache-airflow-providers-smtp/changelog.rst +++ b/docs/apache-airflow-providers-smtp/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-smtp`` ------------------------------------------ + .. include:: ../../airflow/providers/smtp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-snowflake/changelog.rst b/docs/apache-airflow-providers-snowflake/changelog.rst index c3a0cdf18ef9..d0734ce6d7d6 100644 --- a/docs/apache-airflow-providers-snowflake/changelog.rst +++ b/docs/apache-airflow-providers-snowflake/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-snowflake`` ----------------------------------------------- + .. include:: ../../airflow/providers/snowflake/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sqlite/changelog.rst b/docs/apache-airflow-providers-sqlite/changelog.rst index 99d35831de58..04dfe43c6020 100644 --- a/docs/apache-airflow-providers-sqlite/changelog.rst +++ b/docs/apache-airflow-providers-sqlite/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-sqlite`` -------------------------------------------- + .. include:: ../../airflow/providers/sqlite/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-ssh/changelog.rst b/docs/apache-airflow-providers-ssh/changelog.rst index d6ba0ea87983..2252e9bf0ce1 100644 --- a/docs/apache-airflow-providers-ssh/changelog.rst +++ b/docs/apache-airflow-providers-ssh/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-ssh`` ----------------------------------------- + .. include:: ../../airflow/providers/ssh/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-tableau/changelog.rst b/docs/apache-airflow-providers-tableau/changelog.rst index 0e60b899d701..39a7eec68ce4 100644 --- a/docs/apache-airflow-providers-tableau/changelog.rst +++ b/docs/apache-airflow-providers-tableau/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-tableau`` --------------------------------------------- + .. include:: ../../airflow/providers/tableau/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-tabular/changelog.rst b/docs/apache-airflow-providers-tabular/changelog.rst index 6257ac42c7f6..7c12450cea66 100644 --- a/docs/apache-airflow-providers-tabular/changelog.rst +++ b/docs/apache-airflow-providers-tabular/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-tabular`` --------------------------------------------- + .. include:: ../../airflow/providers/tabular/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-telegram/changelog.rst b/docs/apache-airflow-providers-telegram/changelog.rst index d4686e21b62d..86c7a9867260 100644 --- a/docs/apache-airflow-providers-telegram/changelog.rst +++ b/docs/apache-airflow-providers-telegram/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-telegram`` ---------------------------------------------- + .. include:: ../../airflow/providers/telegram/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-trino/changelog.rst b/docs/apache-airflow-providers-trino/changelog.rst index 97f28e55cd18..efd69bd600ed 100644 --- a/docs/apache-airflow-providers-trino/changelog.rst +++ b/docs/apache-airflow-providers-trino/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-trino`` ------------------------------------------- + .. include:: ../../airflow/providers/trino/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-vertica/changelog.rst b/docs/apache-airflow-providers-vertica/changelog.rst index 89424409bca9..447905b4c6c0 100644 --- a/docs/apache-airflow-providers-vertica/changelog.rst +++ b/docs/apache-airflow-providers-vertica/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-vertica`` --------------------------------------------- + .. include:: ../../airflow/providers/vertica/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-zendesk/changelog.rst b/docs/apache-airflow-providers-zendesk/changelog.rst index fe70585d1efe..e32bcaee32f0 100644 --- a/docs/apache-airflow-providers-zendesk/changelog.rst +++ b/docs/apache-airflow-providers-zendesk/changelog.rst @@ -16,7 +16,6 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-zendesk`` --------------------------------------------- + .. include:: ../../airflow/providers/zendesk/CHANGELOG.rst From 68517c1765b4b5a87b7ab82129393f2fd7631e0c Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 5 Jul 2023 21:14:48 +0530 Subject: [PATCH 062/533] Updating pyenv instructions for quick start guide (#32374) Co-authored-by: Amogh Desai --- CONTRIBUTORS_QUICK_START.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CONTRIBUTORS_QUICK_START.rst b/CONTRIBUTORS_QUICK_START.rst index b2aa30c9de72..20e6191b3db9 100644 --- a/CONTRIBUTORS_QUICK_START.rst +++ b/CONTRIBUTORS_QUICK_START.rst @@ -179,17 +179,27 @@ extra while installing airflow. .. code-block:: bash +For Architectures other than MacOS/ARM $ pyenv install --list $ pyenv install 3.8.5 $ pyenv versions +For MacOS/Arm (3.9.1 is the first version of Python to support MacOS/ARM, but 3.8.10 works too) + $ pyenv install --list + $ pyenv install 3.8.10 + $ pyenv versions + 5. Creating new virtual environment named ``airflow-env`` for installed version python. In next chapter virtual environment ``airflow-env`` will be used for installing airflow. .. code-block:: bash +For Architectures other than MacOS/ARM $ pyenv virtualenv 3.8.5 airflow-env +For MacOS/Arm (3.9.1 is the first version of Python to support MacOS/ARM, but 3.8.10 works too) + $ pyenv virtualenv 3.8.10 airflow-env + 6. Entering virtual environment ``airflow-env`` .. code-block:: bash From 383efbcbe00a0ce25251ba3a06a625f0ae40d806 Mon Sep 17 00:00:00 2001 From: Mats Holm <32451087+holmen1@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:39:37 +0200 Subject: [PATCH 063/533] Update pipeline.rst (#32369) Typo in final dag, merge_data. --- docs/apache-airflow/tutorial/pipeline.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/apache-airflow/tutorial/pipeline.rst b/docs/apache-airflow/tutorial/pipeline.rst index c9bfcfa6013f..fca2418a701c 100644 --- a/docs/apache-airflow/tutorial/pipeline.rst +++ b/docs/apache-airflow/tutorial/pipeline.rst @@ -288,7 +288,10 @@ Putting all of the pieces together, we have our completed DAG. FROM employees_temp ) t ON CONFLICT ("Serial Number") DO UPDATE - SET "Serial Number" = excluded."Serial Number"; + SET + "Employee Markme" = excluded."Employee Markme", + "Description" = excluded."Description", + "Leave" = excluded."Leave"; """ try: postgres_hook = PostgresHook(postgres_conn_id="tutorial_pg_conn") From d3f0900bbb68bda36f81475bf64d83d51ee32b22 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Wed, 5 Jul 2023 12:52:14 -0400 Subject: [PATCH 064/533] AIP-56 - Create auth manager interface and very basic implementation of FAB auth manager (#32217) --- airflow/auth/__init__.py | 17 +++++++ airflow/auth/managers/__init__.py | 17 +++++++ airflow/auth/managers/base_auth_manager.py | 35 ++++++++++++++ airflow/auth/managers/fab/__init__.py | 17 +++++++ airflow/auth/managers/fab/fab_auth_manager.py | 41 ++++++++++++++++ airflow/config_templates/config.yml | 7 +++ airflow/config_templates/default_airflow.cfg | 3 ++ airflow/configuration.py | 20 ++++++++ airflow/www/extensions/init_jinja_globals.py | 3 +- .../templates/appbuilder/navbar_right.html | 9 ++-- tests/auh/__init__.py | 16 +++++++ tests/auh/managers/__init__.py | 16 +++++++ tests/auh/managers/fab/__init__.py | 16 +++++++ .../auh/managers/fab/test_fab_auth_manager.py | 48 +++++++++++++++++++ tests/www/views/conftest.py | 1 + 15 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 airflow/auth/__init__.py create mode 100644 airflow/auth/managers/__init__.py create mode 100644 airflow/auth/managers/base_auth_manager.py create mode 100644 airflow/auth/managers/fab/__init__.py create mode 100644 airflow/auth/managers/fab/fab_auth_manager.py create mode 100644 tests/auh/__init__.py create mode 100644 tests/auh/managers/__init__.py create mode 100644 tests/auh/managers/fab/__init__.py create mode 100644 tests/auh/managers/fab/test_fab_auth_manager.py diff --git a/airflow/auth/__init__.py b/airflow/auth/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/auth/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/__init__.py b/airflow/auth/managers/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/auth/managers/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py new file mode 100644 index 000000000000..6a4bb86f1666 --- /dev/null +++ b/airflow/auth/managers/base_auth_manager.py @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from abc import abstractmethod + +from airflow.utils.log.logging_mixin import LoggingMixin + + +class BaseAuthManager(LoggingMixin): + """ + Class to derive in order to implement concrete auth managers. + + Auth managers are responsible for any user management related operation such as login, logout, authz, ... + """ + + @abstractmethod + def get_user_name(self) -> str: + """Return the username associated to the user in session.""" + ... diff --git a/airflow/auth/managers/fab/__init__.py b/airflow/auth/managers/fab/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/auth/managers/fab/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py new file mode 100644 index 000000000000..420385716822 --- /dev/null +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from flask_login import current_user + +from airflow.auth.managers.base_auth_manager import BaseAuthManager + + +class FabAuthManager(BaseAuthManager): + """ + Flask-AppBuilder auth manager. + + This auth manager is responsible for providing a backward compatible user management experience to users. + """ + + def get_user_name(self) -> str: + """ + Return the username associated to the user in session. + + For backward compatibility reasons, the username in FAB auth manager is the concatenation of the + first name and the last name. + """ + first_name = current_user.first_name or "" + last_name = current_user.last_name or "" + return f"{first_name} {last_name}".strip() diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index d588a7f26e3b..273f2a36c76d 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -70,6 +70,13 @@ core: type: string example: ~ default: "SequentialExecutor" + auth_manager: + description: | + The auth manager class that airflow should use. Full import path to the auth manager class. + version_added: ~ + type: string + example: ~ + default: "airflow.auth.managers.fab.fab_auth_manager.FabAuthManager" parallelism: description: | This defines the maximum number of task instances that can run concurrently per scheduler in diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index ae6bdec08524..439944022fe3 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -58,6 +58,9 @@ default_timezone = utc # full import path to the class when using a custom executor. executor = SequentialExecutor +# The auth manager class that airflow should use. Full import path to the auth manager class. +auth_manager = airflow.auth.managers.fab.fab_auth_manager.FabAuthManager + # This defines the maximum number of task instances that can run concurrently per scheduler in # Airflow, regardless of the worker count. Generally this value, multiplied by the number of # schedulers in your cluster, is the maximum number of task instances with the running diff --git a/airflow/configuration.py b/airflow/configuration.py index 288595efbd94..6a55a6c99cfc 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -42,6 +42,7 @@ from typing_extensions import overload +from airflow.auth.managers.base_auth_manager import BaseAuthManager from airflow.exceptions import AirflowConfigException from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend from airflow.utils import yaml @@ -1743,6 +1744,24 @@ def initialize_secrets_backends() -> list[BaseSecretsBackend]: return backend_list +def initialize_auth_manager() -> BaseAuthManager: + """ + Initialize auth manager. + + * import user manager class + * instantiate it and return it + """ + auth_manager_cls = conf.getimport(section="core", key="auth_manager") + + if not auth_manager_cls: + raise AirflowConfigException( + "No auth manager defined in the config. " + "Please specify one using section/key [core/auth_manager]." + ) + + return auth_manager_cls() + + @functools.lru_cache(maxsize=None) def _DEFAULT_CONFIG() -> str: path = _default_config_file_path("default_airflow.cfg") @@ -1808,4 +1827,5 @@ def __getattr__(name): conf = initialize_config() secrets_backend_list = initialize_secrets_backends() +auth_manager = initialize_auth_manager() conf.validate() diff --git a/airflow/www/extensions/init_jinja_globals.py b/airflow/www/extensions/init_jinja_globals.py index 0674a8e4a3ea..9ef948084cc9 100644 --- a/airflow/www/extensions/init_jinja_globals.py +++ b/airflow/www/extensions/init_jinja_globals.py @@ -21,7 +21,7 @@ import pendulum import airflow -from airflow.configuration import conf +from airflow.configuration import auth_manager, conf from airflow.settings import IS_K8S_OR_K8SCELERY_EXECUTOR, STATE_COLORS from airflow.utils.net import get_hostname from airflow.utils.platform import get_airflow_git_version @@ -68,6 +68,7 @@ def prepare_jinja_globals(): "git_version": git_version, "k8s_or_k8scelery_executor": IS_K8S_OR_K8SCELERY_EXECUTOR, "rest_api_enabled": False, + "auth_manager": auth_manager, "config_test_connection": conf.get("core", "test_connection", fallback="Disabled"), } diff --git a/airflow/www/templates/appbuilder/navbar_right.html b/airflow/www/templates/appbuilder/navbar_right.html index 8e606c65b30b..1ade0847a65e 100644 --- a/airflow/www/templates/appbuilder/navbar_right.html +++ b/airflow/www/templates/appbuilder/navbar_right.html @@ -66,12 +66,9 @@ {% if not current_user.is_anonymous %}
  • - - {% if current_user.first_name and current_user.last_name %} - {{ (current_user.first_name[0] + current_user.last_name[0]).upper() }} - {% else %} - person - {% endif %} + + {% set user_names = auth_manager.get_user_name().split(" ", 1) %} + {% for name in user_names %}{{ name[0].upper() }}{% endfor %} diff --git a/tests/auh/__init__.py b/tests/auh/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/auh/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/auh/managers/__init__.py b/tests/auh/managers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/auh/managers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/auh/managers/fab/__init__.py b/tests/auh/managers/fab/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/auh/managers/fab/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/auh/managers/fab/test_fab_auth_manager.py b/tests/auh/managers/fab/test_fab_auth_manager.py new file mode 100644 index 000000000000..16e5151495c9 --- /dev/null +++ b/tests/auh/managers/fab/test_fab_auth_manager.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager +from airflow.www.fab_security.sqla.models import User + + +@pytest.fixture +def auth_manager(): + return FabAuthManager() + + +class TestFabAuthManager: + @pytest.mark.parametrize( + "first_name,last_name,expected", + [ + ("First", "Last", "First Last"), + ("First", None, "First"), + (None, "Last", "Last"), + ], + ) + @mock.patch("flask_login.utils._get_user") + def test_get_user_name(self, mock_current_user, first_name, last_name, expected, auth_manager): + user = User() + user.first_name = first_name + user.last_name = last_name + mock_current_user.return_value = user + + assert auth_manager.get_user_name() == expected diff --git a/tests/www/views/conftest.py b/tests/www/views/conftest.py index 209a106df6a1..25e3a6d20aee 100644 --- a/tests/www/views/conftest.py +++ b/tests/www/views/conftest.py @@ -173,6 +173,7 @@ def local_context(self): "scheduler_job", # airflow.www.views.AirflowBaseView.extra_args "macros", + "auth_manager", ] for key in keys_to_delete: del result[key] From e45bee884068399e7265421511e17fed106ce5b4 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Wed, 5 Jul 2023 11:10:03 -0700 Subject: [PATCH 065/533] D205 Support - Providers: Pagerduty to SMTP (inclusive) (#32358) --- airflow/providers/plexus/hooks/plexus.py | 6 +++--- airflow/providers/plexus/operators/job.py | 3 +-- airflow/providers/presto/hooks/presto.py | 3 +-- .../providers/qubole/operators/qubole_check.py | 9 ++++++--- airflow/providers/salesforce/hooks/salesforce.py | 5 +++-- airflow/providers/segment/hooks/segment.py | 7 ++----- airflow/providers/sftp/operators/sftp.py | 4 ++-- airflow/providers/slack/hooks/slack.py | 6 ++---- airflow/providers/slack/hooks/slack_webhook.py | 7 +++---- airflow/providers/slack/transfers/sql_to_slack.py | 15 +++++++++------ airflow/providers/smtp/hooks/smtp.py | 9 +++++---- 11 files changed, 37 insertions(+), 37 deletions(-) diff --git a/airflow/providers/plexus/hooks/plexus.py b/airflow/providers/plexus/hooks/plexus.py index 09b858ad7c00..9432fe5fb7c8 100644 --- a/airflow/providers/plexus/hooks/plexus.py +++ b/airflow/providers/plexus/hooks/plexus.py @@ -29,9 +29,9 @@ class PlexusHook(BaseHook): """ - Used for jwt token generation and storage to - make Plexus API calls. Requires email and password - Airflow variables be created. + Used for jwt token generation and storage to make Plexus API calls. + + Requires email and password Airflow variables be created. Example: - export AIRFLOW_VAR_EMAIL = user@corescientific.com diff --git a/airflow/providers/plexus/operators/job.py b/airflow/providers/plexus/operators/job.py index 963924bbbbb9..5eb83f7b6bc8 100644 --- a/airflow/providers/plexus/operators/job.py +++ b/airflow/providers/plexus/operators/job.py @@ -131,8 +131,7 @@ def _api_lookup(self, param: str, hook): def construct_job_params(self, hook: Any) -> dict[Any, Any | None]: """ - Creates job_params dict for api call to - launch a Plexus job. + Creates job_params dict for api call to launch a Plexus job. Some parameters required to launch a job are not available to the user in the Plexus diff --git a/airflow/providers/presto/hooks/presto.py b/airflow/providers/presto/hooks/presto.py index e0ed55758d32..477b4cd797de 100644 --- a/airflow/providers/presto/hooks/presto.py +++ b/airflow/providers/presto/hooks/presto.py @@ -220,8 +220,7 @@ def insert_rows( @staticmethod def _serialize_cell(cell: Any, conn: Connection | None = None) -> Any: """ - Presto will adapt all arguments to the execute() method internally, - hence we return cell without any conversion. + Presto will adapt all execute() args internally, hence we return cell without any conversion. :param cell: The cell to insert into the table :param conn: The database connection diff --git a/airflow/providers/qubole/operators/qubole_check.py b/airflow/providers/qubole/operators/qubole_check.py index cf01ae244e42..18b2667bda36 100644 --- a/airflow/providers/qubole/operators/qubole_check.py +++ b/airflow/providers/qubole/operators/qubole_check.py @@ -45,7 +45,8 @@ def get_db_hook(self) -> QuboleCheckHook: def get_hook(self) -> QuboleCheckHook: """ - Reinitialising the hook, as some template fields might have changed + Reinitialising the hook, as some template fields might have changed. + This method overwrites the original QuboleOperator.get_hook() which returns a QuboleHook. """ return QuboleCheckHook( @@ -55,8 +56,9 @@ def get_hook(self) -> QuboleCheckHook: class QuboleCheckOperator(_QuboleCheckOperatorMixin, SQLCheckOperator, QuboleOperator): """ - Performs checks against Qubole Commands. ``QuboleCheckOperator`` expects - a command that will be executed on QDS. + Performs checks against Qubole Commands. + + ``QuboleCheckOperator`` expects a command that will be executed on QDS. By default, each value on first row of the result of this Qubole Command is evaluated using python ``bool`` casting. If any of the values return ``False``, the check is failed and errors out. @@ -129,6 +131,7 @@ def __init__( class QuboleValueCheckOperator(_QuboleCheckOperatorMixin, SQLValueCheckOperator, QuboleOperator): """ Performs a simple value check using Qubole command. + By default, each value on the first row of this Qubole command is compared with a pre-defined value. The check fails and errors out if the output of the command diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index 7c82e46bb444..336c0db0a0f5 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This module contains a Salesforce Hook which allows you to connect to your Salesforce instance, -retrieve data from it, and write that data to a file for other uses. +Connect to your Salesforce instance, retrieve data from it, and write that data to a file for other uses. .. note:: this hook also relies on the simple_salesforce package: https://github.com/simple-salesforce/simple-salesforce @@ -180,6 +179,7 @@ def make_query(self, query: str, include_deleted: bool = False, query_params: di def describe_object(self, obj: str) -> dict: """ Get the description of an object from Salesforce. + This description is the object's schema and some extra metadata that Salesforce stores for each object. @@ -204,6 +204,7 @@ def get_available_fields(self, obj: str) -> list[str]: def get_object_from_salesforce(self, obj: str, fields: Iterable[str]) -> dict: """ Get all instances of the `object` from Salesforce. + For each model, only get the fields specified in fields. All we really do underneath the hood is run: diff --git a/airflow/providers/segment/hooks/segment.py b/airflow/providers/segment/hooks/segment.py index 43055dea6268..258716387d60 100644 --- a/airflow/providers/segment/hooks/segment.py +++ b/airflow/providers/segment/hooks/segment.py @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This module contains a Segment Hook -which allows you to connect to your Segment account, -retrieve data from it or write to that file. +Connect to your Segment account, retrieve data from it or write to that file. NOTE: this hook also relies on the Segment analytics package: https://github.com/segmentio/analytics-python @@ -33,8 +31,7 @@ class SegmentHook(BaseHook): """ - Create new connection to Segment - and allows you to pull data out of Segment or write to it. + Create new connection to Segment and allows you to pull data out of Segment or write to it. You can then use that file with other Airflow operators to move the data around or interact with segment. diff --git a/airflow/providers/sftp/operators/sftp.py b/airflow/providers/sftp/operators/sftp.py index 884411b3cccf..0920387faab4 100644 --- a/airflow/providers/sftp/operators/sftp.py +++ b/airflow/providers/sftp/operators/sftp.py @@ -39,8 +39,8 @@ class SFTPOperation: class SFTPOperator(BaseOperator): """ SFTPOperator for transferring files from remote host to local or vice a versa. - This operator uses sftp_hook to open sftp transport channel that serve as basis - for file transfer. + + This operator uses sftp_hook to open sftp transport channel that serve as basis for file transfer. :param ssh_conn_id: :ref:`ssh connection id` from airflow Connections. `ssh_conn_id` will be ignored if `ssh_hook` diff --git a/airflow/providers/slack/hooks/slack.py b/airflow/providers/slack/hooks/slack.py index 9d0bc588a377..bd8ee74b34da 100644 --- a/airflow/providers/slack/hooks/slack.py +++ b/airflow/providers/slack/hooks/slack.py @@ -38,10 +38,8 @@ def _ensure_prefixes(conn_type): - """ - Remove when provider min airflow version >= 2.5.0 since this is handled by - provider manager from that version. - """ + # TODO: Remove when provider min airflow version >= 2.5.0 since + # this is handled by provider manager from that version. def dec(func): @wraps(func) diff --git a/airflow/providers/slack/hooks/slack_webhook.py b/airflow/providers/slack/hooks/slack_webhook.py index aef5f29e715a..82b21f175e1a 100644 --- a/airflow/providers/slack/hooks/slack_webhook.py +++ b/airflow/providers/slack/hooks/slack_webhook.py @@ -55,10 +55,8 @@ def wrapper(*args, **kwargs) -> Callable: def _ensure_prefixes(conn_type): - """ - Remove when provider min airflow version >= 2.5.0 since this is handled by - provider manager from that version. - """ + # TODO: Remove when provider min airflow version >= 2.5.0 since + # this is handled by provider manager from that version. def dec(func): @wraps(func) @@ -85,6 +83,7 @@ def _ensure_prefix(field): class SlackWebhookHook(BaseHook): """ This class provide a thin wrapper around the ``slack_sdk.WebhookClient``. + This hook allows you to post messages to Slack by using Incoming Webhooks. .. seealso:: diff --git a/airflow/providers/slack/transfers/sql_to_slack.py b/airflow/providers/slack/transfers/sql_to_slack.py index cf5c01b22c9c..caac3eb7d1e2 100644 --- a/airflow/providers/slack/transfers/sql_to_slack.py +++ b/airflow/providers/slack/transfers/sql_to_slack.py @@ -80,12 +80,15 @@ def _get_query_results(self) -> DataFrame: class SqlToSlackOperator(BaseSqlToSlackOperator): """ - Executes an SQL statement in a given SQL connection and sends the results to Slack. The results of the - query are rendered into the 'slack_message' parameter as a Pandas dataframe using a JINJA variable called - '{{ results_df }}'. The 'results_df' variable name can be changed by specifying a different - 'results_df_name' parameter. The Tabulate library is added to the JINJA environment as a filter to - allow the dataframe to be rendered nicely. For example, set 'slack_message' to {{ results_df | - tabulate(tablefmt="pretty", headers="keys") }} to send the results to Slack as an ascii rendered table. + Executes an SQL statement in a given SQL connection and sends the results to Slack. + + The results of the query are rendered into the 'slack_message' parameter as a Pandas + dataframe using a JINJA variable called '{{ results_df }}'. The 'results_df' variable + name can be changed by specifying a different 'results_df_name' parameter. The Tabulate + library is added to the JINJA environment as a filter to allow the dataframe to be + rendered nicely. For example, set 'slack_message' to {{ results_df | + tabulate(tablefmt="pretty", headers="keys") }} to send the results to Slack as an ascii + rendered table. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index f3a2abb35fd1..6279ea4f940b 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. """ -This module provides everything to be able to search in mails for a specific attachment -and also to download it. +Search in emails for a specific attachment and also to download it. + It uses the smtplib library that is already integrated in python 3. """ from __future__ import annotations @@ -302,8 +302,9 @@ def _get_email_address_list(self, addresses: str | Iterable[str]) -> list[str]: def _get_email_list_from_str(self, addresses: str) -> list[str]: """ - Extract a list of email addresses from a string. The string - can contain multiple email addresses separated by + Extract a list of email addresses from a string. + + The string can contain multiple email addresses separated by any of the following delimiters: ',' or ';'. :param addresses: A string containing one or more email addresses. From d6b5b782c6e23a7c48709b30da47b043dac424e2 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 6 Jul 2023 00:42:31 +0530 Subject: [PATCH 066/533] Defining priorityClasses for airflow services and workers (#31615) --------- Co-authored-by: Amogh Co-authored-by: Amogh Desai --- .../priorityclasses/priority-classes.yaml | 34 ++++++++++++++++++ chart/values.schema.json | 36 +++++++++++++++++++ chart/values.yaml | 15 ++++++++ .../airflow_aux/test_basic_helm_chart.py | 33 +++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 chart/templates/priorityclasses/priority-classes.yaml diff --git a/chart/templates/priorityclasses/priority-classes.yaml b/chart/templates/priorityclasses/priority-classes.yaml new file mode 100644 index 000000000000..cc11ea2d6e79 --- /dev/null +++ b/chart/templates/priorityclasses/priority-classes.yaml @@ -0,0 +1,34 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +################################################# +## Priority classes provisioned via the chart values +################################################# +{{- $Global := . }} +{{- range $e := .Values.priorityClasses }} +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ $Global.Release.Name }}-{{ $e.name }} + labels: + release: {{ $Global.Release.Name }} +preemptionPolicy: {{ default "PreemptLowerPriority" $e.preemptionPolicy }} +value: {{ $e.value | required "value is required" }} +{{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 3f9b7356796e..9eb32df26ef0 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -912,6 +912,42 @@ "- configMapRef:\n name: '{{ .Release.Name }}-airflow-variables'" ] }, + "priorityClasses": { + "description": "Priority Classes created by helm charts", + "type": "array", + "x-docsSection": "Kubernetes", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "preemptionPolicy": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + }, + "default": [], + "examples": [ + { + "name": "class1", + "preemptionPolicy": "PreemptLowerPriority", + "value": 10000 + }, + { + "name": "class2", + "preemptionPolicy": "Never", + "value": 100000 + } + ] + }, "extraSecrets": { "description": "Extra secrets that will be managed by the chart.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 70d862b2fb0a..0fae3648d56f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -283,6 +283,21 @@ enableBuiltInSecretEnvVars: AIRFLOW__ELASTICSEARCH__HOST: true AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST: true +# Priority Classes that will be installed by charts. +# Ideally, there should be an entry for dagProcessor, flower, +# pgbouncer, scheduler, statsd, triggerer, webserver, worker. +# The format for priorityClasses is an array with each element having: +# * name is the name of the priorityClass. Ensure the same name is given to the respective section as well +# * preemptionPolicy for the priorityClass +# * value is the preemption value for the priorityClass +priorityClasses: [] +# - name: class1 (if this is for dagProcessor, ensure overriding .Values.dagProcessor.priorityClass too) +# preemptionPolicy: PreemptLowerPriority +# value: 10000 +# - name: class2 +# preemptionPolicy: Never +# value: 100000 + # Extra secrets that will be managed by the chart # (You can use them with extraEnv or extraEnvFrom or some of the extraVolumes values). # The format for secret data is "key/value" where diff --git a/tests/charts/airflow_aux/test_basic_helm_chart.py b/tests/charts/airflow_aux/test_basic_helm_chart.py index 3e182f9ebc29..7c1e9f862d3a 100644 --- a/tests/charts/airflow_aux/test_basic_helm_chart.py +++ b/tests/charts/airflow_aux/test_basic_helm_chart.py @@ -538,6 +538,39 @@ def test_postgres_connection_url_name_override(self): == base64.b64decode(doc["data"]["connection"]).decode("utf-8") ) + def test_priority_classes(self): + pc = [ + {"name": "class1", "preemptionPolicy": "PreemptLowerPriority", "value": 1000}, + {"name": "class2", "preemptionPolicy": "Never", "value": 10000}, + ] + objs = render_chart( + "my-release", + show_only=["templates/priorityclasses/priority-classes.yaml"], + values={"priorityClasses": pc}, + ) + + assert len(objs) == 2 + + for i in range(len(objs)): + assert objs[i]["kind"] == "PriorityClass" + assert objs[i]["apiVersion"] == "scheduling.k8s.io/v1" + assert objs[i]["metadata"]["name"] == ("my-release" + "-" + pc[i]["name"]) + assert objs[i]["preemptionPolicy"] == pc[i]["preemptionPolicy"] + assert objs[i]["value"] == pc[i]["value"] + + def test_priority_classes_default_preemption(self): + obj = render_chart( + "my-release", + show_only=["templates/priorityclasses/priority-classes.yaml"], + values={ + "priorityClasses": [ + {"name": "class1", "value": 10000}, + ] + }, + )[0] + + assert obj["preemptionPolicy"] == "PreemptLowerPriority" + @staticmethod def default_trigger_obj(version): if version == "default": From d01248382fe45a5f5a7fdeed4082a80c5f814ad8 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 5 Jul 2023 21:39:27 +0200 Subject: [PATCH 067/533] Hide sensitive values from extra in connection edit form (#32309) The fields that are sensitive (i.e password field is used to show them in Connection edit view) should also be hidden when they are stored as "extra" in the form extra field. This PR handles both - replacing such values with a placeholder as well as not updating the value if the placeholder has not been modified. --- airflow/providers_manager.py | 8 ++- airflow/www/views.py | 51 ++++++++++++++--- tests/www/views/test_views_connection.py | 71 +++++++++++++++++++++--- 3 files changed, 112 insertions(+), 18 deletions(-) diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 0563abeaf572..668d79a0bcd3 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -223,6 +223,7 @@ class ConnectionFormWidgetInfo(NamedTuple): package_name: str field: Any field_name: str + is_sensitive: bool T = TypeVar("T", bound=Callable) @@ -882,7 +883,12 @@ def _add_widgets(self, package_name: str, hook_class: type, widgets: dict[str, A # In case of inherited hooks this might be happening several times continue self._connection_form_widgets[prefixed_field_name] = ConnectionFormWidgetInfo( - hook_class.__name__, package_name, field, field_identifier + hook_class.__name__, + package_name, + field, + field_identifier, + hasattr(field.field_class.widget, "input_type") + and field.field_class.widget.input_type == "password", ) def _add_customized_fields(self, package_name: str, hook_class: type, customized_fields: dict): diff --git a/airflow/www/views.py b/airflow/www/views.py index b304dd0ddf02..910f7aabf10a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -88,10 +88,12 @@ from airflow.exceptions import ( AirflowConfigException, AirflowException, + AirflowNotFoundException, ParamValidationError, RemovedInAirflow3Warning, ) from airflow.executors.executor_loader import ExecutorLoader +from airflow.hooks.base import BaseHook from airflow.jobs.job import Job from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner @@ -147,6 +149,8 @@ "} else {xLabel = d3.time.format('%H:%M, %d %b')(new Date(parseInt(d)));} return xLabel;}" ) +SENSITIVE_FIELD_PLACEHOLDER = "RATHER_LONG_SENSITIVE_FIELD_PLACEHOLDER" + def sanitize_args(args: dict[str, str]) -> dict[str, str]: """ @@ -4657,14 +4661,22 @@ class ConnectionModelView(AirflowModelView): base_order = ("conn_id", "asc") - def _iter_extra_field_names(self) -> Iterator[tuple[str, str]]: + def _iter_extra_field_names_and_sensitivity(self) -> Iterator[tuple[str, str, bool]]: """Iterate through provider-backed connection fields. Note that this cannot be a property (including a cached property) because Flask-Appbuilder attempts to access all members on startup, and using a property would initialize the providers manager too eagerly. + + Returns tuple of: + + * key + * field_name + * whether the field is sensitive """ - return ((k, v.field_name) for k, v in ProvidersManager().connection_form_widgets.items()) + return ( + (k, v.field_name, v.is_sensitive) for k, v in ProvidersManager().connection_form_widgets.items() + ) @property def add_columns(self) -> list[str]: @@ -4677,7 +4689,10 @@ def add_columns(self) -> list[str]: superfuluous checks done by Flask-Appbuilder on startup). """ if self._add_columns is type(self)._add_columns and has_request_context(): - self._add_columns = [*self._add_columns, *(k for k, _ in self._iter_extra_field_names())] + self._add_columns = [ + *self._add_columns, + *(k for k, _, _ in self._iter_extra_field_names_and_sensitivity()), + ] return self._add_columns @property @@ -4691,7 +4706,10 @@ def edit_columns(self) -> list[str]: superfuluous checks done by Flask-Appbuilder on startup). """ if self._edit_columns is type(self)._edit_columns and has_request_context(): - self._edit_columns = [*self._edit_columns, *(k for k, _ in self._iter_extra_field_names())] + self._edit_columns = [ + *self._edit_columns, + *(k for k, _, _ in self._iter_extra_field_names_and_sensitivity()), + ] return self._edit_columns @action("muldelete", "Delete", "Are you sure you want to delete selected records?", single=False) @@ -4782,7 +4800,6 @@ def process_form(self, form, is_created): """Process form data.""" conn_id = form.data["conn_id"] conn_type = form.data["conn_type"] - # The extra value is the combination of custom fields for this conn_type and the Extra field. # The extra form field with all extra values (including custom fields) is in the form being processed # so we start with those values, and override them with anything in the custom fields. @@ -4807,8 +4824,7 @@ def process_form(self, form, is_created): ) del form.extra del extra_json - - for key, field_name in self._iter_extra_field_names(): + for key, field_name, is_sensitive in self._iter_extra_field_names_and_sensitivity(): if key in form.data and key.startswith("extra__"): conn_type_from_extra_field = key.split("__")[1] if conn_type_from_extra_field == conn_type: @@ -4817,8 +4833,21 @@ def process_form(self, form, is_created): # value isn't an empty string. if value != "": extra[field_name] = value - if extra.keys(): + sensitive_unchanged_keys = set() + for key, value in extra.items(): + if value == SENSITIVE_FIELD_PLACEHOLDER: + sensitive_unchanged_keys.add(key) + if sensitive_unchanged_keys: + try: + conn = BaseHook.get_connection(conn_id) + except AirflowNotFoundException: + conn = None + for key in sensitive_unchanged_keys: + if conn and conn.extra_dejson.get(key): + extra[key] = conn.extra_dejson.get(key) + else: + del extra[key] form.extra.data = json.dumps(extra) def prefill_form(self, form, pk): @@ -4836,7 +4865,7 @@ def prefill_form(self, form, pk): logging.warning("extra field for %s is not a dictionary", form.data.get("conn_id", "")) return - for field_key, field_name in self._iter_extra_field_names(): + for field_key, field_name, is_sensitive in self._iter_extra_field_names_and_sensitivity(): value = extra_dictionary.get(field_name, "") if not value: @@ -4846,6 +4875,10 @@ def prefill_form(self, form, pk): if value: field = getattr(form, field_key) field.data = value + if is_sensitive and field_name in extra_dictionary: + extra_dictionary[field_name] = SENSITIVE_FIELD_PLACEHOLDER + # form.data is a property that builds the dictionary from fields so we have to modify the fields + form.extra.data = json.dumps(extra_dictionary) class PluginView(AirflowBaseView): diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index a3d00befc2b7..f31152586786 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -79,10 +79,29 @@ def test_prefill_form_null_extra(): mock_form.data = {"conn_id": "test", "extra": None, "conn_type": "test"} cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock(return_value=()) + cmv._iter_extra_field_names_and_sensitivity = mock.Mock(return_value=()) cmv.prefill_form(form=mock_form, pk=1) +def test_prefill_form_sensitive_fields_extra(): + mock_form = mock.Mock() + mock_form.data = { + "conn_id": "test", + "extra": json.dumps({"sensitive_extra": "TEST1", "non_sensitive_extra": "TEST2"}), + "conn_type": "test", + } + + cmv = ConnectionModelView() + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( + return_value=[("sensitive_extra_key", "sensitive_extra", True)] + ) + cmv.prefill_form(form=mock_form, pk=1) + assert json.loads(mock_form.extra.data) == { + "sensitive_extra": "RATHER_LONG_SENSITIVE_FIELD_PLACEHOLDER", + "non_sensitive_extra": "TEST2", + } + + @pytest.mark.parametrize( "extras, expected", [ @@ -106,7 +125,9 @@ def test_prefill_form_backcompat(extras, expected): mock_form.data = {"conn_id": "test", "extra": json.dumps(extras), "conn_type": "test"} cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock(return_value=[("extra__test__my_param", "my_param")]) + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( + return_value=[("extra__test__my_param", "my_param", False)] + ) cmv.prefill_form(form=mock_form, pk=1) assert mock_form.extra__test__my_param.data == expected @@ -134,7 +155,9 @@ def test_process_form_extras_both(mock_pm_hooks, mock_import_str, field_name): } cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock(return_value=[("extra__test__custom_field", field_name)]) + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( + return_value=[("extra__test__custom_field", field_name, False)] + ) cmv.process_form(form=mock_form, is_created=True) assert json.loads(mock_form.extra.data) == { field_name: "custom_field_val", @@ -160,7 +183,7 @@ def test_process_form_extras_extra_only(mock_pm_hooks, mock_import_str): } cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock(return_value=()) + cmv._iter_extra_field_names_and_sensitivity = mock.Mock(return_value=()) cmv.process_form(form=mock_form, is_created=True) assert json.loads(mock_form.extra.data) == {"param2": "param2_val"} @@ -186,10 +209,10 @@ def test_process_form_extras_custom_only(mock_pm_hooks, mock_import_str, field_n } cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock( + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( return_value=[ - ("extra__test3__custom_field", field_name), - ("extra__test3__custom_bool_field", False), + ("extra__test3__custom_field", field_name, False), + ("extra__test3__custom_bool_field", False, False), ], ) cmv.process_form(form=mock_form, is_created=True) @@ -217,7 +240,9 @@ def test_process_form_extras_updates(mock_pm_hooks, mock_import_str, field_name) } cmv = ConnectionModelView() - cmv._iter_extra_field_names = mock.Mock(return_value=[("extra__test4__custom_field", field_name)]) + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( + return_value=[("extra__test4__custom_field", field_name, False)] + ) cmv.process_form(form=mock_form, is_created=True) if field_name == "custom_field": @@ -229,6 +254,36 @@ def test_process_form_extras_updates(mock_pm_hooks, mock_import_str, field_name) assert json.loads(mock_form.extra.data) == {"extra__test4__custom_field": "custom_field_val4"} +@mock.patch("airflow.utils.module_loading.import_string") +@mock.patch("airflow.providers_manager.ProvidersManager.hooks", new_callable=PropertyMock) +@mock.patch("airflow.www.views.BaseHook") +def test_process_form_extras_updates_sensitive_placeholder_unchanged( + mock_base_hook, mock_pm_hooks, mock_import_str +): + """ + Test the handling of sensitive unchanged field (where placeholder has not been modified). + """ + + # Testing parameters set in both extra and custom fields (connection updates). + mock_form = mock.Mock() + mock_form.data = { + "conn_type": "test4", + "conn_id": "extras_test4", + "extra": '{"sensitive_extra": "RATHER_LONG_SENSITIVE_FIELD_PLACEHOLDER", "extra__custom": "value"}', + } + mock_base_hook.get_connection.return_value = Connection(extra='{"sensitive_extra": "old_value"}') + cmv = ConnectionModelView() + cmv._iter_extra_field_names_and_sensitivity = mock.Mock( + return_value=[("sensitive_extra_key", "sensitive_extra", True)] + ) + cmv.process_form(form=mock_form, is_created=True) + + assert json.loads(mock_form.extra.data) == { + "extra__custom": "value", + "sensitive_extra": "old_value", + } + + def test_duplicate_connection(admin_client): """Test Duplicate multiple connection with suffix""" conn1 = Connection( From f6115a349df254d61894f6ddfe4add2404da81c7 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Thu, 6 Jul 2023 09:03:28 +0530 Subject: [PATCH 068/533] Show task instance in web UI by custom operator name (#31852) --- ...7_2_7_0_add_custom_operator_name_column.py | 50 + airflow/models/taskinstance.py | 9 + airflow/www/static/js/task_instances.js | 8 +- airflow/www/views.py | 2 + docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 1616 +---------------- docs/apache-airflow/migrations-ref.rst | 4 +- tests/models/test_taskinstance.py | 1 + tests/www/views/test_views_tasks.py | 7 + 9 files changed, 81 insertions(+), 1618 deletions(-) create mode 100644 airflow/migrations/versions/0127_2_7_0_add_custom_operator_name_column.py diff --git a/airflow/migrations/versions/0127_2_7_0_add_custom_operator_name_column.py b/airflow/migrations/versions/0127_2_7_0_add_custom_operator_name_column.py new file mode 100644 index 000000000000..3ac0bd7ad64d --- /dev/null +++ b/airflow/migrations/versions/0127_2_7_0_add_custom_operator_name_column.py @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add custom_operator_name column + +Revision ID: 788397e78828 +Revises: 937cbd173ca1 +Create Date: 2023-06-12 10:46:52.125149 + +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "788397e78828" +down_revision = "937cbd173ca1" +branch_labels = None +depends_on = None +airflow_version = "2.7.0" + +TABLE_NAME = "task_instance" + + +def upgrade(): + """Apply Add custom_operator_name column""" + with op.batch_alter_table(TABLE_NAME) as batch_op: + batch_op.add_column(sa.Column("custom_operator_name", sa.VARCHAR(length=1000), nullable=True)) + + +def downgrade(): + """Unapply Add custom_operator_name column""" + with op.batch_alter_table(TABLE_NAME) as batch_op: + batch_op.drop_column("custom_operator_name") diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index f429e27e1546..65eeacdfd3ec 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -394,6 +394,7 @@ class TaskInstance(Base, LoggingMixin): queue = Column(String(256)) priority_weight = Column(Integer) operator = Column(String(1000)) + custom_operator_name = Column(String(1000)) queued_dttm = Column(UtcDateTime) queued_by_job_id = Column(Integer) pid = Column(Integer) @@ -571,6 +572,7 @@ def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any "max_tries": task.retries, "executor_config": task.executor_config, "operator": task.task_type, + "custom_operator_name": getattr(task, "custom_operator_name", None), "map_index": map_index, } @@ -617,6 +619,11 @@ def prev_attempted_tries(self) -> int: def next_try_number(self) -> int: return self._try_number + 1 + @property + def operator_name(self) -> str | None: + """@property: use a more friendly display name for the operator, if set.""" + return self.custom_operator_name or self.operator + def command_as_list( self, mark_success=False, @@ -858,6 +865,7 @@ def refresh_from_db(self, session: Session = NEW_SESSION, lock_for_update: bool self.queue = ti.queue self.priority_weight = ti.priority_weight self.operator = ti.operator + self.custom_operator_name = ti.custom_operator_name self.queued_dttm = ti.queued_dttm self.queued_by_job_id = ti.queued_by_job_id self.pid = ti.pid @@ -886,6 +894,7 @@ def refresh_from_task(self, task: Operator, pool_override: str | None = None) -> # value that needs to be stored in the db. self.executor_config = task.executor_config self.operator = task.task_type + self.custom_operator_name = getattr(task, "custom_operator_name", None) @provide_session def clear_xcom_data(self, session: Session = NEW_SESSION) -> None: diff --git a/airflow/www/static/js/task_instances.js b/airflow/www/static/js/task_instances.js index c539d5c3ea16..abadfaab0079 100644 --- a/airflow/www/static/js/task_instances.js +++ b/airflow/www/static/js/task_instances.js @@ -109,8 +109,8 @@ export default function tiTooltip(ti, task, { includeTryNumber = false } = {}) { if (ti.map_index >= 0 && !ti.mapped_states) { tt += `Map Index: ${escapeHtml(ti.map_index)}
    `; } - if (ti.operator !== undefined) { - tt += `Operator: ${escapeHtml(ti.operator)}
    `; + if (ti.operator_name !== undefined) { + tt += `Operator: ${escapeHtml(ti.operator_name)}
    `; } if (task && task.trigger_rule) { tt += `Trigger Rule: ${task.trigger_rule}
    `; @@ -170,8 +170,8 @@ export function taskQueuedStateTooltip(ti) { if (ti.run_id !== undefined) { tt += `Run Id: ${escapeHtml(ti.run_id)}
    `; } - if (ti.operator !== undefined) { - tt += `Operator: ${escapeHtml(ti.operator)}
    `; + if (ti.operator_name !== undefined) { + tt += `Operator: ${escapeHtml(ti.operator_name)}
    `; } if (ti.start_date && ti.queued_dttm) { const startDate = diff --git a/airflow/www/views.py b/airflow/www/views.py index 910f7aabf10a..3b9c35e1ac3a 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -3136,6 +3136,8 @@ class GraphForm(DateTimeWithNumRunsWithDagRunsForm): for ti in dag.get_task_instances(dttm, dttm): if ti.task_id not in task_instances: task_instances[ti.task_id] = wwwutils.get_instance_with_map(ti, session) + # Need to add operator_name explicitly because it's not a column in task_instances model. + task_instances[ti.task_id]["operator_name"] = ti.operator_name tasks = { t.task_id: { "dag_id": t.dag_id, diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index b8f3bd2d1892..dc75b8aee675 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -97c45f96cf0854ba57c0bc350d7e5412ae7afdb860c6ad05b08338ba7e390bd4 \ No newline at end of file +902b20991528af2fceec9c967f4a7a59b921e0fc2e39b6bb2c4727b3f61a56b7 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 69577c77e51f..b57b13b8d2fb 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,1618 +4,10 @@ - - + + %3 - - - -ab_permission - -ab_permission - -id - [INTEGER] - NOT NULL - -name - [VARCHAR(100)] - NOT NULL - - - -ab_permission_view - -ab_permission_view - -id - [INTEGER] - NOT NULL - -permission_id - [INTEGER] - -view_menu_id - [INTEGER] - - - -ab_permission--ab_permission_view - -0..N -{0,1} - - - -ab_permission_view_role - -ab_permission_view_role - -id - [INTEGER] - NOT NULL - -permission_view_id - [INTEGER] - -role_id - [INTEGER] - - - -ab_permission_view--ab_permission_view_role - -0..N -{0,1} - - - -ab_view_menu - -ab_view_menu - -id - [INTEGER] - NOT NULL - -name - [VARCHAR(250)] - NOT NULL - - - -ab_view_menu--ab_permission_view - -0..N -{0,1} - - - -ab_role - -ab_role - -id - [INTEGER] - NOT NULL - -name - [VARCHAR(64)] - NOT NULL - - - -ab_role--ab_permission_view_role - -0..N -{0,1} - - - -ab_user_role - -ab_user_role - -id - [INTEGER] - NOT NULL - -role_id - [INTEGER] - -user_id - [INTEGER] - - - -ab_role--ab_user_role - -0..N -{0,1} - - - -ab_register_user - -ab_register_user - -id - [INTEGER] - NOT NULL - -email - [VARCHAR(512)] - NOT NULL - -first_name - [VARCHAR(256)] - NOT NULL - -last_name - [VARCHAR(256)] - NOT NULL - -password - [VARCHAR(256)] - -registration_date - [DATETIME] - -registration_hash - [VARCHAR(256)] - -username - [VARCHAR(512)] - NOT NULL - - - -ab_user - -ab_user - -id - [INTEGER] - NOT NULL - -active - [BOOLEAN] - -changed_by_fk - [INTEGER] - -changed_on - [DATETIME] - -created_by_fk - [INTEGER] - -created_on - [DATETIME] - -email - [VARCHAR(512)] - NOT NULL - -fail_login_count - [INTEGER] - -first_name - [VARCHAR(256)] - NOT NULL - -last_login - [DATETIME] - -last_name - [VARCHAR(256)] - NOT NULL - -login_count - [INTEGER] - -password - [VARCHAR(256)] - -username - [VARCHAR(512)] - NOT NULL - - - -ab_user--ab_user_role - -0..N -{0,1} - - - -ab_user--ab_user - -0..N -{0,1} - - - -ab_user--ab_user - -0..N -{0,1} - - - -dag_run_note - -dag_run_note - -dag_run_id - [INTEGER] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] - - - -ab_user--dag_run_note - -0..N -{0,1} - - - -task_instance_note - -task_instance_note - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] - - - -ab_user--task_instance_note - -0..N -{0,1} - - - -alembic_version - -alembic_version - -version_num - [VARCHAR(32)] - NOT NULL - - - -callback_request - -callback_request - -id - [INTEGER] - NOT NULL - -callback_data - [JSON] - NOT NULL - -callback_type - [VARCHAR(20)] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -priority_weight - [INTEGER] - NOT NULL - -processor_subdir - [VARCHAR(2000)] - - - -connection - -connection - -id - [INTEGER] - NOT NULL - -conn_id - [VARCHAR(250)] - NOT NULL - -conn_type - [VARCHAR(500)] - NOT NULL - -description - [VARCHAR(5000)] - -extra - [TEXT] - -host - [VARCHAR(500)] - -is_encrypted - [BOOLEAN] - -is_extra_encrypted - [BOOLEAN] - -login - [VARCHAR(500)] - -password - [VARCHAR(5000)] - -port - [INTEGER] - -schema - [VARCHAR(500)] - - - -dag - -dag - -dag_id - [VARCHAR(250)] - NOT NULL - -default_view - [VARCHAR(25)] - -description - [TEXT] - -fileloc - [VARCHAR(2000)] - -has_import_errors - [BOOLEAN] - -has_task_concurrency_limits - [BOOLEAN] - NOT NULL - -is_active - [BOOLEAN] - -is_paused - [BOOLEAN] - -is_subdag - [BOOLEAN] - -last_expired - [TIMESTAMP] - -last_parsed_time - [TIMESTAMP] - -last_pickled - [TIMESTAMP] - -max_active_runs - [INTEGER] - -max_active_tasks - [INTEGER] - NOT NULL - -next_dagrun - [TIMESTAMP] - -next_dagrun_create_after - [TIMESTAMP] - -next_dagrun_data_interval_end - [TIMESTAMP] - -next_dagrun_data_interval_start - [TIMESTAMP] - -owners - [VARCHAR(2000)] - -pickle_id - [INTEGER] - -processor_subdir - [VARCHAR(2000)] - -root_dag_id - [VARCHAR(250)] - -schedule_interval - [TEXT] - -scheduler_lock - [BOOLEAN] - -timetable_description - [VARCHAR(1000)] - - - -dag_owner_attributes - -dag_owner_attributes - -dag_id - [VARCHAR(250)] - NOT NULL - -owner - [VARCHAR(500)] - NOT NULL - -link - [VARCHAR(500)] - NOT NULL - - - -dag--dag_owner_attributes - -1 -1 - - - -dag_schedule_dataset_reference - -dag_schedule_dataset_reference - -dag_id - [VARCHAR(250)] - NOT NULL - -dataset_id - [INTEGER] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - - - -dag--dag_schedule_dataset_reference - -1 -1 - - - -dag_tag - -dag_tag - -dag_id - [VARCHAR(250)] - NOT NULL - -name - [VARCHAR(100)] - NOT NULL - - - -dag--dag_tag - -1 -1 - - - -dag_warning - -dag_warning - -dag_id - [VARCHAR(250)] - NOT NULL - -warning_type - [VARCHAR(50)] - NOT NULL - -message - [TEXT] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - - - -dag--dag_warning - -1 -1 - - - -dataset_dag_run_queue - -dataset_dag_run_queue - -dataset_id - [INTEGER] - NOT NULL - -target_dag_id - [VARCHAR(250)] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - - - -dag--dataset_dag_run_queue - -1 -1 - - - -task_outlet_dataset_reference - -task_outlet_dataset_reference - -dag_id - [VARCHAR(250)] - NOT NULL - -dataset_id - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - - - -dag--task_outlet_dataset_reference - -1 -1 - - - -dag_code - -dag_code - -fileloc_hash - [BIGINT] - NOT NULL - -fileloc - [VARCHAR(2000)] - NOT NULL - -last_updated - [TIMESTAMP] - NOT NULL - -source_code - [TEXT] - NOT NULL - - - -dag_pickle - -dag_pickle - -id - [INTEGER] - NOT NULL - -created_dttm - [TIMESTAMP] - -pickle - [BLOB] - -pickle_hash - [BIGINT] - - - -dag_run - -dag_run - -id - [INTEGER] - NOT NULL - -conf - [BLOB] - -creating_job_id - [INTEGER] - -dag_hash - [VARCHAR(32)] - -dag_id - [VARCHAR(250)] - NOT NULL - -data_interval_end - [TIMESTAMP] - -data_interval_start - [TIMESTAMP] - -end_date - [TIMESTAMP] - -execution_date - [TIMESTAMP] - NOT NULL - -external_trigger - [BOOLEAN] - -last_scheduling_decision - [TIMESTAMP] - -log_template_id - [INTEGER] - -queued_at - [TIMESTAMP] - -run_id - [VARCHAR(250)] - NOT NULL - -run_type - [VARCHAR(50)] - NOT NULL - -start_date - [TIMESTAMP] - -state - [VARCHAR(50)] - -updated_at - [TIMESTAMP] - - - -dag_run--dag_run_note - -1 -1 - - - -dagrun_dataset_event - -dagrun_dataset_event - -dag_run_id - [INTEGER] - NOT NULL - -event_id - [INTEGER] - NOT NULL - - - -dag_run--dagrun_dataset_event - -1 -1 - - - -task_instance - -task_instance - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -duration - [FLOAT] - -end_date - [TIMESTAMP] - -executor_config - [BLOB] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -trigger_id - [INTEGER] - -trigger_timeout - [DATETIME] - -try_number - [INTEGER] - -unixname - [VARCHAR(1000)] - -updated_at - [TIMESTAMP] - - - -dag_run--task_instance - -1 -1 - - - -dag_run--task_instance - -1 -1 - - - -task_reschedule - -task_reschedule - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - NOT NULL - -end_date - [TIMESTAMP] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -reschedule_date - [TIMESTAMP] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -try_number - [INTEGER] - NOT NULL - - - -dag_run--task_reschedule - -0..N -1 - - - -dag_run--task_reschedule - -0..N -1 - - - -task_instance--task_instance_note - -1 -1 - - - -task_instance--task_instance_note - -1 -1 - - - -task_instance--task_instance_note - -1 -1 - - - -task_instance--task_instance_note - -1 -1 - - - -task_instance--task_reschedule - -0..N -1 - - - -task_instance--task_reschedule - -0..N -1 - - - -task_instance--task_reschedule - -0..N -1 - - - -task_instance--task_reschedule - -0..N -1 - - - -rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - [JSON] - -rendered_fields - [JSON] - NOT NULL - - - -task_instance--rendered_task_instance_fields - -1 -1 - - - -task_instance--rendered_task_instance_fields - -1 -1 - - - -task_instance--rendered_task_instance_fields - -1 -1 - - - -task_instance--rendered_task_instance_fields - -1 -1 - - - -task_fail - -task_fail - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - -end_date - [TIMESTAMP] - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - -task_id - [VARCHAR(250)] - NOT NULL - - - -task_instance--task_fail - -0..N -1 - - - -task_instance--task_fail - -0..N -1 - - - -task_instance--task_fail - -0..N -1 - - - -task_instance--task_fail - -0..N -1 - - - -task_map - -task_map - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -keys - [JSON] - -length - [INTEGER] - NOT NULL - - - -task_instance--task_map - -1 -1 - - - -task_instance--task_map - -1 -1 - - - -task_instance--task_map - -1 -1 - - - -task_instance--task_map - -1 -1 - - - -xcom - -xcom - -dag_run_id - [INTEGER] - NOT NULL - -key - [VARCHAR(512)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - -value - [BLOB] - - - -task_instance--xcom - -1 -1 - - - -task_instance--xcom - -0..N -1 - - - -task_instance--xcom - -1 -1 - - - -task_instance--xcom - -0..N -1 - - - -log_template - -log_template - -id - [INTEGER] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -elasticsearch_id - [TEXT] - NOT NULL - -filename - [TEXT] - NOT NULL - - - -log_template--dag_run - -0..N -{0,1} - - - -dataset - -dataset - -id - [INTEGER] - NOT NULL - -created_at - [TIMESTAMP] - NOT NULL - -extra - [JSON] - NOT NULL - -is_orphaned - [BOOLEAN] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -uri - [VARCHAR(3000)] - NOT NULL - - - -dataset--dag_schedule_dataset_reference - -1 -1 - - - -dataset--dataset_dag_run_queue - -1 -1 - - - -dataset--task_outlet_dataset_reference - -1 -1 - - - -dataset_event - -dataset_event - -id - [INTEGER] - NOT NULL - -dataset_id - [INTEGER] - NOT NULL - -extra - [JSON] - NOT NULL - -source_dag_id - [VARCHAR(250)] - -source_map_index - [INTEGER] - -source_run_id - [VARCHAR(250)] - -source_task_id - [VARCHAR(250)] - -timestamp - [TIMESTAMP] - NOT NULL - - - -dataset_event--dagrun_dataset_event - -1 -1 - - - -import_error - -import_error - -id - [INTEGER] - NOT NULL - -filename - [VARCHAR(1024)] - -stacktrace - [TEXT] - -timestamp - [TIMESTAMP] - - - -job - -job - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - -end_date - [TIMESTAMP] - -executor_class - [VARCHAR(500)] - -hostname - [VARCHAR(500)] - -job_type - [VARCHAR(30)] - -latest_heartbeat - [TIMESTAMP] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -unixname - [VARCHAR(1000)] - - - -log - -log - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - -dttm - [TIMESTAMP] - -event - [VARCHAR(30)] - -execution_date - [TIMESTAMP] - -extra - [TEXT] - -map_index - [INTEGER] - -owner - [VARCHAR(500)] - -task_id - [VARCHAR(250)] - - - -trigger - -trigger - -id - [INTEGER] - NOT NULL - -classpath - [VARCHAR(1000)] - NOT NULL - -created_date - [TIMESTAMP] - NOT NULL - -kwargs - [JSON] - NOT NULL - -triggerer_id - [INTEGER] - - - -trigger--task_instance - -0..N -{0,1} - - - -serialized_dag - -serialized_dag - -dag_id - [VARCHAR(250)] - NOT NULL - -dag_hash - [VARCHAR(32)] - NOT NULL - -data - [JSON] - -data_compressed - [BLOB] - -fileloc - [VARCHAR(2000)] - NOT NULL - -fileloc_hash - [BIGINT] - NOT NULL - -last_updated - [TIMESTAMP] - NOT NULL - -processor_subdir - [VARCHAR(2000)] - - - -session - -session - -id - [INTEGER] - NOT NULL - -data - [BLOB] - -expiry - [DATETIME] - -session_id - [VARCHAR(255)] - - - -sla_miss - -sla_miss - -dag_id - [VARCHAR(250)] - NOT NULL - -execution_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -description - [TEXT] - -email_sent - [BOOLEAN] - -notification_sent - [BOOLEAN] - -timestamp - [TIMESTAMP] - - - -slot_pool - -slot_pool - -id - [INTEGER] - NOT NULL - -description - [TEXT] - -pool - [VARCHAR(256)] - -slots - [INTEGER] - - - -variable - -variable - -id - [INTEGER] - NOT NULL - -description - [TEXT] - -is_encrypted - [BOOLEAN] - -key - [VARCHAR(250)] - -val - [TEXT] - + diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 7b09a46f8c02..4f5abde251f9 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``937cbd173ca1`` (head) | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table | +| ``788397e78828`` (head) | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``c804e5c76e3e`` | ``98ae134e6fff`` | ``2.6.2`` | Add ``onupdate`` cascade to ``task_map`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index a9b7ea9b089c..48e150905798 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -3114,6 +3114,7 @@ def test_refresh_from_db(self, create_task_instance): "queue": "some_queue_id", "priority_weight": 123, "operator": "some_custom_operator", + "custom_operator_name": "some_custom_operator", "queued_dttm": run_date + datetime.timedelta(hours=1), "queued_by_job_id": 321, "pid": 123, diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 21c573fe5049..9b45fc6622be 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -986,6 +986,7 @@ def test_task_instances(admin_client): assert resp.status_code == 200 assert resp.json == { "also_run_this": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1016,6 +1017,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "run_after_loop": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1046,6 +1048,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "run_this_last": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1076,6 +1079,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "runme_0": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1106,6 +1110,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "runme_1": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1136,6 +1141,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "runme_2": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, @@ -1166,6 +1172,7 @@ def test_task_instances(admin_client): "updated_at": DEFAULT_DATE.isoformat(), }, "this_will_skip": { + "custom_operator_name": None, "dag_id": "example_bash_operator", "duration": None, "end_date": None, From 225e3041d269698d0456e09586924c1898d09434 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Thu, 6 Jul 2023 07:41:36 +0300 Subject: [PATCH 069/533] Prepare docs for July 2023 wave of Providers (RC2) (#32381) --- .../airbyte/.latest-doc-only-change.txt | 2 +- airflow/providers/alibaba/CHANGELOG.rst | 2 ++ airflow/providers/amazon/CHANGELOG.rst | 5 +++++ .../apache/beam/.latest-doc-only-change.txt | 2 +- .../apache/cassandra/.latest-doc-only-change.txt | 2 +- .../apache/drill/.latest-doc-only-change.txt | 2 +- .../apache/druid/.latest-doc-only-change.txt | 2 +- .../apache/flink/.latest-doc-only-change.txt | 1 + .../apache/hdfs/.latest-doc-only-change.txt | 2 +- airflow/providers/apache/hive/CHANGELOG.rst | 14 ++++++++++++++ airflow/providers/apache/hive/__init__.py | 2 +- airflow/providers/apache/hive/provider.yaml | 1 + .../apache/impala/.latest-doc-only-change.txt | 1 + airflow/providers/apache/impala/CHANGELOG.rst | 13 +++++++++++++ airflow/providers/apache/impala/__init__.py | 2 +- airflow/providers/apache/impala/provider.yaml | 1 + airflow/providers/apache/kafka/CHANGELOG.rst | 2 ++ .../apache/kylin/.latest-doc-only-change.txt | 2 +- airflow/providers/apache/livy/CHANGELOG.rst | 7 +++++++ .../apache/pig/.latest-doc-only-change.txt | 2 +- .../apache/pinot/.latest-doc-only-change.txt | 2 +- .../apache/spark/.latest-doc-only-change.txt | 2 +- .../apache/sqoop/.latest-doc-only-change.txt | 2 +- .../arangodb/.latest-doc-only-change.txt | 2 +- .../providers/asana/.latest-doc-only-change.txt | 2 +- .../atlassian/jira/.latest-doc-only-change.txt | 1 + .../providers/celery/.latest-doc-only-change.txt | 2 +- .../cloudant/.latest-doc-only-change.txt | 2 +- airflow/providers/cncf/kubernetes/CHANGELOG.rst | 6 ++++++ airflow/providers/common/sql/CHANGELOG.rst | 6 ++++++ airflow/providers/databricks/CHANGELOG.rst | 7 +++++++ .../datadog/.latest-doc-only-change.txt | 2 +- airflow/providers/dbt/cloud/CHANGELOG.rst | 15 +++++++++++++++ airflow/providers/dbt/cloud/__init__.py | 2 +- airflow/providers/dbt/cloud/provider.yaml | 1 + .../dingding/.latest-doc-only-change.txt | 2 +- .../discord/.latest-doc-only-change.txt | 2 +- .../providers/docker/.latest-doc-only-change.txt | 2 +- airflow/providers/elasticsearch/CHANGELOG.rst | 2 ++ .../providers/exasol/.latest-doc-only-change.txt | 2 +- .../facebook/.latest-doc-only-change.txt | 2 +- .../providers/ftp/.latest-doc-only-change.txt | 2 +- .../providers/github/.latest-doc-only-change.txt | 2 +- airflow/providers/google/CHANGELOG.rst | 3 +++ .../providers/grpc/.latest-doc-only-change.txt | 2 +- airflow/providers/hashicorp/CHANGELOG.rst | 3 +++ .../providers/http/.latest-doc-only-change.txt | 2 +- .../providers/imap/.latest-doc-only-change.txt | 2 +- .../influxdb/.latest-doc-only-change.txt | 2 +- .../providers/jdbc/.latest-doc-only-change.txt | 2 +- .../jenkins/.latest-doc-only-change.txt | 2 +- airflow/providers/microsoft/azure/CHANGELOG.rst | 5 +++++ .../microsoft/mssql/.latest-doc-only-change.txt | 2 +- .../microsoft/psrp/.latest-doc-only-change.txt | 2 +- .../microsoft/winrm/.latest-doc-only-change.txt | 2 +- .../providers/mongo/.latest-doc-only-change.txt | 2 +- .../providers/mysql/.latest-doc-only-change.txt | 2 +- .../providers/neo4j/.latest-doc-only-change.txt | 2 +- .../providers/odbc/.latest-doc-only-change.txt | 2 +- .../openfaas/.latest-doc-only-change.txt | 2 +- .../opsgenie/.latest-doc-only-change.txt | 2 +- .../providers/oracle/.latest-doc-only-change.txt | 2 +- .../pagerduty/.latest-doc-only-change.txt | 2 +- .../papermill/.latest-doc-only-change.txt | 2 +- .../providers/plexus/.latest-doc-only-change.txt | 2 +- airflow/providers/postgres/CHANGELOG.rst | 14 ++++++++++++++ airflow/providers/postgres/__init__.py | 2 +- airflow/providers/postgres/provider.yaml | 1 + .../providers/presto/.latest-doc-only-change.txt | 2 +- .../providers/qubole/.latest-doc-only-change.txt | 2 +- .../providers/redis/.latest-doc-only-change.txt | 2 +- .../salesforce/.latest-doc-only-change.txt | 2 +- .../providers/samba/.latest-doc-only-change.txt | 2 +- .../segment/.latest-doc-only-change.txt | 2 +- .../sendgrid/.latest-doc-only-change.txt | 2 +- .../providers/sftp/.latest-doc-only-change.txt | 2 +- .../singularity/.latest-doc-only-change.txt | 2 +- .../providers/slack/.latest-doc-only-change.txt | 2 +- .../providers/smtp/.latest-doc-only-change.txt | 1 + airflow/providers/snowflake/CHANGELOG.rst | 14 ++++++++++++++ airflow/providers/snowflake/__init__.py | 2 +- airflow/providers/snowflake/provider.yaml | 1 + .../providers/sqlite/.latest-doc-only-change.txt | 2 +- .../providers/ssh/.latest-doc-only-change.txt | 2 +- .../tableau/.latest-doc-only-change.txt | 2 +- .../tabular/.latest-doc-only-change.txt | 2 +- .../telegram/.latest-doc-only-change.txt | 2 +- airflow/providers/trino/CHANGELOG.rst | 2 ++ airflow/providers/vertica/CHANGELOG.rst | 2 ++ .../zendesk/.latest-doc-only-change.txt | 2 +- .../changelog.rst | 2 -- .../apache-airflow-providers-alibaba/commits.rst | 4 +++- .../changelog.rst | 2 -- docs/apache-airflow-providers-amazon/commits.rst | 7 ++++++- .../changelog.rst | 2 -- .../commits.rst | 16 ++++++++++++++++ .../index.rst | 6 +++--- .../changelog.rst | 2 -- .../commits.rst | 14 ++++++++++++++ .../index.rst | 6 +++--- .../changelog.rst | 2 -- .../commits.rst | 4 +++- .../changelog.rst | 2 -- .../commits.rst | 5 ++++- .../changelog.rst | 2 -- .../apache-airflow-providers-apprise/commits.rst | 4 +++- .../changelog.rst | 2 -- .../commits.rst | 5 ++++- .../changelog.rst | 2 -- .../commits.rst | 5 ++++- .../changelog.rst | 2 -- .../commits.rst | 5 ++++- .../changelog.rst | 2 -- .../commits.rst | 16 ++++++++++++++++ .../apache-airflow-providers-dbt-cloud/index.rst | 6 +++--- .../changelog.rst | 2 -- .../commits.rst | 4 +++- .../changelog.rst | 2 -- docs/apache-airflow-providers-google/commits.rst | 5 ++++- .../changelog.rst | 2 -- .../commits.rst | 11 +++++++---- .../changelog.rst | 2 -- .../commits.rst | 7 ++++++- .../changelog.rst | 2 -- .../commits.rst | 15 +++++++++++++++ docs/apache-airflow-providers-postgres/index.rst | 6 +++--- .../changelog.rst | 2 -- .../commits.rst | 15 +++++++++++++++ .../apache-airflow-providers-snowflake/index.rst | 6 +++--- .../apache-airflow-providers-trino/changelog.rst | 2 -- docs/apache-airflow-providers-trino/commits.rst | 4 +++- .../changelog.rst | 2 -- .../apache-airflow-providers-vertica/commits.rst | 4 +++- 133 files changed, 342 insertions(+), 133 deletions(-) create mode 100644 airflow/providers/apache/flink/.latest-doc-only-change.txt create mode 100644 airflow/providers/apache/impala/.latest-doc-only-change.txt create mode 100644 airflow/providers/atlassian/jira/.latest-doc-only-change.txt create mode 100644 airflow/providers/smtp/.latest-doc-only-change.txt diff --git a/airflow/providers/airbyte/.latest-doc-only-change.txt b/airflow/providers/airbyte/.latest-doc-only-change.txt index 370cc38c596d..f15af5bca1f1 100644 --- a/airflow/providers/airbyte/.latest-doc-only-change.txt +++ b/airflow/providers/airbyte/.latest-doc-only-change.txt @@ -1 +1 @@ -0bc689ee6d4b6967d7ae99a202031aac14d181a2 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/airflow/providers/alibaba/CHANGELOG.rst index b55e5aa6ea61..f969da989594 100644 --- a/airflow/providers/alibaba/CHANGELOG.rst +++ b/airflow/providers/alibaba/CHANGELOG.rst @@ -38,6 +38,8 @@ Features appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Airbyte and Alibaba (#32214)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 2.4.1 ..... diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index 8af159f61104..e8227e109e18 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -54,6 +54,7 @@ Bug Fixes * ``bugfix: break down run+wait method in ECS operator (#32104)`` * ``Handle 'UnboundLocalError' while parsing invalid 's3_url' (#32120)`` * ``Fix 'LambdaInvokeFunctionOperator' payload parameter type (#32259)`` +* ``Bug fix GCSToS3Operator: avoid 'ValueError' when 'replace=False' with files already in S3 (#32322)`` Misc ~~~~ @@ -64,6 +65,8 @@ Misc * ``Refactor Eks Create Cluster Operator code (#31960)`` * ``Use a waiter in 'AthenaHook' (#31942)`` * ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` +* ``Add default_deferrable config (#31712)`` +* ``deprecate arbitrary parameter passing to RDS hook (#32352)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): @@ -75,6 +78,8 @@ Misc * ``Minor name change for the util wait method. (#32152)`` * ``Clean up string concatenation (#32129)`` * ``cleanup Amazon CHANGELOG.rst (#32031)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 8.2.0 ..... diff --git a/airflow/providers/apache/beam/.latest-doc-only-change.txt b/airflow/providers/apache/beam/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/beam/.latest-doc-only-change.txt +++ b/airflow/providers/apache/beam/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt +++ b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/drill/.latest-doc-only-change.txt b/airflow/providers/apache/drill/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/apache/drill/.latest-doc-only-change.txt +++ b/airflow/providers/apache/drill/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/druid/.latest-doc-only-change.txt b/airflow/providers/apache/druid/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/druid/.latest-doc-only-change.txt +++ b/airflow/providers/apache/druid/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/flink/.latest-doc-only-change.txt b/airflow/providers/apache/flink/.latest-doc-only-change.txt new file mode 100644 index 000000000000..f15af5bca1f1 --- /dev/null +++ b/airflow/providers/apache/flink/.latest-doc-only-change.txt @@ -0,0 +1 @@ +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt +++ b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/hive/CHANGELOG.rst b/airflow/providers/apache/hive/CHANGELOG.rst index a57689b1d857..62df235a1b45 100644 --- a/airflow/providers/apache/hive/CHANGELOG.rst +++ b/airflow/providers/apache/hive/CHANGELOG.rst @@ -27,6 +27,20 @@ Changelog --------- +6.1.2 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Add proxy_user template check (#32334)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` 6.1.1 ..... diff --git a/airflow/providers/apache/hive/__init__.py b/airflow/providers/apache/hive/__init__.py index 9e331a66eef1..1b2c1a37ea17 100644 --- a/airflow/providers/apache/hive/__init__.py +++ b/airflow/providers/apache/hive/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "6.1.1" +__version__ = "6.1.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index a277010dd6ac..79125e2cc3d8 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 6.1.2 - 6.1.1 - 6.1.0 - 6.0.0 diff --git a/airflow/providers/apache/impala/.latest-doc-only-change.txt b/airflow/providers/apache/impala/.latest-doc-only-change.txt new file mode 100644 index 000000000000..33a07c855e0f --- /dev/null +++ b/airflow/providers/apache/impala/.latest-doc-only-change.txt @@ -0,0 +1 @@ +09d4718d3a46aecf3355d14d3d23022002f4a818 diff --git a/airflow/providers/apache/impala/CHANGELOG.rst b/airflow/providers/apache/impala/CHANGELOG.rst index 3dcd84dba83d..c6196ad9abd9 100644 --- a/airflow/providers/apache/impala/CHANGELOG.rst +++ b/airflow/providers/apache/impala/CHANGELOG.rst @@ -26,6 +26,19 @@ Changelog --------- +1.1.2 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Add kerberos dependency to Impala Provider (#32304)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Improve provider documentation and README structure (#32125)`` + 1.1.1 ..... diff --git a/airflow/providers/apache/impala/__init__.py b/airflow/providers/apache/impala/__init__.py index 4e874c1c0281..5455603a670a 100644 --- a/airflow/providers/apache/impala/__init__.py +++ b/airflow/providers/apache/impala/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "1.1.1" +__version__ = "1.1.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index 4d02db35c9df..8a0e06ab8700 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 1.1.2 - 1.1.1 - 1.1.0 - 1.0.0 diff --git a/airflow/providers/apache/kafka/CHANGELOG.rst b/airflow/providers/apache/kafka/CHANGELOG.rst index 5c77dc4b349c..a432025d59dc 100644 --- a/airflow/providers/apache/kafka/CHANGELOG.rst +++ b/airflow/providers/apache/kafka/CHANGELOG.rst @@ -39,6 +39,8 @@ Bug Fixes appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 1.1.1 ..... diff --git a/airflow/providers/apache/kylin/.latest-doc-only-change.txt b/airflow/providers/apache/kylin/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/apache/kylin/.latest-doc-only-change.txt +++ b/airflow/providers/apache/kylin/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/livy/CHANGELOG.rst b/airflow/providers/apache/livy/CHANGELOG.rst index 36fb6f5cb330..1b5d7419a5c0 100644 --- a/airflow/providers/apache/livy/CHANGELOG.rst +++ b/airflow/providers/apache/livy/CHANGELOG.rst @@ -36,10 +36,17 @@ Bug Fixes * ``Fix 'LivyHook' TypeError exception on 'session_id' log format (#32051)`` +Misc +~~~~ + +* ``Add default_deferrable config (#31712)`` + .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 3.5.1 ..... diff --git a/airflow/providers/apache/pig/.latest-doc-only-change.txt b/airflow/providers/apache/pig/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/apache/pig/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pig/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/pinot/.latest-doc-only-change.txt b/airflow/providers/apache/pinot/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/pinot/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pinot/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/spark/.latest-doc-only-change.txt b/airflow/providers/apache/spark/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/apache/spark/.latest-doc-only-change.txt +++ b/airflow/providers/apache/spark/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/apache/sqoop/.latest-doc-only-change.txt b/airflow/providers/apache/sqoop/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/apache/sqoop/.latest-doc-only-change.txt +++ b/airflow/providers/apache/sqoop/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/arangodb/.latest-doc-only-change.txt b/airflow/providers/arangodb/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/arangodb/.latest-doc-only-change.txt +++ b/airflow/providers/arangodb/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/asana/.latest-doc-only-change.txt b/airflow/providers/asana/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/asana/.latest-doc-only-change.txt +++ b/airflow/providers/asana/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/atlassian/jira/.latest-doc-only-change.txt b/airflow/providers/atlassian/jira/.latest-doc-only-change.txt new file mode 100644 index 000000000000..f15af5bca1f1 --- /dev/null +++ b/airflow/providers/atlassian/jira/.latest-doc-only-change.txt @@ -0,0 +1 @@ +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/celery/.latest-doc-only-change.txt b/airflow/providers/celery/.latest-doc-only-change.txt index 4d0c8bcde51e..f15af5bca1f1 100644 --- a/airflow/providers/celery/.latest-doc-only-change.txt +++ b/airflow/providers/celery/.latest-doc-only-change.txt @@ -1 +1 @@ -8c37b74a208a808d905c1b86d081d69d7a1aa900 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/cloudant/.latest-doc-only-change.txt b/airflow/providers/cloudant/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/cloudant/.latest-doc-only-change.txt +++ b/airflow/providers/cloudant/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index c089b470adda..ace9bacf2006 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -42,10 +42,16 @@ Bug Fixes * ``Fix 'KubernetesPodTrigger' waiting strategy (#31348)`` * ``fix spark-kubernetes-operator compatibality (#31798)`` +Misc +~~~~ +* ``Add default_deferrable config (#31712)`` + .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 7.1.0 ..... diff --git a/airflow/providers/common/sql/CHANGELOG.rst b/airflow/providers/common/sql/CHANGELOG.rst index 566c2e6b4266..5e77f8fca418 100644 --- a/airflow/providers/common/sql/CHANGELOG.rst +++ b/airflow/providers/common/sql/CHANGELOG.rst @@ -31,10 +31,16 @@ Features * ``openlineage, common.sql: provide OL SQL parser as internal OpenLineage provider API (#31398)`` +Misc +~~~~ +* ``Pass SQLAlchemy engine to construct information schema query. (#32371)`` + .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 1.5.2 ..... diff --git a/airflow/providers/databricks/CHANGELOG.rst b/airflow/providers/databricks/CHANGELOG.rst index b21db94e25af..18db97b7011c 100644 --- a/airflow/providers/databricks/CHANGELOG.rst +++ b/airflow/providers/databricks/CHANGELOG.rst @@ -35,10 +35,17 @@ Bug Fixes * ``Modify 'template_fields' of 'DatabricksSqlOperator' to support parent class fields (#32253)`` +Misc +~~~~ + +* ``Add default_deferrable config (#31712)`` + .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 4.3.0 ..... diff --git a/airflow/providers/datadog/.latest-doc-only-change.txt b/airflow/providers/datadog/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/datadog/.latest-doc-only-change.txt +++ b/airflow/providers/datadog/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/dbt/cloud/CHANGELOG.rst b/airflow/providers/dbt/cloud/CHANGELOG.rst index 7d6b8c1e4c01..0ab5c2c0357a 100644 --- a/airflow/providers/dbt/cloud/CHANGELOG.rst +++ b/airflow/providers/dbt/cloud/CHANGELOG.rst @@ -28,6 +28,21 @@ Changelog --------- +3.2.2 +..... + +Misc +~~~~ + +* ``Add default_deferrable config (#31712)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` + * ``Improve provider documentation and README structure (#32125)`` + 3.2.1 ..... diff --git a/airflow/providers/dbt/cloud/__init__.py b/airflow/providers/dbt/cloud/__init__.py index a929ae6a4c97..41f48b0f4369 100644 --- a/airflow/providers/dbt/cloud/__init__.py +++ b/airflow/providers/dbt/cloud/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "3.2.1" +__version__ = "3.2.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index 3ec8a0a36694..c9aa6bdd041d 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 3.2.2 - 3.2.1 - 3.2.0 - 3.1.1 diff --git a/airflow/providers/dingding/.latest-doc-only-change.txt b/airflow/providers/dingding/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/dingding/.latest-doc-only-change.txt +++ b/airflow/providers/dingding/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/discord/.latest-doc-only-change.txt b/airflow/providers/discord/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/discord/.latest-doc-only-change.txt +++ b/airflow/providers/discord/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/docker/.latest-doc-only-change.txt b/airflow/providers/docker/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/docker/.latest-doc-only-change.txt +++ b/airflow/providers/docker/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index b2838c1590aa..33e5bbcd2fa6 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -41,6 +41,8 @@ Breaking changes .. Review and move the new changes to one of the sections above: * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 4.5.1 ..... diff --git a/airflow/providers/exasol/.latest-doc-only-change.txt b/airflow/providers/exasol/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/exasol/.latest-doc-only-change.txt +++ b/airflow/providers/exasol/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/facebook/.latest-doc-only-change.txt b/airflow/providers/facebook/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/facebook/.latest-doc-only-change.txt +++ b/airflow/providers/facebook/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/ftp/.latest-doc-only-change.txt b/airflow/providers/ftp/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/ftp/.latest-doc-only-change.txt +++ b/airflow/providers/ftp/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/airflow/providers/github/.latest-doc-only-change.txt index 4ca04d3bd48b..f15af5bca1f1 100644 --- a/airflow/providers/github/.latest-doc-only-change.txt +++ b/airflow/providers/github/.latest-doc-only-change.txt @@ -1 +1 @@ -d1aa509bbd1941ceb3fe31789efeebbddd58d32f +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index a418a8d7b176..6bf553b3ebd1 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -53,11 +53,14 @@ Misc * ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` * ``Refactor GKECreateClusterOperator's body validation (#31923)`` * ``Optimize deferrable mode execution for 'BigQueryValueCheckOperator' (#31872)`` +* ``Add default_deferrable config (#31712)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Improve provider documentation and README structure (#32125)`` * ``Google provider docstring improvements (#31731)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 10.2.0 ...... diff --git a/airflow/providers/grpc/.latest-doc-only-change.txt b/airflow/providers/grpc/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/grpc/.latest-doc-only-change.txt +++ b/airflow/providers/grpc/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/hashicorp/CHANGELOG.rst b/airflow/providers/hashicorp/CHANGELOG.rst index 4f22ad96d0ab..117b9583bff5 100644 --- a/airflow/providers/hashicorp/CHANGELOG.rst +++ b/airflow/providers/hashicorp/CHANGELOG.rst @@ -38,6 +38,9 @@ Misc .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` 3.4.1 ..... diff --git a/airflow/providers/http/.latest-doc-only-change.txt b/airflow/providers/http/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/http/.latest-doc-only-change.txt +++ b/airflow/providers/http/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/imap/.latest-doc-only-change.txt b/airflow/providers/imap/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/imap/.latest-doc-only-change.txt +++ b/airflow/providers/imap/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/influxdb/.latest-doc-only-change.txt b/airflow/providers/influxdb/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/influxdb/.latest-doc-only-change.txt +++ b/airflow/providers/influxdb/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/jdbc/.latest-doc-only-change.txt b/airflow/providers/jdbc/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/jdbc/.latest-doc-only-change.txt +++ b/airflow/providers/jdbc/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/jenkins/.latest-doc-only-change.txt b/airflow/providers/jenkins/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/jenkins/.latest-doc-only-change.txt +++ b/airflow/providers/jenkins/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index d82b4c128941..df78b4e7a88b 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -42,17 +42,22 @@ Bug Fixes * ``Cancel pipeline if unexpected exception caught (#32238)`` * ``Fix where account url is build if not provided using login (account name) (#32082)`` +* ``refresh connection if an exception is caught in "AzureDataFactory" (#32323)`` Misc ~~~~ * ``Doc changes: Added Transfers section in Azure provider docs (#32241)`` * ``Adds Sensor section in the Azure providers docs (#32299)`` +* ``Add default_deferrable config (#31712)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Improve provider documentation and README structure (#32125)`` * ``invalid args fix (#32326)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` 6.1.2 ..... diff --git a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt b/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/psrp/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/mongo/.latest-doc-only-change.txt b/airflow/providers/mongo/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/mongo/.latest-doc-only-change.txt +++ b/airflow/providers/mongo/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/mysql/.latest-doc-only-change.txt b/airflow/providers/mysql/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/mysql/.latest-doc-only-change.txt +++ b/airflow/providers/mysql/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/neo4j/.latest-doc-only-change.txt b/airflow/providers/neo4j/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/neo4j/.latest-doc-only-change.txt +++ b/airflow/providers/neo4j/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/odbc/.latest-doc-only-change.txt b/airflow/providers/odbc/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/odbc/.latest-doc-only-change.txt +++ b/airflow/providers/odbc/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/openfaas/.latest-doc-only-change.txt b/airflow/providers/openfaas/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/openfaas/.latest-doc-only-change.txt +++ b/airflow/providers/openfaas/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/opsgenie/.latest-doc-only-change.txt b/airflow/providers/opsgenie/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/opsgenie/.latest-doc-only-change.txt +++ b/airflow/providers/opsgenie/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/oracle/.latest-doc-only-change.txt b/airflow/providers/oracle/.latest-doc-only-change.txt index a780a5d07a3c..f15af5bca1f1 100644 --- a/airflow/providers/oracle/.latest-doc-only-change.txt +++ b/airflow/providers/oracle/.latest-doc-only-change.txt @@ -1 +1 @@ -2794c4172e7b7655b813236fe47222cec46f57e4 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/pagerduty/.latest-doc-only-change.txt b/airflow/providers/pagerduty/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/pagerduty/.latest-doc-only-change.txt +++ b/airflow/providers/pagerduty/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/papermill/.latest-doc-only-change.txt b/airflow/providers/papermill/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/papermill/.latest-doc-only-change.txt +++ b/airflow/providers/papermill/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/plexus/.latest-doc-only-change.txt b/airflow/providers/plexus/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/plexus/.latest-doc-only-change.txt +++ b/airflow/providers/plexus/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/postgres/CHANGELOG.rst b/airflow/providers/postgres/CHANGELOG.rst index c86c013a64d6..3d16a7789c5e 100644 --- a/airflow/providers/postgres/CHANGELOG.rst +++ b/airflow/providers/postgres/CHANGELOG.rst @@ -27,6 +27,20 @@ Changelog --------- +5.5.2 +..... + +Misc +~~~~ + +* ``Deprecate 'runtime_parameters' in favor of options in 'hook_params' (#32345)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``Improve provider documentation and README structure (#32125)`` + 5.5.1 ..... diff --git a/airflow/providers/postgres/__init__.py b/airflow/providers/postgres/__init__.py index b4c3d8aeccd3..37d0931ef361 100644 --- a/airflow/providers/postgres/__init__.py +++ b/airflow/providers/postgres/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "5.5.1" +__version__ = "5.5.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 401da190c605..bfa27ecb1f62 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 5.5.2 - 5.5.1 - 5.5.0 - 5.4.0 diff --git a/airflow/providers/presto/.latest-doc-only-change.txt b/airflow/providers/presto/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/presto/.latest-doc-only-change.txt +++ b/airflow/providers/presto/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/qubole/.latest-doc-only-change.txt b/airflow/providers/qubole/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/qubole/.latest-doc-only-change.txt +++ b/airflow/providers/qubole/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/redis/.latest-doc-only-change.txt b/airflow/providers/redis/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/redis/.latest-doc-only-change.txt +++ b/airflow/providers/redis/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/salesforce/.latest-doc-only-change.txt b/airflow/providers/salesforce/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/salesforce/.latest-doc-only-change.txt +++ b/airflow/providers/salesforce/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/samba/.latest-doc-only-change.txt b/airflow/providers/samba/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/samba/.latest-doc-only-change.txt +++ b/airflow/providers/samba/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/segment/.latest-doc-only-change.txt b/airflow/providers/segment/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/segment/.latest-doc-only-change.txt +++ b/airflow/providers/segment/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/sendgrid/.latest-doc-only-change.txt b/airflow/providers/sendgrid/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/sendgrid/.latest-doc-only-change.txt +++ b/airflow/providers/sendgrid/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/sftp/.latest-doc-only-change.txt b/airflow/providers/sftp/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/sftp/.latest-doc-only-change.txt +++ b/airflow/providers/sftp/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/singularity/.latest-doc-only-change.txt b/airflow/providers/singularity/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/singularity/.latest-doc-only-change.txt +++ b/airflow/providers/singularity/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/slack/.latest-doc-only-change.txt b/airflow/providers/slack/.latest-doc-only-change.txt index 33a07c855e0f..5dcc5759cd8d 100644 --- a/airflow/providers/slack/.latest-doc-only-change.txt +++ b/airflow/providers/slack/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/smtp/.latest-doc-only-change.txt b/airflow/providers/smtp/.latest-doc-only-change.txt new file mode 100644 index 000000000000..5dcc5759cd8d --- /dev/null +++ b/airflow/providers/smtp/.latest-doc-only-change.txt @@ -0,0 +1 @@ +e45bee884068399e7265421511e17fed106ce5b4 diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/airflow/providers/snowflake/CHANGELOG.rst index 2fa9008d8a9a..94e11b6bc269 100644 --- a/airflow/providers/snowflake/CHANGELOG.rst +++ b/airflow/providers/snowflake/CHANGELOG.rst @@ -27,6 +27,20 @@ Changelog --------- +4.3.0 +..... + +Features +~~~~~~~~ + +* ``Add Deferrable switch to SnowflakeSqlApiOperator (#31596)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``Improve provider documentation and README structure (#32125)`` + 4.2.0 ..... diff --git a/airflow/providers/snowflake/__init__.py b/airflow/providers/snowflake/__init__.py index 5e403f026188..53291daf0a5a 100644 --- a/airflow/providers/snowflake/__init__.py +++ b/airflow/providers/snowflake/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.2.0" +__version__ = "4.3.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 2cea953ab428..cad5d16d6f4c 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.3.0 - 4.2.0 - 4.1.0 - 4.0.5 diff --git a/airflow/providers/sqlite/.latest-doc-only-change.txt b/airflow/providers/sqlite/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/sqlite/.latest-doc-only-change.txt +++ b/airflow/providers/sqlite/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/ssh/.latest-doc-only-change.txt b/airflow/providers/ssh/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/ssh/.latest-doc-only-change.txt +++ b/airflow/providers/ssh/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/tableau/.latest-doc-only-change.txt b/airflow/providers/tableau/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/tableau/.latest-doc-only-change.txt +++ b/airflow/providers/tableau/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/tabular/.latest-doc-only-change.txt b/airflow/providers/tabular/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/tabular/.latest-doc-only-change.txt +++ b/airflow/providers/tabular/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/telegram/.latest-doc-only-change.txt b/airflow/providers/telegram/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/telegram/.latest-doc-only-change.txt +++ b/airflow/providers/telegram/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/airflow/providers/trino/CHANGELOG.rst b/airflow/providers/trino/CHANGELOG.rst index 3b6ee23a3054..e48b6656f03b 100644 --- a/airflow/providers/trino/CHANGELOG.rst +++ b/airflow/providers/trino/CHANGELOG.rst @@ -38,6 +38,8 @@ Features .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 5.1.1 ..... diff --git a/airflow/providers/vertica/CHANGELOG.rst b/airflow/providers/vertica/CHANGELOG.rst index 7793a3dedeb2..335476f8d106 100644 --- a/airflow/providers/vertica/CHANGELOG.rst +++ b/airflow/providers/vertica/CHANGELOG.rst @@ -39,6 +39,8 @@ Features .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Improve provider documentation and README structure (#32125)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` 3.4.1 ..... diff --git a/airflow/providers/zendesk/.latest-doc-only-change.txt b/airflow/providers/zendesk/.latest-doc-only-change.txt index 33a07c855e0f..f15af5bca1f1 100644 --- a/airflow/providers/zendesk/.latest-doc-only-change.txt +++ b/airflow/providers/zendesk/.latest-doc-only-change.txt @@ -1 +1 @@ -09d4718d3a46aecf3355d14d3d23022002f4a818 +3878fe6fab3ccc1461932b456c48996f2763139f diff --git a/docs/apache-airflow-providers-alibaba/changelog.rst b/docs/apache-airflow-providers-alibaba/changelog.rst index c4368e7ddbca..ee693a0d9d84 100644 --- a/docs/apache-airflow-providers-alibaba/changelog.rst +++ b/docs/apache-airflow-providers-alibaba/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/alibaba/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-alibaba/commits.rst b/docs/apache-airflow-providers-alibaba/commits.rst index bb9559028c2a..d31b72a1bedf 100644 --- a/docs/apache-airflow-providers-alibaba/commits.rst +++ b/docs/apache-airflow-providers-alibaba/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `0bc689ee6d `_ 2023-06-28 ``D205 Support - Providers: Airbyte and Alibaba (#32214)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `f05beb7219 `_ 2023-06-27 ``Add Alibaba Cloud AnalyticDB Spark Support (#31787)`` diff --git a/docs/apache-airflow-providers-amazon/changelog.rst b/docs/apache-airflow-providers-amazon/changelog.rst index ed6e5a92f2f7..ec76763675bf 100644 --- a/docs/apache-airflow-providers-amazon/changelog.rst +++ b/docs/apache-airflow-providers-amazon/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/amazon/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index fdceb0af6634..58362aebb88a 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -31,11 +31,16 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` +`5623a21a1f `_ 2023-07-04 ``deprecate arbitrary parameter passing to RDS hook (#32352)`` +`575bf2f040 `_ 2023-07-04 ``Bug fix GCSToS3Operator: avoid 'ValueError' when 'replace=False' with files already in S3 (#32322)`` `17132ef4c6 `_ 2023-07-04 ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` `5c72befcfd `_ 2023-07-03 ``Fix 'LambdaInvokeFunctionOperator' payload parameter type (#32259)`` `d6e254db68 `_ 2023-06-30 ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` diff --git a/docs/apache-airflow-providers-apache-hive/changelog.rst b/docs/apache-airflow-providers-apache-hive/changelog.rst index d8ad824aff3d..9383cebfce25 100644 --- a/docs/apache-airflow-providers-apache-hive/changelog.rst +++ b/docs/apache-airflow-providers-apache-hive/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apache/hive/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hive/commits.rst b/docs/apache-airflow-providers-apache-hive/commits.rst index 7377b4f31a64..d5a093ddb8fb 100644 --- a/docs/apache-airflow-providers-apache-hive/commits.rst +++ b/docs/apache-airflow-providers-apache-hive/commits.rst @@ -28,6 +28,21 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`1c1dbd872d `_ 2023-07-04 ``Add proxy_user template check (#32334)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== =================================================================== + 6.1.1 ..... @@ -36,6 +51,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================== Commit Committed Subject ================================================================================================= =========== ============================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `6724eeb621 `_ 2023-06-18 ``Sanitize beeline principal parameter (#31983)`` `13890788ae `_ 2023-06-07 ``Apache provider docstring improvements (#31730)`` diff --git a/docs/apache-airflow-providers-apache-hive/index.rst b/docs/apache-airflow-providers-apache-hive/index.rst index 76ea9a6f4ea4..2676761c73f3 100644 --- a/docs/apache-airflow-providers-apache-hive/index.rst +++ b/docs/apache-airflow-providers-apache-hive/index.rst @@ -78,7 +78,7 @@ Package apache-airflow-providers-apache-hive `Apache Hive `__ -Release: 6.1.1 +Release: 6.1.2 Provider package ---------------- @@ -141,5 +141,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-hive 6.1.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-hive 6.1.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hive 6.1.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hive 6.1.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apache-impala/changelog.rst b/docs/apache-airflow-providers-apache-impala/changelog.rst index 0f07c87feef3..39dfdba00f24 100644 --- a/docs/apache-airflow-providers-apache-impala/changelog.rst +++ b/docs/apache-airflow-providers-apache-impala/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apache/impala/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-impala/commits.rst b/docs/apache-airflow-providers-apache-impala/commits.rst index 86c37726cd5d..fd382113afe5 100644 --- a/docs/apache-airflow-providers-apache-impala/commits.rst +++ b/docs/apache-airflow-providers-apache-impala/commits.rst @@ -28,6 +28,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`bc3b2d16d3 `_ 2023-07-04 ``Add kerberos dependency to Impala Provider (#32304)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================ + 1.1.1 ..... @@ -36,6 +49,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ==================================================================== Commit Committed Subject ================================================================================================= =========== ==================================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `547f5846bf `_ 2023-06-20 ``Add note about dropping Python 3.7 for kafka and impala (#32017)`` ================================================================================================= =========== ==================================================================== diff --git a/docs/apache-airflow-providers-apache-impala/index.rst b/docs/apache-airflow-providers-apache-impala/index.rst index c9a6b364dd90..c4c98b67a410 100644 --- a/docs/apache-airflow-providers-apache-impala/index.rst +++ b/docs/apache-airflow-providers-apache-impala/index.rst @@ -75,7 +75,7 @@ Package apache-airflow-providers-apache-impala `Apache Impala `__. -Release: 1.1.1 +Release: 1.1.2 Provider package ---------------- @@ -127,5 +127,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-impala 1.1.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-impala 1.1.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-impala 1.1.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-impala 1.1.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-apache-kafka/changelog.rst b/docs/apache-airflow-providers-apache-kafka/changelog.rst index b1dafd854b3e..5a5ce636b865 100644 --- a/docs/apache-airflow-providers-apache-kafka/changelog.rst +++ b/docs/apache-airflow-providers-apache-kafka/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apache/kafka/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-kafka/commits.rst b/docs/apache-airflow-providers-apache-kafka/commits.rst index 94450c54b479..92e855d0e9ad 100644 --- a/docs/apache-airflow-providers-apache-kafka/commits.rst +++ b/docs/apache-airflow-providers-apache-kafka/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `1b599c9fbf `_ 2023-06-30 ``Break AwaitMessageTrigger execution when finding a message with the desired format (#31803)`` `8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` diff --git a/docs/apache-airflow-providers-apache-livy/changelog.rst b/docs/apache-airflow-providers-apache-livy/changelog.rst index 05f40d97d28b..b6889d94dc47 100644 --- a/docs/apache-airflow-providers-apache-livy/changelog.rst +++ b/docs/apache-airflow-providers-apache-livy/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apache/livy/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-livy/commits.rst b/docs/apache-airflow-providers-apache-livy/commits.rst index f3685d6d9363..9bdc205bc4c5 100644 --- a/docs/apache-airflow-providers-apache-livy/commits.rst +++ b/docs/apache-airflow-providers-apache-livy/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` `8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `58784ade88 `_ 2023-06-27 ``Fix 'LivyHook' TypeError exception on 'session_id' log format (#32051)`` diff --git a/docs/apache-airflow-providers-apprise/changelog.rst b/docs/apache-airflow-providers-apprise/changelog.rst index 75d15ffd0fb0..54ac6f8ff125 100644 --- a/docs/apache-airflow-providers-apprise/changelog.rst +++ b/docs/apache-airflow-providers-apprise/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apprise/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apprise/commits.rst b/docs/apache-airflow-providers-apprise/commits.rst index af71b748389e..c50ceeef495c 100644 --- a/docs/apache-airflow-providers-apprise/commits.rst +++ b/docs/apache-airflow-providers-apprise/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `f4c4b77486 `_ 2023-06-21 ``Add Apprise Provider (#31533)`` diff --git a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst index 758cd4c9abc7..7c7dfed41b6a 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/cncf/kubernetes/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst index 070badf99a5a..7dbbb8dfbef7 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` `df4c8837d0 `_ 2023-07-01 ``Fix KubernetesPodOperator validate xcom json and add retries (#32113)`` `dd937e51fe `_ 2023-06-30 ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` `8f5de83ee6 `_ 2023-06-29 ``Fix 'KubernetesPodTrigger' waiting strategy (#31348)`` diff --git a/docs/apache-airflow-providers-common-sql/changelog.rst b/docs/apache-airflow-providers-common-sql/changelog.rst index be8dae1c039b..9db1e183f855 100644 --- a/docs/apache-airflow-providers-common-sql/changelog.rst +++ b/docs/apache-airflow-providers-common-sql/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/common/sql/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-common-sql/commits.rst b/docs/apache-airflow-providers-common-sql/commits.rst index d09b7c398104..39787e779bce 100644 --- a/docs/apache-airflow-providers-common-sql/commits.rst +++ b/docs/apache-airflow-providers-common-sql/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`ee4a838d49 `_ 2023-07-05 ``Pass SQLAlchemy engine to construct information schema query. (#32371)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `f2e2125b07 `_ 2023-06-29 ``openlineage, common.sql: provide OL SQL parser as internal OpenLineage provider API (#31398)`` `8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` diff --git a/docs/apache-airflow-providers-databricks/changelog.rst b/docs/apache-airflow-providers-databricks/changelog.rst index 6c5cde733297..1a577385566d 100644 --- a/docs/apache-airflow-providers-databricks/changelog.rst +++ b/docs/apache-airflow-providers-databricks/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/databricks/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-databricks/commits.rst b/docs/apache-airflow-providers-databricks/commits.rst index b0df4f06c724..ea474f65ed3c 100644 --- a/docs/apache-airflow-providers-databricks/commits.rst +++ b/docs/apache-airflow-providers-databricks/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` `6b4350e89c `_ 2023-06-29 ``Modify 'template_fields' of 'DatabricksSqlOperator' to support parent class fields (#32253)`` `d1aa509bbd `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` diff --git a/docs/apache-airflow-providers-dbt-cloud/changelog.rst b/docs/apache-airflow-providers-dbt-cloud/changelog.rst index ab244b93a10e..3dd235ba9559 100644 --- a/docs/apache-airflow-providers-dbt-cloud/changelog.rst +++ b/docs/apache-airflow-providers-dbt-cloud/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/dbt/cloud/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-dbt-cloud/commits.rst b/docs/apache-airflow-providers-dbt-cloud/commits.rst index a67dc83fbd6c..69108a345798 100644 --- a/docs/apache-airflow-providers-dbt-cloud/commits.rst +++ b/docs/apache-airflow-providers-dbt-cloud/commits.rst @@ -28,6 +28,21 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` +`d1aa509bbd `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ======================================================================= + 3.2.1 ..... @@ -36,6 +51,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================== Commit Committed Subject ================================================================================================= =========== ============================================================== +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `dc5bf3fd02 `_ 2023-06-02 ``Add discoverability for triggers in provider.yaml (#31576)`` diff --git a/docs/apache-airflow-providers-dbt-cloud/index.rst b/docs/apache-airflow-providers-dbt-cloud/index.rst index 0fe39d64a879..b9e96de88a11 100644 --- a/docs/apache-airflow-providers-dbt-cloud/index.rst +++ b/docs/apache-airflow-providers-dbt-cloud/index.rst @@ -81,7 +81,7 @@ Package apache-airflow-providers-dbt-cloud `dbt Cloud `__ -Release: 3.2.1 +Release: 3.2.2 Provider package ---------------- @@ -135,5 +135,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-dbt-cloud 3.2.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dbt-cloud 3.2.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 3.2.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 3.2.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-elasticsearch/changelog.rst b/docs/apache-airflow-providers-elasticsearch/changelog.rst index 78aa32de23aa..544af0f45549 100644 --- a/docs/apache-airflow-providers-elasticsearch/changelog.rst +++ b/docs/apache-airflow-providers-elasticsearch/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/elasticsearch/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-elasticsearch/commits.rst b/docs/apache-airflow-providers-elasticsearch/commits.rst index 774d5ea55f50..5a44d297a58c 100644 --- a/docs/apache-airflow-providers-elasticsearch/commits.rst +++ b/docs/apache-airflow-providers-elasticsearch/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `d1aa509bbd `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `b28c90354f `_ 2023-06-24 ``Deprecate the 2 non-official elasticsearch libraries (#31920)`` diff --git a/docs/apache-airflow-providers-google/changelog.rst b/docs/apache-airflow-providers-google/changelog.rst index f3c708e00f11..4708a13a7b6a 100644 --- a/docs/apache-airflow-providers-google/changelog.rst +++ b/docs/apache-airflow-providers-google/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/google/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-google/commits.rst b/docs/apache-airflow-providers-google/commits.rst index d69b4d009e78..ae000fb1f329 100644 --- a/docs/apache-airflow-providers-google/commits.rst +++ b/docs/apache-airflow-providers-google/commits.rst @@ -38,11 +38,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` `d6e254db68 `_ 2023-06-30 ``Deprecate 'delimiter' param and source object's wildcards in GCS, introduce 'match_glob' param. (#31261)`` `dd937e51fe `_ 2023-06-30 ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` `c0eaa9b25d `_ 2023-06-29 ``Add deferrable mode to CloudSQLExportInstanceOperator (#30852)`` diff --git a/docs/apache-airflow-providers-hashicorp/changelog.rst b/docs/apache-airflow-providers-hashicorp/changelog.rst index 10da504065ad..8dbea67189bc 100644 --- a/docs/apache-airflow-providers-hashicorp/changelog.rst +++ b/docs/apache-airflow-providers-hashicorp/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/hashicorp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-hashicorp/commits.rst b/docs/apache-airflow-providers-hashicorp/commits.rst index 8dff6dbb5bc4..0023f7878276 100644 --- a/docs/apache-airflow-providers-hashicorp/commits.rst +++ b/docs/apache-airflow-providers-hashicorp/commits.rst @@ -31,14 +31,17 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`1240dcc167 `_ 2023-07-05 ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `d136457686 `_ 2023-06-20 ``Add default retry to hvac client requests (#31073)`` -================================================================================================= =========== ================================================================ +================================================================================================= =========== ================================================================= 3.4.1 ..... diff --git a/docs/apache-airflow-providers-microsoft-azure/changelog.rst b/docs/apache-airflow-providers-microsoft-azure/changelog.rst index 11ea0a86b489..f6c4f31ab8f8 100644 --- a/docs/apache-airflow-providers-microsoft-azure/changelog.rst +++ b/docs/apache-airflow-providers-microsoft-azure/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/microsoft/azure/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/commits.rst b/docs/apache-airflow-providers-microsoft-azure/commits.rst index 04df0a44ec97..10822915c2a8 100644 --- a/docs/apache-airflow-providers-microsoft-azure/commits.rst +++ b/docs/apache-airflow-providers-microsoft-azure/commits.rst @@ -31,11 +31,16 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`1240dcc167 `_ 2023-07-05 ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` +`f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` +`1d6f6131df `_ 2023-07-04 ``refresh connection if an exception is caught in "AzureDataFactory" (#32323)`` `17132ef4c6 `_ 2023-07-04 ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` `985a6cd2a8 `_ 2023-07-03 ``invalid args fix (#32326)`` `c03d7b7c33 `_ 2023-07-03 ``Adds Sensor section in the Azure providers docs (#32299)`` diff --git a/docs/apache-airflow-providers-postgres/changelog.rst b/docs/apache-airflow-providers-postgres/changelog.rst index 4beb42c22798..a2e6d773da0b 100644 --- a/docs/apache-airflow-providers-postgres/changelog.rst +++ b/docs/apache-airflow-providers-postgres/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/postgres/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-postgres/commits.rst b/docs/apache-airflow-providers-postgres/commits.rst index e5852c29e4d4..91a9df90c9cf 100644 --- a/docs/apache-airflow-providers-postgres/commits.rst +++ b/docs/apache-airflow-providers-postgres/commits.rst @@ -28,6 +28,20 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`56eae9c9a4 `_ 2023-07-04 ``Deprecate 'runtime_parameters' in favor of options in 'hook_params' (#32345)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================================ + 5.5.1 ..... @@ -36,6 +50,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` diff --git a/docs/apache-airflow-providers-postgres/index.rst b/docs/apache-airflow-providers-postgres/index.rst index da163c0156d5..82f5e20fbc9a 100644 --- a/docs/apache-airflow-providers-postgres/index.rst +++ b/docs/apache-airflow-providers-postgres/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-postgres `PostgreSQL `__ -Release: 5.5.1 +Release: 5.5.2 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-postgres 5.5.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-postgres 5.5.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-postgres 5.5.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-postgres 5.5.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-snowflake/changelog.rst b/docs/apache-airflow-providers-snowflake/changelog.rst index d0734ce6d7d6..780400cd238d 100644 --- a/docs/apache-airflow-providers-snowflake/changelog.rst +++ b/docs/apache-airflow-providers-snowflake/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/snowflake/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-snowflake/commits.rst b/docs/apache-airflow-providers-snowflake/commits.rst index 63908e1e5122..f9b7e0cad3ef 100644 --- a/docs/apache-airflow-providers-snowflake/commits.rst +++ b/docs/apache-airflow-providers-snowflake/commits.rst @@ -28,6 +28,20 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`891c2e4019 `_ 2023-07-04 ``Add Deferrable switch to SnowflakeSqlApiOperator (#31596)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================ + 4.2.0 ..... @@ -36,6 +50,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `35845f9fb6 `_ 2023-06-11 ``SnowflakeSqlApiOperator - Change the base class (#31751)`` `07ea574fed `_ 2023-06-07 ``Add D400 pydocstyle check (#31742)`` diff --git a/docs/apache-airflow-providers-snowflake/index.rst b/docs/apache-airflow-providers-snowflake/index.rst index 7da44262da78..fa2eddc05fe8 100644 --- a/docs/apache-airflow-providers-snowflake/index.rst +++ b/docs/apache-airflow-providers-snowflake/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-snowflake `Snowflake `__ -Release: 4.2.0 +Release: 4.3.0 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-snowflake 4.2.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-snowflake 4.2.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 4.3.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 4.3.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-trino/changelog.rst b/docs/apache-airflow-providers-trino/changelog.rst index efd69bd600ed..19bae1ed6559 100644 --- a/docs/apache-airflow-providers-trino/changelog.rst +++ b/docs/apache-airflow-providers-trino/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/trino/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-trino/commits.rst b/docs/apache-airflow-providers-trino/commits.rst index 3ddb3cd8ef9c..b4e638ba5ac1 100644 --- a/docs/apache-airflow-providers-trino/commits.rst +++ b/docs/apache-airflow-providers-trino/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `371833e076 `_ 2023-06-24 ``Trino Hook: Add ability to read JWT from file (#31950)`` ================================================================================================= =========== ================================================================ diff --git a/docs/apache-airflow-providers-vertica/changelog.rst b/docs/apache-airflow-providers-vertica/changelog.rst index 447905b4c6c0..bca685c73f09 100644 --- a/docs/apache-airflow-providers-vertica/changelog.rst +++ b/docs/apache-airflow-providers-vertica/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/vertica/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-vertica/commits.rst b/docs/apache-airflow-providers-vertica/commits.rst index 489949baab86..91ff5ead2df7 100644 --- a/docs/apache-airflow-providers-vertica/commits.rst +++ b/docs/apache-airflow-providers-vertica/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `64a3787c24 `_ 2023-06-30 ``Add various Vertica connection parameters (#32089)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` ================================================================================================= =========== ================================================================ From 3318212482c6e11ac5c2e2828f7e467bca5b7245 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 6 Jul 2023 07:50:06 +0200 Subject: [PATCH 070/533] Fix `operator_extra_links` property serialization in mapped tasks (#31904) * reproduce the problem in a unit test * Fix operator_extra_links serialization * replace fget by __get__ --- airflow/serialization/serialized_objects.py | 6 ++- tests/serialization/test_dag_serialization.py | 40 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 4f5c61943e73..60e2d4b02d7d 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -287,7 +287,7 @@ class BaseSerialization: _datetime_types = (datetime.datetime,) # Object types that are always excluded in serialization. - _excluded_types = (logging.Logger, Connection, type) + _excluded_types = (logging.Logger, Connection, type, property) _json_schema: Validator | None = None @@ -822,7 +822,9 @@ def _serialize_node(cls, op: BaseOperator | MappedOperator, include_deps: bool) if op.operator_extra_links: serialize_op["_operator_extra_links"] = cls._serialize_operator_extra_links( - op.operator_extra_links + op.operator_extra_links.__get__(op) + if isinstance(op.operator_extra_links, property) + else op.operator_extra_links ) if include_deps: diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 69a2c9df2295..c0c0f2f0cde8 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -68,7 +68,7 @@ from airflow.utils.task_group import TaskGroup from airflow.utils.xcom import XCOM_RETURN_KEY from tests.test_utils.config import conf_vars -from tests.test_utils.mock_operators import CustomOperator, GoogleLink, MockOperator +from tests.test_utils.mock_operators import AirflowLink2, CustomOperator, GoogleLink, MockOperator from tests.test_utils.timetables import CustomSerializationTimetable, cron_timetable, delta_timetable repo_root = Path(airflow.__file__).parent.parent @@ -2479,3 +2479,41 @@ def tg(a: str) -> None: serde_tg = serde_dag.task_group.children["tg"] assert isinstance(serde_tg, MappedTaskGroup) assert serde_tg._expand_input == DictOfListsExpandInput({"a": [".", ".."]}) + + +def test_mapped_task_with_operator_extra_links_property(): + class _DummyOperator(BaseOperator): + def __init__(self, inputs, **kwargs): + super().__init__(**kwargs) + self.inputs = inputs + + @property + def operator_extra_links(self): + return (AirflowLink2(),) + + with DAG("test-dag", start_date=datetime(2020, 1, 1)) as dag: + _DummyOperator.partial(task_id="task").expand(inputs=[1, 2, 3]) + serialized_dag = SerializedBaseOperator.serialize(dag) + assert serialized_dag["tasks"][0] == { + "task_id": "task", + "expand_input": { + "type": "dict-of-lists", + "value": {"__type": "dict", "__var": {"inputs": [1, 2, 3]}}, + }, + "partial_kwargs": {}, + "_disallow_kwargs_override": False, + "_expand_input_attr": "expand_input", + "downstream_task_ids": [], + "_operator_extra_links": [{"tests.test_utils.mock_operators.AirflowLink2": {}}], + "ui_color": "#fff", + "ui_fgcolor": "#000", + "template_ext": [], + "template_fields": [], + "template_fields_renderers": {}, + "_task_type": "_DummyOperator", + "_task_module": "tests.serialization.test_dag_serialization", + "_is_empty": False, + "_is_mapped": True, + } + deserialized_dag = SerializedDAG.deserialize_dag(serialized_dag) + assert deserialized_dag.task_dict["task"].operator_extra_links == [AirflowLink2()] From 21e8f878a3c91250d0d198c6c3675b4b350fcb61 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Wed, 5 Jul 2023 23:32:40 -0700 Subject: [PATCH 071/533] D205 Support - Providers: Snowflake to Zendesk (inclusive) (#32359) --- .../snowflake/hooks/snowflake_sql_api.py | 25 +++++++++++-------- .../snowflake/operators/snowflake.py | 20 +++++++-------- .../transfers/copy_into_snowflake.py | 5 +--- .../snowflake/transfers/snowflake_to_slack.py | 14 ++++++----- .../snowflake/triggers/snowflake_trigger.py | 6 +---- .../snowflake/utils/sql_api_generate_jwt.py | 6 ++++- airflow/providers/tableau/hooks/tableau.py | 3 +-- .../providers/tableau/operators/tableau.py | 1 + airflow/providers/tabular/hooks/tabular.py | 6 +++-- .../providers/telegram/operators/telegram.py | 1 + airflow/providers/trino/hooks/trino.py | 3 +-- 11 files changed, 47 insertions(+), 43 deletions(-) diff --git a/airflow/providers/snowflake/hooks/snowflake_sql_api.py b/airflow/providers/snowflake/hooks/snowflake_sql_api.py index eec3c7349e1e..b10a3c670f81 100644 --- a/airflow/providers/snowflake/hooks/snowflake_sql_api.py +++ b/airflow/providers/snowflake/hooks/snowflake_sql_api.py @@ -33,10 +33,10 @@ class SnowflakeSqlApiHook(SnowflakeHook): """ - A client to interact with Snowflake using SQL API and allows submitting - multiple SQL statements in a single request. In combination with aiohttp, make post request to submit SQL - statements for execution, poll to check the status of the execution of a statement. Fetch query results - asynchronously. + A client to interact with Snowflake using SQL API and submit multiple SQL statements in a single request. + + In combination with aiohttp, make post request to submit SQL statements for execution, + poll to check the status of the execution of a statement. Fetch query results asynchronously. This hook requires the snowflake_conn_id connection. This hooks mainly uses account, schema, database, warehouse, private_key_file or private_key_content field must be setup in the connection. Other inputs @@ -137,7 +137,10 @@ def execute_query( conn_config = self._get_conn_params() req_id = uuid.uuid4() - url = f"https://{conn_config['account']}.{conn_config['region']}.snowflakecomputing.com/api/v2/statements" + url = ( + f"https://{conn_config['account']}.{conn_config['region']}" + f".snowflakecomputing.com/api/v2/statements" + ) params: dict[str, Any] | None = {"requestId": str(req_id), "async": True, "pageSize": 10} headers = self.get_headers() if bindings is None: @@ -171,9 +174,7 @@ def execute_query( return self.query_ids def get_headers(self) -> dict[str, Any]: - """Based on the private key, and with connection details JWT Token is generated and header - is formed. - """ + """Form JWT Token and header based on the private key, and connection details.""" if not self.private_key: self.get_private_key() conn_config = self._get_conn_params() @@ -206,13 +207,15 @@ def get_request_url_header_params(self, query_id: str) -> tuple[dict[str, Any], req_id = uuid.uuid4() header = self.get_headers() params = {"requestId": str(req_id)} - url = f"https://{conn_config['account']}.{conn_config['region']}.snowflakecomputing.com/api/v2/statements/{query_id}" + url = ( + f"https://{conn_config['account']}.{conn_config['region']}" + f".snowflakecomputing.com/api/v2/statements/{query_id}" + ) return header, params, url def check_query_output(self, query_ids: list[str]) -> None: """ - Based on the query ids passed as the parameter make HTTP request to snowflake SQL API and logs - the response. + Make HTTP request to snowflake SQL API based on the provided query ids and log the response. :param query_ids: statement handles query id for the individual statements. """ diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index db35fa000731..4087397d8304 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -135,10 +135,11 @@ def _process_output(self, results: list[Any], descriptions: list[Sequence[Sequen class SnowflakeCheckOperator(SQLCheckOperator): """ - Performs a check against Snowflake. The ``SnowflakeCheckOperator`` expects - a sql query that will return a single row. Each value on that - first row is evaluated using python ``bool`` casting. If any of the - values return ``False`` the check is failed and errors out. + Performs a check against Snowflake. + + The ``SnowflakeCheckOperator`` expects a sql query that will return a single row. Each + value on that first row is evaluated using python ``bool`` casting. If any of the values + return ``False`` the check is failed and errors out. Note that Python bool casting evals the following as ``False``: @@ -225,8 +226,7 @@ def __init__( class SnowflakeValueCheckOperator(SQLValueCheckOperator): """ - Performs a simple check using sql code against a specified value, within a - certain level of tolerance. + Performs a simple check using sql code against a specified value, within a certain level of tolerance. :param sql: the sql to be executed :param pass_value: the value to check against @@ -293,8 +293,7 @@ def __init__( class SnowflakeIntervalCheckOperator(SQLIntervalCheckOperator): """ - Checks that the values of metrics given as SQL expressions are within - a certain tolerance of the ones from days_back before. + Checks that the metrics given as SQL expressions are within tolerance of the ones from days_back before. This method constructs a query like so :: @@ -479,6 +478,7 @@ def __init__( def execute(self, context: Context) -> None: """ Make a POST API request to snowflake by using SnowflakeSQL and execute the query to get the ids. + By deferring the SnowflakeSqlApiTrigger class passed along with query ids. """ self.log.info("Executing: %s", self.sql) @@ -539,8 +539,8 @@ def poll_on_queries(self): def execute_complete(self, context: Context, event: dict[str, str | list[str]] | None = None) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if "status" in event and event["status"] == "error": diff --git a/airflow/providers/snowflake/transfers/copy_into_snowflake.py b/airflow/providers/snowflake/transfers/copy_into_snowflake.py index 6e1d22cd2c3b..10071add1a42 100644 --- a/airflow/providers/snowflake/transfers/copy_into_snowflake.py +++ b/airflow/providers/snowflake/transfers/copy_into_snowflake.py @@ -15,10 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains abstract operator that child classes -implement "COPY INTO SQL in Snowflake". -""" +"""Abstract operator that child classes implement ``COPY INTO
    SQL in Snowflake``.""" from __future__ import annotations from typing import Any, Sequence diff --git a/airflow/providers/snowflake/transfers/snowflake_to_slack.py b/airflow/providers/snowflake/transfers/snowflake_to_slack.py index b4b441a5856f..50d38b947170 100644 --- a/airflow/providers/snowflake/transfers/snowflake_to_slack.py +++ b/airflow/providers/snowflake/transfers/snowflake_to_slack.py @@ -25,12 +25,14 @@ class SnowflakeToSlackOperator(SqlToSlackOperator): """ - Executes an SQL statement in Snowflake and sends the results to Slack. The results of the query are - rendered into the 'slack_message' parameter as a Pandas dataframe using a JINJA variable called '{{ - results_df }}'. The 'results_df' variable name can be changed by specifying a different - 'results_df_name' parameter. The Tabulate library is added to the JINJA environment as a filter to - allow the dataframe to be rendered nicely. For example, set 'slack_message' to {{ results_df | - tabulate(tablefmt="pretty", headers="keys") }} to send the results to Slack as an ascii rendered table. + Executes an SQL statement in Snowflake and sends the results to Slack. + + The results of the query are rendered into the 'slack_message' parameter as a Pandas dataframe + using a Jinja variable called '{{ results_df }}'. The 'results_df' variable name can be changed + by specifying a different 'results_df_name' parameter. The Tabulate library is added to the + Jinja environment as a filter to allow the dataframe to be rendered nicely. For example, set + 'slack_message' to {{ results_df | tabulate(tablefmt="pretty", headers="keys") }} to send the + results to Slack as an ASCII rendered table. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/snowflake/triggers/snowflake_trigger.py b/airflow/providers/snowflake/triggers/snowflake_trigger.py index 4f1e0cffb299..bb426fa3d0e4 100644 --- a/airflow/providers/snowflake/triggers/snowflake_trigger.py +++ b/airflow/providers/snowflake/triggers/snowflake_trigger.py @@ -93,11 +93,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: yield TriggerEvent({"status": "error", "message": str(e)}) async def get_query_status(self, query_id: str) -> dict[str, Any]: - """ - Async function to check whether the query statement submitted via SQL API is still - running state and returns True if it is still running else - return False. - """ + """Return True if the SQL query is still running otherwise return False.""" hook = SnowflakeSqlApiHook( self.snowflake_conn_id, self.token_life_time, diff --git a/airflow/providers/snowflake/utils/sql_api_generate_jwt.py b/airflow/providers/snowflake/utils/sql_api_generate_jwt.py index 883ef3aae0d1..553d84caf7c8 100644 --- a/airflow/providers/snowflake/utils/sql_api_generate_jwt.py +++ b/airflow/providers/snowflake/utils/sql_api_generate_jwt.py @@ -42,6 +42,7 @@ class JWTGenerator: """ Creates and signs a JWT with the specified private key file, username, and account identifier. + The JWTGenerator keeps the generated token and only regenerates the token if a specified period of time has passed. @@ -92,6 +93,7 @@ def __init__( def prepare_account_name_for_jwt(self, raw_account: str) -> str: """ Prepare the account identifier for use in the JWT. + For the JWT, the account identifier must not include the subdomain or any region or cloud provider information. @@ -113,7 +115,9 @@ def prepare_account_name_for_jwt(self, raw_account: str) -> str: def get_token(self) -> str | None: """ - Generates a new JWT. If a JWT has been already been generated earlier, return the previously + Generates a new JWT. + + If a JWT has been already been generated earlier, return the previously generated token unless the specified renewal time has passed. """ now = datetime.now(timezone.utc) # Fetch the current time diff --git a/airflow/providers/tableau/hooks/tableau.py b/airflow/providers/tableau/hooks/tableau.py index d801ca97a244..5127a3dc9a80 100644 --- a/airflow/providers/tableau/hooks/tableau.py +++ b/airflow/providers/tableau/hooks/tableau.py @@ -162,8 +162,7 @@ def get_job_status(self, job_id: str) -> TableauJobFinishCode: def wait_for_state(self, job_id: str, target_state: TableauJobFinishCode, check_interval: float) -> bool: """ - Wait until the current state of a defined Tableau Job is equal - to target_state or different from PENDING. + Wait until the current state of a defined Tableau Job is target_state or different from PENDING. :param job_id: The id of the job to check. :param target_state: Enum that describe the Tableau job's target state diff --git a/airflow/providers/tableau/operators/tableau.py b/airflow/providers/tableau/operators/tableau.py index 5e13c2d6f42f..318744835e0e 100644 --- a/airflow/providers/tableau/operators/tableau.py +++ b/airflow/providers/tableau/operators/tableau.py @@ -95,6 +95,7 @@ def __init__( def execute(self, context: Context) -> str: """ Executes the Tableau API resource and pushes the job id or downloaded file URI to xcom. + :param context: The task context during execution. :return: the id of the job that executes the extract refresh or downloaded file URI. """ diff --git a/airflow/providers/tabular/hooks/tabular.py b/airflow/providers/tabular/hooks/tabular.py index 5fed3888f9c1..da99cfbb9fd8 100644 --- a/airflow/providers/tabular/hooks/tabular.py +++ b/airflow/providers/tabular/hooks/tabular.py @@ -30,8 +30,10 @@ class TabularHook(BaseHook): """ - This hook acts as a base hook for tabular services. It offers the ability to generate temporary, - short-lived session tokens to use within Airflow submitted jobs. + This hook acts as a base hook for tabular services. + + It offers the ability to generate temporary, short-lived + session tokens to use within Airflow submitted jobs. :param tabular_conn_id: The :ref:`Tabular connection id` which refers to the information to connect to the Tabular OAuth service. diff --git a/airflow/providers/telegram/operators/telegram.py b/airflow/providers/telegram/operators/telegram.py index 840011ce0b25..4c85487e2c0f 100644 --- a/airflow/providers/telegram/operators/telegram.py +++ b/airflow/providers/telegram/operators/telegram.py @@ -31,6 +31,7 @@ class TelegramOperator(BaseOperator): """ This operator allows you to post messages to Telegram using Telegram Bot API. + Takes both Telegram Bot API token directly or connection that has Telegram token in password field. If both supplied, token parameter will be given precedence. diff --git a/airflow/providers/trino/hooks/trino.py b/airflow/providers/trino/hooks/trino.py index 09fbe6efa66b..67ee432ca386 100644 --- a/airflow/providers/trino/hooks/trino.py +++ b/airflow/providers/trino/hooks/trino.py @@ -240,8 +240,7 @@ def insert_rows( @staticmethod def _serialize_cell(cell: Any, conn: Connection | None = None) -> Any: """ - Trino will adapt all arguments to the execute() method internally, - hence we return cell without any conversion. + Trino will adapt all execute() args internally, hence we return cell without any conversion. :param cell: The cell to insert into the table :param conn: The database connection From 2b841f799a74906709b8c0e6d719fb676d946129 Mon Sep 17 00:00:00 2001 From: Alexis BRENON Date: Thu, 6 Jul 2023 09:42:12 +0200 Subject: [PATCH 072/533] Update Airflow user list (#32393) --- INTHEWILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INTHEWILD.md b/INTHEWILD.md index d203db35c5ca..9a1e17fbffcc 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -221,6 +221,7 @@ Currently, **officially** using Airflow: 1. [Grab](https://www.grab.com/sg/) [[@calvintran](https://github.com/canhtran)] 1. [Gradeup](https://gradeup.co) [[@gradeup](https://github.com/gradeup)] 1. [Grand Rounds](https://www.grandrounds.com/) [[@richddr](https://github.com/richddr), [@timz1290](https://github.com/timz1290), [@wenever](https://github.com/@wenever), & [@runongirlrunon](https://github.com/runongirlrunon)] +1. [Greenbids](https://www.greenbids.ai/) [[@greenbids](https://github.com/greenbids), [@AlexisBRENON](https://github.com/AlexisBRENON)] 1. [Greytip](https://www.greytip.com) [[@greytip](https://github.com/greytip)] 1. [Groupalia](http://es.groupalia.com) [[@jesusfcr](https://github.com/jesusfcr)] 1. [Groupon](https://groupon.com) [[@stevencasey](https://github.com/stevencasey)] From e4757d6dfa6e7385eb90c38c60ab8fefa24e7a0e Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:00:19 +0200 Subject: [PATCH 073/533] Fix GCSToGCSOperator copy without wildcard and exact_match=True (#32376) --- .../google/cloud/transfers/gcs_to_gcs.py | 10 ++++- .../google/cloud/transfers/test_gcs_to_gcs.py | 37 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py index 2b39df1c6a79..222a1ecad7f8 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py @@ -370,6 +370,8 @@ def _copy_source_without_wildcard(self, hook, prefix): self.source_bucket, prefix=prefix, delimiter=self.delimiter, match_glob=self.match_glob ) + objects = [obj for obj in objects if self._check_exact_match(obj, prefix)] + if not self.replace: # If we are not replacing, ignore files already existing in source buckets objects = self._ignore_existing_files( @@ -405,7 +407,7 @@ def _copy_file(self, hook, source_object): def _copy_directory(self, hook, source_objects, prefix): _prefix = prefix.rstrip("/") + "/" for source_obj in source_objects: - if self.exact_match and (source_obj != prefix or not source_obj.endswith(prefix)): + if not self._check_exact_match(source_obj, prefix): continue if self.destination_object is None: destination_object = source_obj @@ -417,6 +419,12 @@ def _copy_directory(self, hook, source_objects, prefix): hook=hook, source_object=source_obj, destination_object=destination_object ) + def _check_exact_match(self, source_object: str, prefix: str) -> bool: + """Checks whether source_object's name matches the prefix according to the exact_match flag.""" + if self.exact_match and (source_object != prefix or not source_object.endswith(prefix)): + return False + return True + def _copy_source_with_wildcard(self, hook, prefix): total_wildcards = prefix.count(WILDCARD) if total_wildcards > 1: diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py index 1cf7be116679..3966be50cc08 100644 --- a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py @@ -128,6 +128,7 @@ def test_execute_no_wildcard_with_replace_flag_false(self, mock_hook): def test_copy_file_with_exact_match(self, mock_hook): SOURCE_FILES = [ "test_object.txt", + "test_object.txt.abc", "test_object.txt.copy/", "test_object.txt.folder/", ] @@ -145,6 +146,42 @@ def test_copy_file_with_exact_match(self, mock_hook): mock.call(TEST_BUCKET, prefix="test_object.txt", delimiter=None, match_glob=None), ] mock_hook.return_value.list.assert_has_calls(mock_calls) + mock_hook.return_value.rewrite.assert_has_calls( + [ + mock.call(TEST_BUCKET, "test_object.txt", DESTINATION_BUCKET, "test_object.txt"), + ] + ) + + @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") + def test_copy_file_with_exact_match_destination(self, mock_hook): + SOURCE_FILES = [ + "test_object.txt", + "test_object.txt.abc", + "test_object.txt.copy/", + "test_object.txt.folder/", + ] + DESTINATION_OBJ = f"{DESTINATION_OBJECT_PREFIX}/test_object.txt" + + mock_hook.return_value.list.return_value = SOURCE_FILES + operator = GCSToGCSOperator( + task_id=TASK_ID, + source_bucket=TEST_BUCKET, + source_object=SOURCE_OBJECT_NO_WILDCARD, + destination_bucket=DESTINATION_BUCKET, + destination_object=DESTINATION_OBJ, + exact_match=True, + ) + + operator.execute(None) + mock_calls = [ + mock.call(TEST_BUCKET, prefix="test_object.txt", delimiter=None, match_glob=None), + ] + mock_hook.return_value.list.assert_has_calls(mock_calls) + + mock_calls_rewrite = [ + mock.call(TEST_BUCKET, "test_object.txt", DESTINATION_BUCKET, DESTINATION_OBJ), + ] + mock_hook.return_value.rewrite.assert_has_calls(mock_calls_rewrite) @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") def test_execute_prefix_and_suffix(self, mock_hook): From becfb3c64f42533d6b8c6806c290952d84bc76a0 Mon Sep 17 00:00:00 2001 From: tnk-ysk <61967017+tnk-ysk@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:58:19 +0900 Subject: [PATCH 074/533] Fix endless loop of defer in cloud_build (#32387) --- airflow/providers/google/cloud/triggers/cloud_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow/providers/google/cloud/triggers/cloud_build.py b/airflow/providers/google/cloud/triggers/cloud_build.py index 5ec9eef3cab0..e07dc939070b 100644 --- a/airflow/providers/google/cloud/triggers/cloud_build.py +++ b/airflow/providers/google/cloud/triggers/cloud_build.py @@ -95,6 +95,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] "message": "Build completed", } ) + return elif cloud_build_instance._pb.status in ( Build.Status.WORKING, Build.Status.PENDING, @@ -111,14 +112,17 @@ async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] Build.Status.EXPIRED, ): yield TriggerEvent({"status": "error", "message": cloud_build_instance.status_detail}) + return else: yield TriggerEvent( {"status": "error", "message": "Unidentified status of Cloud Build instance"} ) + return except Exception as e: self.log.exception("Exception occurred while checking for Cloud Build completion") yield TriggerEvent({"status": "error", "message": str(e)}) + return def _get_async_hook(self) -> CloudBuildAsyncHook: return CloudBuildAsyncHook(gcp_conn_id=self.gcp_conn_id) From e7587b3369af30848c3cf1c7eff9e801b1440793 Mon Sep 17 00:00:00 2001 From: Peng Yu Date: Thu, 6 Jul 2023 05:47:59 -0400 Subject: [PATCH 075/533] Fix the gcp_gcs_delete_objects on empty list (#32383) --- airflow/providers/google/cloud/operators/gcs.py | 2 +- tests/providers/google/cloud/operators/test_gcs.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py index 6e50cbad2d16..1025a5038a59 100644 --- a/airflow/providers/google/cloud/operators/gcs.py +++ b/airflow/providers/google/cloud/operators/gcs.py @@ -323,7 +323,7 @@ def execute(self, context: Context) -> None: impersonation_chain=self.impersonation_chain, ) - if self.objects: + if self.objects is not None: objects = self.objects else: objects = hook.list(bucket_name=self.bucket_name, prefix=self.prefix) diff --git a/tests/providers/google/cloud/operators/test_gcs.py b/tests/providers/google/cloud/operators/test_gcs.py index b048aa0c8e7c..4ceaa5292bb4 100644 --- a/tests/providers/google/cloud/operators/test_gcs.py +++ b/tests/providers/google/cloud/operators/test_gcs.py @@ -124,6 +124,14 @@ def test_delete_objects(self, mock_hook): any_order=True, ) + @mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook") + def test_delete_empty_list_of_objects(self, mock_hook): + operator = GCSDeleteObjectsOperator(task_id=TASK_ID, bucket_name=TEST_BUCKET, objects=[]) + + operator.execute(None) + mock_hook.return_value.list.assert_not_called() + mock_hook.return_value.delete.assert_not_called() + @mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook") def test_delete_prefix(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES[1:4] From 9a0f41ba53185031bc2aa56ead2928ae4b20de99 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 6 Jul 2023 15:19:40 +0530 Subject: [PATCH 076/533] Logging from all containers in KubernetesOperatorPod (#31663) * Logging from all containers in KubernetesOperatorPod * review comments from uranusjs * Fixing init logic * Fixing docs * Fixing sphinx logging * Addressing review comments from vincbeck * nits from potiuk * nits from potiuk * reverting return type * comment from uranusjr * fixing tests * fixing tests * review comments from hussein * handling nits from hussein * fixing tests --------- Co-authored-by: Amogh Co-authored-by: Amogh Desai --- .../cncf/kubernetes/operators/pod.py | 20 +++- .../cncf/kubernetes/utils/pod_manager.py | 95 ++++++++++++++++++- .../test_kubernetes_pod_operator.py | 2 +- .../cncf/kubernetes/utils/test_pod_manager.py | 40 ++++++++ 4 files changed, 147 insertions(+), 10 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index e3ac7708e74c..124685e792e7 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -27,7 +27,7 @@ from collections.abc import Container from contextlib import AbstractContextManager from functools import cached_property -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Sequence from kubernetes.client import CoreV1Api, models as k8s from slugify import slugify @@ -62,6 +62,7 @@ get_container_termination_message, ) from airflow.settings import pod_mutation_hook +from airflow.typing_compat import Literal from airflow.utils import yaml from airflow.utils.helpers import prune_dict, validate_key from airflow.utils.timezone import utcnow @@ -178,6 +179,10 @@ class KubernetesPodOperator(BaseOperator): :param labels: labels to apply to the Pod. (templated) :param startup_timeout_seconds: timeout in seconds to startup the pod. :param get_logs: get the stdout of the base container as logs of the tasks. + :param container_logs: list of containers whose logs will be published to stdout + Takes a sequence of containers, a single container name or True. If True, + all the containers logs are published. Works in conjunction with get_logs param. + The default value is the base container. :param image_pull_policy: Specify a policy to cache or always pull an image. :param annotations: non-identifying metadata you can attach to the Pod. Can be a large range of data, and can include characters @@ -278,6 +283,7 @@ def __init__( reattach_on_restart: bool = True, startup_timeout_seconds: int = 120, get_logs: bool = True, + container_logs: Iterable[str] | str | Literal[True] = BASE_CONTAINER_NAME, image_pull_policy: str | None = None, annotations: dict | None = None, container_resources: k8s.V1ResourceRequirements | None = None, @@ -350,6 +356,11 @@ def __init__( self.cluster_context = cluster_context self.reattach_on_restart = reattach_on_restart self.get_logs = get_logs + self.container_logs = container_logs + if self.container_logs == KubernetesPodOperator.BASE_CONTAINER_NAME: + self.container_logs = ( + base_container_name if base_container_name else KubernetesPodOperator.BASE_CONTAINER_NAME + ) self.image_pull_policy = image_pull_policy self.node_selector = node_selector or {} self.annotations = annotations or {} @@ -572,11 +583,10 @@ def execute_sync(self, context: Context): self.await_pod_start(pod=self.pod) if self.get_logs: - self.pod_manager.fetch_container_logs( + self.pod_manager.fetch_requested_container_logs( pod=self.pod, - container_name=self.base_container_name, - follow=True, - post_termination_timeout=self.POST_TERMINATION_TIMEOUT, + container_logs=self.container_logs, + follow_logs=True, ) else: self.pod_manager.await_container_completion( diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 2251f1d438c0..979180f1d92f 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -23,6 +23,7 @@ import math import time import warnings +from collections.abc import Iterable from contextlib import closing, suppress from dataclasses import dataclass from datetime import datetime, timedelta @@ -43,7 +44,7 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.kubernetes.pod_generator import PodDefaults -from airflow.typing_compat import Protocol +from airflow.typing_compat import Literal, Protocol from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.timezone import utcnow @@ -125,6 +126,17 @@ def container_is_running(pod: V1Pod, container_name: str) -> bool: return container_status.state.running is not None +def container_is_completed(pod: V1Pod, container_name: str) -> bool: + """ + Examines V1Pod ``pod`` to determine whether ``container_name`` is completed. + If that container is present and completed, returns True. Returns False otherwise. + """ + container_status = get_container_status(pod, container_name) + if not container_status: + return False + return container_status.state.terminated is not None + + def container_is_terminated(pod: V1Pod, container_name: str) -> bool: """ Examines V1Pod ``pod`` to determine whether ``container_name`` is terminated. @@ -379,11 +391,12 @@ def consume_logs( for raw_line in logs: line = raw_line.decode("utf-8", errors="backslashreplace") timestamp, message = self.parse_log_line(line) - self.log.info(message) + self.log.info("[%s] %s", container_name, message) except BaseHTTPError as e: self.log.warning( - "Reading of logs interrupted with error %r; will retry. " + "Reading of logs interrupted for container %r with error %r; will retry. " "Set log level to DEBUG for traceback.", + container_name, e, ) self.log.debug( @@ -413,6 +426,65 @@ def consume_logs( ) time.sleep(1) + def fetch_requested_container_logs( + self, pod: V1Pod, container_logs: Iterable[str] | str | Literal[True], follow_logs=False + ) -> list[PodLoggingStatus]: + """ + Follow the logs of containers in the pod specified by input parameter and publish + it to airflow logging. Returns when all the containers exit. + """ + pod_logging_statuses = [] + all_containers = self.get_container_names(pod) + if len(all_containers) == 0: + self.log.error("Could not retrieve containers for the pod: %s", pod.metadata.name) + else: + if isinstance(container_logs, str): + # fetch logs only for requested container if only one container is provided + if container_logs in all_containers: + status = self.fetch_container_logs( + pod=pod, container_name=container_logs, follow=follow_logs + ) + pod_logging_statuses.append(status) + else: + self.log.error( + "container %s whose logs were requested not found in the pod %s", + container_logs, + pod.metadata.name, + ) + elif isinstance(container_logs, bool): + # if True is provided, get logs for all the containers + if container_logs is True: + for container_name in all_containers: + status = self.fetch_container_logs( + pod=pod, container_name=container_name, follow=follow_logs + ) + pod_logging_statuses.append(status) + else: + self.log.error( + "False is not a valid value for container_logs", + ) + else: + # if a sequence of containers are provided, iterate for every container in the pod + if isinstance(container_logs, Iterable): + for container in container_logs: + if container in all_containers: + status = self.fetch_container_logs( + pod=pod, container_name=container, follow=follow_logs + ) + pod_logging_statuses.append(status) + else: + self.log.error( + "Container %s whose logs were requests not found in the pod %s", + container, + pod.metadata.name, + ) + else: + self.log.error( + "Invalid type %s specified for container names input parameter", type(container_logs) + ) + + return pod_logging_statuses + def await_container_completion(self, pod: V1Pod, container_name: str) -> None: """ Waits for the given container in the given pod to be completed. @@ -420,7 +492,12 @@ def await_container_completion(self, pod: V1Pod, container_name: str) -> None: :param pod: pod spec that will be monitored :param container_name: name of the container within the pod to monitor """ - while not self.container_is_terminated(pod=pod, container_name=container_name): + while True: + remote_pod = self.read_pod(pod) + terminated = container_is_completed(remote_pod, container_name) + if terminated: + break + self.log.info("Waiting for container '%s' state to be completed", container_name) time.sleep(1) def await_pod_completion(self, pod: V1Pod) -> V1Pod: @@ -513,6 +590,16 @@ def read_pod_logs( post_termination_timeout=post_termination_timeout, ) + @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True) + def get_container_names(self, pod: V1Pod) -> list[str]: + """Return container names from the POD except for the airflow-xcom-sidecar container.""" + pod_info = self.read_pod(pod) + return [ + container_spec.name + for container_spec in pod_info.spec.containers + if container_spec.name != PodDefaults.SIDECAR_CONTAINER_NAME + ] + @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True) def read_pod_events(self, pod: V1Pod) -> CoreV1EventList: """Reads events from the POD.""" diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py index e650932b5385..2b66a3082be5 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes_tests/test_kubernetes_pod_operator.py @@ -500,7 +500,7 @@ def test_volume_mount(self, mock_get_connection): ) context = create_context(k) k.execute(context=context) - mock_logger.info.assert_any_call("retrieved from mount") + mock_logger.info.assert_any_call("[%s] %s", "base", "retrieved from mount") actual_pod = self.api_client.sanitize_for_serialization(k.pod) self.expected_pod["spec"]["containers"][0]["args"] = args self.expected_pod["spec"]["containers"][0]["volumeMounts"] = [ diff --git a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py index 8f28d33dfdea..c915a03a4aa0 100644 --- a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py +++ b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py @@ -317,6 +317,46 @@ def test_fetch_container_done(self, logs_available, container_running, follow): assert ret.last_log_time is None assert ret.running is False + # adds all valid types for container_logs + @pytest.mark.parametrize("follow", [True, False]) + @pytest.mark.parametrize("container_logs", ["base", "alpine", True, ["base", "alpine"]]) + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.container_is_running") + def test_fetch_requested_container_logs(self, container_is_running, container_logs, follow): + mock_pod = MagicMock() + self.pod_manager.read_pod = MagicMock() + self.pod_manager.get_container_names = MagicMock() + self.pod_manager.get_container_names.return_value = ["base", "alpine"] + container_is_running.return_value = False + self.mock_kube_client.read_namespaced_pod_log.return_value = mock.MagicMock( + stream=mock.MagicMock(return_value=[b"2021-01-01 hi"]) + ) + + ret_values = self.pod_manager.fetch_requested_container_logs( + pod=mock_pod, container_logs=container_logs, follow_logs=follow + ) + for ret in ret_values: + assert ret.running is False + + # adds all invalid types for container_logs + @pytest.mark.parametrize("container_logs", [1, None, 6.8, False]) + @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.container_is_running") + def test_fetch_requested_container_logs_invalid(self, container_running, container_logs): + mock_pod = MagicMock() + self.pod_manager.read_pod = MagicMock() + self.pod_manager.get_container_names = MagicMock() + self.pod_manager.get_container_names.return_value = ["base", "alpine"] + container_running.return_value = False + self.mock_kube_client.read_namespaced_pod_log.return_value = mock.MagicMock( + stream=mock.MagicMock(return_value=[b"2021-01-01 hi"]) + ) + + ret_values = self.pod_manager.fetch_requested_container_logs( + pod=mock_pod, + container_logs=container_logs, + ) + + assert len(ret_values) == 0 + @mock.patch("pendulum.now") @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.container_is_running") @mock.patch("airflow.providers.cncf.kubernetes.utils.pod_manager.PodLogsConsumer.logs_available") From 7a3bc8d7c85448447abd39287ef6a3704b237a90 Mon Sep 17 00:00:00 2001 From: Louis Mackie <88324241+luos-fc@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:09:48 +0100 Subject: [PATCH 077/533] Only update crawler tags if present in config dict (#32331) --- airflow/providers/amazon/aws/hooks/glue_crawler.py | 4 +++- .../amazon/aws/hooks/test_glue_crawler.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/glue_crawler.py b/airflow/providers/amazon/aws/hooks/glue_crawler.py index 82bd3f45fde8..e0022c1e49f1 100644 --- a/airflow/providers/amazon/aws/hooks/glue_crawler.py +++ b/airflow/providers/amazon/aws/hooks/glue_crawler.py @@ -87,7 +87,9 @@ def update_crawler(self, **crawler_kwargs) -> bool: crawler_name = crawler_kwargs["Name"] current_crawler = self.get_crawler(crawler_name) - tags_updated = self.update_tags(crawler_name, crawler_kwargs.pop("Tags", {})) + tags_updated = ( + self.update_tags(crawler_name, crawler_kwargs.pop("Tags")) if "Tags" in crawler_kwargs else False + ) update_config = { key: value diff --git a/tests/providers/amazon/aws/hooks/test_glue_crawler.py b/tests/providers/amazon/aws/hooks/test_glue_crawler.py index 6f34b789ff72..e283812fc08e 100644 --- a/tests/providers/amazon/aws/hooks/test_glue_crawler.py +++ b/tests/providers/amazon/aws/hooks/test_glue_crawler.py @@ -160,14 +160,24 @@ def test_remove_all_tags(self, mock_get_conn): ResourceArn=self.crawler_arn, TagsToRemove=["test", "bar"] ) + @mock_sts + @mock.patch.object(GlueCrawlerHook, "get_conn") + def test_update_missing_tags(self, mock_get_conn): + mock_config_missing_tags = deepcopy(mock_config) + mock_config_missing_tags.pop("Tags") + mock_get_conn.return_value.get_crawler.return_value = {"Crawler": mock_config_missing_tags} + + assert self.hook.update_crawler(**mock_config_missing_tags) is False + mock_get_conn.return_value.get_tags.assert_not_called() + mock_get_conn.return_value.tag_resource.assert_not_called() + mock_get_conn.return_value.untag_resource.assert_not_called() + @mock_sts @mock.patch.object(GlueCrawlerHook, "get_conn") def test_replace_tag(self, mock_get_conn): mock_get_conn.return_value.get_crawler.return_value = {"Crawler": mock_config} mock_get_conn.return_value.get_tags.return_value = {"Tags": mock_config["Tags"]} - mock_config_two = deepcopy(mock_config) - mock_config_two.pop("Tags") assert self.hook.update_tags(mock_crawler_name, {"test": "bla", "bar": "test"}) is True mock_get_conn.return_value.get_tags.assert_called_once_with(ResourceArn=self.crawler_arn) mock_get_conn.return_value.untag_resource.assert_not_called() From a7fc784e0dfbfa063c5876b1e22200f2814376ec Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 6 Jul 2023 17:08:44 +0530 Subject: [PATCH 078/533] Adding helm as a pre requisite to local venv (#32391) Co-authored-by: Jarek Potiuk --------- Co-authored-by: Amogh Desai Co-authored-by: Jarek Potiuk --- LOCAL_VIRTUALENV.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 1f541270ec8e..017f042f25d9 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -54,6 +54,7 @@ Homebrew for macOS to install required software packages: * Python (One of: 3.8, 3.9, 3.10, 3.11) * MySQL 5.7+ * libxml +* helm (only for helm chart tests) Refer to the `Dockerfile.ci `__ for a comprehensive list of required packages. @@ -62,6 +63,7 @@ of required packages. - MySql 2.2.0 needs pkgconf to be a pre requisite, refer `here `_ to install pkgconf - MacOs with ARM architectures require graphviz for venv setup, refer `here `_ to install graphviz + - The helm chart tests need helm to be installed as a pre requisite. Refer `here `_ to install and setup helm Extra Packages -------------- From ef0ed1aacc208be9e52a35211d2beaefb735173a Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Thu, 6 Jul 2023 14:20:25 +0200 Subject: [PATCH 079/533] Fix local OpenLineage import in `SQLExecuteQueryOperator`. (#32400) Signed-off-by: Jakub Dardzinski --- airflow/providers/common/sql/operators/sql.py | 6 +++--- .../common/sql/operators/test_sql_execute.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/airflow/providers/common/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py index 53b499f50631..f1784f618e3e 100644 --- a/airflow/providers/common/sql/operators/sql.py +++ b/airflow/providers/common/sql/operators/sql.py @@ -330,12 +330,12 @@ def get_openlineage_facets_on_start(self) -> OperatorLineage | None: return operator_lineage def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage | None: - operator_lineage = self.get_openlineage_facets_on_start() or OperatorLineage() - try: from airflow.providers.openlineage.extractors import OperatorLineage except ImportError: - return operator_lineage + return None + + operator_lineage = self.get_openlineage_facets_on_start() or OperatorLineage() hook = self.get_db_hook() try: diff --git a/tests/providers/common/sql/operators/test_sql_execute.py b/tests/providers/common/sql/operators/test_sql_execute.py index 48f59e0ffea4..ddd137248540 100644 --- a/tests/providers/common/sql/operators/test_sql_execute.py +++ b/tests/providers/common/sql/operators/test_sql_execute.py @@ -18,6 +18,7 @@ from __future__ import annotations from typing import Any, NamedTuple, Sequence +from unittest import mock from unittest.mock import MagicMock import pytest @@ -354,3 +355,17 @@ def get_db_hook(self): ) == lineage_on_complete ) + + +def test_with_no_openlineage_provider(): + import importlib + + def mock__import__(name, globals_=None, locals_=None, fromlist=(), level=0): + if level == 0 and name.startswith("airflow.providers.openlineage"): + raise ImportError("No provider 'apache-airflow-providers-openlineage'") + return importlib.__import__(name, globals=globals_, locals=locals_, fromlist=fromlist, level=level) + + with mock.patch("builtins.__import__", side_effect=mock__import__): + op = SQLExecuteQueryOperator(task_id=TASK_ID, sql="SELECT 1;") + assert op.get_openlineage_facets_on_start() is None + assert op.get_openlineage_facets_on_complete(None) is None From 89d5e8b456c3f8e2cbe2691a4e1fa52ef2c1b6c5 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:36:58 +0300 Subject: [PATCH 080/533] Fix failing main due to new release of jsonschema (#32405) --- scripts/in_container/verify_providers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index bc56a5fa93f2..78cd4c81182d 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -164,6 +164,10 @@ class ProviderPackageDetails(NamedTuple): "urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!", "urllib3 (1.26.12) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!", "Jupyter is migrating its paths to use standard platformdirs given by the platformdirs library.", + ( + "Importing ErrorTree directly from the jsonschema package is deprecated and will become" + " an ImportError. Import it from jsonschema.exceptions instead." + ), } # The set of warning messages generated by direct importing of some deprecated modules. We should only From ef75a3a6757a033586c933f7b62ab86f846af754 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 6 Jul 2023 17:28:29 +0200 Subject: [PATCH 081/533] Add information on ARM MySQL support in Docker images (#32401) The change #29519 added experimental MariaDB client support for MySQL for ARM images but we missed that in the changelog. --- docs/docker-stack/changelog.rst | 1 + docs/docker-stack/index.rst | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst index b40c0ae68fbd..619dfd9e740b 100644 --- a/docs/docker-stack/changelog.rst +++ b/docs/docker-stack/changelog.rst @@ -107,6 +107,7 @@ Airflow 2.3 This can be done by setting ``DOCKER_BUILDKIT=1`` as an environment variable or by installing `the buildx plugin `_ and running ``docker buildx build`` command. + * MySQL is experimentally supported on ARM through MariaDB client library * Add Python 3.10 support * Switch to ``Debian Bullseye`` based images. ``Debian Buster`` is deprecated and only available for customized image building. diff --git a/docs/docker-stack/index.rst b/docs/docker-stack/index.rst index d868e9b02255..1cf264d97b53 100644 --- a/docs/docker-stack/index.rst +++ b/docs/docker-stack/index.rst @@ -124,14 +124,23 @@ Support The reference Docker Image supports the following platforms and database: -* Intel platform (x86_64) - * Postgres Client - * MySQL Client - * MSSQL Client - -* ARM platform (aarch64) - experimental support, might change any time - * Postgres Client - * MSSQL Client +Intel platform (x86_64) +----------------------- + +* Postgres Client +* MySQL Client +* MSSQL Client + +ARM platform (aarch64) +---------------------- + +ARM support is experimental, might change any time. + +* Postgres Client +* MySQL Client (MySQL 8) +* MSSQL Client + +Note that MySQL on arm has experimental support through MariaDB client library. Usage ===== From 9985c3571175d054bfabef02979ecc934e6aae73 Mon Sep 17 00:00:00 2001 From: Dmitry Zhyhimont Date: Thu, 6 Jul 2023 19:06:22 +0300 Subject: [PATCH 082/533] Fix logic of the skip_all_except method (#31153) * Fix logic of the skip_all_except method to work correctly with a mapped branch operator * Address feadback * Add an unit test * Skipp empty tasks list * Fix static checks * Address feadback * Use fully qualified import --------- Co-authored-by: Zhyhimont Dmitry Co-authored-by: zhyhimont Co-authored-by: Tzu-ping Chung --- airflow/models/skipmixin.py | 56 +++++++++++++++++++++------------- tests/models/test_skipmixin.py | 36 ++++++++++++++++++++++ 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/airflow/models/skipmixin.py b/airflow/models/skipmixin.py index d75a4a0e4d9b..849083e38b95 100644 --- a/airflow/models/skipmixin.py +++ b/airflow/models/skipmixin.py @@ -21,18 +21,19 @@ from typing import TYPE_CHECKING, Iterable, Sequence from airflow.exceptions import AirflowException, RemovedInAirflow3Warning +from airflow.models.dagrun import DagRun from airflow.models.taskinstance import TaskInstance from airflow.serialization.pydantic.dag_run import DagRunPydantic from airflow.utils import timezone from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.session import NEW_SESSION, create_session, provide_session +from airflow.utils.sqlalchemy import tuple_in_condition from airflow.utils.state import State if TYPE_CHECKING: from pendulum import DateTime from sqlalchemy import Session - from airflow.models.dagrun import DagRun from airflow.models.operator import Operator from airflow.models.taskmixin import DAGNode from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic @@ -60,24 +61,30 @@ class SkipMixin(LoggingMixin): def _set_state_to_skipped( self, dag_run: DagRun | DagRunPydantic, - tasks: Iterable[Operator], + tasks: Sequence[str] | Sequence[tuple[str, int]], session: Session, ) -> None: """Used internally to set state of task instances to skipped from the same dag run.""" - now = timezone.utcnow() - - session.query(TaskInstance).filter( - TaskInstance.dag_id == dag_run.dag_id, - TaskInstance.run_id == dag_run.run_id, - TaskInstance.task_id.in_(d.task_id for d in tasks), - ).update( - { - TaskInstance.state: State.SKIPPED, - TaskInstance.start_date: now, - TaskInstance.end_date: now, - }, - synchronize_session=False, - ) + if tasks: + now = timezone.utcnow() + TI = TaskInstance + query = session.query(TI).filter( + TI.dag_id == dag_run.dag_id, + TI.run_id == dag_run.run_id, + ) + if isinstance(tasks[0], tuple): + query = query.filter(tuple_in_condition((TI.task_id, TI.map_index), tasks)) + else: + query = query.filter(TI.task_id.in_(tasks)) + + query.update( + { + TaskInstance.state: State.SKIPPED, + TaskInstance.start_date: now, + TaskInstance.end_date: now, + }, + synchronize_session=False, + ) @provide_session def skip( @@ -130,7 +137,8 @@ def skip( if dag_run is None: raise ValueError("dag_run is required") - self._set_state_to_skipped(dag_run, task_list, session) + task_ids_list = [d.task_id for d in task_list] + self._set_state_to_skipped(dag_run, task_ids_list, session) session.commit() # SkipMixin may not necessarily have a task_id attribute. Only store to XCom if one is available. @@ -140,7 +148,7 @@ def skip( XCom.set( key=XCOM_SKIPMIXIN_KEY, - value={XCOM_SKIPMIXIN_SKIPPED: [d.task_id for d in task_list]}, + value={XCOM_SKIPMIXIN_SKIPPED: task_ids_list}, task_id=task_id, dag_id=dag_run.dag_id, run_id=dag_run.run_id, @@ -183,6 +191,7 @@ def skip_all_except( ) dag_run = ti.get_dagrun() + assert isinstance(dag_run, DagRun) # TODO(potiuk): Handle TaskInstancePydantic case differently - we need to figure out the way to # pass task that has been set in LocalTaskJob but in the way that TaskInstancePydantic definition @@ -218,10 +227,15 @@ def skip_all_except( for branch_task_id in list(branch_task_id_set): branch_task_id_set.update(dag.get_task(branch_task_id).get_flat_relative_ids(upstream=False)) - skip_tasks = [t for t in downstream_tasks if t.task_id not in branch_task_id_set] - follow_task_ids = [t.task_id for t in downstream_tasks if t.task_id in branch_task_id_set] + skip_tasks = [ + (t.task_id, downstream_ti.map_index) + for t in downstream_tasks + if (downstream_ti := dag_run.get_task_instance(t.task_id, map_index=ti.map_index)) + and t.task_id not in branch_task_id_set + ] - self.log.info("Skipping tasks %s", [t.task_id for t in skip_tasks]) + follow_task_ids = [t.task_id for t in downstream_tasks if t.task_id in branch_task_id_set] + self.log.info("Skipping tasks %s", skip_tasks) with create_session() as session: self._set_state_to_skipped(dag_run, skip_tasks, session=session) # For some reason, session.commit() needs to happen before xcom_push. diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index c9912a64d419..547dbec5b420 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -24,6 +24,7 @@ import pytest from airflow import settings +from airflow.decorators import task, task_group from airflow.exceptions import AirflowException from airflow.models.skipmixin import SkipMixin from airflow.models.taskinstance import TaskInstance as TI @@ -133,6 +134,41 @@ def get_state(ti): assert executed_states == expected_states + def test_mapped_tasks_skip_all_except(self, dag_maker): + with dag_maker("dag_test_skip_all_except") as dag: + + @task + def branch_op(k): + ... + + @task_group + def task_group_op(k): + branch_a = EmptyOperator(task_id="branch_a") + branch_b = EmptyOperator(task_id="branch_b") + branch_op(k) >> [branch_a, branch_b] + + task_group_op.expand(k=[i for i in range(2)]) + + dag_maker.create_dagrun() + branch_op_ti_0 = TI(dag.get_task("task_group_op.branch_op"), execution_date=DEFAULT_DATE, map_index=0) + branch_op_ti_1 = TI(dag.get_task("task_group_op.branch_op"), execution_date=DEFAULT_DATE, map_index=1) + branch_a_ti_0 = TI(dag.get_task("task_group_op.branch_a"), execution_date=DEFAULT_DATE, map_index=0) + branch_a_ti_1 = TI(dag.get_task("task_group_op.branch_a"), execution_date=DEFAULT_DATE, map_index=1) + branch_b_ti_0 = TI(dag.get_task("task_group_op.branch_b"), execution_date=DEFAULT_DATE, map_index=0) + branch_b_ti_1 = TI(dag.get_task("task_group_op.branch_b"), execution_date=DEFAULT_DATE, map_index=1) + + SkipMixin().skip_all_except(ti=branch_op_ti_0, branch_task_ids="task_group_op.branch_a") + SkipMixin().skip_all_except(ti=branch_op_ti_1, branch_task_ids="task_group_op.branch_b") + + def get_state(ti): + ti.refresh_from_db() + return ti.state + + assert get_state(branch_a_ti_0) == State.NONE + assert get_state(branch_b_ti_0) == State.SKIPPED + assert get_state(branch_a_ti_1) == State.SKIPPED + assert get_state(branch_b_ti_1) == State.NONE + def test_raise_exception_on_not_accepted_branch_task_ids_type(self, dag_maker): with dag_maker("dag_test_skip_all_except_wrong_type"): task = EmptyOperator(task_id="task") From ee38382efa54565c4b389eaeb536f0d45e12d498 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 6 Jul 2023 19:58:54 +0200 Subject: [PATCH 083/533] Use linear time regular expressions (#32303) The standard regexp library can consume > O(n) in certain circumstances. The re2 library does not have this issue. --- .pre-commit-config.yaml | 8 ++ STATIC_CODE_CHECKS.rst | 2 + .../endpoints/provider_endpoint.py | 4 +- airflow/cli/commands/provider_command.py | 4 +- airflow/cli/commands/user_command.py | 4 +- airflow/config_templates/default_celery.py | 7 +- airflow/configuration.py | 23 ++-- airflow/decorators/base.py | 8 +- airflow/kubernetes/pod_generator.py | 4 +- .../kubernetes/pod_generator_deprecated.py | 5 +- airflow/metrics/validators.py | 7 +- airflow/models/dag.py | 8 +- airflow/models/dagrun.py | 4 +- airflow/security/utils.py | 5 +- airflow/serialization/serde.py | 8 +- airflow/utils/cli.py | 4 +- airflow/utils/db_cleanup.py | 5 +- airflow/utils/email.py | 5 +- airflow/utils/file.py | 19 ++- airflow/utils/log/colored_log.py | 4 +- airflow/utils/log/logging_mixin.py | 5 +- airflow/utils/log/secrets_masker.py | 10 +- airflow/utils/task_group.py | 9 +- airflow/www/fab_security/manager.py | 8 +- airflow/www/views.py | 12 +- .../src/airflow_breeze/pre_commit_ids.py | 1 + images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_static-checks.svg | 130 +++++++++--------- kubernetes_tests/test_base.py | 6 +- tests/utils/log/test_secrets_masker.py | 2 +- 30 files changed, 170 insertions(+), 153 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55129ceaa117..efcf6c6e3e24 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -929,6 +929,14 @@ repos: language: python pass_filenames: true files: ^tests/.*\.py$ + - id: check-usage-of-re2-over-re + language: pygrep + name: Use re2 over re + description: Use re2 module instead of re + entry: "^\\s*from re\\s|^\\s*import re\\s" + pass_filenames: true + files: \.py$ + exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^docker_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$ ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: mypy-dev diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 657976939ff5..381dc53c93fb 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -235,6 +235,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-urlparse-usage-in-code | Don't use urlparse in code | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-usage-of-re2-over-re | Use re2 over re | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-xml | Check XML files with xmllint | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | codespell | Run codespell to check for common misspellings in files | | diff --git a/airflow/api_connexion/endpoints/provider_endpoint.py b/airflow/api_connexion/endpoints/provider_endpoint.py index c829d9c968d6..0a8594c28ae3 100644 --- a/airflow/api_connexion/endpoints/provider_endpoint.py +++ b/airflow/api_connexion/endpoints/provider_endpoint.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -import re +import re2 from airflow.api_connexion import security from airflow.api_connexion.schemas.provider_schema import ( @@ -30,7 +30,7 @@ def _remove_rst_syntax(value: str) -> str: - return re.sub("[`_<>]", "", value.strip(" \n.")) + return re2.sub("[`_<>]", "", value.strip(" \n.")) def _provider_mapper(provider: ProviderInfo) -> Provider: diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py index 75c354d68603..876905c1c230 100644 --- a/airflow/cli/commands/provider_command.py +++ b/airflow/cli/commands/provider_command.py @@ -17,7 +17,7 @@ """Providers sub-commands.""" from __future__ import annotations -import re +import re2 from airflow.cli.simple_table import AirflowConsole from airflow.providers_manager import ProvidersManager @@ -27,7 +27,7 @@ def _remove_rst_syntax(value: str) -> str: - return re.sub("[`_<>]", "", value.strip(" \n.")) + return re2.sub("[`_<>]", "", value.strip(" \n.")) @suppress_logs_and_warning diff --git a/airflow/cli/commands/user_command.py b/airflow/cli/commands/user_command.py index 7cf9c2b57f61..a5a5be978756 100644 --- a/airflow/cli/commands/user_command.py +++ b/airflow/cli/commands/user_command.py @@ -22,10 +22,10 @@ import json import os import random -import re import string from typing import Any +import re2 from marshmallow import Schema, fields, validate from marshmallow.exceptions import ValidationError @@ -164,7 +164,7 @@ def users_export(args): # In the User model the first and last name fields have underscores, # but the corresponding parameters in the CLI don't def remove_underscores(s): - return re.sub("_", "", s) + return re2.sub("_", "", s) users = [ { diff --git a/airflow/config_templates/default_celery.py b/airflow/config_templates/default_celery.py index b0390a82cf46..933d2bbdba48 100644 --- a/airflow/config_templates/default_celery.py +++ b/airflow/config_templates/default_celery.py @@ -19,9 +19,10 @@ from __future__ import annotations import logging -import re import ssl +import re2 + from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException @@ -88,7 +89,7 @@ def _broker_supports_visibility_timeout(url): "ca_certs": conf.get("celery", "SSL_CACERT"), "cert_reqs": ssl.CERT_REQUIRED, } - elif broker_url and re.search("rediss?://|sentinel://", broker_url): + elif broker_url and re2.search("rediss?://|sentinel://", broker_url): broker_use_ssl = { "ssl_keyfile": conf.get("celery", "SSL_KEY"), "ssl_certfile": conf.get("celery", "SSL_CERT"), @@ -114,7 +115,7 @@ def _broker_supports_visibility_timeout(url): f"all necessary certs and key ({e})." ) -if re.search("rediss?://|amqp://|rpc://", result_backend): +if re2.search("rediss?://|amqp://|rpc://", result_backend): log.warning( "You have configured a result_backend of %s, it is highly recommended " "to use an alternative result_backend (i.e. a database).", diff --git a/airflow/configuration.py b/airflow/configuration.py index 6a55a6c99cfc..8cb5971cd5e8 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -23,7 +23,6 @@ import multiprocessing import os import pathlib -import re import shlex import stat import subprocess @@ -36,10 +35,10 @@ from configparser import _UNSET, ConfigParser, NoOptionError, NoSectionError # type: ignore from contextlib import contextmanager, suppress from json.decoder import JSONDecodeError -from re import Pattern -from typing import IO, Any, Dict, Iterable, Set, Tuple, Union +from typing import IO, Any, Dict, Iterable, Pattern, Set, Tuple, Union from urllib.parse import urlsplit +import re2 from typing_extensions import overload from airflow.auth.managers.base_auth_manager import BaseAuthManager @@ -56,7 +55,7 @@ warnings.filterwarnings(action="default", category=DeprecationWarning, module="airflow") warnings.filterwarnings(action="default", category=PendingDeprecationWarning, module="airflow") -_SQLITE3_VERSION_PATTERN = re.compile(r"(?P^\d+(?:\.\d+)*)\D?.*$") +_SQLITE3_VERSION_PATTERN = re2.compile(r"(?P^\d+(?:\.\d+)*)\D?.*$") ConfigType = Union[str, int, float, bool] ConfigOptionsDictType = Dict[str, ConfigType] @@ -270,36 +269,36 @@ def inversed_deprecated_sections(self): # about. Mapping of section -> setting -> { old, replace, by_version } deprecated_values: dict[str, dict[str, tuple[Pattern, str, str]]] = { "core": { - "hostname_callable": (re.compile(r":"), r".", "2.1"), + "hostname_callable": (re2.compile(r":"), r".", "2.1"), }, "webserver": { - "navbar_color": (re.compile(r"\A#007A87\Z", re.IGNORECASE), "#fff", "2.1"), - "dag_default_view": (re.compile(r"^tree$"), "grid", "3.0"), + "navbar_color": (re2.compile(r"(?i)\A#007A87\z"), "#fff", "2.1"), + "dag_default_view": (re2.compile(r"^tree$"), "grid", "3.0"), }, "email": { "email_backend": ( - re.compile(r"^airflow\.contrib\.utils\.sendgrid\.send_email$"), + re2.compile(r"^airflow\.contrib\.utils\.sendgrid\.send_email$"), r"airflow.providers.sendgrid.utils.emailer.send_email", "2.1", ), }, "logging": { "log_filename_template": ( - re.compile(re.escape("{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log")), + re2.compile(re2.escape("{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log")), "XX-set-after-default-config-loaded-XX", "3.0", ), }, "api": { "auth_backends": ( - re.compile(r"^airflow\.api\.auth\.backend\.deny_all$|^$"), + re2.compile(r"^airflow\.api\.auth\.backend\.deny_all$|^$"), "airflow.api.auth.backend.session", "3.0", ), }, "elasticsearch": { "log_id_template": ( - re.compile("^" + re.escape("{dag_id}-{task_id}-{execution_date}-{try_number}") + "$"), + re2.compile("^" + re2.escape("{dag_id}-{task_id}-{execution_date}-{try_number}") + "$"), "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}", "3.0", ) @@ -426,7 +425,7 @@ def _upgrade_postgres_metastore_conn(self): FutureWarning, ) self.upgraded_values[(section, key)] = old_value - new_value = re.sub("^" + re.escape(f"{parsed.scheme}://"), f"{good_scheme}://", old_value) + new_value = re2.sub("^" + re2.escape(f"{parsed.scheme}://"), f"{good_scheme}://", old_value) self._update_env_var(section=section, name=key, new_value=new_value) # if the old value is set via env var, we need to wipe it diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py index df890c036d56..7e3c8ba2c9e3 100644 --- a/airflow/decorators/base.py +++ b/airflow/decorators/base.py @@ -17,7 +17,6 @@ from __future__ import annotations import inspect -import re import warnings from functools import cached_property from itertools import chain @@ -38,6 +37,7 @@ ) import attr +import re2 import typing_extensions from sqlalchemy.orm import Session @@ -144,15 +144,15 @@ def get_unique_task_id( return task_id def _find_id_suffixes(dag: DAG) -> Iterator[int]: - prefix = re.split(r"__\d+$", tg_task_id)[0] + prefix = re2.split(r"__\d+$", tg_task_id)[0] for task_id in dag.task_ids: - match = re.match(rf"^{prefix}__(\d+)$", task_id) + match = re2.match(rf"^{prefix}__(\d+)$", task_id) if match is None: continue yield int(match.group(1)) yield 0 # Default if there's no matching task ID. - core = re.split(r"__\d+$", task_id)[0] + core = re2.split(r"__\d+$", task_id)[0] return f"{core}__{max(_find_id_suffixes(dag)) + 1}" diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py index 91e079683abc..844a1abfd8a8 100644 --- a/airflow/kubernetes/pod_generator.py +++ b/airflow/kubernetes/pod_generator.py @@ -28,10 +28,10 @@ import datetime import logging import os -import re import warnings from functools import reduce +import re2 from dateutil import parser from kubernetes.client import models as k8s from kubernetes.client.api_client import ApiClient @@ -65,7 +65,7 @@ def make_safe_label_value(string: str) -> str: way from the original value sent to this function, then we need to truncate to 53 chars, and append it with a unique hash. """ - safe_label = re.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) + safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) if len(safe_label) > MAX_LABEL_LEN or string != safe_label: safe_hash = md5(string.encode()).hexdigest()[:9] diff --git a/airflow/kubernetes/pod_generator_deprecated.py b/airflow/kubernetes/pod_generator_deprecated.py index 652c993d1ce9..8876556a8d74 100644 --- a/airflow/kubernetes/pod_generator_deprecated.py +++ b/airflow/kubernetes/pod_generator_deprecated.py @@ -25,9 +25,9 @@ from __future__ import annotations import copy -import re import uuid +import re2 from kubernetes.client import models as k8s from airflow.utils.hashlib_wrapper import md5 @@ -70,7 +70,7 @@ def make_safe_label_value(string): way from the original value sent to this function, then we need to truncate to 53 chars, and append it with a unique hash. """ - safe_label = re.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) + safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) if len(safe_label) > MAX_LABEL_LEN or string != safe_label: safe_hash = md5(string.encode()).hexdigest()[:9] @@ -151,7 +151,6 @@ def __init__( extract_xcom: bool = False, priority_class_name: str | None = None, ): - self.pod = k8s.V1Pod() self.pod.api_version = "v1" self.pod.kind = "Pod" diff --git a/airflow/metrics/validators.py b/airflow/metrics/validators.py index 0965b534ff46..0fd5fd1adef8 100644 --- a/airflow/metrics/validators.py +++ b/airflow/metrics/validators.py @@ -21,12 +21,13 @@ import abc import logging -import re import string import warnings from functools import partial, wraps from typing import Callable, Iterable, Pattern, cast +import re2 + from airflow.configuration import conf from airflow.exceptions import InvalidStatsNameException @@ -78,7 +79,7 @@ class MetricNameLengthExemptionWarning(Warning): r"^dagrun\.schedule_delay\.(?P.*)$", r"^dagrun\.(?P.*)\.first_task_scheduling_delay$", } -BACK_COMPAT_METRIC_NAMES: set[Pattern[str]] = {re.compile(name) for name in BACK_COMPAT_METRIC_NAME_PATTERNS} +BACK_COMPAT_METRIC_NAMES: set[Pattern[str]] = {re2.compile(name) for name in BACK_COMPAT_METRIC_NAME_PATTERNS} OTEL_NAME_MAX_LENGTH = 63 @@ -132,7 +133,7 @@ def stat_name_otel_handler( # If the name is in the exceptions list, do not fail it for being too long. # It may still be deemed invalid for other reasons below. for exemption in BACK_COMPAT_METRIC_NAMES: - if re.match(exemption, stat_name): + if re2.match(exemption, stat_name): # There is a back-compat exception for this name; proceed name_length_exemption = True matched_exemption = exemption.pattern diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 11468b59ea70..1ec78b64c379 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -52,7 +52,7 @@ import jinja2 import pendulum -import re2 as re +import re2 from dateutil.relativedelta import relativedelta from pendulum.tz.timezone import Timezone from sqlalchemy import ( @@ -2361,7 +2361,7 @@ def partial_subset( dag = copy.deepcopy(self, memo) # type: ignore if isinstance(task_ids_or_regex, (str, Pattern)): - matched_tasks = [t for t in self.tasks if re.findall(task_ids_or_regex, t.task_id)] + matched_tasks = [t for t in self.tasks if re2.findall(task_ids_or_regex, t.task_id)] else: matched_tasks = [t for t in self.tasks if t.task_id in task_ids_or_regex] @@ -2828,8 +2828,8 @@ def create_dagrun( regex = airflow_conf.get("scheduler", "allowed_run_id_pattern") - if run_id and not re.match(RUN_ID_REGEX, run_id): - if not regex.strip() or not re.match(regex.strip(), run_id): + if run_id and not re2.match(RUN_ID_REGEX, run_id): + if not regex.strip() or not re2.match(regex.strip(), run_id): raise AirflowException( f"The provided run ID '{run_id}' is invalid. It does not match either " f"the configured pattern: '{regex}' or the built-in pattern: '{RUN_ID_REGEX}'" diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index b0b56f3c6c5a..f0b29881a8f3 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -24,7 +24,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, NamedTuple, Sequence, TypeVar, overload -import re2 as re +import re2 from sqlalchemy import ( Boolean, Column, @@ -248,7 +248,7 @@ def validate_run_id(self, key: str, run_id: str) -> str | None: if not run_id: return None regex = airflow_conf.get("scheduler", "allowed_run_id_pattern") - if not re.match(regex, run_id) and not re.match(RUN_ID_REGEX, run_id): + if not re2.match(regex, run_id) and not re2.match(RUN_ID_REGEX, run_id): raise ValueError( f"The run_id provided '{run_id}' does not match the pattern '{regex}' or '{RUN_ID_REGEX}'" ) diff --git a/airflow/security/utils.py b/airflow/security/utils.py index 1ffe0e0d29ba..139e96a13c92 100644 --- a/airflow/security/utils.py +++ b/airflow/security/utils.py @@ -34,9 +34,10 @@ # limitations under the License. # """Various security-related utils.""" -import re import socket +import re2 + from airflow.utils.net import get_hostname @@ -49,7 +50,7 @@ def get_components(principal) -> list[str] | None: """ if not principal: return None - return re.split(r"[/@]", str(principal)) + return re2.split(r"[/@]", str(principal)) def replace_hostname_pattern(components, host=None): diff --git a/airflow/serialization/serde.py b/airflow/serialization/serde.py index d996cb9b162f..8d0ed100ac07 100644 --- a/airflow/serialization/serde.py +++ b/airflow/serialization/serde.py @@ -21,13 +21,13 @@ import enum import functools import logging -import re import sys from importlib import import_module from types import ModuleType -from typing import Any, TypeVar, Union, cast +from typing import Any, Pattern, TypeVar, Union, cast import attr +import re2 import airflow.serialization.serializers from airflow.configuration import conf @@ -357,9 +357,9 @@ def _register(): @functools.lru_cache(maxsize=None) -def _get_patterns() -> list[re.Pattern]: +def _get_patterns() -> list[Pattern]: patterns = conf.get("core", "allowed_deserialization_classes").split() - return [re.compile(re.sub(r"(\w)\.", r"\1\..", p)) for p in patterns] + return [re2.compile(re2.sub(r"(\w)\.", r"\1\..", p)) for p in patterns] _register() diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index 56ac166b5441..c800ed71fcd8 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -21,7 +21,6 @@ import functools import logging import os -import re import socket import sys import threading @@ -32,6 +31,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable, TypeVar, cast +import re2 from sqlalchemy.orm import Session from airflow import settings @@ -252,7 +252,7 @@ def get_dags(subdir: str | None, dag_id: str, use_regex: bool = False): if not use_regex: return [get_dag(subdir, dag_id)] dagbag = DagBag(process_subdir(subdir)) - matched_dags = [dag for dag in dagbag.dags.values() if re.search(dag_id, dag.dag_id)] + matched_dags = [dag for dag in dagbag.dags.values() if re2.search(dag_id, dag.dag_id)] if not matched_dags: raise AirflowException( f"dag_id could not be found with regex: {dag_id}. Either the dag did not exist or " diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 4e9135257229..22fb0020e72a 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -141,13 +141,14 @@ def _dump_table_to_file(*, target_table, file_path, export_format, session): def _do_delete(*, query, orm_model, skip_archive, session): - import re from datetime import datetime + import re2 + print("Performing Delete...") # using bulk delete # create a new table and copy the rows there - timestamp_str = re.sub(r"[^\d]", "", datetime.utcnow().isoformat())[:14] + timestamp_str = re2.sub(r"[^\d]", "", datetime.utcnow().isoformat())[:14] target_table_name = f"{ARCHIVE_TABLE_PREFIX}{orm_model.name}__{timestamp_str}" print(f"Moving data to table {target_table_name}") bind = session.get_bind() diff --git a/airflow/utils/email.py b/airflow/utils/email.py index ee570399e7f2..e807b8f75d9f 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -20,7 +20,6 @@ import collections.abc import logging import os -import re import smtplib import warnings from email.mime.application import MIMEApplication @@ -29,6 +28,8 @@ from email.utils import formatdate from typing import Any, Iterable +import re2 + from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException, RemovedInAirflow3Warning @@ -328,4 +329,4 @@ def _get_email_list_from_str(addresses: str) -> list[str]: :return: A list of email addresses. """ pattern = r"\s*[,;]\s*" - return [address for address in re.split(pattern, addresses)] + return [address for address in re2.split(pattern, addresses)] diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 4a54adbe48b4..08b4048ad58a 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -21,12 +21,12 @@ import io import logging import os -import re import zipfile from collections import OrderedDict from pathlib import Path -from typing import Generator, NamedTuple, Protocol, overload +from typing import Generator, NamedTuple, Pattern, Protocol, overload +import re2 from pathspec.patterns import GitWildMatchPattern from airflow.configuration import conf @@ -53,15 +53,15 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool: class _RegexpIgnoreRule(NamedTuple): """Typed namedtuple with utility functions for regexp ignore rules.""" - pattern: re.Pattern + pattern: Pattern base_dir: Path @staticmethod def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule | None: """Build an ignore rule from the supplied regexp pattern and log a useful warning if it is invalid.""" try: - return _RegexpIgnoreRule(re.compile(pattern), base_dir) - except re.error as e: + return _RegexpIgnoreRule(re2.compile(pattern), base_dir) + except re2.error as e: log.warning("Ignoring invalid regex '%s' from %s: %s", pattern, definition_file, e) return None @@ -79,7 +79,7 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool: class _GlobIgnoreRule(NamedTuple): """Typed namedtuple with utility functions for glob ignore rules.""" - pattern: re.Pattern + pattern: Pattern raw_pattern: str include: bool | None = None relative_to: Path | None = None @@ -150,7 +150,7 @@ def mkdirs(path, mode): Path(path).mkdir(mode=mode, parents=True, exist_ok=True) -ZIP_REGEX = re.compile(rf"((.*\.zip){re.escape(os.sep)})?(.*)") +ZIP_REGEX = re2.compile(rf"((.*\.zip){re2.escape(os.sep)})?(.*)") @overload @@ -191,7 +191,6 @@ def open_maybe_zipped(fileloc, mode="r"): if archive and zipfile.is_zipfile(archive): return io.TextIOWrapper(zipfile.ZipFile(archive, mode=mode).open(filename)) else: - return open(fileloc, mode=mode) @@ -217,7 +216,7 @@ def _find_path_from_directory( ignore_file_path = Path(root) / ignore_file_name if ignore_file_path.is_file(): with open(ignore_file_path) as ifile: - lines_no_comments = [re.sub(r"\s*#.*", "", line) for line in ifile.read().split("\n")] + lines_no_comments = [re2.sub(r"\s*#.*", "", line) for line in ifile.read().split("\n")] # append new patterns and filter out "None" objects, which are invalid patterns patterns += [ p @@ -327,7 +326,7 @@ def find_dag_file_paths(directory: str | os.PathLike[str], safe_mode: bool) -> l return file_paths -COMMENT_PATTERN = re.compile(r"\s*#.*") +COMMENT_PATTERN = re2.compile(r"\s*#.*") def might_contain_dag(file_path: str, safe_mode: bool, zip_file: zipfile.ZipFile | None = None) -> bool: diff --git a/airflow/utils/log/colored_log.py b/airflow/utils/log/colored_log.py index 6df2c98fcd13..aea7c255e618 100644 --- a/airflow/utils/log/colored_log.py +++ b/airflow/utils/log/colored_log.py @@ -18,11 +18,11 @@ """Class responsible for colouring logs based on log level.""" from __future__ import annotations -import re import sys from logging import LogRecord from typing import Any +import re2 from colorlog import TTYColoredFormatter from colorlog.escape_codes import esc, escape_codes @@ -61,7 +61,7 @@ def _color_arg(arg: Any) -> str | float | int: @staticmethod def _count_number_of_arguments_in_message(record: LogRecord) -> int: - matches = re.findall(r"%.", record.msg) + matches = re2.findall(r"%.", record.msg) return len(matches) if matches else 0 def _color_record_args(self, record: LogRecord) -> LogRecord: diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 02f89f281217..646c73f1b4bc 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -20,14 +20,15 @@ import abc import enum import logging -import re import sys from io import IOBase from logging import Handler, Logger, StreamHandler from typing import IO, Any, TypeVar, cast +import re2 + # 7-bit C1 ANSI escape sequences -ANSI_ESCAPE = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") +ANSI_ESCAPE = re2.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") # Private: A sentinel objects diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py index 54547c1e6e6e..2c5e4c47e0d4 100644 --- a/airflow/utils/log/secrets_masker.py +++ b/airflow/utils/log/secrets_masker.py @@ -19,7 +19,6 @@ import collections.abc import logging -import re import sys from functools import cached_property from typing import ( @@ -31,12 +30,15 @@ Iterable, Iterator, List, + Pattern, TextIO, Tuple, TypeVar, Union, ) +import re2 + from airflow import settings from airflow.compat.functools import cache from airflow.typing_compat import TypeGuard @@ -144,7 +146,7 @@ def _is_v1_env_var(v: Any) -> TypeGuard[V1EnvVar]: class SecretsMasker(logging.Filter): """Redact secrets from logs.""" - replacer: re.Pattern | None = None + replacer: Pattern | None = None patterns: set[str] ALREADY_FILTERED_FLAG = "__SecretsMasker_filtered" @@ -332,13 +334,13 @@ def add_mask(self, secret: str | dict | Iterable, name: str | None = None): new_mask = False for s in self._adaptations(secret): if s: - pattern = re.escape(s) + pattern = re2.escape(s) if pattern not in self.patterns and (not name or should_hide_value_for_key(name)): self.patterns.add(pattern) new_mask = True if new_mask: - self.replacer = re.compile("|".join(self.patterns)) + self.replacer = re2.compile("|".join(self.patterns)) elif isinstance(secret, collections.abc.Iterable): for v in secret: diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index 58af85bb7511..d59ecaf40b4d 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -24,10 +24,11 @@ import copy import functools import operator -import re import weakref from typing import TYPE_CHECKING, Any, Generator, Iterator, Sequence +import re2 + from airflow.compat.functools import cache from airflow.exceptions import ( AirflowDagCycleException, @@ -166,11 +167,11 @@ def _check_for_group_id_collisions(self, add_suffix_on_collision: bool): if self._group_id in self.used_group_ids: if not add_suffix_on_collision: raise DuplicateTaskIdFound(f"group_id '{self._group_id}' has already been added to the DAG") - base = re.split(r"__\d+$", self._group_id)[0] + base = re2.split(r"__\d+$", self._group_id)[0] suffixes = sorted( - int(re.split(r"^.+__", used_group_id)[1]) + int(re2.split(r"^.+__", used_group_id)[1]) for used_group_id in self.used_group_ids - if used_group_id is not None and re.match(rf"^{base}__\d+$", used_group_id) + if used_group_id is not None and re2.match(rf"^{base}__\d+$", used_group_id) ) if not suffixes: self._group_id += "__1" diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index 7145128e21b5..be0c4c63e39f 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -22,11 +22,11 @@ import datetime import json import logging -import re from functools import cached_property from typing import Any from uuid import uuid4 +import re2 from flask import Flask, current_app, g, session, url_for from flask_appbuilder import AppBuilder from flask_appbuilder.const import ( @@ -703,7 +703,7 @@ def get_oauth_user_info(self, provider, resp): def _azure_parse_jwt(self, id_token): jwt_token_parts = r"^([^\.\s]*)\.([^\.\s]+)\.([^\.\s]*)$" - matches = re.search(jwt_token_parts, id_token) + matches = re2.search(jwt_token_parts, id_token) if not matches or len(matches.groups()) < 3: log.error("Unable to parse token.") return {} @@ -1375,8 +1375,8 @@ def auth_user_oauth(self, userinfo): def _has_access_builtin_roles(self, role, action_name: str, resource_name: str) -> bool: """Checks permission on builtin role.""" perms = self.builtin_roles.get(role.name, []) - for (_resource_name, _action_name) in perms: - if re.match(_resource_name, resource_name) and re.match(_action_name, action_name): + for _resource_name, _action_name in perms: + if re2.match(_resource_name, resource_name) and re2.match(_action_name, action_name): return True return False diff --git a/airflow/www/views.py b/airflow/www/views.py index 3b9c35e1ac3a..370e15ee007d 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -24,7 +24,6 @@ import json import logging import math -import re import sys import traceback import warnings @@ -39,6 +38,7 @@ import flask.json import lazy_object_proxy import nvd3 +import re2 import sqlalchemy as sqla from croniter import croniter from flask import ( @@ -2105,8 +2105,8 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): return redirect(origin) regex = conf.get("scheduler", "allowed_run_id_pattern") - if run_id and not re.match(RUN_ID_REGEX, run_id): - if not regex.strip() or not re.match(regex.strip(), run_id): + if run_id and not re2.match(RUN_ID_REGEX, run_id): + if not regex.strip() or not re2.match(regex.strip(), run_id): flash( f"The provided run ID '{run_id}' is invalid. It does not match either " f"the configured pattern: '{regex}' or the built-in pattern: '{RUN_ID_REGEX}'", @@ -4743,7 +4743,7 @@ def action_mulduplicate(self, connections, session: Session = NEW_SESSION): """Duplicate Multiple connections.""" for selected_conn in connections: new_conn_id = selected_conn.conn_id - match = re.search(r"_copy(\d+)$", selected_conn.conn_id) + match = re2.search(r"_copy(\d+)$", selected_conn.conn_id) base_conn_id = selected_conn.conn_id if match: @@ -5002,8 +5002,8 @@ def _build_link(match_obj): return Markup(f'{text}') cd = escape(description) - cd = re.sub(r"`(.*)[\s+]+<(.*)>`__", _build_link, cd) - cd = re.sub(r"\n", r"
    ", cd) + cd = re2.sub(r"`(.*)[\s+]+<(.*)>`__", _build_link, cd) + cd = re2.sub(r"\n", r"
    ", cd) return Markup(cd) diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index b4c713360ed8..1349a962107e 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -71,6 +71,7 @@ "check-system-tests-tocs", "check-tests-unittest-testcase", "check-urlparse-usage-in-code", + "check-usage-of-re2-over-re", "check-xml", "codespell", "compile-www-assets", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 61e50641f2b6..241dc9bd43f6 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -60,7 +60,7 @@ setup:version:123b462a421884dc2320ffc5e54b2478 setup:201c30ea237ea013a9a209a77092a2e8 shell:13f90c5749811e2f00e24d95e44e946d start-airflow:22c118d58b13a9d190e966bed5bb8ed8 -static-checks:9985d1db64592e29ab71b8a000ce302e +static-checks:a0f6ae35129b99b88ce65aede2c3a150 testing:docker-compose-tests:70167e67853cacd9ca784695d65a7846 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b testing:integration-tests:35f0ac57157bf8fe227fd080cf216622 diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index e9f5238acee0..e735b2afc306 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -35,8 +35,8 @@ .breeze-static-checks-r1 { fill: #c5c8c6;font-weight: bold } .breeze-static-checks-r2 { fill: #c5c8c6 } .breeze-static-checks-r3 { fill: #d0b344;font-weight: bold } -.breeze-static-checks-r4 { fill: #68a0b3;font-weight: bold } -.breeze-static-checks-r5 { fill: #868887 } +.breeze-static-checks-r4 { fill: #868887 } +.breeze-static-checks-r5 { fill: #68a0b3;font-weight: bold } .breeze-static-checks-r6 { fill: #98a84b;font-weight: bold } .breeze-static-checks-r7 { fill: #8d7b39 } @@ -256,72 +256,72 @@ -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... Run static checks. -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run.                                             │ -│(all | black | blacken-docs | check-aiobotocore-optional |                       │ -│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ -│check-apache-license-rat | check-base-operator-partial-arguments |               │ -│check-base-operator-usage | check-boring-cyborg-configuration |                  │ -│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ -│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ -│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ -│| check-docstring-param-types | check-example-dags-urls |                        │ -│check-executables-have-shebangs | check-extra-packages-references |              │ -│check-extras-order | check-for-inclusive-language | check-hooks-apply |          │ -│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ -│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ -│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ -│check-no-relative-imports | check-only-new-session-with-provide-session |        │ -│check-persist-credentials-disabled-in-github-workflows |                         │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ -│check-provider-yaml-valid | check-providers-init-file-missing |                  │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ -│check-start-date-not-used-in-defaults | check-system-tests-present |             │ -│check-system-tests-tocs | check-tests-unittest-testcase |                        │ -│check-urlparse-usage-in-code | check-xml | codespell | compile-www-assets |      │ -│compile-www-assets-dev | create-missing-init-py-files-tests | debug-statements | │ -│detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |  │ -│identity | insert-license | lint-chart-schema | lint-css | lint-dockerfile |     │ -│lint-helm-chart | lint-json-schema | lint-markdown | lint-openapi |              │ -│mixed-line-ending | mypy-core | mypy-dev | mypy-docs | mypy-providers |          │ -│pretty-format-json | python-no-log-warn | replace-bad-characters | rst-backticks │ -│| ruff | shellcheck | trailing-whitespace | ts-compile-format-lint-www |         │ -│update-black-version | update-breeze-cmd-output |                                │ -│update-breeze-readme-config-hash | update-common-sql-api-stubs |                 │ -│update-er-diagram | update-extras | update-in-the-wild-to-be-sorted |            │ -│update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |    │ -│update-local-yml-file | update-migration-references |                            │ -│update-providers-dependencies | update-spelling-wordlist-to-be-sorted |          │ -│update-supported-versions | update-vendored-in-k8s-json-schema | update-version |│ -│yamllint)                                                                        │ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--initialize-environmentInitialize environment before running checks.│ -│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ -│(INTEGER RANGE)                                                       │ -│[default: 3; 1<=x<=10]                                                │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ -│exclusive with --last-commit.                                                                │ -│(TEXT)                                                                                       │ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ -│branch and HEAD of your branch.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run.                                             │ +│(all | black | blacken-docs | check-aiobotocore-optional |                       │ +│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ +│check-apache-license-rat | check-base-operator-partial-arguments |               │ +│check-base-operator-usage | check-boring-cyborg-configuration |                  │ +│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ +│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ +│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ +│| check-docstring-param-types | check-example-dags-urls |                        │ +│check-executables-have-shebangs | check-extra-packages-references |              │ +│check-extras-order | check-for-inclusive-language | check-hooks-apply |          │ +│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ +│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ +│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ +│check-no-relative-imports | check-only-new-session-with-provide-session |        │ +│check-persist-credentials-disabled-in-github-workflows |                         │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ +│check-provider-yaml-valid | check-providers-init-file-missing |                  │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ +│check-start-date-not-used-in-defaults | check-system-tests-present |             │ +│check-system-tests-tocs | check-tests-unittest-testcase |                        │ +│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ +│| compile-www-assets | compile-www-assets-dev |                                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ +│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ +│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ +│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ +│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ +│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ +│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ +│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ +│update-installed-providers-to-be-sorted | update-local-yml-file |                │ +│update-migration-references | update-providers-dependencies |                    │ +│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ +│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--initialize-environmentInitialize environment before running checks.│ +│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ +│(INTEGER RANGE)                                                       │ +│[default: 3; 1<=x<=10]                                                │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ +│exclusive with --last-commit.                                                                │ +│(TEXT)                                                                                       │ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ +│branch and HEAD of your branch.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index dfbf640fd648..5b45969c3d28 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -17,7 +17,6 @@ from __future__ import annotations import os -import re import subprocess import tempfile import time @@ -26,6 +25,7 @@ from subprocess import check_call, check_output import pytest +import re2 import requests import requests.exceptions from requests.adapters import HTTPAdapter @@ -103,7 +103,7 @@ def _describe_resources(self, namespace: str): def _num_pods_in_namespace(namespace): air_pod = check_output(["kubectl", "get", "pods", "-n", namespace]).decode() air_pod = air_pod.split("\n") - names = [re.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" in x] + names = [re2.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" in x] return len(names) @staticmethod @@ -111,7 +111,7 @@ def _delete_airflow_pod(name=""): suffix = "-" + name if name else "" air_pod = check_output(["kubectl", "get", "pods"]).decode() air_pod = air_pod.split("\n") - names = [re.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" + suffix in x] + names = [re2.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" + suffix in x] if names: check_call(["kubectl", "delete", "pod", names[0]]) diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py index 9dad28bf4518..e47cd5cee57e 100644 --- a/tests/utils/log/test_secrets_masker.py +++ b/tests/utils/log/test_secrets_masker.py @@ -393,7 +393,7 @@ class TestMaskSecretAdapter: def reset_secrets_masker_and_skip_escape(self): self.secrets_masker = SecretsMasker() with patch("airflow.utils.log.secrets_masker._secrets_masker", return_value=self.secrets_masker): - with patch("airflow.utils.log.secrets_masker.re.escape", lambda x: x): + with patch("airflow.utils.log.secrets_masker.re2.escape", lambda x: x): yield def test_calling_mask_secret_adds_adaptations_for_returned_str(self): From 358e6e8fa18166084fc17b23e75c6c29a37f245f Mon Sep 17 00:00:00 2001 From: Sumesh Premraj <3002040+sumeshpremraj@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:59:04 +0200 Subject: [PATCH 084/533] Fix headers passed into HttpAsyncHook (#32409) --- airflow/providers/http/hooks/http.py | 2 +- tests/providers/http/hooks/test_http.py | 34 +++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 5bff4716a954..0b0443efc484 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -369,7 +369,7 @@ async def run( url, json=data if self.method in ("POST", "PATCH") else None, params=data if self.method == "GET" else None, - headers=headers, + headers=_headers, auth=auth, **extra_options, ) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 8d2bc8691d9a..e702dded9c10 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -82,7 +82,6 @@ def test_get_request_with_port(self, mock_session, mock_request): ): expected_url = "http://test.com:1234/some/endpoint" for endpoint in ["some/endpoint", "/some/endpoint"]: - try: self.get_hook.run(endpoint) except MissingSchema: @@ -175,7 +174,6 @@ def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, r @mock.patch("airflow.providers.http.hooks.http.requests.Session") def test_retry_on_conn_error(self, mocked_session): - retry_args = dict( wait=tenacity.wait_none(), stop=tenacity.stop_after_attempt(7), @@ -192,7 +190,6 @@ def send_and_raise(unused_request, **kwargs): assert self.get_hook._retry_obj.stop.max_attempt_number + 1 == mocked_session.call_count def test_run_with_advanced_retry(self, requests_mock): - requests_mock.get("http://test:8080/v1/test", status_code=200, reason="OK") retry_args = dict( @@ -298,7 +295,6 @@ def test_requests_ca_bundle_env_var(self, mock_session_send): with mock.patch( "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ): - self.get_hook.run("/some/endpoint") mock_session_send.assert_called_once_with( @@ -317,7 +313,6 @@ def test_verify_respects_requests_ca_bundle_env_var(self, mock_session_send): with mock.patch( "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ): - self.get_hook.run("/some/endpoint", extra_options={"verify": True}) mock_session_send.assert_called_once_with( @@ -530,3 +525,32 @@ async def test_async_post_request_with_error_code(aioresponse): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): with pytest.raises(AirflowException): await hook.run("v1/test") + + +@pytest.mark.asyncio +async def test_async_request_uses_connection_extra(aioresponse): + """Test api call asynchronously with a connection that has extra field.""" + + connection_extra = {"bareer": "test"} + connection_id = "http_default" + + def get_airflow_connection_with_extra(unused_conn_id=None): + return Connection( + conn_id=connection_id, conn_type="http", host="test:8080/", extra=json.dumps(connection_extra) + ) + + aioresponse.post( + "http://test:8080/v1/test", + status=200, + payload='{"status":{"status": 200}}', + reason="OK", + ) + + with mock.patch( + "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_extra + ): + hook = HttpAsyncHook() + with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: + await hook.run("v1/test") + headers = mocked_function.call_args.kwargs.get("headers") + assert all(key in headers and headers[key] == value for key, value in connection_extra.items()) From bb97bf21fd320c593b77246590d4f8d2b0369c24 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 7 Jul 2023 00:27:06 +0200 Subject: [PATCH 085/533] Limit click until we fix mypy issues (#32413) Click 8.1.4 breaks our mypy checks. The upper limit can be lifted when the #32412 issue is resolved --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 547e1221d37e..90673ef86dfb 100644 --- a/setup.py +++ b/setup.py @@ -266,7 +266,9 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve doc = [ "astroid>=2.12.3", "checksumdir", - "click>=8.0", + # Click 8.1.4 breaks our mypy checks. The upper limit can be lifted when the + # https://github.com/apache/airflow/issues/32412 issue is resolved + "click>=8.0,<8.1.4", # Docutils 0.17.0 converts generated
    into
    and breaks our doc formatting # By adding a lot of whitespace separation. This limit can be lifted when we update our doc to handle #
    tags for sections From b3db4de4985eccb859a30a07a2350499370c6a9a Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Fri, 7 Jul 2023 04:40:33 +0530 Subject: [PATCH 086/533] Updates health check endpoint to include dag_processor status. (#32382) --- airflow/api/common/airflow_health.py | 19 +++++++++++++++++++ airflow/api_connexion/openapi/v1.yaml | 18 ++++++++++++++++++ .../api_connexion/schemas/health_schema.py | 7 +++++++ airflow/www/static/js/types/api-generated.ts | 17 +++++++++++++++++ .../logging-monitoring/check-health.rst | 8 ++++++++ tests/api/common/test_airflow_health.py | 17 ++++++++++++++++- 6 files changed, 85 insertions(+), 1 deletion(-) diff --git a/airflow/api/common/airflow_health.py b/airflow/api/common/airflow_health.py index 3174add64842..5d37de540a49 100644 --- a/airflow/api/common/airflow_health.py +++ b/airflow/api/common/airflow_health.py @@ -18,6 +18,7 @@ from typing import Any +from airflow.jobs.dag_processor_job_runner import DagProcessorJobRunner from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner @@ -30,8 +31,10 @@ def get_airflow_health() -> dict[str, Any]: metadatabase_status = HEALTHY latest_scheduler_heartbeat = None latest_triggerer_heartbeat = None + latest_dag_processor_heartbeat = None scheduler_status = UNHEALTHY triggerer_status: str | None = UNHEALTHY + dag_processor_status: str | None = UNHEALTHY try: latest_scheduler_job = SchedulerJobRunner.most_recent_job() @@ -55,6 +58,18 @@ def get_airflow_health() -> dict[str, Any]: except Exception: metadatabase_status = UNHEALTHY + try: + latest_dag_processor_job = DagProcessorJobRunner.most_recent_job() + + if latest_dag_processor_job: + latest_dag_processor_heartbeat = latest_dag_processor_job.latest_heartbeat.isoformat() + if latest_dag_processor_job.is_alive(): + dag_processor_status = HEALTHY + else: + dag_processor_status = None + except Exception: + metadatabase_status = UNHEALTHY + airflow_health_status = { "metadatabase": {"status": metadatabase_status}, "scheduler": { @@ -65,6 +80,10 @@ def get_airflow_health() -> dict[str, Any]: "status": triggerer_status, "latest_triggerer_heartbeat": latest_triggerer_heartbeat, }, + "dag_processor": { + "status": dag_processor_status, + "latest_dag_processor_heartbeat": latest_dag_processor_heartbeat, + }, } return airflow_health_status diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 8bf976aa4627..3e1dcdedd740 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3084,6 +3084,8 @@ components: $ref: '#/components/schemas/SchedulerStatus' triggerer: $ref: '#/components/schemas/TriggererStatus' + dag_processor: + $ref: '#/components/schemas/DagProcessorStatus' MetadatabaseStatus: type: object @@ -3121,6 +3123,22 @@ components: readOnly: true nullable: true + DagProcessorStatus: + type: object + description: | + The status and the latest dag processor heartbeat. + + *New in version 2.6.3* + properties: + status: + $ref: '#/components/schemas/HealthStatus' + latest_dag_processor_heartbeat: + description: The time the dag processor last did a heartbeat. + type: string + format: datetime + readOnly: true + nullable: true + Pool: description: The pool type: object diff --git a/airflow/api_connexion/schemas/health_schema.py b/airflow/api_connexion/schemas/health_schema.py index 5391ce5ae235..5e184171fb39 100644 --- a/airflow/api_connexion/schemas/health_schema.py +++ b/airflow/api_connexion/schemas/health_schema.py @@ -41,12 +41,19 @@ class TriggererInfoSchema(BaseInfoSchema): latest_triggerer_heartbeat = fields.String(dump_only=True) +class DagProcessorInfoSchema(BaseInfoSchema): + """Schema for DagProcessor info.""" + + latest_dag_processor_heartbeat = fields.String(dump_only=True) + + class HealthInfoSchema(Schema): """Schema for the Health endpoint.""" metadatabase = fields.Nested(MetaDatabaseInfoSchema) scheduler = fields.Nested(SchedulerInfoSchema) triggerer = fields.Nested(TriggererInfoSchema) + dag_processor = fields.Nested(DagProcessorInfoSchema) health_schema = HealthInfoSchema() diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 347f76e08f1e..aa9bebf4d6de 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1192,6 +1192,7 @@ export interface components { metadatabase?: components["schemas"]["MetadatabaseStatus"]; scheduler?: components["schemas"]["SchedulerStatus"]; triggerer?: components["schemas"]["TriggererStatus"]; + dag_processor?: components["schemas"]["DagProcessorStatus"]; }; /** @description The status of the metadatabase. */ MetadatabaseStatus: { @@ -1219,6 +1220,19 @@ export interface components { */ latest_triggerer_heartbeat?: string | null; }; + /** + * @description The status and the latest dag processor heartbeat. + * + * *New in version 2.6.3* + */ + DagProcessorStatus: { + status?: components["schemas"]["HealthStatus"]; + /** + * Format: datetime + * @description The time the dag processor last did a heartbeat. + */ + latest_dag_processor_heartbeat?: string | null; + }; /** @description The pool */ Pool: { /** @description The name of pool. */ @@ -4667,6 +4681,9 @@ export type SchedulerStatus = CamelCasedPropertiesDeep< export type TriggererStatus = CamelCasedPropertiesDeep< components["schemas"]["TriggererStatus"] >; +export type DagProcessorStatus = CamelCasedPropertiesDeep< + components["schemas"]["DagProcessorStatus"] +>; export type Pool = CamelCasedPropertiesDeep; export type PoolCollection = CamelCasedPropertiesDeep< components["schemas"]["PoolCollection"] diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst index 798343f9d1e0..58e883407866 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst @@ -51,6 +51,10 @@ To check the health status of your Airflow instance, you can simply access the e "triggerer":{ "status":"healthy", "latest_triggerer_heartbeat":"2018-12-26 17:16:12+00:00" + }, + "dag_processor":{ + "status":"healthy", + "latest_dag_processor_heartbeat":"2018-12-26 17:16:12+00:00" } } @@ -71,6 +75,10 @@ To check the health status of your Airflow instance, you can simply access the e Note that the ``status`` and ``latest_triggerer_heartbeat`` fields in the health check response will be null for deployments that do not include a ``triggerer`` component. + * The status of the ``dag_processor`` behaves exactly like that of the ``scheduler`` as described above. + Note that the ``status`` and ``latest_dag_processor_heartbeat`` fields in the health check response will be null for + deployments that do not include a ``dag_processor`` component. + Please keep in mind that the HTTP response code of ``/health`` endpoint **should not** be used to determine the health status of the application. The return code is only indicative of the state of the rest call (200 for success). diff --git a/tests/api/common/test_airflow_health.py b/tests/api/common/test_airflow_health.py index 1f778f7a2507..20bd101cabe8 100644 --- a/tests/api/common/test_airflow_health.py +++ b/tests/api/common/test_airflow_health.py @@ -22,6 +22,7 @@ from airflow.api.common.airflow_health import ( HEALTHY, UNHEALTHY, + DagProcessorJobRunner, SchedulerJobRunner, TriggererJobRunner, get_airflow_health, @@ -31,13 +32,14 @@ def test_get_airflow_health_only_metadatabase_healthy(): SchedulerJobRunner.most_recent_job = MagicMock(return_value=None) TriggererJobRunner.most_recent_job = MagicMock(return_value=None) - + DagProcessorJobRunner.most_recent_job = MagicMock(return_value=None) health_status = get_airflow_health() expected_status = { "metadatabase": {"status": HEALTHY}, "scheduler": {"status": UNHEALTHY, "latest_scheduler_heartbeat": None}, "triggerer": {"status": None, "latest_triggerer_heartbeat": None}, + "dag_processor": {"status": None, "latest_dag_processor_heartbeat": None}, } assert health_status == expected_status @@ -46,6 +48,7 @@ def test_get_airflow_health_only_metadatabase_healthy(): def test_get_airflow_health_metadatabase_unhealthy(): SchedulerJobRunner.most_recent_job = MagicMock(side_effect=Exception) TriggererJobRunner.most_recent_job = MagicMock(side_effect=Exception) + DagProcessorJobRunner.most_recent_job = MagicMock(side_effect=Exception) health_status = get_airflow_health() @@ -53,6 +56,7 @@ def test_get_airflow_health_metadatabase_unhealthy(): "metadatabase": {"status": UNHEALTHY}, "scheduler": {"status": UNHEALTHY, "latest_scheduler_heartbeat": None}, "triggerer": {"status": UNHEALTHY, "latest_triggerer_heartbeat": None}, + "dag_processor": {"status": UNHEALTHY, "latest_dag_processor_heartbeat": None}, } assert health_status == expected_status @@ -64,6 +68,7 @@ def test_get_airflow_health_scheduler_healthy_no_triggerer(): latest_scheduler_job_mock.is_alive = MagicMock(return_value=True) SchedulerJobRunner.most_recent_job = MagicMock(return_value=latest_scheduler_job_mock) TriggererJobRunner.most_recent_job = MagicMock(return_value=None) + DagProcessorJobRunner.most_recent_job = MagicMock(return_value=None) health_status = get_airflow_health() @@ -74,6 +79,7 @@ def test_get_airflow_health_scheduler_healthy_no_triggerer(): "latest_scheduler_heartbeat": latest_scheduler_job_mock.latest_heartbeat.isoformat(), }, "triggerer": {"status": None, "latest_triggerer_heartbeat": None}, + "dag_processor": {"status": None, "latest_dag_processor_heartbeat": None}, } assert health_status == expected_status @@ -83,8 +89,13 @@ def test_get_airflow_health_triggerer_healthy_no_scheduler_job_record(): latest_triggerer_job_mock = MagicMock() latest_triggerer_job_mock.latest_heartbeat = datetime.now() latest_triggerer_job_mock.is_alive = MagicMock(return_value=True) + latest_dag_processor_job_mock = MagicMock() + latest_dag_processor_job_mock.latest_heartbeat = datetime.now() + latest_dag_processor_job_mock.is_alive = MagicMock(return_value=True) + SchedulerJobRunner.most_recent_job = MagicMock(return_value=None) TriggererJobRunner.most_recent_job = MagicMock(return_value=latest_triggerer_job_mock) + DagProcessorJobRunner.most_recent_job = MagicMock(return_value=latest_dag_processor_job_mock) health_status = get_airflow_health() @@ -95,6 +106,10 @@ def test_get_airflow_health_triggerer_healthy_no_scheduler_job_record(): "status": HEALTHY, "latest_triggerer_heartbeat": latest_triggerer_job_mock.latest_heartbeat.isoformat(), }, + "dag_processor": { + "status": HEALTHY, + "latest_dag_processor_heartbeat": latest_dag_processor_job_mock.latest_heartbeat.isoformat(), + }, } assert health_status == expected_status From 2d690de110825ba09b9445967b47c44edd8f151c Mon Sep 17 00:00:00 2001 From: Avinash Holla Pandeshwar Date: Fri, 7 Jul 2023 04:42:37 +0530 Subject: [PATCH 087/533] Fix BigQuery transfer operators to respect project_id arguments (#32232) --- .../google/cloud/transfers/bigquery_to_gcs.py | 6 +- .../google/cloud/transfers/gcs_to_bigquery.py | 31 +++---- .../cloud/transfers/test_bigquery_to_gcs.py | 12 ++- .../cloud/transfers/test_gcs_to_bigquery.py | 92 +++++++++++++------ 4 files changed, 86 insertions(+), 55 deletions(-) diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py index 8836e3ee354e..467751115151 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py @@ -146,7 +146,7 @@ def _handle_job_error(job: BigQueryJob | UnknownJob) -> None: def _prepare_configuration(self): source_project, source_dataset, source_table = self.hook.split_tablename( table_input=self.source_project_dataset_table, - default_project_id=self.project_id or self.hook.project_id, + default_project_id=self.hook.project_id, var_name="source_project_dataset_table", ) @@ -184,7 +184,7 @@ def _submit_job( return hook.insert_job( configuration=configuration, - project_id=configuration["extract"]["sourceTable"]["projectId"], + project_id=self.project_id or hook.project_id, location=self.location, job_id=job_id, timeout=self.result_timeout, @@ -255,7 +255,7 @@ def execute(self, context: Context): trigger=BigQueryInsertJobTrigger( conn_id=self.gcp_conn_id, job_id=job_id, - project_id=self.hook.project_id, + project_id=self.project_id or self.hook.project_id, ), method_name="execute_complete", ) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index da462eae74df..46c5adc95b88 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -301,7 +301,7 @@ def _submit_job( # Submit a new job without waiting for it to complete. return hook.insert_job( configuration=self.configuration, - project_id=self.project_id, + project_id=self.project_id or hook.project_id, location=self.location, job_id=job_id, timeout=self.result_timeout, @@ -359,7 +359,7 @@ def execute(self, context: Context): if self.external_table: self.log.info("Creating a new BigQuery table for storing data...") - table_obj_api_repr = self._create_empty_table() + table_obj_api_repr = self._create_external_table() BigQueryTableLink.persist( context=context, @@ -381,7 +381,7 @@ def execute(self, context: Context): except Conflict: # If the job already exists retrieve it job = self.hook.get_job( - project_id=self.hook.project_id, + project_id=self.project_id or self.hook.project_id, location=self.location, job_id=job_id, ) @@ -414,12 +414,12 @@ def execute(self, context: Context): persist_kwargs = { "context": context, "task_instance": self, - "project_id": self.hook.project_id, "table_id": table, } if not isinstance(table, str): persist_kwargs["table_id"] = table["tableId"] persist_kwargs["dataset_id"] = table["datasetId"] + persist_kwargs["project_id"] = table["projectId"] BigQueryTableLink.persist(**persist_kwargs) self.job_id = job.job_id @@ -430,7 +430,7 @@ def execute(self, context: Context): trigger=BigQueryInsertJobTrigger( conn_id=self.gcp_conn_id, job_id=self.job_id, - project_id=self.hook.project_id, + project_id=self.project_id or self.hook.project_id, ), method_name="execute_complete", ) @@ -475,7 +475,9 @@ def _find_max_value_in_column(self): } } try: - job_id = hook.insert_job(configuration=self.configuration, project_id=hook.project_id) + job_id = hook.insert_job( + configuration=self.configuration, project_id=self.project_id or hook.project_id + ) rows = list(hook.get_job(job_id=job_id, location=self.location).result()) except BadRequest as e: if "Unrecognized name:" in e.message: @@ -498,12 +500,7 @@ def _find_max_value_in_column(self): else: raise RuntimeError(f"The {select_command} returned no rows!") - def _create_empty_table(self): - self.project_id, dataset_id, table_id = self.hook.split_tablename( - table_input=self.destination_project_dataset_table, - default_project_id=self.project_id or self.hook.project_id, - ) - + def _create_external_table(self): external_config_api_repr = { "autodetect": self.autodetect, "sourceFormat": self.source_format, @@ -549,7 +546,7 @@ def _create_empty_table(self): # build table definition table = Table( - table_ref=TableReference.from_string(self.destination_project_dataset_table, self.project_id) + table_ref=TableReference.from_string(self.destination_project_dataset_table, self.hook.project_id) ) table.external_data_configuration = external_config if self.labels: @@ -567,7 +564,7 @@ def _create_empty_table(self): self.log.info("Creating external table: %s", self.destination_project_dataset_table) self.hook.create_empty_table( table_resource=table_obj_api_repr, - project_id=self.project_id, + project_id=self.project_id or self.hook.project_id, location=self.location, exists_ok=True, ) @@ -575,9 +572,9 @@ def _create_empty_table(self): return table_obj_api_repr def _use_existing_table(self): - self.project_id, destination_dataset, destination_table = self.hook.split_tablename( + destination_project_id, destination_dataset, destination_table = self.hook.split_tablename( table_input=self.destination_project_dataset_table, - default_project_id=self.project_id or self.hook.project_id, + default_project_id=self.hook.project_id, var_name="destination_project_dataset_table", ) @@ -597,7 +594,7 @@ def _use_existing_table(self): "autodetect": self.autodetect, "createDisposition": self.create_disposition, "destinationTable": { - "projectId": self.project_id, + "projectId": destination_project_id, "datasetId": destination_dataset, "tableId": destination_table, }, diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py b/tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py index b1d0563f58eb..b7bf8bef62d8 100644 --- a/tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py @@ -31,6 +31,7 @@ TEST_DATASET = "test-dataset" TEST_TABLE_ID = "test-table-id" PROJECT_ID = "test-project-id" +JOB_PROJECT_ID = "job-project-id" class TestBigQueryToGCSOperator: @@ -66,7 +67,7 @@ def test_execute(self, mock_hook): mock_hook.return_value.split_tablename.return_value = (PROJECT_ID, TEST_DATASET, TEST_TABLE_ID) mock_hook.return_value.generate_job_id.return_value = real_job_id mock_hook.return_value.insert_job.return_value = MagicMock(job_id="real_job_id", error_result=False) - mock_hook.return_value.project_id = PROJECT_ID + mock_hook.return_value.project_id = JOB_PROJECT_ID operator = BigQueryToGCSOperator( task_id=TASK_ID, @@ -77,13 +78,14 @@ def test_execute(self, mock_hook): field_delimiter=field_delimiter, print_header=print_header, labels=labels, + project_id=JOB_PROJECT_ID, ) operator.execute(context=mock.MagicMock()) mock_hook.return_value.insert_job.assert_called_once_with( job_id="123456_hash", configuration=expected_configuration, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, location=None, timeout=None, retry=DEFAULT_RETRY, @@ -122,10 +124,10 @@ def test_execute_deferrable_mode(self, mock_hook): mock_hook.return_value.split_tablename.return_value = (PROJECT_ID, TEST_DATASET, TEST_TABLE_ID) mock_hook.return_value.generate_job_id.return_value = real_job_id mock_hook.return_value.insert_job.return_value = MagicMock(job_id="real_job_id", error_result=False) - mock_hook.return_value.project_id = PROJECT_ID + mock_hook.return_value.project_id = JOB_PROJECT_ID operator = BigQueryToGCSOperator( - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, task_id=TASK_ID, source_project_dataset_table=source_project_dataset_table, destination_cloud_storage_uris=destination_cloud_storage_uris, @@ -146,7 +148,7 @@ def test_execute_deferrable_mode(self, mock_hook): mock_hook.return_value.insert_job.assert_called_once_with( configuration=expected_configuration, job_id="123456_hash", - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, location=None, timeout=None, retry=DEFAULT_RETRY, diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py b/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py index 80958f281f63..1a0f38183754 100644 --- a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py +++ b/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py @@ -40,6 +40,7 @@ PROJECT_ID = "test-project" DATASET = "dataset" TABLE = "table" +JOB_PROJECT_ID = "job-project-id" WRITE_DISPOSITION = "WRITE_TRUNCATE" MAX_ID_KEY = "id" TEST_DATASET_LOCATION = "US" @@ -85,6 +86,7 @@ def test_max_value_external_table_should_execute_successfully(self, hook): schema_fields=SCHEMA_FIELDS, max_id_key=MAX_ID_KEY, external_table=True, + project_id=JOB_PROJECT_ID, ) result = operator.execute(context=MagicMock()) @@ -93,7 +95,7 @@ def test_max_value_external_table_should_execute_successfully(self, hook): hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -123,7 +125,7 @@ def test_max_value_external_table_should_execute_successfully(self, hook): "schemaUpdateOptions": [], } }, - project_id=hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) @@ -145,6 +147,7 @@ def test_max_value_without_external_table_should_execute_successfully(self, hook max_id_key=MAX_ID_KEY, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) result = operator.execute(context=MagicMock()) @@ -172,7 +175,7 @@ def test_max_value_without_external_table_should_execute_successfully(self, hook job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ), @@ -184,7 +187,7 @@ def test_max_value_without_external_table_should_execute_successfully(self, hook "schemaUpdateOptions": [], } }, - project_id=hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ), ] @@ -208,6 +211,7 @@ def test_max_value_should_throw_ex_when_query_returns_no_rows(self, hook): max_id_key=MAX_ID_KEY, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -233,7 +237,7 @@ def test_max_value_should_throw_ex_when_query_returns_no_rows(self, hook): job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ), @@ -245,7 +249,7 @@ def test_max_value_should_throw_ex_when_query_returns_no_rows(self, hook): "schemaUpdateOptions": [], } }, - project_id=hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ), ] @@ -269,13 +273,14 @@ def test_labels_external_table_should_execute_successfully(self, hook): write_disposition=WRITE_DISPOSITION, external_table=True, labels=LABELS, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": LABELS, @@ -316,6 +321,7 @@ def test_labels_without_external_table_should_execute_successfully(self, hook): schema_fields=SCHEMA_FIELDS, external_table=False, labels=LABELS, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -342,7 +348,7 @@ def test_labels_without_external_table_should_execute_successfully(self, hook): job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ) @@ -368,13 +374,14 @@ def test_description_external_table_should_execute_successfully(self, hook): schema_fields=SCHEMA_FIELDS, description=DESCRIPTION, external_table=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -416,6 +423,7 @@ def test_description_without_external_table_should_execute_successfully(self, ho write_disposition=WRITE_DISPOSITION, external_table=False, description=DESCRIPTION, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -441,7 +449,7 @@ def test_description_without_external_table_should_execute_successfully(self, ho fieldDelimiter=",", ), }, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, location=None, job_id=pytest.real_job_id, timeout=None, @@ -467,6 +475,7 @@ def test_source_objs_as_list_external_table_should_execute_successfully(self, ho write_disposition=WRITE_DISPOSITION, destination_project_dataset_table=TEST_EXPLICIT_DEST, external_table=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -474,7 +483,7 @@ def test_source_objs_as_list_external_table_should_execute_successfully(self, ho hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -515,6 +524,7 @@ def test_source_objs_as_list_without_external_table_should_execute_successfully( write_disposition=WRITE_DISPOSITION, destination_project_dataset_table=TEST_EXPLICIT_DEST, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -545,7 +555,7 @@ def test_source_objs_as_list_without_external_table_should_execute_successfully( job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ) @@ -569,6 +579,7 @@ def test_source_objs_as_string_external_table_should_execute_successfully(self, write_disposition=WRITE_DISPOSITION, destination_project_dataset_table=TEST_EXPLICIT_DEST, external_table=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -576,7 +587,7 @@ def test_source_objs_as_string_external_table_should_execute_successfully(self, hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -615,6 +626,7 @@ def test_source_objs_as_string_without_external_table_should_execute_successfull destination_project_dataset_table=TEST_EXPLICIT_DEST, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -645,7 +657,7 @@ def test_source_objs_as_string_without_external_table_should_execute_successfull job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ) @@ -672,6 +684,7 @@ def test_schema_obj_external_table_should_execute_successfully(self, bq_hook, gc write_disposition=WRITE_DISPOSITION, destination_project_dataset_table=TEST_EXPLICIT_DEST, external_table=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -679,7 +692,7 @@ def test_schema_obj_external_table_should_execute_successfully(self, bq_hook, gc bq_hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -723,6 +736,7 @@ def test_schema_obj_without_external_table_should_execute_successfully(self, bq_ destination_project_dataset_table=TEST_EXPLICIT_DEST, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -746,7 +760,7 @@ def test_schema_obj_without_external_table_should_execute_successfully(self, bq_ "encoding": "UTF-8", } }, - project_id=bq_hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, location=None, job_id=pytest.real_job_id, timeout=None, @@ -774,6 +788,7 @@ def test_autodetect_none_external_table_should_execute_successfully(self, hook): destination_project_dataset_table=TEST_EXPLICIT_DEST, external_table=True, autodetect=None, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -781,7 +796,7 @@ def test_autodetect_none_external_table_should_execute_successfully(self, hook): hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -820,6 +835,7 @@ def test_autodetect_none_without_external_table_should_execute_successfully(self write_disposition=WRITE_DISPOSITION, autodetect=None, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -842,7 +858,7 @@ def test_autodetect_none_without_external_table_should_execute_successfully(self "encoding": "UTF-8", } }, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, location=None, job_id=pytest.real_job_id, timeout=None, @@ -870,6 +886,7 @@ def test_execute_should_throw_ex_when_no_bucket_specified(self, hook): max_id_key=MAX_ID_KEY, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -890,6 +907,7 @@ def test_execute_should_throw_ex_when_no_source_objects_specified(self, hook): max_id_key=MAX_ID_KEY, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -912,6 +930,7 @@ def test_execute_should_throw_ex_when_no_destination_project_dataset_table_speci max_id_key=MAX_ID_KEY, write_disposition=WRITE_DISPOSITION, external_table=False, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -942,6 +961,7 @@ def test_source_format_check_should_throw_ex_when_incorrect_source_type( external_table=False, autodetect=False, source_format="incorrect", + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -973,6 +993,7 @@ def test_schema_fields_integer_scanner_external_table_should_execute_successfull max_id_key=MAX_ID_KEY, external_table=True, autodetect=True, + project_id=JOB_PROJECT_ID, ) result = operator.execute(context=MagicMock()) @@ -981,7 +1002,7 @@ def test_schema_fields_integer_scanner_external_table_should_execute_successfull bq_hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -1010,7 +1031,7 @@ def test_schema_fields_integer_scanner_external_table_should_execute_successfull "schemaUpdateOptions": [], } }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ) @mock.patch(GCS_TO_BQ_PATH.format("GCSHook")) @@ -1041,6 +1062,7 @@ def test_schema_fields_integer_scanner_without_external_table_should_execute_suc max_id_key=MAX_ID_KEY, external_table=False, autodetect=True, + project_id=JOB_PROJECT_ID, ) result = operator.execute(context=MagicMock()) @@ -1067,7 +1089,7 @@ def test_schema_fields_integer_scanner_without_external_table_should_execute_suc job_id=pytest.real_job_id, location=None, nowait=True, - project_id=bq_hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ), @@ -1079,7 +1101,7 @@ def test_schema_fields_integer_scanner_without_external_table_should_execute_suc "schemaUpdateOptions": [], } }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ), ] @@ -1104,6 +1126,7 @@ def test_schema_fields_without_external_table_should_execute_successfully(self, schema_fields=SCHEMA_FIELDS_INT, external_table=False, autodetect=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) @@ -1129,7 +1152,7 @@ def test_schema_fields_without_external_table_should_execute_successfully(self, job_id=pytest.real_job_id, location=None, nowait=True, - project_id=hook.return_value.split_tablename.return_value[0], + project_id=JOB_PROJECT_ID, retry=DEFAULT_RETRY, timeout=None, ) @@ -1156,13 +1179,14 @@ def test_schema_fields_external_table_should_execute_successfully(self, hook): schema_fields=SCHEMA_FIELDS_INT, external_table=True, autodetect=True, + project_id=JOB_PROJECT_ID, ) operator.execute(context=MagicMock()) hook.return_value.create_empty_table.assert_called_once_with( exists_ok=True, location=None, - project_id=PROJECT_ID, + project_id=JOB_PROJECT_ID, table_resource={ "tableReference": {"projectId": PROJECT_ID, "datasetId": DATASET, "tableId": TABLE}, "labels": {}, @@ -1208,6 +1232,7 @@ def test_execute_without_external_table_async_should_execute_successfully(self, external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(TaskDeferred) as exc: @@ -1233,6 +1258,7 @@ def test_execute_without_external_table_async_should_throw_ex_when_event_status_ external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) operator.execute_complete( context=None, event={"status": "error", "message": "test failure message"} @@ -1253,6 +1279,7 @@ def test_execute_logging_without_external_table_async_should_execute_successfull external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with mock.patch.object(operator.log, "info") as mock_log_info: operator.execute_complete( @@ -1286,6 +1313,7 @@ def test_execute_without_external_table_generate_job_id_async_should_execute_suc external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(TaskDeferred): @@ -1326,6 +1354,7 @@ def test_execute_without_external_table_reattach_async_should_execute_successful external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(TaskDeferred): @@ -1334,7 +1363,7 @@ def test_execute_without_external_table_reattach_async_should_execute_successful hook.return_value.get_job.assert_called_once_with( location=TEST_DATASET_LOCATION, job_id=pytest.real_job_id, - project_id=hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ) job._begin.assert_called_once_with() @@ -1365,6 +1394,7 @@ def test_execute_without_external_table_force_rerun_async_should_execute_success external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(AirflowException) as exc: @@ -1381,7 +1411,7 @@ def test_execute_without_external_table_force_rerun_async_should_execute_success hook.return_value.get_job.assert_called_once_with( location=TEST_DATASET_LOCATION, job_id=pytest.real_job_id, - project_id=hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ) @mock.patch(GCS_TO_BQ_PATH.format("GCSHook")) @@ -1406,6 +1436,7 @@ def test_schema_fields_without_external_table_async_should_execute_successfully( external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(TaskDeferred): @@ -1437,7 +1468,7 @@ def test_schema_fields_without_external_table_async_should_execute_successfully( schema={"fields": SCHEMA_FIELDS}, ), }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, location=None, job_id=pytest.real_job_id, timeout=None, @@ -1452,7 +1483,7 @@ def test_schema_fields_without_external_table_async_should_execute_successfully( "schemaUpdateOptions": [], } }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ), ] @@ -1482,6 +1513,7 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu external_table=False, autodetect=True, deferrable=True, + project_id=JOB_PROJECT_ID, ) with pytest.raises(TaskDeferred): @@ -1512,7 +1544,7 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu encoding="UTF-8", ), }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, location=None, job_id=pytest.real_job_id, timeout=None, @@ -1527,7 +1559,7 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu "schemaUpdateOptions": [], } }, - project_id=bq_hook.return_value.project_id, + project_id=JOB_PROJECT_ID, ), ] From d49d8856eb33c0c01c0ea70311ff7ca64b7c3e22 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 7 Jul 2023 07:34:39 +0100 Subject: [PATCH 088/533] Refactor SetupTeardownContext Manager (#32235) * Refactor SetupTeardownContext Manager This refactor also includes preventing unreasonable usage of the context manager and typing of the class * Fix typing * Use else --- airflow/decorators/setup_teardown.py | 2 +- airflow/models/taskmixin.py | 6 - airflow/utils/setup_teardown.py | 176 +++++++++++++----------- tests/decorators/test_setup_teardown.py | 97 ++++++++++++- 4 files changed, 188 insertions(+), 93 deletions(-) diff --git a/airflow/decorators/setup_teardown.py b/airflow/decorators/setup_teardown.py index e810c75a8812..e9bb19b5c390 100644 --- a/airflow/decorators/setup_teardown.py +++ b/airflow/decorators/setup_teardown.py @@ -72,7 +72,7 @@ def __enter__(self): # means we have XComArgs operators = [task.operator for task in self.tasks] SetupTeardownContext.push_setup_teardown_task(operators) - return self + return SetupTeardownContext def __exit__(self, exc_type, exc_val, exc_tb): SetupTeardownContext.set_work_task_roots_and_leaves() diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py index 0e8343664293..658759a9b763 100644 --- a/airflow/models/taskmixin.py +++ b/airflow/models/taskmixin.py @@ -123,21 +123,15 @@ def set_setup_teardown_ctx_dependencies(self, other: DependencyMixin | Sequence[ op1 = self if isinstance(self, PlainXComArg): op1 = self.operator - if getattr(op1, "is_setup") or getattr(op1, "is_teardown"): - return SetupTeardownContext.update_context_map(op1) if isinstance(other, Sequence): for op in other: if isinstance(op, PlainXComArg): op = op.operator - if getattr(op, "is_setup") or getattr(op, "is_teardown"): - continue SetupTeardownContext.update_context_map(op) return if isinstance(other, PlainXComArg): other = other.operator - if getattr(other, "is_setup") or getattr(other, "is_teardown"): - return SetupTeardownContext.update_context_map(other) diff --git a/airflow/utils/setup_teardown.py b/airflow/utils/setup_teardown.py index 03eada60b6bc..a82fbe739e14 100644 --- a/airflow/utils/setup_teardown.py +++ b/airflow/utils/setup_teardown.py @@ -17,12 +17,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from airflow.exceptions import AirflowException if TYPE_CHECKING: - from airflow.models.operator import Operator + from airflow.models.abstractoperator import AbstractOperator + from airflow.models.taskmixin import DependencyMixin + from airflow.models.xcom_arg import PlainXComArg class BaseSetupTeardownContext: @@ -31,27 +33,27 @@ class BaseSetupTeardownContext: :meta private: """ - _context_managed_setup_task: Operator | list[Operator] | None = None - _previous_context_managed_setup_task: list[Operator | list[Operator]] = [] - _context_managed_teardown_task: Operator | list[Operator] | None = None - _previous_context_managed_teardown_task: list[Operator | list[Operator]] = [] active: bool = False - context_map: dict[Operator | tuple[Operator], list[Operator]] = {} + context_map: dict[AbstractOperator | tuple[AbstractOperator], list[AbstractOperator]] = {} + _context_managed_setup_task: AbstractOperator | list[AbstractOperator] = [] + _previous_context_managed_setup_task: list[AbstractOperator | list[AbstractOperator]] = [] + _context_managed_teardown_task: AbstractOperator | list[AbstractOperator] = [] + _previous_context_managed_teardown_task: list[AbstractOperator | list[AbstractOperator]] = [] @classmethod - def push_context_managed_setup_task(cls, task: Operator | list[Operator]): + def push_context_managed_setup_task(cls, task: AbstractOperator | list[AbstractOperator]): if cls._context_managed_setup_task: cls._previous_context_managed_setup_task.append(cls._context_managed_setup_task) cls._context_managed_setup_task = task @classmethod - def push_context_managed_teardown_task(cls, task: Operator | list[Operator]): + def push_context_managed_teardown_task(cls, task: AbstractOperator | list[AbstractOperator]): if cls._context_managed_teardown_task: cls._previous_context_managed_teardown_task.append(cls._context_managed_teardown_task) cls._context_managed_teardown_task = task @classmethod - def pop_context_managed_setup_task(cls) -> Operator | list[Operator] | None: + def pop_context_managed_setup_task(cls) -> AbstractOperator | list[AbstractOperator]: old_setup_task = cls._context_managed_setup_task if cls._previous_context_managed_setup_task: cls._context_managed_setup_task = cls._previous_context_managed_setup_task.pop() @@ -63,29 +65,37 @@ def pop_context_managed_setup_task(cls) -> Operator | list[Operator] | None: else: setup_task.set_downstream(old_setup_task) else: - cls._context_managed_setup_task = None + cls._context_managed_setup_task = [] return old_setup_task @classmethod - def update_context_map(cls, operator): - ctx = BaseSetupTeardownContext.context_map - if setup_task := BaseSetupTeardownContext.get_context_managed_setup_task(): + def update_context_map(cls, task: DependencyMixin): + from airflow.models.abstractoperator import AbstractOperator + + task_ = cast(AbstractOperator, task) + if task_.is_setup or task_.is_teardown: + return + ctx = cls.context_map + + def _get_or_set_item(item): + if ctx.get(item) is None: + ctx[item] = [task_] + else: + ctx[item].append(task_) + + if setup_task := cls.get_context_managed_setup_task(): if isinstance(setup_task, list): - setup_task = tuple(setup_task) - if ctx.get(setup_task) is None: - ctx[setup_task] = [operator] + _get_or_set_item(tuple(setup_task)) else: - ctx[setup_task].append(operator) - if teardown_task := BaseSetupTeardownContext.get_context_managed_teardown_task(): + _get_or_set_item(setup_task) + if teardown_task := cls.get_context_managed_teardown_task(): if isinstance(teardown_task, list): - teardown_task = tuple(teardown_task) - if ctx.get(teardown_task) is None: - ctx[teardown_task] = [operator] + _get_or_set_item(tuple(teardown_task)) else: - ctx[teardown_task].append(operator) + _get_or_set_item(teardown_task) @classmethod - def pop_context_managed_teardown_task(cls) -> Operator | list[Operator] | None: + def pop_context_managed_teardown_task(cls) -> AbstractOperator | list[AbstractOperator]: old_teardown_task = cls._context_managed_teardown_task if cls._previous_context_managed_teardown_task: cls._context_managed_teardown_task = cls._previous_context_managed_teardown_task.pop() @@ -97,70 +107,64 @@ def pop_context_managed_teardown_task(cls) -> Operator | list[Operator] | None: else: teardown_task.set_upstream(old_teardown_task) else: - cls._context_managed_teardown_task = None + cls._context_managed_teardown_task = [] return old_teardown_task @classmethod - def get_context_managed_setup_task(cls) -> Operator | list[Operator] | None: + def get_context_managed_setup_task(cls) -> AbstractOperator | list[AbstractOperator]: return cls._context_managed_setup_task @classmethod - def get_context_managed_teardown_task(cls) -> Operator | list[Operator] | None: + def get_context_managed_teardown_task(cls) -> AbstractOperator | list[AbstractOperator]: return cls._context_managed_teardown_task @classmethod - def push_setup_teardown_task(cls, operator: Operator | list[Operator]): + def push_setup_teardown_task(cls, operator: AbstractOperator | list[AbstractOperator]): if isinstance(operator, list): - first_task: Operator = operator[0] - if first_task.is_teardown: - if not all(task.is_teardown == first_task.is_teardown for task in operator): - raise ValueError("All tasks in the list must be either setup or teardown tasks") - upstream_tasks = first_task.upstream_list - for task in upstream_tasks: - if not task.is_setup and not task.is_teardown: - raise ValueError( - "All upstream tasks in the context manager must be a setup or teardown task" - ) - BaseSetupTeardownContext.push_context_managed_teardown_task(operator) - upstream_setup: list[Operator] = [task for task in upstream_tasks if task.is_setup] - if upstream_setup: - BaseSetupTeardownContext.push_context_managed_setup_task(upstream_setup) - elif first_task.is_setup: - if not all(task.is_setup == first_task.is_setup for task in operator): - raise ValueError("All tasks in the list must be either setup or teardown tasks") - for task in first_task.upstream_list: - if not task.is_setup and not task.is_teardown: - raise ValueError( - "All upstream tasks in the context manager must be a setup or teardown task" - ) - BaseSetupTeardownContext.push_context_managed_setup_task(operator) - downstream_teardown: list[Operator] = [ - task for task in first_task.downstream_list if task.is_teardown - ] - if downstream_teardown: - BaseSetupTeardownContext.push_context_managed_teardown_task(downstream_teardown) + if operator[0].is_teardown: + cls._push_tasks(operator) + elif operator[0].is_setup: + cls._push_tasks(operator, setup=True) elif operator.is_teardown: - upstream_tasks = operator.upstream_list - for task in upstream_tasks: - if not task.is_setup and not task.is_teardown: - raise ValueError( - "All upstream tasks in the context manager must be a setup or teardown task" - ) - BaseSetupTeardownContext.push_context_managed_teardown_task(operator) - upstream_setup = [task for task in upstream_tasks if task.is_setup] - if upstream_setup: - BaseSetupTeardownContext.push_context_managed_setup_task(upstream_setup) + cls._push_tasks(operator) elif operator.is_setup: - for task in operator.upstream_list: - if not task.is_setup and not task.is_teardown: - raise ValueError( - "All upstream tasks in the context manager must be a setup or teardown task" + cls._push_tasks(operator, setup=True) + cls.active = True + + @classmethod + def _push_tasks(cls, operator: AbstractOperator | list[AbstractOperator], setup: bool = False): + if isinstance(operator, list): + upstream_tasks = operator[0].upstream_list + downstream_list = operator[0].downstream_list + if not all(task.is_setup == operator[0].is_setup for task in operator): + cls.error("All tasks in the list must be either setup or teardown tasks") + else: + upstream_tasks = operator.upstream_list + downstream_list = operator.downstream_list + if setup: + if upstream_tasks: + cls.error("Setup tasks cannot have upstreams set manually on the context manager") + cls.push_context_managed_setup_task(operator) + for task in downstream_list: + if not task.is_teardown: + cls.error( + "Downstream tasks to a setup task must be a teardown task on the context manager" ) - BaseSetupTeardownContext.push_context_managed_setup_task(operator) - downstream_teardown = [task for task in operator.downstream_list if task.is_teardown] - if downstream_teardown: - BaseSetupTeardownContext.push_context_managed_teardown_task(downstream_teardown) - BaseSetupTeardownContext.active = True + if task.downstream_list: + cls.error("Multiple shifts are not allowed in the context manager") + if downstream_list: + cls.push_context_managed_teardown_task(list(downstream_list)) + else: + for task in upstream_tasks: + if not task.is_setup: + cls.error("Upstream tasks to a teardown task must be a setup task on the context manager") + if task.upstream_list: + cls.error("Multiple shifts are not allowed in the context manager") + if downstream_list: + cls.error("Downstream to a teardown task cannot be set manually on the context manager") + cls.push_context_managed_teardown_task(operator) + if upstream_tasks: + cls.push_context_managed_setup_task(list(upstream_tasks)) @classmethod def set_work_task_roots_and_leaves(cls): @@ -190,22 +194,32 @@ def set_work_task_roots_and_leaves(cls): task << leaves else: teardown_task << leaves - setup_task = BaseSetupTeardownContext.pop_context_managed_setup_task() - teardown_task = BaseSetupTeardownContext.pop_context_managed_teardown_task() + setup_task = cls.pop_context_managed_setup_task() + teardown_task = cls.pop_context_managed_teardown_task() if isinstance(setup_task, list): setup_task = tuple(setup_task) if isinstance(teardown_task, list): teardown_task = tuple(teardown_task) - BaseSetupTeardownContext.active = False - BaseSetupTeardownContext.context_map.pop(setup_task, None) - BaseSetupTeardownContext.context_map.pop(teardown_task, None) + cls.active = False + cls.context_map.pop(setup_task, None) + cls.context_map.pop(teardown_task, None) + + @classmethod + def error(cls, message: str): + cls.active = False + cls.context_map.clear() + cls._context_managed_setup_task = [] + cls._context_managed_teardown_task = [] + cls._previous_context_managed_setup_task = [] + cls._previous_context_managed_teardown_task = [] + raise ValueError(message) class SetupTeardownContext(BaseSetupTeardownContext): """Context manager for setup and teardown tasks.""" @staticmethod - def add_task(task): + def add_task(task: AbstractOperator | PlainXComArg): """Add task to context manager.""" from airflow.models.xcom_arg import PlainXComArg diff --git a/tests/decorators/test_setup_teardown.py b/tests/decorators/test_setup_teardown.py index e93216eaa923..8163c42b69ac 100644 --- a/tests/decorators/test_setup_teardown.py +++ b/tests/decorators/test_setup_teardown.py @@ -25,6 +25,36 @@ from airflow.operators.bash import BashOperator +def make_task(name, type_, setup_=False, teardown_=False): + if type_ == "classic" and setup_: + return BashOperator(task_id=name, bash_command="echo 1").as_setup() + elif type_ == "classic" and teardown_: + return BashOperator(task_id=name, bash_command="echo 1").as_teardown() + elif type_ == "classic": + return BashOperator(task_id=name, bash_command="echo 1") + elif setup_: + + @setup + def setuptask(): + pass + + return setuptask.override(task_id=name)() + elif teardown_: + + @teardown + def teardowntask(): + pass + + return teardowntask.override(task_id=name)() + else: + + @task + def my_task(): + pass + + return my_task.override(task_id=name)() + + class TestSetupTearDownTask: def test_marking_functions_as_setup_task(self, dag_maker): @setup @@ -1114,7 +1144,7 @@ def teardowntask(): print("teardowntask") with pytest.raises( - ValueError, match="All upstream tasks in the context manager must be a setup or teardown task" + ValueError, match="Upstream tasks to a teardown task must be a setup task on the context manager" ): with dag_maker(): with setuptask() >> mytask() >> teardowntask(): @@ -1130,7 +1160,7 @@ def setuptask(): print("setuptask") with pytest.raises( - ValueError, match="All upstream tasks in the context manager must be a setup or teardown task" + ValueError, match="Setup tasks cannot have upstreams set manually on the context manager" ): with dag_maker(): with mytask() >> setuptask(): @@ -1150,7 +1180,7 @@ def teardowntask2(): print("teardowntask") with pytest.raises( - ValueError, match="All upstream tasks in the context manager must be a setup or teardown task" + ValueError, match="Upstream tasks to a teardown task must be a setup task on the context manager" ): with dag_maker(): with mytask() >> context_wrapper([teardowntask(), teardowntask2()]): @@ -1170,10 +1200,11 @@ def setuptask2(): print("setuptask") with pytest.raises( - ValueError, match="All upstream tasks in the context manager must be a setup or teardown task" + ValueError, + match="Downstream tasks to a setup task must be a teardown task on the context manager", ): with dag_maker(): - with mytask() >> context_wrapper([setuptask(), setuptask2()]): + with mytask() << context_wrapper([setuptask(), setuptask2()]): ... def test_tasks_decorators_called_outside_context_manager_can_link_up(self, dag_maker): @@ -1359,3 +1390,59 @@ def mytask(): assert dag.task_group.children["mytask2"].upstream_task_ids == {"setuptask2"} assert dag.task_group.children["mytask2"].downstream_task_ids == {"teardowntask2"} assert dag.task_group.children["teardowntask2"].upstream_task_ids == {"mytask2", "setuptask2"} + + def test_prevent_bad_usage_of_contextmanager(self, dag_maker): + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + with pytest.raises( + ValueError, + match="Downstream to a teardown task cannot be set manually on the context manager", + ): + with setuptask << teardowntask: + ... + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + setuptask2 = make_task("setuptask2", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + with pytest.raises(ValueError, match="Multiple shifts are not allowed in the context manager"): + with setuptask >> setuptask2 >> teardowntask: + ... + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + with pytest.raises( + ValueError, match="Setup tasks cannot have upstreams set manually on the context manager" + ): + with teardowntask >> setuptask: + ... + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + teardowntask2 = make_task("teardowntask2", type_="decorated", teardown_=True) + with pytest.raises(ValueError, match="Multiple shifts are not allowed in the context manager"): + with teardowntask2 << teardowntask << setuptask: + ... + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + setuptask2 = make_task("setuptask2", type_="decorated", setup_=True) + with pytest.raises( + ValueError, match="Setup tasks cannot have upstreams set manually on the context manager" + ): + with teardowntask >> context_wrapper([setuptask2, setuptask]): + ... + + with dag_maker(): + setuptask = make_task("setuptask", type_="decorated", setup_=True) + teardowntask = make_task("teardowntask", type_="decorated", teardown_=True) + setuptask2 = make_task("setuptask2", type_="decorated", setup_=True) + teardowntask2 = make_task("teardowntask2", type_="decorated", teardown_=True) + with pytest.raises(ValueError, match="Multiple shifts are not allowed in the context manager"): + with setuptask >> setuptask2 >> context_wrapper([teardowntask, teardowntask2]): + ... From 4bbb633830f8380d7776d12ddd3557dcaaa3133f Mon Sep 17 00:00:00 2001 From: Bruce Date: Thu, 6 Jul 2023 23:54:53 -0700 Subject: [PATCH 089/533] Use WebEncoder to encode DagRun.conf in DagRun's list view (#32385) * Use WebEncoder to encode DagRun.conf in DagRun's list view * test case for json_f with WebEncoder * Always use WebEncoder in web views --- airflow/www/utils.py | 3 ++- tests/www/test_utils.py | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index b31f9326d988..4aaeda595bb2 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -49,6 +49,7 @@ from airflow.utils import timezone from airflow.utils.code_utils import get_python_source from airflow.utils.helpers import alchemy_to_dict +from airflow.utils.json import WebEncoder from airflow.utils.state import State, TaskInstanceState from airflow.www.forms import DateTimeWithTimezoneField from airflow.www.widgets import AirflowDateTimePickerWidget @@ -481,7 +482,7 @@ def json_f(attr_name): def json_(attr): f = attr.get(attr_name) - serialized = json.dumps(f) + serialized = json.dumps(f, cls=WebEncoder) return Markup("{}").format(serialized) return json_ diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index dd09a24b3f09..12fe017c6270 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -19,13 +19,15 @@ import re from datetime import datetime +from unittest.mock import Mock from urllib.parse import parse_qs from bs4 import BeautifulSoup +from markupsafe import Markup from airflow.utils import json as utils_json from airflow.www import utils -from airflow.www.utils import wrapped_markdown +from airflow.www.utils import json_f, wrapped_markdown class TestUtils: @@ -243,6 +245,26 @@ def test_get_dag_run_conf(self): ) assert expected_encoded_dag_run_conf == encoded_dag_run_conf + def test_json_f_webencoder(self): + dag_run_conf = { + "1": "string", + "2": b"bytes", + "3": 123, + "4": "à".encode("latin"), + "5": datetime(2023, 1, 1), + } + expected_encoded_dag_run_conf = ( + # HTML sanitization is insane + '{"1": "string", "2": "bytes", "3": 123, "4": "\\u00e0", "5": "2023-01-01T00:00:00+00:00"}' + ) + expected_markup = Markup("{}").format(expected_encoded_dag_run_conf) + + formatter = json_f("conf") + dagrun = Mock() + dagrun.get = Mock(return_value=dag_run_conf) + + assert formatter(dagrun) == expected_markup + class TestWrappedMarkdown: def test_wrapped_markdown_with_docstring_curly_braces(self): From b1c1f97744e735c5230fc658cf8deb71aa770f9b Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Fri, 7 Jul 2023 11:19:27 +0300 Subject: [PATCH 090/533] Add explanation about tag creation and cleanup (#32406) * Add explanation about tag creation and cleanup --- docs/apache-airflow/howto/add-dag-tags.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/apache-airflow/howto/add-dag-tags.rst b/docs/apache-airflow/howto/add-dag-tags.rst index 7e75cec734bc..5b7db76229f0 100644 --- a/docs/apache-airflow/howto/add-dag-tags.rst +++ b/docs/apache-airflow/howto/add-dag-tags.rst @@ -38,3 +38,7 @@ In your Dag file, pass a list of tags you want to add to DAG object: **Screenshot**: .. image:: ../img/add-dag-tags.png + +Tags are registered as part of dag parsing. +In case of stale tags, you can purge old data with the Airflow CLI command ``airflow db clean``. +See :ref:`db clean usage` for more details. From 53d9421905c3c268744c3e43601db63bc7b6fa87 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 7 Jul 2023 18:33:25 +0800 Subject: [PATCH 091/533] build(pre-commit): check deferrable default value (#32370) --------- Co-authored-by: Tzu-ping Chung --- .pre-commit-config.yaml | 6 + STATIC_CODE_CHECKS.rst | 2 + .../google/cloud/operators/datafusion.py | 3 +- .../snowflake/operators/snowflake.py | 7 +- .../src/airflow_breeze/pre_commit_ids.py | 1 + images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-commands.svg | 104 ++++++------- images/breeze/output_setup.svg | 32 ++-- ...utput_setup_check-all-params-in-groups.svg | 54 +++---- ...output_setup_regenerate-command-images.svg | 62 ++++---- images/breeze/output_static-checks.svg | 140 +++++++++--------- .../pre_commit_check_deferrable_default.py | 108 ++++++++++++++ 12 files changed, 321 insertions(+), 200 deletions(-) create mode 100755 scripts/ci/pre_commit/pre_commit_check_deferrable_default.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efcf6c6e3e24..afac835f2bb9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -937,6 +937,12 @@ repos: pass_filenames: true files: \.py$ exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^docker_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$ + - id: check-deferrable-default-value + name: Check default value of deferrable attribute + language: python + entry: ./scripts/ci/pre_commit/pre_commit_check_deferrable_default.py + pass_filenames: false + files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$ ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: mypy-dev diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 381dc53c93fb..a9ec8c106aa1 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -171,6 +171,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-decorated-operator-implements-custom-name | Check @task decorator implements custom_operator_name | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-deferrable-default-value | Check default value of deferrable attribute | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-docstring-param-types | Check that docstrings do not specify param types | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-example-dags-urls | Check that example dags url include provider versions | | diff --git a/airflow/providers/google/cloud/operators/datafusion.py b/airflow/providers/google/cloud/operators/datafusion.py index b214c36d7844..8ae64e1a4b72 100644 --- a/airflow/providers/google/cloud/operators/datafusion.py +++ b/airflow/providers/google/cloud/operators/datafusion.py @@ -24,6 +24,7 @@ from googleapiclient.errors import HttpError from airflow import AirflowException +from airflow.configuration import conf from airflow.providers.google.cloud.hooks.datafusion import SUCCESS_STATES, DataFusionHook, PipelineStates from airflow.providers.google.cloud.links.datafusion import ( DataFusionInstanceLink, @@ -759,7 +760,7 @@ def __init__( gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, asynchronous=False, - deferrable=False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), poll_interval=3.0, **kwargs, ) -> None: diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 4087397d8304..61e60b5781fb 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Sequence, SupportsAbs, cast from airflow import AirflowException +from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.common.sql.operators.sql import ( SQLCheckOperator, @@ -30,9 +31,7 @@ SQLIntervalCheckOperator, SQLValueCheckOperator, ) -from airflow.providers.snowflake.hooks.snowflake_sql_api import ( - SnowflakeSqlApiHook, -) +from airflow.providers.snowflake.hooks.snowflake_sql_api import SnowflakeSqlApiHook from airflow.providers.snowflake.triggers.snowflake_trigger import SnowflakeSqlApiTrigger if TYPE_CHECKING: @@ -451,7 +450,7 @@ def __init__( token_life_time: timedelta = LIFETIME, token_renewal_delta: timedelta = RENEWAL_DELTA, bindings: dict[str, Any] | None = None, - deferrable: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs: Any, ) -> None: self.snowflake_conn_id = snowflake_conn_id diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 1349a962107e..ed7a06d4a466 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -39,6 +39,7 @@ "check-core-deprecation-classes", "check-daysago-import-from-utils", "check-decorated-operator-implements-custom-name", + "check-deferrable-default-value", "check-docstring-param-types", "check-example-dags-urls", "check-executables-have-shebangs", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 241dc9bd43f6..88363d44e93c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -60,7 +60,7 @@ setup:version:123b462a421884dc2320ffc5e54b2478 setup:201c30ea237ea013a9a209a77092a2e8 shell:13f90c5749811e2f00e24d95e44e946d start-airflow:22c118d58b13a9d190e966bed5bb8ed8 -static-checks:a0f6ae35129b99b88ce65aede2c3a150 +static-checks:521edd8e7669b666b288ad0243b45be7 testing:docker-compose-tests:70167e67853cacd9ca784695d65a7846 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b testing:integration-tests:35f0ac57157bf8fe227fd080cf216622 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index f313cb8abe78..e5b2130124d5 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #868887 } -.breeze-help-r5 { fill: #68a0b3;font-weight: bold } +.breeze-help-r4 { fill: #68a0b3;font-weight: bold } +.breeze-help-r5 { fill: #868887 } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -211,57 +211,57 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg index 566cad3ff843..3698b389cb23 100644 --- a/images/breeze/output_setup.svg +++ b/images/breeze/output_setup.svg @@ -35,8 +35,8 @@ .breeze-setup-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-r2 { fill: #c5c8c6 } .breeze-setup-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-r4 { fill: #868887 } -.breeze-setup-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-r5 { fill: #868887 } .breeze-setup-r6 { fill: #98a84b;font-weight: bold } @@ -108,23 +108,23 @@ -Usage: breeze setup [OPTIONS] COMMAND [ARGS]... +Usage: breeze setup [OPTIONS] COMMAND [ARGS]... Tools that developers can use to configure Breeze -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Setup ──────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│autocomplete                    Enables autocompletion of breeze commands.                                        │ -│self-upgrade                    Self upgrade Breeze.                                                              │ -│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ -│regenerate-command-images       Regenerate breeze command images.                                                 │ -│version                         Print information about version of apache-airflow-breeze.                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│check-all-params-in-groups                 Check that all parameters are put in groups.                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Setup ──────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│autocomplete                    Enables autocompletion of breeze commands.                                        │ +│self-upgrade                    Self upgrade Breeze.                                                              │ +│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ +│regenerate-command-images       Regenerate breeze command images.                                                 │ +│version                         Print information about version of apache-airflow-breeze.                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│check-all-params-in-groups                 Check that all parameters are put in groups.                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_check-all-params-in-groups.svg b/images/breeze/output_setup_check-all-params-in-groups.svg index fbdca6ae752f..c684f857c9f9 100644 --- a/images/breeze/output_setup_check-all-params-in-groups.svg +++ b/images/breeze/output_setup_check-all-params-in-groups.svg @@ -35,8 +35,8 @@ .breeze-setup-check-all-params-in-groups-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-check-all-params-in-groups-r2 { fill: #c5c8c6 } .breeze-setup-check-all-params-in-groups-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-check-all-params-in-groups-r4 { fill: #868887 } -.breeze-setup-check-all-params-in-groups-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-check-all-params-in-groups-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-check-all-params-in-groups-r5 { fill: #868887 } .breeze-setup-check-all-params-in-groups-r6 { fill: #8d7b39 } .breeze-setup-check-all-params-in-groups-r7 { fill: #98a84b;font-weight: bold } @@ -142,34 +142,34 @@ -Usage: breeze setup check-all-params-in-groups [OPTIONS] +Usage: breeze setup check-all-params-in-groups [OPTIONS] Check that all parameters are put in groups. -╭─ Check all params in groups flags ───────────────────────────────────────────────────────────────────────────────────╮ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ -│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |     │ -│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |      │ -│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster |  │ -│k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests | k8s:setup-env |│ -│k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | │ -│prod-image:verify | prod-image | release-management:create-minor-branch |                              │ -│release-management:generate-constraints | release-management:generate-issue-content-providers |        │ -│release-management:generate-providers-metadata | release-management:install-provider-packages |        │ -│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |       │ -│release-management:prepare-provider-packages | release-management:release-prod-images |                │ -│release-management:start-rc-process | release-management:start-release |                               │ -│release-management:verify-provider-packages | release-management | sbom:generate-provider-requirements │ -│| sbom:update-sbom-information | sbom | setup:autocomplete | setup:check-all-params-in-groups |        │ -│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |  │ -│start-airflow | static-checks | testing:docker-compose-tests | testing:helm-tests |                    │ -│testing:integration-tests | testing:tests | testing)                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Check all params in groups flags ───────────────────────────────────────────────────────────────────────────────────╮ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                      │ +│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |     │ +│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |      │ +│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster |  │ +│k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests | k8s:setup-env |│ +│k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | │ +│prod-image:verify | prod-image | release-management:create-minor-branch |                              │ +│release-management:generate-constraints | release-management:generate-issue-content-providers |        │ +│release-management:generate-providers-metadata | release-management:install-provider-packages |        │ +│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |       │ +│release-management:prepare-provider-packages | release-management:release-prod-images |                │ +│release-management:start-rc-process | release-management:start-release |                               │ +│release-management:verify-provider-packages | release-management | sbom:generate-provider-requirements │ +│| sbom:update-sbom-information | sbom | setup:autocomplete | setup:check-all-params-in-groups |        │ +│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |  │ +│start-airflow | static-checks | testing:docker-compose-tests | testing:helm-tests |                    │ +│testing:integration-tests | testing:tests | testing)                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_regenerate-command-images.svg b/images/breeze/output_setup_regenerate-command-images.svg index 00dc4b90232d..dd7ee821952b 100644 --- a/images/breeze/output_setup_regenerate-command-images.svg +++ b/images/breeze/output_setup_regenerate-command-images.svg @@ -35,8 +35,8 @@ .breeze-setup-regenerate-command-images-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-regenerate-command-images-r2 { fill: #c5c8c6 } .breeze-setup-regenerate-command-images-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-regenerate-command-images-r4 { fill: #868887 } -.breeze-setup-regenerate-command-images-r5 { fill: #68a0b3;font-weight: bold } +.breeze-setup-regenerate-command-images-r4 { fill: #68a0b3;font-weight: bold } +.breeze-setup-regenerate-command-images-r5 { fill: #868887 } .breeze-setup-regenerate-command-images-r6 { fill: #8d7b39 } .breeze-setup-regenerate-command-images-r7 { fill: #98a84b;font-weight: bold } @@ -154,38 +154,38 @@ -Usage: breeze setup regenerate-command-images [OPTIONS] +Usage: breeze setup regenerate-command-images [OPTIONS] Regenerate breeze command images. -╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--forceForces regeneration of all images│ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ -│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |  │ -│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |   │ -│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster │ -│| k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests |           │ -│k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build |│ -│prod-image:pull | prod-image:verify | prod-image | release-management:create-minor-branch |         │ -│release-management:generate-constraints | release-management:generate-issue-content-providers |     │ -│release-management:generate-providers-metadata | release-management:install-provider-packages |     │ -│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ -│release-management:prepare-provider-packages | release-management:release-prod-images |             │ -│release-management:start-rc-process | release-management:start-release |                            │ -│release-management:verify-provider-packages | release-management |                                  │ -│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |    │ -│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ -│setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks |                │ -│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ -│testing)                                                                                            │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--forceForces regeneration of all images│ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ +│(main | build-docs | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check |  │ +│ci:selective-check | ci | ci-image:build | ci-image:pull | ci-image:verify | ci-image | cleanup |   │ +│compile-www-assets | down | exec | k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster │ +│| k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s | k8s:logs | k8s:run-complete-tests |           │ +│k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | k8s | prod-image:build |│ +│prod-image:pull | prod-image:verify | prod-image | release-management:create-minor-branch |         │ +│release-management:generate-constraints | release-management:generate-issue-content-providers |     │ +│release-management:generate-providers-metadata | release-management:install-provider-packages |     │ +│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ +│release-management:prepare-provider-packages | release-management:release-prod-images |             │ +│release-management:start-rc-process | release-management:start-release |                            │ +│release-management:verify-provider-packages | release-management |                                  │ +│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |    │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ +│setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks |                │ +│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ +│testing)                                                                                            │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index e735b2afc306..b372524f7a48 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - + - + @@ -243,9 +243,12 @@ + + + - Command: static-checks + Command: static-checks @@ -256,72 +259,73 @@ -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... Run static checks. -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run.                                             │ -│(all | black | blacken-docs | check-aiobotocore-optional |                       │ -│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ -│check-apache-license-rat | check-base-operator-partial-arguments |               │ -│check-base-operator-usage | check-boring-cyborg-configuration |                  │ -│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ -│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ -│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ -│| check-docstring-param-types | check-example-dags-urls |                        │ -│check-executables-have-shebangs | check-extra-packages-references |              │ -│check-extras-order | check-for-inclusive-language | check-hooks-apply |          │ -│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ -│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ -│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ -│check-no-relative-imports | check-only-new-session-with-provide-session |        │ -│check-persist-credentials-disabled-in-github-workflows |                         │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ -│check-provider-yaml-valid | check-providers-init-file-missing |                  │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ -│check-start-date-not-used-in-defaults | check-system-tests-present |             │ -│check-system-tests-tocs | check-tests-unittest-testcase |                        │ -│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ -│| compile-www-assets | compile-www-assets-dev |                                  │ -│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ -│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ -│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ -│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ -│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ -│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ -│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ -│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ -│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ -│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-local-yml-file |                │ -│update-migration-references | update-providers-dependencies |                    │ -│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ -│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--initialize-environmentInitialize environment before running checks.│ -│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ -│(INTEGER RANGE)                                                       │ -│[default: 3; 1<=x<=10]                                                │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ -│exclusive with --last-commit.                                                                │ -│(TEXT)                                                                                       │ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ -│branch and HEAD of your branch.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run.                                             │ +│(all | black | blacken-docs | check-aiobotocore-optional |                       │ +│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ +│check-apache-license-rat | check-base-operator-partial-arguments |               │ +│check-base-operator-usage | check-boring-cyborg-configuration |                  │ +│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ +│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ +│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ +│| check-deferrable-default-value | check-docstring-param-types |                 │ +│check-example-dags-urls | check-executables-have-shebangs |                      │ +│check-extra-packages-references | check-extras-order |                           │ +│check-for-inclusive-language | check-hooks-apply |                               │ +│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ +│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ +│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ +│check-no-relative-imports | check-only-new-session-with-provide-session |        │ +│check-persist-credentials-disabled-in-github-workflows |                         │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ +│check-provider-yaml-valid | check-providers-init-file-missing |                  │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ +│check-start-date-not-used-in-defaults | check-system-tests-present |             │ +│check-system-tests-tocs | check-tests-unittest-testcase |                        │ +│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ +│| compile-www-assets | compile-www-assets-dev |                                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ +│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ +│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ +│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ +│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ +│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ +│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ +│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ +│update-installed-providers-to-be-sorted | update-local-yml-file |                │ +│update-migration-references | update-providers-dependencies |                    │ +│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ +│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--initialize-environmentInitialize environment before running checks.│ +│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ +│(INTEGER RANGE)                                                       │ +│[default: 3; 1<=x<=10]                                                │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ +│exclusive with --last-commit.                                                                │ +│(TEXT)                                                                                       │ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ +│branch and HEAD of your branch.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py b/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py new file mode 100755 index 000000000000..784d25d5221e --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_check_deferrable_default.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import ast +import glob +import itertools +import os +import sys +from typing import Iterator + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) + +DEFERRABLE_DOC = ( + "https://github.com/apache/airflow/blob/main/docs/apache-airflow/" + "authoring-and-scheduling/deferring.rst#writing-deferrable-operators" +) + + +def _is_valid_deferrable_default(default: ast.AST) -> bool: + """Check whether default is 'conf.getboolean("operators", "default_deferrable", fallback=False)'""" + if not isinstance(default, ast.Call): + return False # Not a function call. + + # Check the function callee is exactly 'conf.getboolean'. + call_to_conf_getboolean = ( + isinstance(default.func, ast.Attribute) + and isinstance(default.func.value, ast.Name) + and default.func.value.id == "conf" + and default.func.attr == "getboolean" + ) + if not call_to_conf_getboolean: + return False + + # Check arguments. + return ( + len(default.args) == 2 + and isinstance(default.args[0], ast.Constant) + and default.args[0].value == "operators" + and isinstance(default.args[1], ast.Constant) + and default.args[1].value == "default_deferrable" + and len(default.keywords) == 1 + and default.keywords[0].arg == "fallback" + and isinstance(default.keywords[0].value, ast.Constant) + and default.keywords[0].value.value is False + ) + + +def iter_check_deferrable_default_errors(module_filename: str) -> Iterator[str]: + ast_obj = ast.parse(open(module_filename).read()) + cls_nodes = (node for node in ast.iter_child_nodes(ast_obj) if isinstance(node, ast.ClassDef)) + init_method_nodes = ( + node + for cls_node in cls_nodes + for node in ast.iter_child_nodes(cls_node) + if isinstance(node, ast.FunctionDef) and node.name == "__init__" + ) + + for node in init_method_nodes: + args = node.args + arguments = reversed([*args.args, *args.kwonlyargs]) + defaults = reversed([*args.defaults, *args.kw_defaults]) + for argument, default in itertools.zip_longest(arguments, defaults, fillvalue=None): + if argument is None or default is None: + continue + if argument.arg != "deferrable" or _is_valid_deferrable_default(default): + continue + yield f"{module_filename}:{default.lineno}" + + +def main() -> int: + modules = itertools.chain( + glob.glob(f"{ROOT_DIR}/airflow/**/sensors/**.py", recursive=True), + glob.glob(f"{ROOT_DIR}/airflow/**/operators/**.py", recursive=True), + ) + + errors = [error for module in modules for error in iter_check_deferrable_default_errors(module)] + if errors: + print("Incorrect deferrable default values detected at:") + for error in errors: + print(f" {error}") + print( + """Please set the default value of deferrbale to """ + """"conf.getboolean("operators", "default_deferrable", fallback=False)"\n""" + f"See: {DEFERRABLE_DOC}\n" + ) + + return len(errors) + + +if __name__ == "__main__": + sys.exit(main()) From 26bb408d17d6eecb6f67e31e536f052a37a66db8 Mon Sep 17 00:00:00 2001 From: Phani Kumar <94376113+phanikumv@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:52:06 +0530 Subject: [PATCH 092/533] Refactor Sqlalchemy queries to 2.0 style (Part 4) (#32339) --- airflow/cli/commands/connection_command.py | 14 ++++++++------ airflow/secrets/metastore.py | 5 +++-- airflow/www/utils.py | 17 +++++++++-------- tests/cli/commands/test_connection_command.py | 4 +--- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py index 63888c0ec96e..e7b83e342e29 100644 --- a/airflow/cli/commands/connection_command.py +++ b/airflow/cli/commands/connection_command.py @@ -26,6 +26,7 @@ from typing import Any from urllib.parse import urlsplit, urlunsplit +from sqlalchemy import select from sqlalchemy.orm import exc from airflow.cli.simple_table import AirflowConsole @@ -77,9 +78,10 @@ def connections_get(args): def connections_list(args): """Lists all connections at the command line.""" with create_session() as session: - query = session.query(Connection) + query = select(Connection) if args.conn_id: - query = query.filter(Connection.conn_id == args.conn_id) + query = query.where(Connection.conn_id == args.conn_id) + query = session.scalars(query) conns = query.all() AirflowConsole().print_as( @@ -177,7 +179,7 @@ def connections_export(args): raise SystemExit("Option `--serialization-format` may only be used with file type `env`.") with create_session() as session: - connections = session.query(Connection).order_by(Connection.conn_id).all() + connections = session.scalars(select(Connection).order_by(Connection.conn_id)).all() msg = _format_connections( conns=connections, @@ -265,7 +267,7 @@ def connections_add(args): new_conn.set_extra(args.conn_extra) with create_session() as session: - if not session.query(Connection).filter(Connection.conn_id == new_conn.conn_id).first(): + if not session.scalar(select(Connection).where(Connection.conn_id == new_conn.conn_id).limit(1)): session.add(new_conn) msg = "Successfully added `conn_id`={conn_id} : {uri}" msg = msg.format( @@ -293,7 +295,7 @@ def connections_delete(args): """Deletes connection from DB.""" with create_session() as session: try: - to_delete = session.query(Connection).filter(Connection.conn_id == args.conn_id).one() + to_delete = session.scalars(select(Connection).where(Connection.conn_id == args.conn_id)).one() except exc.NoResultFound: raise SystemExit(f"Did not find a connection with `conn_id`={args.conn_id}") except exc.MultipleResultsFound: @@ -326,7 +328,7 @@ def _import_helper(file_path: str, overwrite: bool) -> None: print(f"Could not import connection. {e}") continue - existing_conn_id = session.query(Connection.id).filter(Connection.conn_id == conn_id).scalar() + existing_conn_id = session.scalar(select(Connection.id).where(Connection.conn_id == conn_id)) if existing_conn_id is not None: if not overwrite: print(f"Could not import connection {conn_id}: connection already exists.") diff --git a/airflow/secrets/metastore.py b/airflow/secrets/metastore.py index dc81675d771f..160c063dff0f 100644 --- a/airflow/secrets/metastore.py +++ b/airflow/secrets/metastore.py @@ -21,6 +21,7 @@ import warnings from typing import TYPE_CHECKING +from sqlalchemy import select from sqlalchemy.orm import Session from airflow.exceptions import RemovedInAirflow3Warning @@ -38,7 +39,7 @@ class MetastoreBackend(BaseSecretsBackend): def get_connection(self, conn_id: str, session: Session = NEW_SESSION) -> Connection | None: from airflow.models.connection import Connection - conn = session.query(Connection).filter(Connection.conn_id == conn_id).first() + conn = session.scalar(select(Connection).where(Connection.conn_id == conn_id).limit(1)) session.expunge_all() return conn @@ -65,7 +66,7 @@ def get_variable(self, key: str, session: Session = NEW_SESSION) -> str | None: """ from airflow.models.variable import Variable - var_value = session.query(Variable).filter(Variable.key == key).first() + var_value = session.scalar(select(Variable).where(Variable.key == key).limit(1)) session.expunge_all() if var_value: return var_value.val diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 4aaeda595bb2..1ca628912249 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -37,7 +37,7 @@ from pygments import highlight, lexers from pygments.formatters import HtmlFormatter from pygments.lexer import Lexer -from sqlalchemy import delete, func, types +from sqlalchemy import delete, func, select, types from sqlalchemy.ext.associationproxy import AssociationProxy from sqlalchemy.sql import Select @@ -68,16 +68,15 @@ def datetime_to_string(value: DateTime | None) -> str | None: def get_mapped_instances(task_instance, session): - return ( - session.query(TaskInstance) - .filter( + return session.scalars( + select(TaskInstance) + .where( TaskInstance.dag_id == task_instance.dag_id, TaskInstance.run_id == task_instance.run_id, TaskInstance.task_id == task_instance.task_id, ) .order_by(TaskInstance.map_index) - .all() - ) + ).all() def get_instance_with_map(task_instance, session): @@ -179,14 +178,16 @@ def encode_dag_run( def check_import_errors(fileloc, session): # Check dag import errors - import_errors = session.query(errors.ImportError).filter(errors.ImportError.filename == fileloc).all() + import_errors = session.scalars( + select(errors.ImportError).where(errors.ImportError.filename == fileloc) + ).all() if import_errors: for import_error in import_errors: flash("Broken DAG: [{ie.filename}] {ie.stacktrace}".format(ie=import_error), "dag_import_error") def check_dag_warnings(dag_id, session): - dag_warnings = session.query(DagWarning).filter(DagWarning.dag_id == dag_id).all() + dag_warnings = session.scalars(select(DagWarning).where(DagWarning.dag_id == dag_id)).all() if dag_warnings: for dag_warning in dag_warnings: flash(dag_warning.message, "warning") diff --git a/tests/cli/commands/test_connection_command.py b/tests/cli/commands/test_connection_command.py index 12b3de6170c0..0c48cd099b72 100644 --- a/tests/cli/commands/test_connection_command.py +++ b/tests/cli/commands/test_connection_command.py @@ -167,9 +167,7 @@ def test_cli_connections_export_should_raise_error_if_fetching_connections_fails def my_side_effect(_): raise Exception("dummy exception") - mock_session.return_value.__enter__.return_value.query.return_value.order_by.side_effect = ( - my_side_effect - ) + mock_session.return_value.__enter__.return_value.scalars.side_effect = my_side_effect args = self.parser.parse_args(["connections", "export", output_filepath.as_posix()]) with pytest.raises(Exception, match=r"dummy exception"): connection_command.connections_export(args) From 3ed5b7ad3e9144b11563bea74c8d619b33ba1c6c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 7 Jul 2023 13:54:47 +0200 Subject: [PATCH 093/533] Cover wider set of excluded tests for re2 check (#32424) We are adding more tests following *_test folder convention (for example helm_tests) and re2 check should exclude all of those. --- .pre-commit-config.yaml | 4 ++-- STATIC_CODE_CHECKS.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afac835f2bb9..15e1399a0c77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -931,12 +931,12 @@ repos: files: ^tests/.*\.py$ - id: check-usage-of-re2-over-re language: pygrep - name: Use re2 over re + name: Use re2 module instead of re description: Use re2 module instead of re entry: "^\\s*from re\\s|^\\s*import re\\s" pass_filenames: true files: \.py$ - exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^docker_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$ + exclude: ^airflow/providers|^dev/.*\.py$|^scripts/.*\.py$|^tests/|^\w+_tests/|^docs/.*\.py$|^airflow/utils/helpers.py$ - id: check-deferrable-default-value name: Check default value of deferrable attribute language: python diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index a9ec8c106aa1..4bcba4bb55e2 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -237,7 +237,7 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-urlparse-usage-in-code | Don't use urlparse in code | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ -| check-usage-of-re2-over-re | Use re2 over re | | +| check-usage-of-re2-over-re | Use re2 module instead of re | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-xml | Check XML files with xmllint | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ From 907aeeb4d5068ac1dc7a59dc26196ee0a53f31f8 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 7 Jul 2023 19:48:40 +0530 Subject: [PATCH 094/533] Migrating helm chart tests to helm_tests to prevent test DB init (#32394) --------- Co-authored-by: Amogh Desai --- .dockerignore | 2 ++ Dockerfile.ci | 4 ++-- TESTING.rst | 10 ++++---- .../src/airflow_breeze/global_constants.py | 4 ++-- .../utils/docker_command_utils.py | 1 + .../airflow_breeze/utils/selective_checks.py | 2 +- .../airflow_aux => helm_tests}/__init__.py | 0 .../airflow_aux}/__init__.py | 0 .../airflow_aux/test_airflow_common.py | 6 ++--- .../airflow_aux/test_annotations.py | 4 ++++ .../airflow_aux/test_basic_helm_chart.py | 10 ++++---- .../test_celery_kubernetes_executor.py | 2 ++ .../airflow_aux/test_chart_quality.py | 4 +++- .../airflow_aux/test_cleanup_pods.py | 4 ++++ .../airflow_aux/test_configmap.py | 4 +++- .../airflow_aux/test_create_user_job.py | 4 ++++ .../airflow_aux/test_extra_env_env_from.py | 2 ++ .../test_logs_persistent_volume_claim.py | 2 ++ .../airflow_aux/test_migrate_database_job.py | 2 ++ .../airflow_aux/test_pod_launcher_role.py | 2 ++ .../airflow_aux/test_pod_template_file.py | 6 +++-- .../airflow_core}/__init__.py | 0 .../airflow_core/test_dag_processor.py | 14 ++++++----- .../airflow_core/test_pdb_scheduler.py | 2 ++ .../airflow_core/test_scheduler.py | 16 ++++++++++--- .../airflow_core/test_triggerer.py | 12 +++++++--- .../airflow_core/test_worker.py | 18 ++++++++++---- .../security => helm_tests/other}/__init__.py | 0 .../test_dags_persistent_volume_claim.py | 2 ++ .../other/test_flower.py | 8 +++++++ .../other/test_git_sync_scheduler.py | 2 ++ .../other/test_git_sync_triggerer.py | 2 ++ .../other/test_git_sync_webserver.py | 2 ++ .../other/test_git_sync_worker.py | 2 ++ .../charts => helm_tests}/other/test_keda.py | 22 +++++++---------- .../other/test_limit_ranges.py | 2 ++ .../other/test_pdb_pgbouncer.py | 2 ++ .../other/test_pgbouncer.py | 6 +++++ .../charts => helm_tests}/other/test_redis.py | 2 ++ .../other/test_resource_quota.py | 2 ++ .../other/test_statsd.py | 2 ++ .../security}/__init__.py | 0 .../security/test_elasticsearch_secret.py | 2 ++ .../security/test_extra_configmaps_secrets.py | 2 ++ .../security/test_kerberos.py | 2 ++ .../test_metadata_connection_secret.py | 1 + .../security/test_rbac.py | 2 ++ .../security/test_rbac_pod_log_reader.py | 2 ++ .../test_result_backend_connection_secret.py | 2 ++ .../security/test_scc_rolebinding.py | 2 ++ .../security/test_security_context.py | 4 ++++ helm_tests/webserver/__init__.py | 16 +++++++++++++ .../webserver/test_ingress_flower.py | 2 ++ .../webserver/test_ingress_web.py | 2 ++ .../webserver/test_pdb_webserver.py | 2 ++ .../webserver/test_webserver.py | 12 +++++++++- images/breeze/output_testing.svg | 24 +++++++++---------- scripts/ci/docker-compose/local.yml | 3 +++ scripts/docker/entrypoint_ci.sh | 4 ++-- 59 files changed, 211 insertions(+), 66 deletions(-) rename {tests/charts/airflow_aux => helm_tests}/__init__.py (100%) rename {tests/charts/airflow_core => helm_tests/airflow_aux}/__init__.py (100%) rename {tests/charts => helm_tests}/airflow_aux/test_airflow_common.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_annotations.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_basic_helm_chart.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_celery_kubernetes_executor.py (97%) rename {tests/charts => helm_tests}/airflow_aux/test_chart_quality.py (94%) rename {tests/charts => helm_tests}/airflow_aux/test_cleanup_pods.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_configmap.py (98%) rename {tests/charts => helm_tests}/airflow_aux/test_create_user_job.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_extra_env_env_from.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_logs_persistent_volume_claim.py (98%) rename {tests/charts => helm_tests}/airflow_aux/test_migrate_database_job.py (99%) rename {tests/charts => helm_tests}/airflow_aux/test_pod_launcher_role.py (98%) rename {tests/charts => helm_tests}/airflow_aux/test_pod_template_file.py (99%) rename {tests/charts/other => helm_tests/airflow_core}/__init__.py (100%) rename {tests/charts => helm_tests}/airflow_core/test_dag_processor.py (98%) rename {tests/charts => helm_tests}/airflow_core/test_pdb_scheduler.py (98%) rename {tests/charts => helm_tests}/airflow_core/test_scheduler.py (98%) rename {tests/charts => helm_tests}/airflow_core/test_triggerer.py (98%) rename {tests/charts => helm_tests}/airflow_core/test_worker.py (98%) rename {tests/charts/security => helm_tests/other}/__init__.py (100%) rename {tests/charts => helm_tests}/other/test_dags_persistent_volume_claim.py (99%) rename {tests/charts => helm_tests}/other/test_flower.py (99%) rename {tests/charts => helm_tests}/other/test_git_sync_scheduler.py (99%) rename {tests/charts => helm_tests}/other/test_git_sync_triggerer.py (97%) rename {tests/charts => helm_tests}/other/test_git_sync_webserver.py (99%) rename {tests/charts => helm_tests}/other/test_git_sync_worker.py (99%) rename {tests/charts => helm_tests}/other/test_keda.py (92%) rename {tests/charts => helm_tests}/other/test_limit_ranges.py (97%) rename {tests/charts => helm_tests}/other/test_pdb_pgbouncer.py (98%) rename {tests/charts => helm_tests}/other/test_pgbouncer.py (99%) rename {tests/charts => helm_tests}/other/test_redis.py (99%) rename {tests/charts => helm_tests}/other/test_resource_quota.py (98%) rename {tests/charts => helm_tests}/other/test_statsd.py (99%) rename {tests/charts/webserver => helm_tests/security}/__init__.py (100%) rename {tests/charts => helm_tests}/security/test_elasticsearch_secret.py (99%) rename {tests/charts => helm_tests}/security/test_extra_configmaps_secrets.py (99%) rename {tests/charts => helm_tests}/security/test_kerberos.py (99%) rename {tests/charts => helm_tests}/security/test_metadata_connection_secret.py (99%) rename {tests/charts => helm_tests}/security/test_rbac.py (99%) rename {tests/charts => helm_tests}/security/test_rbac_pod_log_reader.py (98%) rename {tests/charts => helm_tests}/security/test_result_backend_connection_secret.py (99%) rename {tests/charts => helm_tests}/security/test_scc_rolebinding.py (99%) rename {tests/charts => helm_tests}/security/test_security_context.py (99%) create mode 100644 helm_tests/webserver/__init__.py rename {tests/charts => helm_tests}/webserver/test_ingress_flower.py (99%) rename {tests/charts => helm_tests}/webserver/test_ingress_web.py (99%) rename {tests/charts => helm_tests}/webserver/test_pdb_webserver.py (98%) rename {tests/charts => helm_tests}/webserver/test_webserver.py (99%) diff --git a/.dockerignore b/.dockerignore index c6f143200bd3..3a6fd4e2f1e0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,6 +42,8 @@ # Add tests and kubernetes_tests to context. !tests !kubernetes_tests +!helm_tests +!docker_tests !.coveragerc !.rat-excludes diff --git a/Dockerfile.ci b/Dockerfile.ci index 6df47d6a8a49..e30f1a9a7567 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1031,7 +1031,7 @@ else "tests/utils" ) WWW_TESTS=("tests/www") - HELM_CHART_TESTS=("tests/charts") + HELM_CHART_TESTS=("helm_tests") INTEGRATION_TESTS=("tests/integration") SYSTEM_TESTS=("tests/system") ALL_TESTS=("tests") @@ -1068,7 +1068,7 @@ else SELECTED_TESTS=("${WWW_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then if [[ ${HELM_TEST_PACKAGE=} != "" ]]; then - SELECTED_TESTS=("tests/charts/${HELM_TEST_PACKAGE}") + SELECTED_TESTS=("helm_tests/${HELM_TEST_PACKAGE}") else SELECTED_TESTS=("${HELM_CHART_TESTS[@]}") fi diff --git a/TESTING.rst b/TESTING.rst index bab7207956c5..94b4ff743859 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -686,7 +686,7 @@ Helm Unit Tests On the Airflow Project, we have decided to stick with pythonic testing for our Helm chart. This makes our chart easier to test, easier to modify, and able to run with the same testing infrastructure. To add Helm unit tests -add them in ``tests/charts``. +add them in ``helm_tests``. .. code-block:: python @@ -738,7 +738,7 @@ so rather than running all tests, you can run only tests from a selected package breeze testing helm-tests --helm-test-package basic -Will run all tests from ``tests/charts/basic`` package. +Will run all tests from ``tests-charts/basic`` package. You can also run Helm tests individually via the usual ``breeze`` command. Just enter breeze and run the @@ -755,19 +755,19 @@ This enters breeze container. .. code-block:: bash - pytest tests/charts -n auto + pytest helm_tests -n auto This runs all chart tests using all processors you have available. .. code-block:: bash - pytest tests/charts/test_airflow_common.py -n auto + pytest helm_tests/test_airflow_common.py -n auto This will run all tests from ``tests_airflow_common.py`` file using all processors you have available. .. code-block:: bash - pytest tests/charts/test_airflow_common.py + pytest helm_tests/test_airflow_common.py This will run all tests from ``tests_airflow_common.py`` file sequentially. diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 0a66f7d98ced..7b79af5ad4b8 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -120,8 +120,8 @@ def all_helm_test_packages() -> list[str]: return sorted( [ candidate.name - for candidate in (AIRFLOW_SOURCES_ROOT / "tests" / "charts").iterdir() - if candidate.is_dir() + for candidate in (AIRFLOW_SOURCES_ROOT / "helm_tests").iterdir() + if candidate.is_dir() and candidate.name != "__pycache__" ] ) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 13cd8f060d32..8a85c6b4d6d3 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -102,6 +102,7 @@ ("setup.cfg", "/opt/airflow/setup.cfg"), ("setup.py", "/opt/airflow/setup.py"), ("tests", "/opt/airflow/tests"), + ("helm_tests", "/opt/airflow/helm_tests"), ("kubernetes_tests", "/opt/airflow/kubernetes_tests"), ("docker_tests", "/opt/airflow/docker_tests"), ("chart", "/opt/airflow/chart"), diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 7c7d622dab61..43c0b8b94bb6 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -126,7 +126,7 @@ def __hash__(self): r"^chart", r"^airflow/kubernetes", r"^tests/kubernetes", - r"^tests/charts", + r"^helm_tests", ], FileGroupForCi.SETUP_FILES: [ r"^pyproject.toml", diff --git a/tests/charts/airflow_aux/__init__.py b/helm_tests/__init__.py similarity index 100% rename from tests/charts/airflow_aux/__init__.py rename to helm_tests/__init__.py diff --git a/tests/charts/airflow_core/__init__.py b/helm_tests/airflow_aux/__init__.py similarity index 100% rename from tests/charts/airflow_core/__init__.py rename to helm_tests/airflow_aux/__init__.py diff --git a/tests/charts/airflow_aux/test_airflow_common.py b/helm_tests/airflow_aux/test_airflow_common.py similarity index 99% rename from tests/charts/airflow_aux/test_airflow_common.py rename to helm_tests/airflow_aux/test_airflow_common.py index 9a14b21a51f4..ebc17d20638d 100644 --- a/tests/charts/airflow_aux/test_airflow_common.py +++ b/helm_tests/airflow_aux/test_airflow_common.py @@ -25,7 +25,7 @@ class TestAirflowCommon: """ This class holds tests that apply to more than 1 Airflow component so - we don't have to repeat tests everywhere + we don't have to repeat tests everywhere. The one general exception will be the KubernetesExecutor PodTemplateFile, as it requires extra test setup. @@ -162,9 +162,7 @@ def test_annotations(self): assert "true" in annotations["test-annotation/safe-to-evict"] def test_global_affinity_tolerations_topology_spread_constraints_and_node_selector(self): - """ - Test affinity, tolerations, etc are correctly applied on all pods created - """ + """Test affinity, tolerations, etc are correctly applied on all pods created.""" k8s_objects = render_chart( values={ "cleanup": {"enabled": True}, diff --git a/tests/charts/airflow_aux/test_annotations.py b/helm_tests/airflow_aux/test_annotations.py similarity index 99% rename from tests/charts/airflow_aux/test_annotations.py rename to helm_tests/airflow_aux/test_annotations.py index a4392c20f4d0..8f297bf73b3e 100644 --- a/tests/charts/airflow_aux/test_annotations.py +++ b/helm_tests/airflow_aux/test_annotations.py @@ -36,6 +36,8 @@ def get_object_annotations(obj): class TestServiceAccountAnnotations: + """Tests Service Account Annotations.""" + @pytest.mark.parametrize( "values,show_only,expected_annotations", [ @@ -383,6 +385,8 @@ def test_annotations_are_added(self, values, show_only, expected_annotations): ], ) class TestPerComponentPodAnnotations: + """Tests Per Component Pod Annotations.""" + def test_annotations_are_added(self, values, show_only, expected_annotations): k8s_objects = render_chart( values=values, diff --git a/tests/charts/airflow_aux/test_basic_helm_chart.py b/helm_tests/airflow_aux/test_basic_helm_chart.py similarity index 99% rename from tests/charts/airflow_aux/test_basic_helm_chart.py rename to helm_tests/airflow_aux/test_basic_helm_chart.py index 7c1e9f862d3a..3cf03707f3ea 100644 --- a/tests/charts/airflow_aux/test_basic_helm_chart.py +++ b/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -31,6 +31,8 @@ class TestBaseChartTest: + """Tests basic helm chart tests.""" + def _get_values_with_version(self, values, version): if version != "default": values["airflowVersion"] = version @@ -255,7 +257,7 @@ def test_network_policies_are_valid(self): assert kind_name in kind_names_tuples def test_labels_are_valid(self): - """Test labels are correctly applied on all objects created by this chart""" + """Test labels are correctly applied on all objects created by this chart.""" release_name = "test-basic" k8s_objects = render_chart( name=release_name, @@ -368,7 +370,7 @@ def test_labels_are_valid(self): warnings.warn(f"Unchecked objects: {kind_k8s_obj_labels_tuples.keys()}") def test_labels_are_valid_on_job_templates(self): - """Test labels are correctly applied on all job templates created by this chart""" + """Test labels are correctly applied on all job templates created by this chart.""" release_name = "test-basic" k8s_objects = render_chart( name=release_name, @@ -441,7 +443,7 @@ def test_chart_is_consistent_with_official_airflow_image(self): def get_k8s_objs_with_image(obj: list[Any] | dict[str, Any]) -> list[dict[str, Any]]: """ Recursive helper to retrieve all the k8s objects that have an "image" key - inside k8s obj or list of k8s obj + inside k8s obj or list of k8s obj. """ out = [] if isinstance(obj, list): @@ -511,7 +513,7 @@ def test_invalid_dags_access_mode(self): @pytest.mark.parametrize("namespace", ["abc", "123", "123abc", "123-abc"]) def test_namespace_names(self, namespace): - """Test various namespace names to make sure they render correctly in templates""" + """Test various namespace names to make sure they render correctly in templates.""" render_chart(namespace=namespace) def test_postgres_connection_url_no_override(self): diff --git a/tests/charts/airflow_aux/test_celery_kubernetes_executor.py b/helm_tests/airflow_aux/test_celery_kubernetes_executor.py similarity index 97% rename from tests/charts/airflow_aux/test_celery_kubernetes_executor.py rename to helm_tests/airflow_aux/test_celery_kubernetes_executor.py index 382ada608c5e..4c34cf59411b 100644 --- a/tests/charts/airflow_aux/test_celery_kubernetes_executor.py +++ b/helm_tests/airflow_aux/test_celery_kubernetes_executor.py @@ -22,6 +22,8 @@ class TestCeleryKubernetesExecutor: + """Tests celery kubernetes executor.""" + def test_should_create_a_worker_deployment_with_the_celery_executor(self): docs = render_chart( values={ diff --git a/tests/charts/airflow_aux/test_chart_quality.py b/helm_tests/airflow_aux/test_chart_quality.py similarity index 94% rename from tests/charts/airflow_aux/test_chart_quality.py rename to helm_tests/airflow_aux/test_chart_quality.py index a44bf656ea97..a4246c03d51a 100644 --- a/tests/charts/airflow_aux/test_chart_quality.py +++ b/helm_tests/airflow_aux/test_chart_quality.py @@ -22,10 +22,12 @@ import yaml from jsonschema import validate -CHART_DIR = Path(__file__).resolve().parents[3] / "chart" +CHART_DIR = Path(__file__).parents[2] / "chart" class TestChartQuality: + """Tests chart quality.""" + def test_values_validate_schema(self): values = yaml.safe_load((CHART_DIR / "values.yaml").read_text()) schema = json.loads((CHART_DIR / "values.schema.json").read_text()) diff --git a/tests/charts/airflow_aux/test_cleanup_pods.py b/helm_tests/airflow_aux/test_cleanup_pods.py similarity index 99% rename from tests/charts/airflow_aux/test_cleanup_pods.py rename to helm_tests/airflow_aux/test_cleanup_pods.py index c7d3956506ca..60859ac3d432 100644 --- a/tests/charts/airflow_aux/test_cleanup_pods.py +++ b/helm_tests/airflow_aux/test_cleanup_pods.py @@ -23,6 +23,8 @@ class TestCleanupPods: + """Tests cleanup of pods.""" + def test_should_create_cronjob_for_enabled_cleanup(self): docs = render_chart( values={ @@ -296,6 +298,8 @@ def test_airflow_local_settings(self): class TestCleanupServiceAccount: + """Tests cleanup of service accounts.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/tests/charts/airflow_aux/test_configmap.py b/helm_tests/airflow_aux/test_configmap.py similarity index 98% rename from tests/charts/airflow_aux/test_configmap.py rename to helm_tests/airflow_aux/test_configmap.py index 86f35e6c6ea2..3560b19c180c 100644 --- a/tests/charts/airflow_aux/test_configmap.py +++ b/helm_tests/airflow_aux/test_configmap.py @@ -23,6 +23,8 @@ class TestConfigmap: + """Tests configmaps.""" + def test_single_annotation(self): docs = render_chart( values={ @@ -103,7 +105,7 @@ def test_pod_template_is_templated(self): apiVersion: v1 kind: Pod metadata: - name: dummy-name + name: example-name labels: mylabel: {{ .Release.Name }} """, diff --git a/tests/charts/airflow_aux/test_create_user_job.py b/helm_tests/airflow_aux/test_create_user_job.py similarity index 99% rename from tests/charts/airflow_aux/test_create_user_job.py rename to helm_tests/airflow_aux/test_create_user_job.py index 8f8ddcd65e51..76c32f4e19be 100644 --- a/tests/charts/airflow_aux/test_create_user_job.py +++ b/helm_tests/airflow_aux/test_create_user_job.py @@ -23,6 +23,8 @@ class TestCreateUserJob: + """Tests create user job.""" + def test_should_run_by_default(self): docs = render_chart(show_only=["templates/jobs/create-user-job.yaml"]) assert "Job" == docs[0]["kind"] @@ -380,6 +382,8 @@ def test_airflow_local_settings(self): class TestCreateUserJobServiceAccount: + """Tests create user job service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/tests/charts/airflow_aux/test_extra_env_env_from.py b/helm_tests/airflow_aux/test_extra_env_env_from.py similarity index 99% rename from tests/charts/airflow_aux/test_extra_env_env_from.py rename to helm_tests/airflow_aux/test_extra_env_env_from.py index 0dc5ff845a35..e7b61560ce1c 100644 --- a/tests/charts/airflow_aux/test_extra_env_env_from.py +++ b/helm_tests/airflow_aux/test_extra_env_env_from.py @@ -73,6 +73,8 @@ class TestExtraEnvEnvFrom: + """Tests extra env from.""" + k8s_objects: list[dict[str, Any]] k8s_objects_by_key: dict[tuple[str, str], dict[str, Any]] diff --git a/tests/charts/airflow_aux/test_logs_persistent_volume_claim.py b/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py similarity index 98% rename from tests/charts/airflow_aux/test_logs_persistent_volume_claim.py rename to helm_tests/airflow_aux/test_logs_persistent_volume_claim.py index 8ca356116b6b..f815b3d21791 100644 --- a/tests/charts/airflow_aux/test_logs_persistent_volume_claim.py +++ b/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py @@ -22,6 +22,8 @@ class TestLogsPersistentVolumeClaim: + """Tests logs PVC.""" + def test_should_not_generate_a_document_if_persistence_is_disabled(self): docs = render_chart( values={"logs": {"persistence": {"enabled": False}}}, diff --git a/tests/charts/airflow_aux/test_migrate_database_job.py b/helm_tests/airflow_aux/test_migrate_database_job.py similarity index 99% rename from tests/charts/airflow_aux/test_migrate_database_job.py rename to helm_tests/airflow_aux/test_migrate_database_job.py index 6b6936958d76..359c035b9e27 100644 --- a/tests/charts/airflow_aux/test_migrate_database_job.py +++ b/helm_tests/airflow_aux/test_migrate_database_job.py @@ -23,6 +23,8 @@ class TestMigrateDatabaseJob: + """Tests migrate DB job.""" + def test_should_run_by_default(self): docs = render_chart(show_only=["templates/jobs/migrate-database-job.yaml"]) assert "Job" == docs[0]["kind"] diff --git a/tests/charts/airflow_aux/test_pod_launcher_role.py b/helm_tests/airflow_aux/test_pod_launcher_role.py similarity index 98% rename from tests/charts/airflow_aux/test_pod_launcher_role.py rename to helm_tests/airflow_aux/test_pod_launcher_role.py index 63614b42ab37..64131fa6f204 100644 --- a/tests/charts/airflow_aux/test_pod_launcher_role.py +++ b/helm_tests/airflow_aux/test_pod_launcher_role.py @@ -23,6 +23,8 @@ class TestPodLauncher: + """Tests pod launcher.""" + @pytest.mark.parametrize( "executor, rbac, allow, expected_accounts", [ diff --git a/tests/charts/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py similarity index 99% rename from tests/charts/airflow_aux/test_pod_template_file.py rename to helm_tests/airflow_aux/test_pod_template_file.py index a14c713107ba..4d67b91b1965 100644 --- a/tests/charts/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -29,7 +29,7 @@ @pytest.fixture(scope="class", autouse=True) def isolate_chart(request): - chart_dir = Path(__file__).parents[3] / "chart" + chart_dir = Path(__file__).parents[2] / "chart" with TemporaryDirectory(prefix=request.cls.__name__) as tmp_dir: temp_chart_dir = Path(tmp_dir) / "chart" copytree(chart_dir, temp_chart_dir) @@ -42,6 +42,8 @@ def isolate_chart(request): class TestPodTemplateFile: + """Tests pod template file.""" + def test_should_work(self): docs = render_chart( values={}, @@ -430,7 +432,7 @@ def test_should_create_valid_affinity_tolerations_topology_spread_constraints_an ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and worker affinity etc, worker affinity etc is used""" + """When given both global and worker affinity etc, worker affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { diff --git a/tests/charts/other/__init__.py b/helm_tests/airflow_core/__init__.py similarity index 100% rename from tests/charts/other/__init__.py rename to helm_tests/airflow_core/__init__.py diff --git a/tests/charts/airflow_core/test_dag_processor.py b/helm_tests/airflow_core/test_dag_processor.py similarity index 98% rename from tests/charts/airflow_core/test_dag_processor.py rename to helm_tests/airflow_core/test_dag_processor.py index 37bba48cc53c..dbde3f7b33de 100644 --- a/tests/charts/airflow_core/test_dag_processor.py +++ b/helm_tests/airflow_core/test_dag_processor.py @@ -24,6 +24,8 @@ class TestDagProcessor: + """Tests DAG processor.""" + @pytest.mark.parametrize( "airflow_version, num_docs", [ @@ -32,7 +34,7 @@ class TestDagProcessor: ], ) def test_only_exists_on_new_airflow_versions(self, airflow_version, num_docs): - """Standalone Dag Processor was only added from Airflow 2.3 onwards""" + """Standalone Dag Processor was only added from Airflow 2.3 onwards.""" docs = render_chart( values={ "airflowVersion": airflow_version, @@ -44,9 +46,7 @@ def test_only_exists_on_new_airflow_versions(self, airflow_version, num_docs): assert num_docs == len(docs) def test_can_be_disabled(self): - """ - Standalone Dag Processor is disabled by default. - """ + """Standalone Dag Processor is disabled by default.""" docs = render_chart( values={"dagProcessor": {"enabled": False}}, show_only=["templates/dag-processor/dag-processor-deployment.yaml"], @@ -223,7 +223,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and triggerer affinity etc, triggerer affinity etc is used""" + """When given both global and triggerer affinity etc, triggerer affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -411,7 +411,7 @@ def test_resources_are_not_added_by_default(self): ], ) def test_strategy(self, strategy, expected_strategy): - """strategy should be used when we aren't using both LocalExecutor and workers.persistence""" + """Strategy should be used when we aren't using both LocalExecutor and workers.persistence.""" docs = render_chart( values={ "dagProcessor": {"enabled": True, "strategy": strategy}, @@ -563,5 +563,7 @@ def test_should_add_component_specific_annotations(self): class TestDagProcessorLogGroomer(LogGroomerTestBase): + """DAG processor log groomer.""" + obj_name = "dag-processor" folder = "dag-processor" diff --git a/tests/charts/airflow_core/test_pdb_scheduler.py b/helm_tests/airflow_core/test_pdb_scheduler.py similarity index 98% rename from tests/charts/airflow_core/test_pdb_scheduler.py rename to helm_tests/airflow_core/test_pdb_scheduler.py index a96777941ec0..917b0514c6d8 100644 --- a/tests/charts/airflow_core/test_pdb_scheduler.py +++ b/helm_tests/airflow_core/test_pdb_scheduler.py @@ -22,6 +22,8 @@ class TestSchedulerPdb: + """Tests Scheduler PDB.""" + def test_should_pass_validation_with_just_pdb_enabled_v1(self): render_chart( values={"scheduler": {"podDisruptionBudget": {"enabled": True}}}, diff --git a/tests/charts/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py similarity index 98% rename from tests/charts/airflow_core/test_scheduler.py rename to helm_tests/airflow_core/test_scheduler.py index 32a8aa292170..faf666854b55 100644 --- a/tests/charts/airflow_core/test_scheduler.py +++ b/helm_tests/airflow_core/test_scheduler.py @@ -24,6 +24,8 @@ class TestScheduler: + """Tests scheduler.""" + @pytest.mark.parametrize( "executor, persistence, kind", [ @@ -243,7 +245,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and scheduler affinity etc, scheduler affinity etc is used""" + """When given both global and scheduler affinity etc, scheduler affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -506,7 +508,7 @@ def test_airflow_local_settings(self): def test_scheduler_update_strategy( self, executor, persistence, update_strategy, expected_update_strategy ): - """updateStrategy should only be used when we have a local executor and workers.persistence""" + """UpdateStrategy should only be used when we have a local executor and workers.persistence.""" docs = render_chart( values={ "executor": executor, @@ -538,7 +540,7 @@ def test_scheduler_update_strategy( ], ) def test_scheduler_strategy(self, executor, persistence, strategy, expected_strategy): - """strategy should be used when we aren't using both a local executor and workers.persistence""" + """Strategy should be used when we aren't using both a local executor and workers.persistence.""" docs = render_chart( values={ "executor": executor, @@ -698,6 +700,8 @@ def test_scheduler_pod_hostaliases(self): class TestSchedulerNetworkPolicy: + """Tests scheduler network policy.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ @@ -714,11 +718,15 @@ def test_should_add_component_specific_labels(self): class TestSchedulerLogGroomer(LogGroomerTestBase): + """Scheduler log groomer.""" + obj_name = "scheduler" folder = "scheduler" class TestSchedulerService: + """Tests scheduler service.""" + @pytest.mark.parametrize( "executor, creates_service", [ @@ -762,6 +770,8 @@ def test_should_add_component_specific_labels(self): class TestSchedulerServiceAccount: + """Tests scheduler service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/tests/charts/airflow_core/test_triggerer.py b/helm_tests/airflow_core/test_triggerer.py similarity index 98% rename from tests/charts/airflow_core/test_triggerer.py rename to helm_tests/airflow_core/test_triggerer.py index c91721fbeaf1..0154c8f8c0c9 100644 --- a/tests/charts/airflow_core/test_triggerer.py +++ b/helm_tests/airflow_core/test_triggerer.py @@ -24,6 +24,8 @@ class TestTriggerer: + """Tests triggerer.""" + @pytest.mark.parametrize( "airflow_version, num_docs", [ @@ -32,7 +34,7 @@ class TestTriggerer: ], ) def test_only_exists_on_new_airflow_versions(self, airflow_version, num_docs): - """Trigger was only added from Airflow 2.2 onwards""" + """Trigger was only added from Airflow 2.2 onwards.""" docs = render_chart( values={"airflowVersion": airflow_version}, show_only=["templates/triggerer/triggerer-deployment.yaml"], @@ -43,7 +45,7 @@ def test_only_exists_on_new_airflow_versions(self, airflow_version, num_docs): def test_can_be_disabled(self): """ Triggerer should be able to be disabled if the users desires - (e.g. Python 3.6 or doesn't want to use async tasks) + (e.g. Python 3.6 or doesn't want to use async tasks). """ docs = render_chart( values={"triggerer": {"enabled": False}}, @@ -244,7 +246,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and triggerer affinity etc, triggerer affinity etc is used""" + """When given both global and triggerer affinity etc, triggerer affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -556,6 +558,8 @@ def test_should_add_component_specific_annotations(self): class TestTriggererServiceAccount: + """Tests triggerer service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ @@ -572,5 +576,7 @@ def test_should_add_component_specific_labels(self): class TestTriggererLogGroomer(LogGroomerTestBase): + """Triggerer log groomer.""" + obj_name = "triggerer" folder = "triggerer" diff --git a/tests/charts/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py similarity index 98% rename from tests/charts/airflow_core/test_worker.py rename to helm_tests/airflow_core/test_worker.py index f3127b3bdf46..20cc851ae917 100644 --- a/tests/charts/airflow_core/test_worker.py +++ b/helm_tests/airflow_core/test_worker.py @@ -24,6 +24,8 @@ class TestWorker: + """Tests worker.""" + @pytest.mark.parametrize( "executor, persistence, kind", [ @@ -34,9 +36,7 @@ class TestWorker: ], ) def test_worker_kind(self, executor, persistence, kind): - """ - Test worker kind is StatefulSet when worker persistence is enabled. - """ + """Test worker kind is StatefulSet when worker persistence is enabled.""" docs = render_chart( values={ "executor": executor, @@ -296,7 +296,7 @@ def test_should_create_valid_affinity_tolerations_and_node_selector(self): ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and worker affinity etc, worker affinity etc is used""" + """When given both global and worker affinity etc, worker affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -617,11 +617,15 @@ def test_should_add_component_specific_annotations(self): class TestWorkerLogGroomer(LogGroomerTestBase): + """Worker groomer.""" + obj_name = "worker" folder = "workers" class TestWorkerKedaAutoScaler: + """Tests worker keda auto scaler.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ @@ -697,6 +701,8 @@ def test_should_use_keda_query(self, query, executor, expected_query): class TestWorkerNetworkPolicy: + """Tests worker network policy.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ @@ -714,6 +720,8 @@ def test_should_add_component_specific_labels(self): class TestWorkerService: + """Tests worker service.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ @@ -730,6 +738,8 @@ def test_should_add_component_specific_labels(self): class TestWorkerServiceAccount: + """Tests worker service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/tests/charts/security/__init__.py b/helm_tests/other/__init__.py similarity index 100% rename from tests/charts/security/__init__.py rename to helm_tests/other/__init__.py diff --git a/tests/charts/other/test_dags_persistent_volume_claim.py b/helm_tests/other/test_dags_persistent_volume_claim.py similarity index 99% rename from tests/charts/other/test_dags_persistent_volume_claim.py rename to helm_tests/other/test_dags_persistent_volume_claim.py index 38237c7fc7a7..527a1d859a57 100644 --- a/tests/charts/other/test_dags_persistent_volume_claim.py +++ b/helm_tests/other/test_dags_persistent_volume_claim.py @@ -22,6 +22,8 @@ class TestDagsPersistentVolumeClaim: + """Tests DAGs PVC.""" + def test_should_not_generate_a_document_if_persistence_is_disabled(self): docs = render_chart( values={"dags": {"persistence": {"enabled": False}}}, diff --git a/tests/charts/other/test_flower.py b/helm_tests/other/test_flower.py similarity index 99% rename from tests/charts/other/test_flower.py rename to helm_tests/other/test_flower.py index 779e3a495599..337399439e59 100644 --- a/tests/charts/other/test_flower.py +++ b/helm_tests/other/test_flower.py @@ -23,6 +23,8 @@ class TestFlowerDeployment: + """Tests flower deployment.""" + @pytest.mark.parametrize( "executor,flower_enabled,created", [ @@ -352,6 +354,8 @@ def test_should_add_component_specific_annotations(self): class TestFlowerService: + """Tests flower service.""" + @pytest.mark.parametrize( "executor,flower_enabled,created", [ @@ -457,6 +461,8 @@ def test_should_add_component_specific_labels(self): class TestFlowerNetworkPolicy: + """Tests flower network policy.""" + def test_off_by_default(self): docs = render_chart( show_only=["templates/flower/flower-networkpolicy.yaml"], @@ -556,6 +562,8 @@ def test_should_add_component_specific_labels(self): class TestFlowerServiceAccount: + """Tests flower service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/tests/charts/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py similarity index 99% rename from tests/charts/other/test_git_sync_scheduler.py rename to helm_tests/other/test_git_sync_scheduler.py index 597458ae9805..91039954f8d6 100644 --- a/tests/charts/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -22,6 +22,8 @@ class TestGitSyncSchedulerTest: + """Test git sync scheduler.""" + def test_should_add_dags_volume(self): docs = render_chart( values={"dags": {"gitSync": {"enabled": True}}}, diff --git a/tests/charts/other/test_git_sync_triggerer.py b/helm_tests/other/test_git_sync_triggerer.py similarity index 97% rename from tests/charts/other/test_git_sync_triggerer.py rename to helm_tests/other/test_git_sync_triggerer.py index d64f3eec6f5c..96f7092666eb 100644 --- a/tests/charts/other/test_git_sync_triggerer.py +++ b/helm_tests/other/test_git_sync_triggerer.py @@ -22,6 +22,8 @@ class TestGitSyncTriggerer: + """Test git sync triggerer.""" + def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): docs = render_chart( values={ diff --git a/tests/charts/other/test_git_sync_webserver.py b/helm_tests/other/test_git_sync_webserver.py similarity index 99% rename from tests/charts/other/test_git_sync_webserver.py rename to helm_tests/other/test_git_sync_webserver.py index ea264673d7af..b574abcc639e 100644 --- a/tests/charts/other/test_git_sync_webserver.py +++ b/helm_tests/other/test_git_sync_webserver.py @@ -23,6 +23,8 @@ class TestGitSyncWebserver: + """Test git sync webserver.""" + def test_should_add_dags_volume_to_the_webserver_if_git_sync_and_persistence_is_enabled(self): docs = render_chart( values={ diff --git a/tests/charts/other/test_git_sync_worker.py b/helm_tests/other/test_git_sync_worker.py similarity index 99% rename from tests/charts/other/test_git_sync_worker.py rename to helm_tests/other/test_git_sync_worker.py index 177fc8fed76f..31589a78f3ba 100644 --- a/tests/charts/other/test_git_sync_worker.py +++ b/helm_tests/other/test_git_sync_worker.py @@ -22,6 +22,8 @@ class TestGitSyncWorker: + """Test git sync worker.""" + def test_should_add_dags_volume_to_the_worker_if_git_sync_and_persistence_is_enabled(self): docs = render_chart( values={ diff --git a/tests/charts/other/test_keda.py b/helm_tests/other/test_keda.py similarity index 92% rename from tests/charts/other/test_keda.py rename to helm_tests/other/test_keda.py index 7a0326070a9a..774b617b50c3 100644 --- a/tests/charts/other/test_keda.py +++ b/helm_tests/other/test_keda.py @@ -23,8 +23,10 @@ class TestKeda: + """Tests keda.""" + def test_keda_disabled_by_default(self): - """disabled by default""" + """Disabled by default.""" docs = render_chart( values={}, show_only=["templates/workers/worker-kedaautoscaler.yaml"], @@ -39,8 +41,8 @@ def test_keda_disabled_by_default(self): ], ) def test_keda_enabled(self, executor, is_created): - """ - ScaledObject should only be created when set to enabled and executor is Celery or CeleryKubernetes + """ScaledObject should only be created when set to enabled and + executor is Celery or CeleryKubernetes. """ docs = render_chart( values={ @@ -56,9 +58,7 @@ def test_keda_enabled(self, executor, is_created): @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryKubernetesExecutor"]) def test_keda_advanced(self, executor): - """ - Verify keda advanced config. - """ + """Verify keda advanced config.""" expected_advanced = { "horizontalPodAutoscalerConfig": { "behavior": { @@ -85,7 +85,7 @@ def test_keda_advanced(self, executor): @staticmethod def build_query(executor, concurrency=16, queue=None): - """Builds the query used by KEDA autoscaler to determine how many workers there should be""" + """Builds the query used by KEDA autoscaler to determine how many workers there should be.""" query = ( f"SELECT ceil(COUNT(*)::decimal / {concurrency}) " "FROM task_instance WHERE (state='running' OR state='queued')" @@ -104,9 +104,7 @@ def build_query(executor, concurrency=16, queue=None): ], ) def test_keda_concurrency(self, executor, concurrency): - """ - Verify keda sql query uses configured concurrency - """ + """Verify keda sql query uses configured concurrency.""" docs = render_chart( values={ "workers": {"keda": {"enabled": True}, "persistence": {"enabled": False}}, @@ -154,9 +152,7 @@ def test_keda_query_kubernetes_queue(self, executor, queue, should_filter): ], ) def test_persistence(self, enabled, kind): - """ - If worker persistence is enabled, scaleTargetRef should be StatefulSet else Deployment. - """ + """If worker persistence is enabled, scaleTargetRef should be StatefulSet else Deployment.""" is_enabled = enabled == "enabled" docs = render_chart( values={ diff --git a/tests/charts/other/test_limit_ranges.py b/helm_tests/other/test_limit_ranges.py similarity index 97% rename from tests/charts/other/test_limit_ranges.py rename to helm_tests/other/test_limit_ranges.py index 722574501033..92b7b862a180 100644 --- a/tests/charts/other/test_limit_ranges.py +++ b/helm_tests/other/test_limit_ranges.py @@ -22,6 +22,8 @@ class TestLimitRanges: + """Tests limit ranges.""" + def test_limit_ranges_template(self): docs = render_chart( values={"limits": [{"max": {"cpu": "500m"}, "min": {"min": "200m"}, "type": "Container"}]}, diff --git a/tests/charts/other/test_pdb_pgbouncer.py b/helm_tests/other/test_pdb_pgbouncer.py similarity index 98% rename from tests/charts/other/test_pdb_pgbouncer.py rename to helm_tests/other/test_pdb_pgbouncer.py index 4d61f907d21c..42c47e9a72ec 100644 --- a/tests/charts/other/test_pdb_pgbouncer.py +++ b/helm_tests/other/test_pdb_pgbouncer.py @@ -20,6 +20,8 @@ class TestPgbouncerPdb: + """Tests PgBouncer PDB.""" + def test_should_pass_validation_with_just_pdb_enabled_v1(self): render_chart( values={"pgbouncer": {"enabled": True, "podDisruptionBudget": {"enabled": True}}}, diff --git a/tests/charts/other/test_pgbouncer.py b/helm_tests/other/test_pgbouncer.py similarity index 99% rename from tests/charts/other/test_pgbouncer.py rename to helm_tests/other/test_pgbouncer.py index 4445539ea244..7846476cc00c 100644 --- a/tests/charts/other/test_pgbouncer.py +++ b/helm_tests/other/test_pgbouncer.py @@ -25,6 +25,8 @@ class TestPgbouncer: + """Tests PgBouncer.""" + @pytest.mark.parametrize("yaml_filename", ["pgbouncer-deployment", "pgbouncer-service"]) def test_pgbouncer_resources_not_created_by_default(self, yaml_filename): docs = render_chart( @@ -363,6 +365,8 @@ def test_should_add_component_specific_annotations(self): class TestPgbouncerConfig: + """Tests PgBouncer config.""" + def test_config_not_created_by_default(self): docs = render_chart( show_only=["templates/secrets/pgbouncer-config-secret.yaml"], @@ -523,6 +527,8 @@ def test_extra_ini_configs(self): class TestPgbouncerExporter: + """Tests PgBouncer exporter.""" + def test_secret_not_created_by_default(self): docs = render_chart( show_only=["templates/secrets/pgbouncer-stats-secret.yaml"], diff --git a/tests/charts/other/test_redis.py b/helm_tests/other/test_redis.py similarity index 99% rename from tests/charts/other/test_redis.py rename to helm_tests/other/test_redis.py index 5062de57cc7c..4236d829657a 100644 --- a/tests/charts/other/test_redis.py +++ b/helm_tests/other/test_redis.py @@ -40,6 +40,8 @@ class TestRedis: + """Tests redis.""" + @staticmethod def get_broker_url_in_broker_url_secret(k8s_obj_by_key): broker_url_in_obj = b64decode( diff --git a/tests/charts/other/test_resource_quota.py b/helm_tests/other/test_resource_quota.py similarity index 98% rename from tests/charts/other/test_resource_quota.py rename to helm_tests/other/test_resource_quota.py index 1775479f7a4c..b63c111a4dc3 100644 --- a/tests/charts/other/test_resource_quota.py +++ b/helm_tests/other/test_resource_quota.py @@ -22,6 +22,8 @@ class TestResourceQuota: + """Tests resource quota.""" + def test_resource_quota_template(self): docs = render_chart( values={ diff --git a/tests/charts/other/test_statsd.py b/helm_tests/other/test_statsd.py similarity index 99% rename from tests/charts/other/test_statsd.py rename to helm_tests/other/test_statsd.py index 04b121be4617..8c048e2141a4 100644 --- a/tests/charts/other/test_statsd.py +++ b/helm_tests/other/test_statsd.py @@ -24,6 +24,8 @@ class TestStatsd: + """Tests statsd.""" + def test_should_create_statsd_default(self): docs = render_chart(show_only=["templates/statsd/statsd-deployment.yaml"]) diff --git a/tests/charts/webserver/__init__.py b/helm_tests/security/__init__.py similarity index 100% rename from tests/charts/webserver/__init__.py rename to helm_tests/security/__init__.py diff --git a/tests/charts/security/test_elasticsearch_secret.py b/helm_tests/security/test_elasticsearch_secret.py similarity index 99% rename from tests/charts/security/test_elasticsearch_secret.py rename to helm_tests/security/test_elasticsearch_secret.py index 468921290a63..c8e20f57699b 100644 --- a/tests/charts/security/test_elasticsearch_secret.py +++ b/helm_tests/security/test_elasticsearch_secret.py @@ -26,6 +26,8 @@ class TestElasticsearchSecret: + """Tests elasticsearch secret.""" + def test_should_not_generate_a_document_if_elasticsearch_disabled(self): docs = render_chart( diff --git a/tests/charts/security/test_extra_configmaps_secrets.py b/helm_tests/security/test_extra_configmaps_secrets.py similarity index 99% rename from tests/charts/security/test_extra_configmaps_secrets.py rename to helm_tests/security/test_extra_configmaps_secrets.py index bc22f26cfe08..8e50adab5af1 100644 --- a/tests/charts/security/test_extra_configmaps_secrets.py +++ b/helm_tests/security/test_extra_configmaps_secrets.py @@ -29,6 +29,8 @@ class TestExtraConfigMapsSecrets: + """Tests extra configmaps and secrets.""" + def test_extra_configmaps(self): values_str = textwrap.dedent( """ diff --git a/tests/charts/security/test_kerberos.py b/helm_tests/security/test_kerberos.py similarity index 99% rename from tests/charts/security/test_kerberos.py rename to helm_tests/security/test_kerberos.py index 27b9d02a883f..278faa80c8fe 100644 --- a/tests/charts/security/test_kerberos.py +++ b/helm_tests/security/test_kerberos.py @@ -24,6 +24,8 @@ class TestKerberos: + """Tests kerberos.""" + def test_kerberos_not_mentioned_in_render_if_disabled(self): # the name is deliberately shorter as we look for "kerberos" in the rendered chart k8s_objects = render_chart(name="no-krbros", values={"kerberos": {"enabled": False}}) diff --git a/tests/charts/security/test_metadata_connection_secret.py b/helm_tests/security/test_metadata_connection_secret.py similarity index 99% rename from tests/charts/security/test_metadata_connection_secret.py rename to helm_tests/security/test_metadata_connection_secret.py index 1c910a977bb6..4a64f3ab5b2b 100644 --- a/tests/charts/security/test_metadata_connection_secret.py +++ b/helm_tests/security/test_metadata_connection_secret.py @@ -24,6 +24,7 @@ class TestMetadataConnectionSecret: + """Tests metadata connection secret.""" non_chart_database_values = { "user": "someuser", diff --git a/tests/charts/security/test_rbac.py b/helm_tests/security/test_rbac.py similarity index 99% rename from tests/charts/security/test_rbac.py rename to helm_tests/security/test_rbac.py index 82d69cc535c1..12ffcb73e6bb 100644 --- a/tests/charts/security/test_rbac.py +++ b/helm_tests/security/test_rbac.py @@ -108,6 +108,8 @@ class TestRBAC: + """Tests RBAC.""" + def _get_values_with_version(self, values, version): if version != "default": values["airflowVersion"] = version diff --git a/tests/charts/security/test_rbac_pod_log_reader.py b/helm_tests/security/test_rbac_pod_log_reader.py similarity index 98% rename from tests/charts/security/test_rbac_pod_log_reader.py rename to helm_tests/security/test_rbac_pod_log_reader.py index ed3753ab02fc..82ec13fcc133 100644 --- a/tests/charts/security/test_rbac_pod_log_reader.py +++ b/helm_tests/security/test_rbac_pod_log_reader.py @@ -23,6 +23,8 @@ class TestPodReader: + """Tests RBAC Pod Reader.""" + @pytest.mark.parametrize( "triggerer, webserver, expected", [ diff --git a/tests/charts/security/test_result_backend_connection_secret.py b/helm_tests/security/test_result_backend_connection_secret.py similarity index 99% rename from tests/charts/security/test_result_backend_connection_secret.py rename to helm_tests/security/test_result_backend_connection_secret.py index 70a27baa5d5c..f23dfe83bd05 100644 --- a/tests/charts/security/test_result_backend_connection_secret.py +++ b/helm_tests/security/test_result_backend_connection_secret.py @@ -25,6 +25,8 @@ class TestResultBackendConnectionSecret: + """Tests result backend connection secret.""" + def _get_values_with_version(self, values, version): if version != "default": values["airflowVersion"] = version diff --git a/tests/charts/security/test_scc_rolebinding.py b/helm_tests/security/test_scc_rolebinding.py similarity index 99% rename from tests/charts/security/test_scc_rolebinding.py rename to helm_tests/security/test_scc_rolebinding.py index 25ff32956b6b..cb674f931e95 100644 --- a/tests/charts/security/test_scc_rolebinding.py +++ b/helm_tests/security/test_scc_rolebinding.py @@ -23,6 +23,8 @@ class TestSCCActivation: + """Tests SCCs.""" + @pytest.mark.parametrize( "rbac_enabled,scc_enabled,created", [ diff --git a/tests/charts/security/test_security_context.py b/helm_tests/security/test_security_context.py similarity index 99% rename from tests/charts/security/test_security_context.py rename to helm_tests/security/test_security_context.py index 66c06023d22b..f7693af19805 100644 --- a/tests/charts/security/test_security_context.py +++ b/helm_tests/security/test_security_context.py @@ -22,6 +22,8 @@ class TestSCBackwardsCompatibility: + """Tests SC Backward Compatibility.""" + def test_check_deployments_and_jobs(self): docs = render_chart( values={ @@ -97,6 +99,8 @@ def test_gitsync_sidecar_and_init_container(self): class TestSecurityContext: + """Tests security context.""" + # Test securityContext setting for Pods and Containers def test_check_default_setting(self): docs = render_chart( diff --git a/helm_tests/webserver/__init__.py b/helm_tests/webserver/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/helm_tests/webserver/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/charts/webserver/test_ingress_flower.py b/helm_tests/webserver/test_ingress_flower.py similarity index 99% rename from tests/charts/webserver/test_ingress_flower.py rename to helm_tests/webserver/test_ingress_flower.py index c915619aa8c2..0bd95bf8a2ab 100644 --- a/tests/charts/webserver/test_ingress_flower.py +++ b/helm_tests/webserver/test_ingress_flower.py @@ -25,6 +25,8 @@ class TestIngressFlower: + """Tests ingress flower.""" + def test_should_pass_validation_with_just_ingress_enabled_v1(self): render_chart( values={"flower": {"enabled": True}, "ingress": {"flower": {"enabled": True}}}, diff --git a/tests/charts/webserver/test_ingress_web.py b/helm_tests/webserver/test_ingress_web.py similarity index 99% rename from tests/charts/webserver/test_ingress_web.py rename to helm_tests/webserver/test_ingress_web.py index 13fbbdc3e7ce..b90eee352de6 100644 --- a/tests/charts/webserver/test_ingress_web.py +++ b/helm_tests/webserver/test_ingress_web.py @@ -23,6 +23,8 @@ class TestIngressWeb: + """Tests ingress web.""" + def test_should_pass_validation_with_just_ingress_enabled_v1(self): render_chart( values={"ingress": {"web": {"enabled": True}}}, diff --git a/tests/charts/webserver/test_pdb_webserver.py b/helm_tests/webserver/test_pdb_webserver.py similarity index 98% rename from tests/charts/webserver/test_pdb_webserver.py rename to helm_tests/webserver/test_pdb_webserver.py index 35347eb5a532..ce88b2a7da13 100644 --- a/tests/charts/webserver/test_pdb_webserver.py +++ b/helm_tests/webserver/test_pdb_webserver.py @@ -22,6 +22,8 @@ class TestWebserverPdb: + """Tests webserver pdb.""" + def test_should_pass_validation_with_just_pdb_enabled_v1(self): render_chart( values={"webserver": {"podDisruptionBudget": {"enabled": True}}}, diff --git a/tests/charts/webserver/test_webserver.py b/helm_tests/webserver/test_webserver.py similarity index 99% rename from tests/charts/webserver/test_webserver.py rename to helm_tests/webserver/test_webserver.py index 4b1e7ec3750a..4d6c1181b838 100644 --- a/tests/charts/webserver/test_webserver.py +++ b/helm_tests/webserver/test_webserver.py @@ -23,6 +23,8 @@ class TestWebserverDeployment: + """Tests webserver deployment.""" + def test_should_add_host_header_to_liveness_and_readiness_probes(self): docs = render_chart( values={ @@ -349,7 +351,7 @@ def test_should_create_default_affinity(self): ) def test_affinity_tolerations_topology_spread_constraints_and_node_selector_precedence(self): - """When given both global and webserver affinity etc, webserver affinity etc is used""" + """When given both global and webserver affinity etc, webserver affinity etc is used.""" expected_affinity = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -760,6 +762,8 @@ def test_webserver_pod_hostaliases(self): class TestWebserverService: + """Tests webserver service.""" + def test_default_service(self): docs = render_chart( show_only=["templates/webserver/webserver-service.yaml"], @@ -869,6 +873,8 @@ def test_nodeport_service(self, ports, expected_ports): class TestWebserverConfigmap: + """Tests webserver configmap.""" + def test_no_webserver_config_configmap_by_default(self): docs = render_chart(show_only=["templates/configmaps/webserver-configmap.yaml"]) assert 0 == len(docs) @@ -900,6 +906,8 @@ def test_webserver_config_configmap(self): class TestWebserverNetworkPolicy: + """Tests webserver network policy.""" + def test_off_by_default(self): docs = render_chart( show_only=["templates/webserver/webserver-networkpolicy.yaml"], @@ -992,6 +1000,8 @@ def test_should_add_component_specific_labels(self): class TestWebserverServiceAccount: + """Tests webserver service account.""" + def test_should_add_component_specific_labels(self): docs = render_chart( values={ diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index 691a33058236..db5690e0ad3e 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #868887 } -.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } +.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } +.breeze-testing-r5 { fill: #868887 } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -96,19 +96,19 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integratio tests.                                      │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│integration-tests                        Run the specified integratio tests.                                      │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index a292dba6a455..bc9fb047f8ef 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -114,6 +114,9 @@ services: - type: bind source: ../../../tests target: /opt/airflow/tests + - type: bind + source: ../../../helm_tests + target: /opt/airflow/helm_tests - type: bind source: ../../../kubernetes_tests target: /opt/airflow/kubernetes_tests diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 25af9ca64262..287bcbca0e09 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -461,7 +461,7 @@ else "tests/utils" ) WWW_TESTS=("tests/www") - HELM_CHART_TESTS=("tests/charts") + HELM_CHART_TESTS=("helm_tests") INTEGRATION_TESTS=("tests/integration") SYSTEM_TESTS=("tests/system") ALL_TESTS=("tests") @@ -498,7 +498,7 @@ else SELECTED_TESTS=("${WWW_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then if [[ ${HELM_TEST_PACKAGE=} != "" ]]; then - SELECTED_TESTS=("tests/charts/${HELM_TEST_PACKAGE}") + SELECTED_TESTS=("helm_tests/${HELM_TEST_PACKAGE}") else SELECTED_TESTS=("${HELM_CHART_TESTS[@]}") fi From 3a14e845640c83850e1a43748903559ca1fcacd8 Mon Sep 17 00:00:00 2001 From: Alexis BRENON Date: Fri, 7 Jul 2023 18:09:26 +0200 Subject: [PATCH 095/533] Document cron and delta timetables (#32392) Add a comparison between delta and cron data interval timetable to highlight differences when catchup is False. --- .../authoring-and-scheduling/timetable.rst | 86 ++++++++++++++++++- docs/apache-airflow/core-concepts/dag-run.rst | 6 ++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/timetable.rst b/docs/apache-airflow/authoring-and-scheduling/timetable.rst index 490e27d4f208..6366e18d2288 100644 --- a/docs/apache-airflow/authoring-and-scheduling/timetable.rst +++ b/docs/apache-airflow/authoring-and-scheduling/timetable.rst @@ -114,6 +114,11 @@ DeltaDataIntervalTimetable Schedules data intervals with a time delta. Can be selected by providing a :class:`datetime.timedelta` or ``dateutil.relativedelta.relativedelta`` to the ``schedule`` parameter of a DAG. +This timetable is more focused on the data interval value and does not necessarily align execution dates with +arbitrary bounds such as start of day or of hour. + +.. seealso:: `Differences between the cron and delta data interval timetables`_ + .. code-block:: python @dag(schedule=datetime.timedelta(minutes=30)) @@ -129,6 +134,7 @@ A timetable that accepts a cron expression, creates data intervals according to trigger points, and triggers a DAG run at the end of each data interval. .. seealso:: `Differences between the two cron timetables`_ +.. seealso:: `Differences between the cron and delta data interval timetables`_ This can be selected by providing a string that is a valid cron expression to the ``schedule`` parameter of a DAG as described in the :doc:`../core-concepts/dags` documentation. @@ -171,10 +177,15 @@ first) event for the data interval, otherwise manual runs will run with a ``data def example_dag(): pass + +Timetables comparisons +---------------------- + + .. _Differences between the two cron timetables: Differences between the two cron timetables -------------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are two timetables `CronTriggerTimetable`_ and `CronDataIntervalTimetable`_ that accepts a cron expression. There are some differences between the two: @@ -183,7 +194,7 @@ There are some differences between the two: expect cron to behave than that of `CronDataIntervalTimetable`_ (when ``catchup`` is ``False``). Whether taking care of *Data Interval* -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ `CronTriggerTimetable`_ *does not* care the idea of *data interval*. It means the value of ``data_interval_start``, ``data_interval_end`` and legacy ``execution_date`` are the same - the time when a DAG run is triggered. @@ -193,7 +204,7 @@ On the other hand, `CronDataIntervalTimetable`_ *does* care the idea of *data in and end of the interval respectively. The time when a DAG run is triggered -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There is no difference between the two when ``catchup`` is ``True``. :ref:`dag-catchup` tells you how DAG runs are triggered when ``catchup`` is ``True``. @@ -217,3 +228,72 @@ is immediately triggered after you re-enable the DAG. By these examples, you see how `CronTriggerTimetable`_ triggers DAG runs is more intuitive and more similar to what people expect cron to behave than how `CronDataIntervalTimetable`_ does. + + +.. _Differences between the cron and delta data interval timetables: + +Differences between the cron and delta data interval timetables: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Choosing between `DeltaDataIntervalTimetable`_ and `CronDataIntervalTimetable`_ depends on your use case. +If you enable a DAG at 01:05 on February 1st, the following table summarizes the DAG runs created (and the +data interval that they cover), depending on 3 arguments: ``schedule``, ``start_date`` and ``catchup``. + +.. list-table:: + :header-rows: 1 + + * - ``schedule`` + - ``start_date`` + - ``catchup`` + - Intervals covered + - Remarks + + * - ``*/30 * * * *`` + - ``year-02-01`` + - ``True`` + - * 00:00 - 00:30 + * 00:30 - 01:00 + - Same behavior than using the timedelta object. + + * - ``*/30 * * * *`` + - ``year-02-01`` + - ``False`` + - * 00:30 - 01:00 + - + + * - ``*/30 * * * *`` + - ``year-02-01 00:10`` + - ``True`` + - * 00:30 - 01:00 + - Interval 00:00 - 00:30 is not after the start date, and so is skipped. + + * - ``*/30 * * * *`` + - ``year-02-01 00:10`` + - ``False`` + - * 00:30 - 01:00 + - Whatever the start date, the data intervals are aligned with hour/day/etc. boundaries. + + * - ``datetime.timedelta(minutes=30)`` + - ``year-02-01`` + - ``True`` + - * 00:00 - 00:30 + * 00:30 - 01:00 + - Same behavior than using the cron expression. + + * - ``datetime.timedelta(minutes=30)`` + - ``year-02-01`` + - ``False`` + - * 00:35 - 01:05 + - Interval is not aligned with start date but with the current time. + + * - ``datetime.timedelta(minutes=30)`` + - ``year-02-01 00:10`` + - ``True`` + - * 00:10 - 00:40 + - Interval is aligned with start date. Next one will be triggered in 5 minutes covering 00:40 - 01:10. + + * - ``datetime.timedelta(minutes=30)`` + - ``year-02-01 00:10`` + - ``False`` + - * 00:35 - 01:05 + - Interval is aligned with current time. Next run will be triggered in 30 minutes. diff --git a/docs/apache-airflow/core-concepts/dag-run.rst b/docs/apache-airflow/core-concepts/dag-run.rst index 391fe958ff45..f9ae98e3511a 100644 --- a/docs/apache-airflow/core-concepts/dag-run.rst +++ b/docs/apache-airflow/core-concepts/dag-run.rst @@ -161,6 +161,12 @@ with a data between 2016-01-01 and 2016-01-02, and the next one will be created just after midnight on the morning of 2016-01-03 with a data interval between 2016-01-02 and 2016-01-03. +Be aware that using a ``datetime.timedelta`` object as schedule can lead to a different behavior. +In such a case, the single DAG Run created will cover data between 2016-01-01 06:00 and +2016-01-02 06:00 (one schedule interval ending now). For a more detailed description of the +differences between a cron and a delta based schedule, take a look at the +:ref:`timetables comparison ` + If the ``dag.catchup`` value had been ``True`` instead, the scheduler would have created a DAG Run for each completed interval between 2015-12-01 and 2016-01-02 (but not yet one for 2016-01-02, as that interval hasn't completed) and the scheduler will execute them sequentially. From 6c854dcb164caffea640b368eda94a2a9166fa97 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 7 Jul 2023 21:44:55 +0530 Subject: [PATCH 096/533] Migrating Google AutoML example_dags to sys tests (#32368) --------- Co-authored-by: Amogh Desai --- .../example_automl_nl_text_classification.py | 23 +++++++++++++------ .../example_automl_nl_text_sentiment.py | 20 ++++++++++++---- .../automl}/example_automl_translation.py | 22 ++++++++++++++---- ...utoml_video_intelligence_classification.py | 20 ++++++++++++---- ...mple_automl_video_intelligence_tracking.py | 19 +++++++++++---- .../example_automl_vision_object_detection.py | 20 ++++++++++++---- 6 files changed, 93 insertions(+), 31 deletions(-) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_nl_text_classification.py (87%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_nl_text_sentiment.py (88%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_translation.py (88%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_video_intelligence_classification.py (88%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_video_intelligence_tracking.py (88%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_vision_object_detection.py (88%) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py similarity index 87% rename from airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py rename to tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py index afe28a9ff82e..39129162097e 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py @@ -39,13 +39,9 @@ GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_TEXT_CLS_BUCKET = os.environ.get("GCP_AUTOML_TEXT_CLS_BUCKET", "gs://INVALID BUCKET NAME") -# Example values -DATASET_ID = "" - # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "text_classification_model_metadata": {}, } @@ -55,6 +51,7 @@ "text_classification_dataset_metadata": {"classification_type": "MULTICLASS"}, } + IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TEXT_CLS_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id @@ -65,12 +62,13 @@ start_date=datetime(2021, 1, 1), catchup=False, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -78,11 +76,9 @@ location=GCP_AUTOML_LOCATION, input_config=IMPORT_INPUT_CONFIG, ) - MODEL["dataset_id"] = dataset_id create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = cast(str, XComArg(create_model, key="model_id")) delete_model_task = AutoMLDeleteModelOperator( @@ -99,10 +95,23 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: # create_dataset_task >> import_dataset_task # create_dataset_task >> create_model # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py similarity index 88% rename from airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py rename to tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py index a823b8af8953..6bbe0d2bfbea 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py @@ -39,13 +39,9 @@ GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") GCP_AUTOML_SENTIMENT_BUCKET = os.environ.get("GCP_AUTOML_SENTIMENT_BUCKET", "gs://INVALID BUCKET NAME") -# Example values -DATASET_ID = "" - # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "text_sentiment_model_metadata": {}, } @@ -66,12 +62,13 @@ catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -100,7 +97,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -108,3 +107,14 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_translation.py b/tests/system/providers/google/cloud/automl/example_automl_translation.py similarity index 88% rename from airflow/providers/google/cloud/example_dags/example_automl_translation.py rename to tests/system/providers/google/cloud/automl/example_automl_translation.py index 2bef20caabee..87bf7166dcd1 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_translation.py +++ b/tests/system/providers/google/cloud/automl/example_automl_translation.py @@ -41,13 +41,9 @@ "GCP_AUTOML_TRANSLATION_BUCKET", "gs://INVALID BUCKET NAME/file" ) -# Example values -DATASET_ID = "TRL123456789" - # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "translation_model_metadata": {}, } @@ -60,6 +56,7 @@ }, } + IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TRANSLATION_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id @@ -69,10 +66,11 @@ with models.DAG( "example_automl_translation", start_date=datetime(2021, 1, 1), + schedule="@once", catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) @@ -106,7 +104,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -114,3 +114,15 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py similarity index 88% rename from airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py rename to tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py index 2b55c42a8ab9..63447a123059 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py @@ -41,13 +41,9 @@ "GCP_AUTOML_VIDEO_BUCKET", "gs://INVALID BUCKET NAME/hmdb_split1.csv" ) -# Example values -DATASET_ID = "VCN123455678" - # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "video_classification_model_metadata": {}, } @@ -69,12 +65,13 @@ catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -103,7 +100,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -111,3 +110,14 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py similarity index 88% rename from airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py rename to tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py index daed5748b239..5ac8fa7457fa 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py @@ -42,13 +42,10 @@ "gs://INVALID BUCKET NAME/youtube_8m_videos_animal_tiny.csv", ) -# Example values -DATASET_ID = "VOT123456789" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "video_object_tracking_model_metadata": {}, } @@ -70,12 +67,13 @@ catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -104,7 +102,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -112,3 +112,14 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py similarity index 88% rename from airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py rename to tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py index 8b9ae271d112..933c2760cbbb 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py +++ b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py @@ -41,13 +41,9 @@ "GCP_AUTOML_DETECTION_BUCKET", "gs://INVALID BUCKET NAME/img/openimage/csv/salads_ml_use.csv" ) -# Example values -DATASET_ID = "" - # Example model MODEL = { "display_name": "auto_model", - "dataset_id": DATASET_ID, "image_object_detection_model_metadata": {}, } @@ -69,12 +65,13 @@ catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -103,7 +100,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -111,3 +110,14 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 05f1acfcb708a6785d5b60dd6a2a1ef930a73a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Fri, 7 Jul 2023 11:50:48 -0700 Subject: [PATCH 097/533] Introduce a base class for aws triggers (#32274) --- airflow/providers/amazon/aws/hooks/athena.py | 2 +- .../providers/amazon/aws/operators/batch.py | 8 +- airflow/providers/amazon/aws/operators/ecs.py | 16 +- airflow/providers/amazon/aws/operators/eks.py | 13 +- airflow/providers/amazon/aws/operators/emr.py | 6 +- .../amazon/aws/operators/glue_crawler.py | 2 +- .../amazon/aws/operators/redshift_cluster.py | 20 +- airflow/providers/amazon/aws/sensors/batch.py | 14 +- airflow/providers/amazon/aws/sensors/emr.py | 10 +- .../providers/amazon/aws/triggers/athena.py | 58 +- airflow/providers/amazon/aws/triggers/base.py | 130 ++++ .../providers/amazon/aws/triggers/batch.py | 106 +-- airflow/providers/amazon/aws/triggers/ecs.py | 99 +-- airflow/providers/amazon/aws/triggers/eks.py | 253 +++---- airflow/providers/amazon/aws/triggers/emr.py | 334 ++++----- .../amazon/aws/triggers/glue_crawler.py | 74 +- .../amazon/aws/triggers/redshift_cluster.py | 408 ++++------- airflow/providers/amazon/provider.yaml | 3 + .../amazon/aws/operators/test_athena.py | 1 - .../amazon/aws/operators/test_batch.py | 9 +- .../amazon/aws/operators/test_ecs.py | 2 - .../amazon/aws/operators/test_eks.py | 4 +- .../amazon/aws/sensors/test_batch.py | 24 +- .../amazon/aws/triggers/test_athena.py | 36 +- .../amazon/aws/triggers/test_base.py | 89 +++ .../amazon/aws/triggers/test_batch.py | 216 +----- .../providers/amazon/aws/triggers/test_ecs.py | 91 +-- .../providers/amazon/aws/triggers/test_eks.py | 470 ++----------- .../providers/amazon/aws/triggers/test_emr.py | 572 ++-------------- .../amazon/aws/triggers/test_emr_trigger.py | 53 +- .../amazon/aws/triggers/test_glue.py | 19 + .../aws/triggers/test_redshift_cluster.py | 648 ++---------------- 32 files changed, 1121 insertions(+), 2669 deletions(-) create mode 100644 airflow/providers/amazon/aws/triggers/base.py create mode 100644 tests/providers/amazon/aws/triggers/test_base.py diff --git a/airflow/providers/amazon/aws/hooks/athena.py b/airflow/providers/amazon/aws/hooks/athena.py index 7341ebcf70f2..27e194d3aa8d 100644 --- a/airflow/providers/amazon/aws/hooks/athena.py +++ b/airflow/providers/amazon/aws/hooks/athena.py @@ -253,7 +253,7 @@ def poll_query_status( try: wait( waiter=self.get_waiter("query_complete"), - waiter_delay=sleep_time or self.sleep_time, + waiter_delay=self.sleep_time if sleep_time is None else sleep_time, waiter_max_attempts=max_polling_attempts or 120, args={"QueryExecutionId": query_execution_id}, failure_message=f"Error while waiting for query {query_execution_id} to complete", diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index 9dd954d05cd8..e6221ae3e0af 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -41,7 +41,7 @@ from airflow.providers.amazon.aws.links.logs import CloudWatchEventsLink from airflow.providers.amazon.aws.triggers.batch import ( BatchCreateComputeEnvironmentTrigger, - BatchOperatorTrigger, + BatchJobTrigger, ) from airflow.providers.amazon.aws.utils import trim_none_values from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher @@ -221,12 +221,12 @@ def execute(self, context: Context): if self.deferrable: self.defer( timeout=self.execution_timeout, - trigger=BatchOperatorTrigger( + trigger=BatchJobTrigger( job_id=self.job_id, - max_retries=self.max_retries or 10, + waiter_max_attempts=self.max_retries or 10, aws_conn_id=self.aws_conn_id, region_name=self.region_name, - poll_interval=self.poll_interval, + waiter_delay=self.poll_interval, ), method_name="execute_complete", ) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index e5833bf4c3d5..6df72c6264f0 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -33,7 +33,11 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.hooks.ecs import EcsClusterStates, EcsHook, should_retry_eni from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook -from airflow.providers.amazon.aws.triggers.ecs import ClusterWaiterTrigger, TaskDoneTrigger +from airflow.providers.amazon.aws.triggers.ecs import ( + ClusterActiveTrigger, + ClusterInactiveTrigger, + TaskDoneTrigger, +) from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher from airflow.utils.helpers import prune_dict from airflow.utils.session import provide_session @@ -139,13 +143,12 @@ def execute(self, context: Context): self.log.info("Cluster %r in state: %r.", self.cluster_name, cluster_state) elif self.deferrable: self.defer( - trigger=ClusterWaiterTrigger( - waiter_name="cluster_active", + trigger=ClusterActiveTrigger( cluster_arn=cluster_details["clusterArn"], waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, - region=self.region, + region_name=self.region, ), method_name="_complete_exec_with_cluster_desc", # timeout is set to ensure that if a trigger dies, the timeout does not restart @@ -217,13 +220,12 @@ def execute(self, context: Context): self.log.info("Cluster %r in state: %r.", self.cluster_name, cluster_state) elif self.deferrable: self.defer( - trigger=ClusterWaiterTrigger( - waiter_name="cluster_inactive", + trigger=ClusterInactiveTrigger( cluster_arn=cluster_details["clusterArn"], waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, - region=self.region, + region_name=self.region, ), method_name="_complete_exec_with_cluster_desc", # timeout is set to ensure that if a trigger dies, the timeout does not restart diff --git a/airflow/providers/amazon/aws/operators/eks.py b/airflow/providers/amazon/aws/operators/eks.py index 56e9269f88d7..6858f801121b 100644 --- a/airflow/providers/amazon/aws/operators/eks.py +++ b/airflow/providers/amazon/aws/operators/eks.py @@ -31,8 +31,9 @@ from airflow.providers.amazon.aws.hooks.eks import EksHook from airflow.providers.amazon.aws.triggers.eks import ( EksCreateFargateProfileTrigger, + EksCreateNodegroupTrigger, EksDeleteFargateProfileTrigger, - EksNodegroupTrigger, + EksDeleteNodegroupTrigger, ) from airflow.providers.amazon.aws.utils.waiter_with_logging import wait from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction @@ -413,12 +414,11 @@ def execute(self, context: Context): if self.deferrable: self.defer( - trigger=EksNodegroupTrigger( - waiter_name="nodegroup_active", + trigger=EksCreateNodegroupTrigger( cluster_name=self.cluster_name, nodegroup_name=self.nodegroup_name, aws_conn_id=self.aws_conn_id, - region=self.region, + region_name=self.region, waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, ), @@ -711,12 +711,11 @@ def execute(self, context: Context): eks_hook.delete_nodegroup(clusterName=self.cluster_name, nodegroupName=self.nodegroup_name) if self.deferrable: self.defer( - trigger=EksNodegroupTrigger( - waiter_name="nodegroup_deleted", + trigger=EksDeleteNodegroupTrigger( cluster_name=self.cluster_name, nodegroup_name=self.nodegroup_name, aws_conn_id=self.aws_conn_id, - region=self.region, + region_name=self.region, waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, ), diff --git a/airflow/providers/amazon/aws/operators/emr.py b/airflow/providers/amazon/aws/operators/emr.py index 8330a586e442..4cb070da733b 100644 --- a/airflow/providers/amazon/aws/operators/emr.py +++ b/airflow/providers/amazon/aws/operators/emr.py @@ -577,7 +577,7 @@ def execute(self, context: Context) -> str | None: virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, - poll_interval=self.poll_interval, + waiter_delay=self.poll_interval, ), method_name="execute_complete", ) @@ -943,8 +943,8 @@ def execute(self, context: Context) -> None: self.defer( trigger=EmrTerminateJobFlowTrigger( job_flow_id=self.job_flow_id, - poll_interval=self.waiter_delay, - max_attempts=self.waiter_max_attempts, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", diff --git a/airflow/providers/amazon/aws/operators/glue_crawler.py b/airflow/providers/amazon/aws/operators/glue_crawler.py index 71e2607039c3..c9fb298ee7c0 100644 --- a/airflow/providers/amazon/aws/operators/glue_crawler.py +++ b/airflow/providers/amazon/aws/operators/glue_crawler.py @@ -96,7 +96,7 @@ def execute(self, context: Context): self.defer( trigger=GlueCrawlerCompleteTrigger( crawler_name=crawler_name, - poll_interval=self.poll_interval, + waiter_delay=self.poll_interval, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", diff --git a/airflow/providers/amazon/aws/operators/redshift_cluster.py b/airflow/providers/amazon/aws/operators/redshift_cluster.py index cde4a32226e9..ed6aa79e9f2d 100644 --- a/airflow/providers/amazon/aws/operators/redshift_cluster.py +++ b/airflow/providers/amazon/aws/operators/redshift_cluster.py @@ -267,8 +267,8 @@ def execute(self, context: Context): self.defer( trigger=RedshiftCreateClusterTrigger( cluster_identifier=self.cluster_identifier, - poll_interval=self.poll_interval, - max_attempt=self.max_attempt, + waiter_delay=self.poll_interval, + waiter_max_attempts=self.max_attempt, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", @@ -361,8 +361,8 @@ def execute(self, context: Context) -> Any: self.defer( trigger=RedshiftCreateClusterSnapshotTrigger( cluster_identifier=self.cluster_identifier, - poll_interval=self.poll_interval, - max_attempts=self.max_attempt, + waiter_delay=self.poll_interval, + waiter_max_attempts=self.max_attempt, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", @@ -510,8 +510,8 @@ def execute(self, context: Context): self.defer( trigger=RedshiftResumeClusterTrigger( cluster_identifier=self.cluster_identifier, - poll_interval=self.poll_interval, - max_attempts=self.max_attempts, + waiter_delay=self.poll_interval, + waiter_max_attempts=self.max_attempts, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", @@ -598,8 +598,8 @@ def execute(self, context: Context): self.defer( trigger=RedshiftPauseClusterTrigger( cluster_identifier=self.cluster_identifier, - poll_interval=self.poll_interval, - max_attempts=self.max_attempts, + waiter_delay=self.poll_interval, + waiter_max_attempts=self.max_attempts, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", @@ -690,8 +690,8 @@ def execute(self, context: Context): timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60), trigger=RedshiftDeleteClusterTrigger( cluster_identifier=self.cluster_identifier, - poll_interval=self.poll_interval, - max_attempts=self.max_attempts, + waiter_delay=self.poll_interval, + waiter_max_attempts=self.max_attempts, aws_conn_id=self.aws_conn_id, ), method_name="execute_complete", diff --git a/airflow/providers/amazon/aws/sensors/batch.py b/airflow/providers/amazon/aws/sensors/batch.py index 32da5b4cf252..454d123f9471 100644 --- a/airflow/providers/amazon/aws/sensors/batch.py +++ b/airflow/providers/amazon/aws/sensors/batch.py @@ -25,7 +25,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook -from airflow.providers.amazon.aws.triggers.batch import BatchSensorTrigger +from airflow.providers.amazon.aws.triggers.batch import BatchJobTrigger from airflow.sensors.base import BaseSensorOperator if TYPE_CHECKING: @@ -98,11 +98,12 @@ def execute(self, context: Context) -> None: ) self.defer( timeout=timeout, - trigger=BatchSensorTrigger( + trigger=BatchJobTrigger( job_id=self.job_id, aws_conn_id=self.aws_conn_id, region_name=self.region_name, - poke_interval=self.poke_interval, + waiter_delay=int(self.poke_interval), + waiter_max_attempts=self.max_retries, ), method_name="execute_complete", ) @@ -113,9 +114,10 @@ def execute_complete(self, context: Context, event: dict[str, Any]) -> None: Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ - if "status" in event and event["status"] == "failure": - raise AirflowException(event["message"]) - self.log.info(event["message"]) + if event["status"] != "success": + raise AirflowException(f"Error while running job: {event}") + job_id = event["job_id"] + self.log.info("Batch Job %s complete", job_id) @deprecated(reason="use `hook` property instead.") def get_hook(self) -> BatchClientHook: diff --git a/airflow/providers/amazon/aws/sensors/emr.py b/airflow/providers/amazon/aws/sensors/emr.py index 9953dfa78260..71935e0b9b6d 100644 --- a/airflow/providers/amazon/aws/sensors/emr.py +++ b/airflow/providers/amazon/aws/sensors/emr.py @@ -316,7 +316,7 @@ def execute(self, context: Context): virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, - poll_interval=self.poll_interval, + waiter_delay=self.poll_interval, ), method_name="execute_complete", ) @@ -501,9 +501,9 @@ def execute(self, context: Context) -> None: timeout=timedelta(seconds=self.poke_interval * self.max_attempts), trigger=EmrTerminateJobFlowTrigger( job_flow_id=self.job_flow_id, - max_attempts=self.max_attempts, + waiter_max_attempts=self.max_attempts, aws_conn_id=self.aws_conn_id, - poll_interval=int(self.poke_interval), + waiter_delay=int(self.poke_interval), ), method_name="execute_complete", ) @@ -628,9 +628,9 @@ def execute(self, context: Context) -> None: trigger=EmrStepSensorTrigger( job_flow_id=self.job_flow_id, step_id=self.step_id, + waiter_delay=int(self.poke_interval), + waiter_max_attempts=self.max_attempts, aws_conn_id=self.aws_conn_id, - max_attempts=self.max_attempts, - poke_interval=int(self.poke_interval), ), method_name="execute_complete", ) diff --git a/airflow/providers/amazon/aws/triggers/athena.py b/airflow/providers/amazon/aws/triggers/athena.py index efae559470a2..636c1350598e 100644 --- a/airflow/providers/amazon/aws/triggers/athena.py +++ b/airflow/providers/amazon/aws/triggers/athena.py @@ -16,14 +16,12 @@ # under the License. from __future__ import annotations -from typing import Any - from airflow.providers.amazon.aws.hooks.athena import AthenaHook -from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait -from airflow.triggers.base import BaseTrigger, TriggerEvent +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger -class AthenaTrigger(BaseTrigger): +class AthenaTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftCreateClusterOperator. @@ -31,46 +29,30 @@ class AthenaTrigger(BaseTrigger): Redshift cluster to be in the `available` state. :param query_execution_id: ID of the Athena query execution to watch - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempt: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, query_execution_id: str, - poll_interval: int, - max_attempt: int, + waiter_delay: int, + waiter_max_attempts: int, aws_conn_id: str, ): - self.query_execution_id = query_execution_id - self.poll_interval = poll_interval - self.max_attempt = max_attempt - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "query_execution_id": str(self.query_execution_id), - "poll_interval": str(self.poll_interval), - "max_attempt": str(self.max_attempt), - "aws_conn_id": str(self.aws_conn_id), - }, + super().__init__( + serialized_fields={"query_execution_id": query_execution_id}, + waiter_name="query_complete", + waiter_args={"QueryExecutionId": query_execution_id}, + failure_message=f"Error while waiting for query {query_execution_id} to complete", + status_message=f"Query execution id: {query_execution_id}", + status_queries=["QueryExecution.Status"], + return_value=query_execution_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - async def run(self): - hook = AthenaHook(self.aws_conn_id) - async with hook.async_conn as client: - waiter = hook.get_waiter("query_complete", deferrable=True, client=client) - await async_wait( - waiter=waiter, - waiter_delay=self.poll_interval, - waiter_max_attempts=self.max_attempt, - args={"QueryExecutionId": self.query_execution_id}, - failure_message=f"Error while waiting for query {self.query_execution_id} to complete", - status_message=f"Query execution id: {self.query_execution_id}, " - "Query is still in non-terminal state", - status_args=["QueryExecution.Status.State"], - ) - yield TriggerEvent({"status": "success", "value": self.query_execution_id}) + def hook(self) -> AwsGenericHook: + return AthenaHook(self.aws_conn_id) diff --git a/airflow/providers/amazon/aws/triggers/base.py b/airflow/providers/amazon/aws/triggers/base.py new file mode 100644 index 000000000000..41f7d2dc33d7 --- /dev/null +++ b/airflow/providers/amazon/aws/triggers/base.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, AsyncIterator + +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook +from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait +from airflow.triggers.base import BaseTrigger, TriggerEvent + + +class AwsBaseWaiterTrigger(BaseTrigger): + """ + Base class for all AWS Triggers that follow the "standard" model of just waiting on a waiter. + + Subclasses need to implement the hook() method. + + :param serialized_fields: Fields that are specific to the subclass trigger and need to be serialized + to be passed to the __init__ method on deserialization. + The conn id, region, and waiter delay & attempts are always serialized. + format: {: } + + :param waiter_name: The name of the (possibly custom) boto waiter to use. + + :param waiter_args: The arguments to pass to the waiter. + :param failure_message: The message to log if a failure state is reached. + :param status_message: The message logged when printing the status of the service. + :param status_queries: A list containing the JMESPath queries to retrieve status information from + the waiter response. See https://jmespath.org/tutorial.html + + :param return_key: The key to use for the return_value in the TriggerEvent this emits on success. + Defaults to "value". + :param return_value: A value that'll be returned in the return_key field of the TriggerEvent. + Set to None if there is nothing to return. + + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. To be used to build the hook. + :param region_name: The AWS region where the resources to watch are. To be used to build the hook. + """ + + def __init__( + self, + *, + serialized_fields: dict[str, Any], + waiter_name: str, + waiter_args: dict[str, Any], + failure_message: str, + status_message: str, + status_queries: list[str], + return_key: str = "value", + return_value: Any, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str | None, + region_name: str | None = None, + ): + # parameters that should be hardcoded in the child's implem + self.serialized_fields = serialized_fields + + self.waiter_name = waiter_name + self.waiter_args = waiter_args + self.failure_message = failure_message + self.status_message = status_message + self.status_queries = status_queries + + self.return_key = return_key + self.return_value = return_value + + # parameters that should be passed directly from the child's parameters + self.waiter_delay = waiter_delay + self.attempts = waiter_max_attempts + self.aws_conn_id = aws_conn_id + self.region_name = region_name + + def serialize(self) -> tuple[str, dict[str, Any]]: + # here we put together the "common" params, + # and whatever extras we got from the subclass in serialized_fields + params = dict( + { + "waiter_delay": self.waiter_delay, + "waiter_max_attempts": self.attempts, + "aws_conn_id": self.aws_conn_id, + }, + **self.serialized_fields, + ) + if self.region_name: + # if we serialize the None value from this, it breaks subclasses that don't have it in their ctor. + params["region_name"] = self.region_name + return ( + # remember that self is an instance of the subclass here, not of this class. + self.__class__.__module__ + "." + self.__class__.__qualname__, + params, + ) + + @abstractmethod + def hook(self) -> AwsGenericHook: + """Override in subclasses to return the right hook.""" + ... + + async def run(self) -> AsyncIterator[TriggerEvent]: + hook = self.hook() + async with hook.async_conn as client: + waiter = hook.get_waiter(self.waiter_name, deferrable=True, client=client) + await async_wait( + waiter, + self.waiter_delay, + self.attempts, + self.waiter_args, + self.failure_message, + self.status_message, + self.status_queries, + ) + yield TriggerEvent({"status": "success", self.return_key: self.return_value}) diff --git a/airflow/providers/amazon/aws/triggers/batch.py b/airflow/providers/amazon/aws/triggers/batch.py index f7d335280d6a..87be13933293 100644 --- a/airflow/providers/amazon/aws/triggers/batch.py +++ b/airflow/providers/amazon/aws/triggers/batch.py @@ -21,12 +21,15 @@ from typing import Any from botocore.exceptions import WaiterError +from deprecated import deprecated +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook -from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.triggers.base import BaseTrigger, TriggerEvent +@deprecated(reason="use BatchJobTrigger instead") class BatchOperatorTrigger(BaseTrigger): """ Asynchronously poll the boto3 API and wait for the Batch job to be in the `SUCCEEDED` state. @@ -106,6 +109,7 @@ async def run(self): yield TriggerEvent({"status": "success", "job_id": self.job_id}) +@deprecated(reason="use BatchJobTrigger instead") class BatchSensorTrigger(BaseTrigger): """ Checks for the status of a submitted job_id to AWS Batch until it reaches a failure or a success state. @@ -189,56 +193,78 @@ async def run(self): ) -class BatchCreateComputeEnvironmentTrigger(BaseTrigger): +class BatchJobTrigger(AwsBaseWaiterTrigger): + """ + Checks for the status of a submitted job_id to AWS Batch until it reaches a failure or a success state. + + :param job_id: the job ID, to poll for job completion or not + :param region_name: AWS region name to use + Override the region_name in connection (if provided) + :param aws_conn_id: connection id of AWS credentials / region name. If None, + credential boto3 strategy will be used + :param waiter_delay: polling period in seconds to check for the status of the job + :param waiter_max_attempts: The maximum number of attempts to be made. + """ + + def __init__( + self, + job_id: str | None, + region_name: str | None, + aws_conn_id: str | None = "aws_default", + waiter_delay: int = 5, + waiter_max_attempts: int = 720, + ): + super().__init__( + serialized_fields={"job_id": job_id}, + waiter_name="batch_job_complete", + waiter_args={"jobs": [job_id]}, + failure_message=f"Failure while running batch job {job_id}", + status_message=f"Batch job {job_id} not ready yet", + status_queries=["jobs[].status", "computeEnvironments[].statusReason"], + return_key="job_id", + return_value=job_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) + + def hook(self) -> AwsGenericHook: + return BatchClientHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class BatchCreateComputeEnvironmentTrigger(AwsBaseWaiterTrigger): """ Asynchronously poll the boto3 API and wait for the compute environment to be ready. - :param job_id: A unique identifier for the cluster. - :param max_retries: The maximum number of attempts to be made. + :param compute_env_arn: The ARN of the compute env. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. :param region_name: region name to use in AWS Hook - :param poll_interval: The amount of time in seconds to wait between attempts. + :param waiter_delay: The amount of time in seconds to wait between attempts. """ def __init__( self, - compute_env_arn: str | None = None, - poll_interval: int = 30, - max_retries: int = 10, + compute_env_arn: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 10, aws_conn_id: str | None = "aws_default", region_name: str | None = None, ): - super().__init__() - self.compute_env_arn = compute_env_arn - self.max_retries = max_retries - self.aws_conn_id = aws_conn_id - self.region_name = region_name - self.poll_interval = poll_interval - - def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes BatchOperatorTrigger arguments and classpath.""" - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "compute_env_arn": self.compute_env_arn, - "max_retries": self.max_retries, - "aws_conn_id": self.aws_conn_id, - "region_name": self.region_name, - "poll_interval": self.poll_interval, - }, + super().__init__( + serialized_fields={"compute_env_arn": compute_env_arn}, + waiter_name="compute_env_ready", + waiter_args={"computeEnvironments": [compute_env_arn]}, + failure_message="Failure while creating Compute Environment", + status_message="Compute Environment not ready yet", + status_queries=["computeEnvironments[].status", "computeEnvironments[].statusReason"], + return_value=compute_env_arn, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) - async def run(self): - hook = BatchClientHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) - async with hook.async_conn as client: - waiter = hook.get_waiter("compute_env_ready", deferrable=True, client=client) - await async_wait( - waiter=waiter, - waiter_delay=self.poll_interval, - waiter_max_attempts=self.max_retries, - args={"computeEnvironments": [self.compute_env_arn]}, - failure_message="Failure while creating Compute Environment", - status_message="Compute Environment not ready yet", - status_args=["computeEnvironments[].status", "computeEnvironments[].statusReason"], - ) - yield TriggerEvent({"status": "success", "value": self.compute_env_arn}) + def hook(self) -> AwsGenericHook: + return BatchClientHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) diff --git a/airflow/providers/amazon/aws/triggers/ecs.py b/airflow/providers/amazon/aws/triggers/ecs.py index c8977d33e960..29ad22e13e79 100644 --- a/airflow/providers/amazon/aws/triggers/ecs.py +++ b/airflow/providers/amazon/aws/triggers/ecs.py @@ -22,68 +22,89 @@ from botocore.exceptions import ClientError, WaiterError +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.ecs import EcsHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher -from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait from airflow.triggers.base import BaseTrigger, TriggerEvent -class ClusterWaiterTrigger(BaseTrigger): +class ClusterActiveTrigger(AwsBaseWaiterTrigger): """ - Polls the status of a cluster using a given waiter. Can be used to poll for an active or inactive cluster. + Polls the status of a cluster until it's active. - :param waiter_name: Name of the waiter to use, for instance 'cluster_active' or 'cluster_inactive' :param cluster_arn: ARN of the cluster to watch. :param waiter_delay: The amount of time in seconds to wait between attempts. :param waiter_max_attempts: The number of times to ping for status. Will fail after that many unsuccessful attempts. :param aws_conn_id: The Airflow connection used for AWS credentials. - :param region: The AWS region where the cluster is located. + :param region_name: The AWS region where the cluster is located. """ def __init__( self, - waiter_name: str, cluster_arn: str, - waiter_delay: int | None, - waiter_max_attempts: int | None, + waiter_delay: int, + waiter_max_attempts: int, aws_conn_id: str | None, - region: str | None, + region_name: str | None, ): - self.cluster_arn = cluster_arn - self.waiter_name = waiter_name - self.waiter_delay = waiter_delay if waiter_delay is not None else 15 # written like this to allow 0 - self.attempts = waiter_max_attempts or 999999999 - self.aws_conn_id = aws_conn_id - self.region = region + super().__init__( + serialized_fields={"cluster_arn": cluster_arn}, + waiter_name="cluster_active", + waiter_args={"clusters": [cluster_arn]}, + failure_message="Failure while waiting for cluster to be available", + status_message="Cluster is not ready yet", + status_queries=["clusters[].status", "failures"], + return_key="arn", + return_value=cluster_arn, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "waiter_name": self.waiter_name, - "cluster_arn": self.cluster_arn, - "waiter_delay": self.waiter_delay, - "waiter_max_attempts": self.attempts, - "aws_conn_id": self.aws_conn_id, - "region": self.region, - }, + def hook(self) -> AwsGenericHook: + return EcsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class ClusterInactiveTrigger(AwsBaseWaiterTrigger): + """ + Polls the status of a cluster until it's inactive. + + :param cluster_arn: ARN of the cluster to watch. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The number of times to ping for status. + Will fail after that many unsuccessful attempts. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: The AWS region where the cluster is located. + """ + + def __init__( + self, + cluster_arn: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str | None, + region_name: str | None, + ): + super().__init__( + serialized_fields={"cluster_arn": cluster_arn}, + waiter_name="cluster_inactive", + waiter_args={"clusters": [cluster_arn]}, + failure_message="Failure while waiting for cluster to be deactivated", + status_message="Cluster deactivation is not done yet", + status_queries=["clusters[].status", "failures"], + return_value=cluster_arn, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) - async def run(self) -> AsyncIterator[TriggerEvent]: - async with EcsHook(aws_conn_id=self.aws_conn_id, region_name=self.region).async_conn as client: - waiter = client.get_waiter(self.waiter_name) - await async_wait( - waiter, - self.waiter_delay, - self.attempts, - {"clusters": [self.cluster_arn]}, - "error when checking cluster status", - "Status of cluster", - ["clusters[].status"], - ) - yield TriggerEvent({"status": "success", "arn": self.cluster_arn}) + def hook(self) -> AwsGenericHook: + return EcsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) class TaskDoneTrigger(BaseTrigger): diff --git a/airflow/providers/amazon/aws/triggers/eks.py b/airflow/providers/amazon/aws/triggers/eks.py index d01c88dc88c8..a6fb75eb80fa 100644 --- a/airflow/providers/amazon/aws/triggers/eks.py +++ b/airflow/providers/amazon/aws/triggers/eks.py @@ -16,18 +16,15 @@ # under the License. from __future__ import annotations -import asyncio -from typing import Any +import warnings -from botocore.exceptions import WaiterError - -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.eks import EksHook -from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait -from airflow.triggers.base import BaseTrigger, TriggerEvent +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger -class EksCreateFargateProfileTrigger(BaseTrigger): +class EksCreateFargateProfileTrigger(AwsBaseWaiterTrigger): """ Asynchronously wait for the fargate profile to be created. @@ -46,57 +43,35 @@ def __init__( waiter_max_attempts: int, aws_conn_id: str, region: str | None = None, + region_name: str | None = None, ): - self.cluster_name = cluster_name - self.fargate_profile_name = fargate_profile_name - self.waiter_delay = waiter_delay - self.waiter_max_attempts = waiter_max_attempts - self.aws_conn_id = aws_conn_id - self.region = region - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "cluster_name": self.cluster_name, - "fargate_profile_name": self.fargate_profile_name, - "waiter_delay": str(self.waiter_delay), - "waiter_max_attempts": str(self.waiter_max_attempts), - "aws_conn_id": self.aws_conn_id, - "region": self.region, - }, + if region is not None: + warnings.warn( + "please use region_name param instead of region", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + region_name = region + + super().__init__( + serialized_fields={"cluster_name": cluster_name, "fargate_profile_name": fargate_profile_name}, + waiter_name="fargate_profile_active", + waiter_args={"clusterName": cluster_name, "fargateProfileName": fargate_profile_name}, + failure_message="Failure while creating Fargate profile", + status_message="Fargate profile not created yet", + status_queries=["fargateProfile.status"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) - async def run(self): - self.hook = EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region) - async with self.hook.async_conn as client: - attempt = 0 - waiter = client.get_waiter("fargate_profile_active") - while attempt < int(self.waiter_max_attempts): - attempt += 1 - try: - await waiter.wait( - clusterName=self.cluster_name, - fargateProfileName=self.fargate_profile_name, - WaiterConfig={"Delay": int(self.waiter_delay), "MaxAttempts": 1}, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - raise AirflowException(f"Create Fargate Profile failed: {error}") - self.log.info( - "Status of fargate profile is %s", error.last_response["fargateProfile"]["status"] - ) - await asyncio.sleep(int(self.waiter_delay)) - if attempt >= int(self.waiter_max_attempts): - raise AirflowException( - f"Create Fargate Profile failed - max attempts reached: {self.waiter_max_attempts}" - ) - else: - yield TriggerEvent({"status": "success", "message": "Fargate Profile Created"}) + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) -class EksDeleteFargateProfileTrigger(BaseTrigger): +class EksDeleteFargateProfileTrigger(AwsBaseWaiterTrigger): """ Asynchronously wait for the fargate profile to be deleted. @@ -115,59 +90,37 @@ def __init__( waiter_max_attempts: int, aws_conn_id: str, region: str | None = None, + region_name: str | None = None, ): - self.cluster_name = cluster_name - self.fargate_profile_name = fargate_profile_name - self.waiter_delay = waiter_delay - self.waiter_max_attempts = waiter_max_attempts - self.aws_conn_id = aws_conn_id - self.region = region - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "cluster_name": self.cluster_name, - "fargate_profile_name": self.fargate_profile_name, - "waiter_delay": str(self.waiter_delay), - "waiter_max_attempts": str(self.waiter_max_attempts), - "aws_conn_id": self.aws_conn_id, - "region": self.region, - }, + if region is not None: + warnings.warn( + "please use region_name param instead of region", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + region_name = region + + super().__init__( + serialized_fields={"cluster_name": cluster_name, "fargate_profile_name": fargate_profile_name}, + waiter_name="fargate_profile_deleted", + waiter_args={"clusterName": cluster_name, "fargateProfileName": fargate_profile_name}, + failure_message="Failure while deleting Fargate profile", + status_message="Fargate profile not deleted yet", + status_queries=["fargateProfile.status"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) - async def run(self): - self.hook = EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region) - async with self.hook.async_conn as client: - attempt = 0 - waiter = client.get_waiter("fargate_profile_deleted") - while attempt < int(self.waiter_max_attempts): - attempt += 1 - try: - await waiter.wait( - clusterName=self.cluster_name, - fargateProfileName=self.fargate_profile_name, - WaiterConfig={"Delay": int(self.waiter_delay), "MaxAttempts": 1}, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - raise AirflowException(f"Delete Fargate Profile failed: {error}") - self.log.info( - "Status of fargate profile is %s", error.last_response["fargateProfile"]["status"] - ) - await asyncio.sleep(int(self.waiter_delay)) - if attempt >= int(self.waiter_max_attempts): - raise AirflowException( - f"Delete Fargate Profile failed - max attempts reached: {self.waiter_max_attempts}" - ) - else: - yield TriggerEvent({"status": "success", "message": "Fargate Profile Deleted"}) + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) -class EksNodegroupTrigger(BaseTrigger): +class EksCreateNodegroupTrigger(AwsBaseWaiterTrigger): """ - Trigger for EksCreateNodegroupOperator and EksDeleteNodegroupOperator. + Trigger for EksCreateNodegroupOperator. The trigger will asynchronously poll the boto3 API and wait for the nodegroup to be in the state specified by the waiter. @@ -184,54 +137,70 @@ class EksNodegroupTrigger(BaseTrigger): def __init__( self, - waiter_name: str, cluster_name: str, nodegroup_name: str, waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region: str | None, + region_name: str | None, ): - self.waiter_name = waiter_name - self.cluster_name = cluster_name - self.nodegroup_name = nodegroup_name - self.aws_conn_id = aws_conn_id - self.waiter_delay = waiter_delay - self.waiter_max_attempts = waiter_max_attempts - self.region = region - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "waiter_name": self.waiter_name, - "cluster_name": self.cluster_name, - "nodegroup_name": self.nodegroup_name, - "waiter_delay": str(self.waiter_delay), - "waiter_max_attempts": str(self.waiter_max_attempts), - "aws_conn_id": self.aws_conn_id, - "region": self.region, - }, + super().__init__( + serialized_fields={"cluster_name": cluster_name, "nodegroup_name": nodegroup_name}, + waiter_name="nodegroup_active", + waiter_args={"clusterName": cluster_name, "nodegroupName": nodegroup_name}, + failure_message="Error creating nodegroup", + status_message="Nodegroup status is", + status_queries=["nodegroup.status", "nodegroup.health.issues"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) - async def run(self): - self.hook = EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region) - async with self.hook.async_conn as client: - waiter = client.get_waiter(self.waiter_name) - await async_wait( - waiter=waiter, - waiter_max_attempts=int(self.waiter_max_attempts), - waiter_delay=int(self.waiter_delay), - args={"clusterName": self.cluster_name, "nodegroupName": self.nodegroup_name}, - failure_message="Error checking nodegroup", - status_message="Nodegroup status is", - status_args=["nodegroup.status"], - ) + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class EksDeleteNodegroupTrigger(AwsBaseWaiterTrigger): + """ + Trigger for EksDeleteNodegroupOperator. + + The trigger will asynchronously poll the boto3 API and wait for the + nodegroup to be in the state specified by the waiter. - yield TriggerEvent( - { - "status": "success", - "cluster_name": self.cluster_name, - "nodegroup_name": self.nodegroup_name, - } + :param waiter_name: Name of the waiter to use, for instance 'nodegroup_active' or 'nodegroup_deleted' + :param cluster_name: The name of the EKS cluster associated with the node group. + :param nodegroup_name: The name of the nodegroup to check. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region: Which AWS region the connection should use. (templated) + If this is None or empty then the default boto3 behaviour is used. + """ + + def __init__( + self, + cluster_name: str, + nodegroup_name: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + ): + super().__init__( + serialized_fields={"cluster_name": cluster_name, "nodegroup_name": nodegroup_name}, + waiter_name="nodegroup_deleted", + waiter_args={"clusterName": cluster_name, "nodegroupName": nodegroup_name}, + failure_message="Error deleting nodegroup", + status_message="Nodegroup status is", + status_queries=["nodegroup.status", "nodegroup.health.issues"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, ) + + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) diff --git a/airflow/providers/amazon/aws/triggers/emr.py b/airflow/providers/amazon/aws/triggers/emr.py index dbf620e9cbec..7deadc1f3746 100644 --- a/airflow/providers/amazon/aws/triggers/emr.py +++ b/airflow/providers/amazon/aws/triggers/emr.py @@ -17,16 +17,16 @@ from __future__ import annotations import asyncio -from functools import cached_property -from typing import Any, AsyncIterator +import warnings +from typing import Any from botocore.exceptions import WaiterError -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook -from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.triggers.base import BaseTrigger, TriggerEvent -from airflow.utils.helpers import prune_dict class EmrAddStepsTrigger(BaseTrigger): @@ -102,157 +102,113 @@ async def run(self): yield TriggerEvent({"status": "success", "message": "Steps completed", "step_ids": self.step_ids}) -class EmrCreateJobFlowTrigger(BaseTrigger): +class EmrCreateJobFlowTrigger(AwsBaseWaiterTrigger): """ Asynchronously poll the boto3 API and wait for the JobFlow to finish executing. :param job_flow_id: The id of the job flow to wait for. - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, job_flow_id: str, - poll_interval: int, - max_attempts: int, - aws_conn_id: str, + poll_interval: int | None = None, # deprecated + max_attempts: int | None = None, # deprecated + aws_conn_id: str | None = None, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, ): - self.job_flow_id = job_flow_id - self.poll_interval = poll_interval - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "job_flow_id": self.job_flow_id, - "poll_interval": str(self.poll_interval), - "max_attempts": str(self.max_attempts), - "aws_conn_id": self.aws_conn_id, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempts", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"job_flow_id": job_flow_id}, + waiter_name="job_flow_waiting", + waiter_args={"ClusterId": job_flow_id}, + failure_message="JobFlow creation failed", + status_message="JobFlow creation in progress", + status_queries=[ + "Cluster.Status.State", + "Cluster.Status.StateChangeReason", + "Cluster.Status.ErrorDetails", + ], + return_key="job_flow_id", + return_value=job_flow_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - async def run(self): - self.hook = EmrHook(aws_conn_id=self.aws_conn_id) - async with self.hook.async_conn as client: - attempt = 0 - waiter = self.hook.get_waiter("job_flow_waiting", deferrable=True, client=client) - while attempt < int(self.max_attempts): - attempt = attempt + 1 - try: - await waiter.wait( - ClusterId=self.job_flow_id, - WaiterConfig=prune_dict( - { - "Delay": self.poll_interval, - "MaxAttempts": 1, - } - ), - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - raise AirflowException(f"JobFlow creation failed: {error}") - self.log.info( - "Status of jobflow is %s - %s", - error.last_response["Cluster"]["Status"]["State"], - error.last_response["Cluster"]["Status"]["StateChangeReason"], - ) - await asyncio.sleep(int(self.poll_interval)) - if attempt >= int(self.max_attempts): - raise AirflowException(f"JobFlow creation failed - max attempts reached: {self.max_attempts}") - else: - yield TriggerEvent( - { - "status": "success", - "message": "JobFlow completed successfully", - "job_flow_id": self.job_flow_id, - } - ) + def hook(self) -> AwsGenericHook: + return EmrHook(aws_conn_id=self.aws_conn_id) -class EmrTerminateJobFlowTrigger(BaseTrigger): +class EmrTerminateJobFlowTrigger(AwsBaseWaiterTrigger): """ Asynchronously poll the boto3 API and wait for the JobFlow to finish terminating. :param job_flow_id: ID of the EMR Job Flow to terminate - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, job_flow_id: str, - poll_interval: int, - max_attempts: int, - aws_conn_id: str, + poll_interval: int | None = None, # deprecated + max_attempts: int | None = None, # deprecated + aws_conn_id: str | None = None, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, ): - self.job_flow_id = job_flow_id - self.poll_interval = poll_interval - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "job_flow_id": self.job_flow_id, - "poll_interval": str(self.poll_interval), - "max_attempts": str(self.max_attempts), - "aws_conn_id": self.aws_conn_id, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempts", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"job_flow_id": job_flow_id}, + waiter_name="job_flow_terminated", + waiter_args={"ClusterId": job_flow_id}, + failure_message="JobFlow termination failed", + status_message="JobFlow termination in progress", + status_queries=[ + "Cluster.Status.State", + "Cluster.Status.StateChangeReason", + "Cluster.Status.ErrorDetails", + ], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - async def run(self): - self.hook = EmrHook(aws_conn_id=self.aws_conn_id) - async with self.hook.async_conn as client: - attempt = 0 - waiter = self.hook.get_waiter("job_flow_terminated", deferrable=True, client=client) - while attempt < int(self.max_attempts): - attempt = attempt + 1 - try: - await waiter.wait( - ClusterId=self.job_flow_id, - WaiterConfig=prune_dict( - { - "Delay": self.poll_interval, - "MaxAttempts": 1, - } - ), - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - raise AirflowException(f"JobFlow termination failed: {error}") - self.log.info( - "Status of jobflow is %s - %s", - error.last_response["Cluster"]["Status"]["State"], - error.last_response["Cluster"]["Status"]["StateChangeReason"], - ) - await asyncio.sleep(int(self.poll_interval)) - if attempt >= int(self.max_attempts): - raise AirflowException(f"JobFlow termination failed - max attempts reached: {self.max_attempts}") - else: - yield TriggerEvent( - { - "status": "success", - "message": "JobFlow terminated successfully", - } - ) + def hook(self) -> AwsGenericHook: + return EmrHook(aws_conn_id=self.aws_conn_id) -class EmrContainerTrigger(BaseTrigger): +class EmrContainerTrigger(AwsBaseWaiterTrigger): """ Poll for the status of EMR container until reaches terminal state. :param virtual_cluster_id: Reference Emr cluster id :param job_id: job_id to check the state :param aws_conn_id: Reference to AWS connection id - :param poll_interval: polling period in seconds to check for the status + :param waiter_delay: polling period in seconds to check for the status """ def __init__( @@ -260,116 +216,70 @@ def __init__( virtual_cluster_id: str, job_id: str, aws_conn_id: str = "aws_default", - poll_interval: int = 30, - **kwargs: Any, + poll_interval: int | None = None, # deprecated + waiter_delay: int = 30, + waiter_max_attempts: int = 600, ): - self.virtual_cluster_id = virtual_cluster_id - self.job_id = job_id - self.aws_conn_id = aws_conn_id - self.poll_interval = poll_interval - super().__init__(**kwargs) - - @cached_property - def hook(self) -> EmrContainerHook: - return EmrContainerHook(self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) - - def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes EmrContainerTrigger arguments and classpath.""" - return ( - "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger", - { - "virtual_cluster_id": self.virtual_cluster_id, - "job_id": self.job_id, - "aws_conn_id": self.aws_conn_id, - "poll_interval": self.poll_interval, - }, + if poll_interval is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + super().__init__( + serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, + waiter_name="container_job_complete", + waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, + failure_message="Job failed", + status_message="Job in progress", + status_queries=["jobRun.state", "jobRun.failureReason"], + return_key="job_id", + return_value=job_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - async def run(self) -> AsyncIterator[TriggerEvent]: - async with self.hook.async_conn as client: - waiter = self.hook.get_waiter("container_job_complete", deferrable=True, client=client) - attempt = 0 - while True: - attempt = attempt + 1 - try: - await waiter.wait( - id=self.job_id, - virtualClusterId=self.virtual_cluster_id, - WaiterConfig={ - "Delay": self.poll_interval, - "MaxAttempts": 1, - }, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - yield TriggerEvent({"status": "failure", "message": f"Job Failed: {error}"}) - break - self.log.info( - "Job status is %s. Retrying attempt %s", - error.last_response["jobRun"]["state"], - attempt, - ) - await asyncio.sleep(int(self.poll_interval)) - - yield TriggerEvent({"status": "success", "job_id": self.job_id}) + def hook(self) -> AwsGenericHook: + return EmrContainerHook(self.aws_conn_id) -class EmrStepSensorTrigger(BaseTrigger): +class EmrStepSensorTrigger(AwsBaseWaiterTrigger): """ Poll for the status of EMR container until reaches terminal state. :param job_flow_id: job_flow_id which contains the step check the state of :param step_id: step to check the state of + :param waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made :param aws_conn_id: Reference to AWS connection id - :param max_attempts: The maximum number of attempts to be made - :param poke_interval: polling period in seconds to check for the status """ def __init__( self, job_flow_id: str, step_id: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, aws_conn_id: str = "aws_default", - max_attempts: int = 60, - poke_interval: int = 30, - **kwargs: Any, ): - self.job_flow_id = job_flow_id - self.step_id = step_id - self.aws_conn_id = aws_conn_id - self.max_attempts = max_attempts - self.poke_interval = poke_interval - super().__init__(**kwargs) - - @cached_property - def hook(self) -> EmrHook: - return EmrHook(self.aws_conn_id) - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.emr.EmrStepSensorTrigger", - { - "job_flow_id": self.job_flow_id, - "step_id": self.step_id, - "aws_conn_id": self.aws_conn_id, - "max_attempts": self.max_attempts, - "poke_interval": self.poke_interval, - }, + super().__init__( + serialized_fields={"job_flow_id": job_flow_id, "step_id": step_id}, + waiter_name="step_wait_for_terminal", + waiter_args={"ClusterId": job_flow_id, "StepId": step_id}, + failure_message=f"Error while waiting for step {step_id} to complete", + status_message=f"Step id: {step_id}, Step is still in non-terminal state", + status_queries=[ + "Step.Status.State", + "Step.Status.FailureDetails", + "Step.Status.StateChangeReason", + ], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - async def run(self) -> AsyncIterator[TriggerEvent]: - - async with self.hook.async_conn as client: - waiter = client.get_waiter("step_wait_for_terminal", deferrable=True, client=client) - await async_wait( - waiter=waiter, - waiter_delay=self.poke_interval, - waiter_max_attempts=self.max_attempts, - args={"ClusterId": self.job_flow_id, "StepId": self.step_id}, - failure_message=f"Error while waiting for step {self.step_id} to complete", - status_message=f"Step id: {self.step_id}, Step is still in non-terminal state", - status_args=["Step.Status.State"], - ) - - yield TriggerEvent({"status": "success"}) + def hook(self) -> AwsGenericHook: + return EmrHook(self.aws_conn_id) diff --git a/airflow/providers/amazon/aws/triggers/glue_crawler.py b/airflow/providers/amazon/aws/triggers/glue_crawler.py index e891e87627f7..092b05242983 100644 --- a/airflow/providers/amazon/aws/triggers/glue_crawler.py +++ b/airflow/providers/amazon/aws/triggers/glue_crawler.py @@ -16,17 +16,15 @@ # under the License. from __future__ import annotations -import asyncio -from functools import cached_property -from typing import AsyncIterator - -from botocore.exceptions import WaiterError +import warnings +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.glue_crawler import GlueCrawlerHook -from airflow.triggers.base import BaseTrigger, TriggerEvent +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger -class GlueCrawlerCompleteTrigger(BaseTrigger): +class GlueCrawlerCompleteTrigger(AwsBaseWaiterTrigger): """ Watches for a glue crawl, triggers when it finishes. @@ -35,41 +33,33 @@ class GlueCrawlerCompleteTrigger(BaseTrigger): :param aws_conn_id: The Airflow connection used for AWS credentials. """ - def __init__(self, crawler_name: str, poll_interval: int, aws_conn_id: str): - super().__init__() - self.crawler_name = crawler_name - self.poll_interval = poll_interval - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict]: - return ( - # dynamically generate the fully qualified name of the class - self.__class__.__module__ + "." + self.__class__.__qualname__, - { - "crawler_name": self.crawler_name, - "poll_interval": self.poll_interval, - "aws_conn_id": self.aws_conn_id, - }, + def __init__( + self, + crawler_name: str, + poll_interval: int | None = None, + aws_conn_id: str = "aws_default", + waiter_delay: int = 5, + waiter_max_attempts: int = 1500, + ): + if poll_interval is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + super().__init__( + serialized_fields={"crawler_name": crawler_name}, + waiter_name="crawler_ready", + waiter_args={"Name": crawler_name}, + failure_message="Error while waiting for glue crawl to complete", + status_message="Status of glue crawl is", + status_queries=["Crawler.State", "Crawler.LastCrawl"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self) -> GlueCrawlerHook: + def hook(self) -> AwsGenericHook: return GlueCrawlerHook(aws_conn_id=self.aws_conn_id) - - async def run(self) -> AsyncIterator[TriggerEvent]: - async with self.hook.async_conn as client: - waiter = self.hook.get_waiter("crawler_ready", deferrable=True, client=client) - while True: - try: - await waiter.wait( - Name=self.crawler_name, - WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 1}, - ) - break # we reach this point only if the waiter met a success criteria - except WaiterError as error: - if "terminal failure" in str(error): - raise - self.log.info("Status of glue crawl is %s", error.last_response["Crawler"]["State"]) - await asyncio.sleep(int(self.poll_interval)) - - yield TriggerEvent({"status": "success", "message": "Crawl Complete"}) diff --git a/airflow/providers/amazon/aws/triggers/redshift_cluster.py b/airflow/providers/amazon/aws/triggers/redshift_cluster.py index b3770bd395c6..678f435364c0 100644 --- a/airflow/providers/amazon/aws/triggers/redshift_cluster.py +++ b/airflow/providers/amazon/aws/triggers/redshift_cluster.py @@ -16,17 +16,15 @@ # under the License. from __future__ import annotations -import asyncio -from functools import cached_property -from typing import Any, AsyncIterator - -from botocore.exceptions import WaiterError +import warnings +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftHook -from airflow.triggers.base import BaseTrigger, TriggerEvent +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger -class RedshiftCreateClusterTrigger(BaseTrigger): +class RedshiftCreateClusterTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftCreateClusterOperator. @@ -34,51 +32,47 @@ class RedshiftCreateClusterTrigger(BaseTrigger): Redshift cluster to be in the `available` state. :param cluster_identifier: A unique identifier for the cluster. - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempt: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, cluster_identifier: str, - poll_interval: int, - max_attempt: int, - aws_conn_id: str, + poll_interval: int | None = None, + max_attempt: int | None = None, + aws_conn_id: str = "aws_default", + waiter_delay: int = 15, + waiter_max_attempts: int = 999999, ): - self.cluster_identifier = cluster_identifier - self.poll_interval = poll_interval - self.max_attempt = max_attempt - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftCreateClusterTrigger", - { - "cluster_identifier": str(self.cluster_identifier), - "poll_interval": str(self.poll_interval), - "max_attempt": str(self.max_attempt), - "aws_conn_id": str(self.aws_conn_id), - }, + if poll_interval is not None or max_attempt is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempt.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempt or waiter_max_attempts + super().__init__( + serialized_fields={"cluster_identifier": cluster_identifier}, + waiter_name="cluster_available", + waiter_args={"ClusterIdentifier": cluster_identifier}, + failure_message="Error while creating the redshift cluster", + status_message="Redshift cluster creation in progress", + status_queries=["Clusters[].ClusterStatus"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self) -> RedshiftHook: + def hook(self) -> AwsGenericHook: return RedshiftHook(aws_conn_id=self.aws_conn_id) - async def run(self): - async with self.hook.async_conn as client: - await client.get_waiter("cluster_available").wait( - ClusterIdentifier=self.cluster_identifier, - WaiterConfig={ - "Delay": int(self.poll_interval), - "MaxAttempts": int(self.max_attempt), - }, - ) - yield TriggerEvent({"status": "success", "message": "Cluster Created"}) - -class RedshiftPauseClusterTrigger(BaseTrigger): +class RedshiftPauseClusterTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftPauseClusterOperator. @@ -86,70 +80,47 @@ class RedshiftPauseClusterTrigger(BaseTrigger): Redshift cluster to be in the `paused` state. :param cluster_identifier: A unique identifier for the cluster. - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, cluster_identifier: str, - poll_interval: int, - max_attempts: int, - aws_conn_id: str, + poll_interval: int | None = None, + max_attempts: int | None = None, + aws_conn_id: str = "aws_default", + waiter_delay: int = 15, + waiter_max_attempts: int = 999999, ): - self.cluster_identifier = cluster_identifier - self.poll_interval = poll_interval - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftPauseClusterTrigger", - { - "cluster_identifier": self.cluster_identifier, - "poll_interval": str(self.poll_interval), - "max_attempts": str(self.max_attempts), - "aws_conn_id": self.aws_conn_id, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempt.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"cluster_identifier": cluster_identifier}, + waiter_name="cluster_paused", + waiter_args={"ClusterIdentifier": cluster_identifier}, + failure_message="Error while pausing the redshift cluster", + status_message="Redshift cluster pausing in progress", + status_queries=["Clusters[].ClusterStatus"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self) -> RedshiftHook: + def hook(self) -> AwsGenericHook: return RedshiftHook(aws_conn_id=self.aws_conn_id) - async def run(self): - async with self.hook.async_conn as client: - attempt = 0 - waiter = self.hook.get_waiter("cluster_paused", deferrable=True, client=client) - while attempt < int(self.max_attempts): - attempt = attempt + 1 - try: - await waiter.wait( - ClusterIdentifier=self.cluster_identifier, - WaiterConfig={ - "Delay": int(self.poll_interval), - "MaxAttempts": 1, - }, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - yield TriggerEvent({"status": "failure", "message": f"Pause Cluster Failed: {error}"}) - break - self.log.info( - "Status of cluster is %s", error.last_response["Clusters"][0]["ClusterStatus"] - ) - await asyncio.sleep(int(self.poll_interval)) - if attempt >= int(self.max_attempts): - yield TriggerEvent( - {"status": "failure", "message": "Pause Cluster Failed - max attempts reached."} - ) - else: - yield TriggerEvent({"status": "success", "message": "Cluster paused"}) - -class RedshiftCreateClusterSnapshotTrigger(BaseTrigger): +class RedshiftCreateClusterSnapshotTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftCreateClusterSnapshotOperator. @@ -157,75 +128,47 @@ class RedshiftCreateClusterSnapshotTrigger(BaseTrigger): Redshift cluster snapshot to be in the `available` state. :param cluster_identifier: A unique identifier for the cluster. - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, cluster_identifier: str, - poll_interval: int, - max_attempts: int, - aws_conn_id: str, + poll_interval: int | None = None, + max_attempts: int | None = None, + aws_conn_id: str = "aws_default", + waiter_delay: int = 15, + waiter_max_attempts: int = 999999, ): - self.cluster_identifier = cluster_identifier - self.poll_interval = poll_interval - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftCreateClusterSnapshotTrigger", - { - "cluster_identifier": self.cluster_identifier, - "poll_interval": str(self.poll_interval), - "max_attempts": str(self.max_attempts), - "aws_conn_id": self.aws_conn_id, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempt.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"cluster_identifier": cluster_identifier}, + waiter_name="snapshot_available", + waiter_args={"ClusterIdentifier": cluster_identifier}, + failure_message="Create Cluster Snapshot Failed", + status_message="Redshift Cluster Snapshot in progress", + status_queries=["Clusters[].ClusterStatus"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self) -> RedshiftHook: + def hook(self) -> AwsGenericHook: return RedshiftHook(aws_conn_id=self.aws_conn_id) - async def run(self): - async with self.hook.async_conn as client: - attempt = 0 - waiter = client.get_waiter("snapshot_available") - while attempt < int(self.max_attempts): - attempt = attempt + 1 - try: - await waiter.wait( - ClusterIdentifier=self.cluster_identifier, - WaiterConfig={ - "Delay": int(self.poll_interval), - "MaxAttempts": 1, - }, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - yield TriggerEvent( - {"status": "failure", "message": f"Create Cluster Snapshot Failed: {error}"} - ) - break - self.log.info( - "Status of cluster snapshot is %s", error.last_response["Snapshots"][0]["Status"] - ) - await asyncio.sleep(int(self.poll_interval)) - if attempt >= int(self.max_attempts): - yield TriggerEvent( - { - "status": "failure", - "message": "Create Cluster Snapshot Cluster Failed - max attempts reached.", - } - ) - else: - yield TriggerEvent({"status": "success", "message": "Cluster Snapshot Created"}) - -class RedshiftResumeClusterTrigger(BaseTrigger): +class RedshiftResumeClusterTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftResumeClusterOperator. @@ -233,141 +176,86 @@ class RedshiftResumeClusterTrigger(BaseTrigger): Redshift cluster to be in the `available` state. :param cluster_identifier: A unique identifier for the cluster. - :param poll_interval: The amount of time in seconds to wait between attempts. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. """ def __init__( self, cluster_identifier: str, - poll_interval: int, - max_attempts: int, - aws_conn_id: str, + poll_interval: int | None = None, + max_attempts: int | None = None, + aws_conn_id: str = "aws_default", + waiter_delay: int = 15, + waiter_max_attempts: int = 999999, ): - self.cluster_identifier = cluster_identifier - self.poll_interval = poll_interval - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftResumeClusterTrigger", - { - "cluster_identifier": self.cluster_identifier, - "poll_interval": str(self.poll_interval), - "max_attempts": str(self.max_attempts), - "aws_conn_id": self.aws_conn_id, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempt.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"cluster_identifier": cluster_identifier}, + waiter_name="cluster_resumed", + waiter_args={"ClusterIdentifier": cluster_identifier}, + failure_message="Resume Cluster Snapshot Failed", + status_message="Redshift Cluster resuming in progress", + status_queries=["Clusters[].ClusterStatus"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self) -> RedshiftHook: + def hook(self) -> AwsGenericHook: return RedshiftHook(aws_conn_id=self.aws_conn_id) - async def run(self): - async with self.hook.async_conn as client: - attempt = 0 - waiter = self.hook.get_waiter("cluster_resumed", deferrable=True, client=client) - while attempt < int(self.max_attempts): - attempt = attempt + 1 - try: - await waiter.wait( - ClusterIdentifier=self.cluster_identifier, - WaiterConfig={ - "Delay": int(self.poll_interval), - "MaxAttempts": 1, - }, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - yield TriggerEvent( - {"status": "failure", "message": f"Resume Cluster Failed: {error}"} - ) - break - self.log.info( - "Status of cluster is %s", error.last_response["Clusters"][0]["ClusterStatus"] - ) - await asyncio.sleep(int(self.poll_interval)) - if attempt >= int(self.max_attempts): - yield TriggerEvent( - {"status": "failure", "message": "Resume Cluster Failed - max attempts reached."} - ) - else: - yield TriggerEvent({"status": "success", "message": "Cluster resumed"}) - -class RedshiftDeleteClusterTrigger(BaseTrigger): +class RedshiftDeleteClusterTrigger(AwsBaseWaiterTrigger): """ Trigger for RedshiftDeleteClusterOperator. :param cluster_identifier: A unique identifier for the cluster. - :param max_attempts: The maximum number of attempts to be made. + :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. - :param poll_interval: The amount of time in seconds to wait between attempts. + :param waiter_delay: The amount of time in seconds to wait between attempts. """ def __init__( self, cluster_identifier: str, - max_attempts: int = 30, + poll_interval: int | None = None, + max_attempts: int | None = None, aws_conn_id: str = "aws_default", - poll_interval: int = 30, + waiter_delay: int = 30, + waiter_max_attempts: int = 30, ): - super().__init__() - self.cluster_identifier = cluster_identifier - self.max_attempts = max_attempts - self.aws_conn_id = aws_conn_id - self.poll_interval = poll_interval - - def serialize(self) -> tuple[str, dict[str, Any]]: - return ( - "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger", - { - "cluster_identifier": self.cluster_identifier, - "max_attempts": self.max_attempts, - "aws_conn_id": self.aws_conn_id, - "poll_interval": self.poll_interval, - }, + if poll_interval is not None or max_attempts is not None: + warnings.warn( + "please use waiter_delay instead of poll_interval " + "and waiter_max_attempts instead of max_attempt.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + waiter_delay = poll_interval or waiter_delay + waiter_max_attempts = max_attempts or waiter_max_attempts + super().__init__( + serialized_fields={"cluster_identifier": cluster_identifier}, + waiter_name="cluster_deleted", + waiter_args={"ClusterIdentifier": cluster_identifier}, + failure_message="Delete Cluster Failed", + status_message="Redshift Cluster deletion in progress", + status_queries=["Clusters[].ClusterStatus"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, ) - @cached_property - def hook(self): + def hook(self) -> AwsGenericHook: return RedshiftHook(aws_conn_id=self.aws_conn_id) - - async def run(self) -> AsyncIterator[TriggerEvent]: - async with self.hook.async_conn as client: - attempt = 0 - waiter = client.get_waiter("cluster_deleted") - while attempt < self.max_attempts: - attempt = attempt + 1 - try: - await waiter.wait( - ClusterIdentifier=self.cluster_identifier, - WaiterConfig={ - "Delay": self.poll_interval, - "MaxAttempts": 1, - }, - ) - break - except WaiterError as error: - if "terminal failure" in str(error): - yield TriggerEvent( - {"status": "failure", "message": f"Delete Cluster Failed: {error}"} - ) - break - self.log.info( - "Cluster status is %s. Retrying attempt %s/%s", - error.last_response["Clusters"][0]["ClusterStatus"], - attempt, - self.max_attempts, - ) - await asyncio.sleep(int(self.poll_interval)) - - if attempt >= self.max_attempts: - yield TriggerEvent( - {"status": "failure", "message": "Delete Cluster Failed - max attempts reached."} - ) - else: - yield TriggerEvent({"status": "success", "message": "Cluster deleted."}) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 390a5866368a..3622896504da 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -516,6 +516,9 @@ hooks: - airflow.providers.amazon.aws.hooks.appflow triggers: + - integration-name: Amazon Web Services + python-modules: + - airflow.providers.amazon.aws.triggers.base - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.triggers.athena diff --git a/tests/providers/amazon/aws/operators/test_athena.py b/tests/providers/amazon/aws/operators/test_athena.py index 9e528525204c..3eb634985079 100644 --- a/tests/providers/amazon/aws/operators/test_athena.py +++ b/tests/providers/amazon/aws/operators/test_athena.py @@ -169,4 +169,3 @@ def test_is_deferred(self, mock_run_query): self.athena.execute(None) assert isinstance(deferred.value.trigger, AthenaTrigger) - assert deferred.value.trigger.query_execution_id == ATHENA_QUERY_ID diff --git a/tests/providers/amazon/aws/operators/test_batch.py b/tests/providers/amazon/aws/operators/test_batch.py index 3aace0bb3e5b..7255035b9d55 100644 --- a/tests/providers/amazon/aws/operators/test_batch.py +++ b/tests/providers/amazon/aws/operators/test_batch.py @@ -29,7 +29,7 @@ # Use dummy AWS credentials from airflow.providers.amazon.aws.triggers.batch import ( BatchCreateComputeEnvironmentTrigger, - BatchOperatorTrigger, + BatchJobTrigger, ) AWS_REGION = "eu-west-1" @@ -276,7 +276,7 @@ def test_defer_if_deferrable_param_set(self, mock_client): with pytest.raises(TaskDeferred) as exc: batch.execute(context=None) - assert isinstance(exc.value.trigger, BatchOperatorTrigger), "Trigger is not a BatchOperatorTrigger" + assert isinstance(exc.value.trigger, BatchJobTrigger) @mock.patch.object(BatchClientHook, "get_job_description") @mock.patch.object(BatchClientHook, "wait_for_job") @@ -349,6 +349,5 @@ def test_defer(self, client_mock): operator.execute(None) assert isinstance(deferred.value.trigger, BatchCreateComputeEnvironmentTrigger) - assert deferred.value.trigger.compute_env_arn == "my_arn" - assert deferred.value.trigger.poll_interval == 456789 - assert deferred.value.trigger.max_retries == 123456 + assert deferred.value.trigger.waiter_delay == 456789 + assert deferred.value.trigger.attempts == 123456 diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index b89ea59c561b..f32491817192 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -724,7 +724,6 @@ def test_execute_deferrable(self, mock_client: MagicMock): with pytest.raises(TaskDeferred) as defer: op.execute(None) - assert defer.value.trigger.cluster_arn == "my arn" assert defer.value.trigger.waiter_delay == 12 assert defer.value.trigger.attempts == 34 @@ -789,7 +788,6 @@ def test_execute_deferrable(self, mock_client: MagicMock): with pytest.raises(TaskDeferred) as defer: op.execute(None) - assert defer.value.trigger.cluster_arn == "my arn" assert defer.value.trigger.waiter_delay == 12 assert defer.value.trigger.attempts == 34 diff --git a/tests/providers/amazon/aws/operators/test_eks.py b/tests/providers/amazon/aws/operators/test_eks.py index 9ea8ab72d72f..8381a26ef151 100644 --- a/tests/providers/amazon/aws/operators/test_eks.py +++ b/tests/providers/amazon/aws/operators/test_eks.py @@ -36,8 +36,8 @@ ) from airflow.providers.amazon.aws.triggers.eks import ( EksCreateFargateProfileTrigger, + EksCreateNodegroupTrigger, EksDeleteFargateProfileTrigger, - EksNodegroupTrigger, ) from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction from airflow.typing_compat import TypedDict @@ -489,7 +489,7 @@ def test_create_nodegroup_deferrable(self, mock_create_nodegroup): ) with pytest.raises(TaskDeferred) as exc: operator.execute({}) - assert isinstance(exc.value.trigger, EksNodegroupTrigger), "Trigger is not a EksNodegroupTrigger" + assert isinstance(exc.value.trigger, EksCreateNodegroupTrigger) def test_create_nodegroup_deferrable_versus_wait_for_completion(self): op_kwargs = {**self.create_nodegroup_params} diff --git a/tests/providers/amazon/aws/sensors/test_batch.py b/tests/providers/amazon/aws/sensors/test_batch.py index 42e9bffb5b68..74b348381e8e 100644 --- a/tests/providers/amazon/aws/sensors/test_batch.py +++ b/tests/providers/amazon/aws/sensors/test_batch.py @@ -27,7 +27,7 @@ BatchJobQueueSensor, BatchSensor, ) -from airflow.providers.amazon.aws.triggers.batch import BatchSensorTrigger +from airflow.providers.amazon.aws.triggers.batch import BatchJobTrigger TASK_ID = "batch_job_sensor" JOB_ID = "8222a1c2-b246-4e19-b1b8-0039bb4407c0" @@ -210,26 +210,10 @@ def test_batch_sensor_async(self): with pytest.raises(TaskDeferred) as exc: self.TASK.execute({}) - assert isinstance(exc.value.trigger, BatchSensorTrigger), "Trigger is not a BatchSensorTrigger" + assert isinstance(exc.value.trigger, BatchJobTrigger), "Trigger is not a BatchJobTrigger" def test_batch_sensor_async_execute_failure(self): """Tests that an AirflowException is raised in case of error event""" - with pytest.raises(AirflowException) as exc_info: - self.TASK.execute_complete( - context={}, event={"status": "failure", "message": "test failure message"} - ) - - assert str(exc_info.value) == "test failure message" - - @pytest.mark.parametrize( - "event", - [{"status": "success", "message": f"AWS Batch job ({JOB_ID}) succeeded"}], - ) - def test_batch_sensor_async_execute_complete(self, caplog, event): - """Tests that execute_complete method returns None and that it prints expected log""" - - with mock.patch.object(self.TASK.log, "info") as mock_log_info: - assert self.TASK.execute_complete(context={}, event=event) is None - - mock_log_info.assert_called_with(event["message"]) + with pytest.raises(AirflowException): + self.TASK.execute_complete(context={}, event={"status": "failure"}) diff --git a/tests/providers/amazon/aws/triggers/test_athena.py b/tests/providers/amazon/aws/triggers/test_athena.py index 04e601f4392c..d18bdc1553f7 100644 --- a/tests/providers/amazon/aws/triggers/test_athena.py +++ b/tests/providers/amazon/aws/triggers/test_athena.py @@ -16,38 +16,20 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - -import pytest -from botocore.exceptions import WaiterError - -from airflow.providers.amazon.aws.hooks.athena import AthenaHook from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger class TestAthenaTrigger: - @pytest.mark.asyncio - @mock.patch.object(AthenaHook, "get_waiter") - @mock.patch.object(AthenaHook, "async_conn") # LatestBoto step of CI fails without this - async def test_run_with_error(self, conn_mock, waiter_mock): - waiter_mock.side_effect = WaiterError("name", "reason", {}) - - trigger = AthenaTrigger("query_id", 0, 5, None) + def test_serialize_recreate(self): + trigger = AthenaTrigger("query_id", 1, 5, "aws connection") - with pytest.raises(WaiterError): - generator = trigger.run() - await generator.asend(None) + class_path, args = trigger.serialize() - @pytest.mark.asyncio - @mock.patch.object(AthenaHook, "get_waiter") - @mock.patch.object(AthenaHook, "async_conn") # LatestBoto step of CI fails without this - async def test_run_success(self, conn_mock, waiter_mock): - waiter_mock().wait = AsyncMock() - trigger = AthenaTrigger("my_query_id", 0, 5, None) + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) - generator = trigger.run() - event = await generator.asend(None) + class_path2, args2 = instance.serialize() - assert event.payload["status"] == "success" - assert event.payload["value"] == "my_query_id" + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_base.py b/tests/providers/amazon/aws/triggers/test_base.py new file mode 100644 index 000000000000..aad9ce45ec4a --- /dev/null +++ b/tests/providers/amazon/aws/triggers/test_base.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import MagicMock + +import pytest + +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger +from airflow.triggers.base import TriggerEvent + + +class TestImplem(AwsBaseWaiterTrigger): + """An empty implementation that allows instantiation for tests.""" + + __test__ = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def hook(self) -> AwsGenericHook: + return MagicMock() + + +class TestAwsBaseWaiterTrigger: + def setup_method(self): + self.trigger = TestImplem( + serialized_fields={}, + waiter_name="", + waiter_args={}, + failure_message="", + status_message="", + status_queries=[], + return_value=None, + waiter_delay=0, + waiter_max_attempts=0, + aws_conn_id="", + ) + + def test_region_serialized(self): + self.trigger.region_name = "my_region" + _, args = self.trigger.serialize() + + assert "region_name" in args + assert args["region_name"] == "my_region" + + def test_region_not_serialized_if_omitted(self): + _, args = self.trigger.serialize() + + assert "region_name" not in args + + def test_serialize_extra_fields(self): + self.trigger.serialized_fields = {"foo": "bar", "foz": "baz"} + + _, args = self.trigger.serialize() + + assert "foo" in args + assert args["foo"] == "bar" + assert "foz" in args + assert args["foz"] == "baz" + + @pytest.mark.asyncio + @mock.patch("airflow.providers.amazon.aws.triggers.base.async_wait") + async def test_run(self, wait_mock: MagicMock): + self.trigger.return_key = "hello" + self.trigger.return_value = "world" + + generator = self.trigger.run() + res: TriggerEvent = await generator.asend(None) + + wait_mock.assert_called_once() + assert res.payload["status"] == "success" + assert res.payload["hello"] == "world" diff --git a/tests/providers/amazon/aws/triggers/test_batch.py b/tests/providers/amazon/aws/triggers/test_batch.py index e33736076237..6ceee613321d 100644 --- a/tests/providers/amazon/aws/triggers/test_batch.py +++ b/tests/providers/amazon/aws/triggers/test_batch.py @@ -16,20 +16,12 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - import pytest -from botocore.exceptions import WaiterError -from airflow import AirflowException -from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook from airflow.providers.amazon.aws.triggers.batch import ( BatchCreateComputeEnvironmentTrigger, - BatchOperatorTrigger, - BatchSensorTrigger, + BatchJobTrigger, ) -from airflow.triggers.base import TriggerEvent BATCH_JOB_ID = "job_id" POLL_INTERVAL = 5 @@ -39,186 +31,34 @@ pytest.importorskip("aiobotocore") -class TestBatchOperatorTrigger: - def test_batch_operator_trigger_serialize(self): - batch_trigger = BatchOperatorTrigger( - job_id=BATCH_JOB_ID, - poll_interval=POLL_INTERVAL, - max_retries=MAX_ATTEMPT, - aws_conn_id=AWS_CONN_ID, - region_name=AWS_REGION, - ) - class_path, args = batch_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.batch.BatchOperatorTrigger" - assert args["job_id"] == BATCH_JOB_ID - assert args["poll_interval"] == POLL_INTERVAL - assert args["max_retries"] == MAX_ATTEMPT - assert args["aws_conn_id"] == AWS_CONN_ID - assert args["region_name"] == AWS_REGION - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.async_conn") - async def test_batch_job_trigger_run(self, mock_async_conn, mock_get_waiter): - the_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = the_mock - - mock_get_waiter().wait = AsyncMock() - - batch_trigger = BatchOperatorTrigger( - job_id=BATCH_JOB_ID, - poll_interval=POLL_INTERVAL, - max_retries=MAX_ATTEMPT, - aws_conn_id=AWS_CONN_ID, - region_name=AWS_REGION, - ) - - generator = batch_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "job_id": BATCH_JOB_ID}) - - -class TestBatchSensorTrigger: - TRIGGER = BatchSensorTrigger( - job_id=BATCH_JOB_ID, - region_name=AWS_REGION, - aws_conn_id=AWS_CONN_ID, - poke_interval=POLL_INTERVAL, +class TestBatchTrigger: + @pytest.mark.parametrize( + "trigger", + [ + BatchJobTrigger( + job_id=BATCH_JOB_ID, + waiter_delay=POLL_INTERVAL, + waiter_max_attempts=MAX_ATTEMPT, + aws_conn_id=AWS_CONN_ID, + region_name=AWS_REGION, + ), + BatchCreateComputeEnvironmentTrigger( + compute_env_arn="my_arn", + waiter_delay=POLL_INTERVAL, + waiter_max_attempts=MAX_ATTEMPT, + aws_conn_id=AWS_CONN_ID, + region_name=AWS_REGION, + ), + ], ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() - def test_batch_sensor_trigger_serialization(self): - """ - Asserts that the BatchSensorTrigger correctly serializes its arguments - and classpath. - """ - - classpath, kwargs = self.TRIGGER.serialize() - assert classpath == "airflow.providers.amazon.aws.triggers.batch.BatchSensorTrigger" - assert kwargs == { - "job_id": BATCH_JOB_ID, - "region_name": AWS_REGION, - "aws_conn_id": AWS_CONN_ID, - "poke_interval": POLL_INTERVAL, - } - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.async_conn") - async def test_batch_job_trigger_run(self, mock_async_conn, mock_get_waiter): - the_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = the_mock - - mock_get_waiter().wait = AsyncMock() - - batch_trigger = BatchOperatorTrigger( - job_id=BATCH_JOB_ID, - poll_interval=POLL_INTERVAL, - max_retries=MAX_ATTEMPT, - aws_conn_id=AWS_CONN_ID, - region_name=AWS_REGION, - ) - - generator = batch_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "job_id": BATCH_JOB_ID}) - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.async_conn") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_job_description") - async def test_batch_sensor_trigger_completed(self, mock_response, mock_async_conn, mock_get_waiter): - """Test if the success event is returned from trigger.""" - mock_response.return_value = {"status": "SUCCEEDED"} - - the_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = the_mock - - mock_get_waiter().wait = AsyncMock() - - trigger = BatchSensorTrigger( - job_id=BATCH_JOB_ID, - region_name=AWS_REGION, - aws_conn_id=AWS_CONN_ID, - ) - generator = trigger.run() - actual_response = await generator.asend(None) - assert ( - TriggerEvent( - {"status": "success", "job_id": BATCH_JOB_ID, "message": f"Job {BATCH_JOB_ID} Succeeded"} - ) - == actual_response - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.get_job_description") - @mock.patch("airflow.providers.amazon.aws.hooks.batch_client.BatchClientHook.async_conn") - async def test_batch_sensor_trigger_failure( - self, mock_async_conn, mock_response, mock_get_waiter, mock_sleep - ): - """Test if the failure event is returned from trigger.""" - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_response.return_value = {"status": "failed"} - - name = "batch_job_complete" - reason = ( - "An error occurred (UnrecognizedClientException): The security token included in the " - "request is invalid. " - ) - last_response = ({"Error": {"Message": "The security token included in the request is invalid."}},) - - error_failed = WaiterError( - name=name, - reason=reason, - last_response=last_response, - ) - - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error_failed]) - mock_sleep.return_value = True - - trigger = BatchSensorTrigger(job_id=BATCH_JOB_ID, region_name=AWS_REGION, aws_conn_id=AWS_CONN_ID) - generator = trigger.run() - actual_response = await generator.asend(None) - assert actual_response == TriggerEvent( - {"status": "failure", "message": f"Job Failed: Waiter {name} failed: {reason}"} - ) - - -class TestBatchCreateComputeEnvironmentTrigger: - @pytest.mark.asyncio - @mock.patch.object(BatchClientHook, "async_conn") - @mock.patch.object(BatchClientHook, "get_waiter") - async def test_success(self, get_waiter_mock, conn_mock): - get_waiter_mock().wait = AsyncMock( - side_effect=[ - WaiterError( - "situation normal", "first try", {"computeEnvironments": [{"status": "my_status"}]} - ), - {}, - ] - ) - trigger = BatchCreateComputeEnvironmentTrigger("my_arn", poll_interval=0, max_retries=3) - - generator = trigger.run() - response: TriggerEvent = await generator.asend(None) - - assert response.payload["status"] == "success" - assert response.payload["value"] == "my_arn" + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) - @pytest.mark.asyncio - @mock.patch.object(BatchClientHook, "async_conn") - @mock.patch.object(BatchClientHook, "get_waiter") - async def test_failure(self, get_waiter_mock, conn_mock): - get_waiter_mock().wait = AsyncMock( - side_effect=[WaiterError("terminal failure", "terminal failure reason", {})] - ) - trigger = BatchCreateComputeEnvironmentTrigger("my_arn", poll_interval=0, max_retries=3) + class_path2, args2 = instance.serialize() - with pytest.raises(AirflowException): - generator = trigger.run() - await generator.asend(None) + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_ecs.py b/tests/providers/amazon/aws/triggers/test_ecs.py index 09b5decbe631..551ab39a4486 100644 --- a/tests/providers/amazon/aws/triggers/test_ecs.py +++ b/tests/providers/amazon/aws/triggers/test_ecs.py @@ -22,64 +22,16 @@ import pytest from botocore.exceptions import WaiterError -from airflow import AirflowException from airflow.providers.amazon.aws.hooks.ecs import EcsHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook -from airflow.providers.amazon.aws.triggers.ecs import ClusterWaiterTrigger, TaskDoneTrigger +from airflow.providers.amazon.aws.triggers.ecs import ( + ClusterActiveTrigger, + ClusterInactiveTrigger, + TaskDoneTrigger, +) from airflow.triggers.base import TriggerEvent -class TestClusterWaiterTrigger: - @pytest.mark.asyncio - @mock.patch.object(EcsHook, "async_conn") - async def test_run_max_attempts(self, client_mock): - a_mock = mock.MagicMock() - client_mock.__aenter__.return_value = a_mock - wait_mock = AsyncMock() - wait_mock.side_effect = WaiterError("name", "reason", {"clusters": [{"status": "my_status"}]}) - a_mock.get_waiter().wait = wait_mock - - max_attempts = 5 - trigger = ClusterWaiterTrigger("my_waiter", "cluster_arn", 0, max_attempts, None, None) - - with pytest.raises(AirflowException): - generator = trigger.run() - await generator.asend(None) - - assert wait_mock.call_count == max_attempts - - @pytest.mark.asyncio - @mock.patch.object(EcsHook, "async_conn") - async def test_run_success(self, client_mock): - a_mock = mock.MagicMock() - client_mock.__aenter__.return_value = a_mock - wait_mock = AsyncMock() - a_mock.get_waiter().wait = wait_mock - - trigger = ClusterWaiterTrigger("my_waiter", "cluster_arn", 0, 5, None, None) - - generator = trigger.run() - response: TriggerEvent = await generator.asend(None) - - assert response.payload["status"] == "success" - assert response.payload["arn"] == "cluster_arn" - - @pytest.mark.asyncio - @mock.patch.object(EcsHook, "async_conn") - async def test_run_error(self, client_mock): - a_mock = mock.MagicMock() - client_mock.__aenter__.return_value = a_mock - wait_mock = AsyncMock() - wait_mock.side_effect = WaiterError("terminal failure", "reason", {}) - a_mock.get_waiter().wait = wait_mock - - trigger = ClusterWaiterTrigger("my_waiter", "cluster_arn", 0, 5, None, None) - - with pytest.raises(AirflowException): - generator = trigger.run() - await generator.asend(None) - - class TestTaskDoneTrigger: @pytest.mark.asyncio @mock.patch.object(EcsHook, "async_conn") @@ -121,3 +73,36 @@ async def test_run_success(self, _, client_mock): assert response.payload["status"] == "success" assert response.payload["task_arn"] == "my_task_arn" + + +class TestClusterTriggers: + @pytest.mark.parametrize( + "trigger", + [ + ClusterActiveTrigger( + cluster_arn="my_arn", + aws_conn_id="my_conn", + waiter_delay=1, + waiter_max_attempts=2, + region_name="my_region", + ), + ClusterInactiveTrigger( + cluster_arn="my_arn", + aws_conn_id="my_conn", + waiter_delay=1, + waiter_max_attempts=2, + region_name="my_region", + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_eks.py b/tests/providers/amazon/aws/triggers/test_eks.py index 0b94e957db6c..045519aea57e 100644 --- a/tests/providers/amazon/aws/triggers/test_eks.py +++ b/tests/providers/amazon/aws/triggers/test_eks.py @@ -16,20 +16,14 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - import pytest -from botocore.exceptions import WaiterError -from airflow.exceptions import AirflowException -from airflow.providers.amazon.aws.hooks.eks import EksHook from airflow.providers.amazon.aws.triggers.eks import ( EksCreateFargateProfileTrigger, + EksCreateNodegroupTrigger, EksDeleteFargateProfileTrigger, - EksNodegroupTrigger, + EksDeleteNodegroupTrigger, ) -from airflow.triggers.base import TriggerEvent TEST_CLUSTER_IDENTIFIER = "test-cluster" TEST_FARGATE_PROFILE_NAME = "test-fargate-profile" @@ -40,416 +34,50 @@ TEST_REGION = "test-region" -class TestEksCreateFargateProfileTrigger: - def test_eks_create_fargate_profile_serialize(self): - eks_create_fargate_profile_trigger = EksCreateFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - class_path, args = eks_create_fargate_profile_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.eks.EksCreateFargateProfileTrigger" - assert args["cluster_name"] == TEST_CLUSTER_IDENTIFIER - assert args["fargate_profile_name"] == TEST_FARGATE_PROFILE_NAME - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["waiter_delay"] == str(TEST_WAITER_DELAY) - assert args["waiter_max_attempts"] == str(TEST_WAITER_MAX_ATTEMPTS) - - @pytest.mark.asyncio - @mock.patch.object(EksHook, "async_conn") - async def test_eks_create_fargate_profile_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock.get_waiter().wait = AsyncMock() - - eks_create_fargate_profile_trigger = EksCreateFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - generator = eks_create_fargate_profile_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Fargate Profile Created"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_create_fargate_profile_trigger_run_multiple_attempts( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "CREATING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - eks_create_fargate_profile_trigger = EksCreateFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - generator = eks_create_fargate_profile_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Fargate Profile Created"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_create_fargate_profile_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "CREATING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - eks_create_fargate_profile_trigger = EksCreateFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=2, - ) - with pytest.raises(AirflowException) as exc: - generator = eks_create_fargate_profile_trigger.run() - await generator.asend(None) - assert "Create Fargate Profile failed - max attempts reached:" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_create_fargate_profile_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_creating = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "CREATING"}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"fargateProfile": {"status": "CREATE_FAILED"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error_creating, error_creating, error_failed]) - mock_sleep.return_value = True - - eks_create_fargate_profile_trigger = EksCreateFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - with pytest.raises(AirflowException) as exc: - generator = eks_create_fargate_profile_trigger.run() - await generator.asend(None) - assert f"Create Fargate Profile failed: {error_failed}" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 3 - - -class TestEksDeleteFargateProfileTrigger: - def test_eks_delete_fargate_profile_serialize(self): - eks_delete_fargate_profile_trigger = EksDeleteFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - class_path, args = eks_delete_fargate_profile_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.eks.EksDeleteFargateProfileTrigger" - assert args["cluster_name"] == TEST_CLUSTER_IDENTIFIER - assert args["fargate_profile_name"] == TEST_FARGATE_PROFILE_NAME - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["waiter_delay"] == str(TEST_WAITER_DELAY) - assert args["waiter_max_attempts"] == str(TEST_WAITER_MAX_ATTEMPTS) - - @pytest.mark.asyncio - @mock.patch.object(EksHook, "async_conn") - async def test_eks_delete_fargate_profile_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock.get_waiter().wait = AsyncMock() - - eks_delete_fargate_profile_trigger = EksDeleteFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - generator = eks_delete_fargate_profile_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Fargate Profile Deleted"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_delete_fargate_profile_trigger_run_multiple_attempts( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "DELETING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - eks_delete_fargate_profile_trigger = EksDeleteFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - - generator = eks_delete_fargate_profile_trigger.run() - response = await generator.asend(None) - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Fargate Profile Deleted"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_delete_fargate_profile_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "DELETING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - mock_sleep.return_value = True - - eks_delete_fargate_profile_trigger = EksDeleteFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=2, - ) - with pytest.raises(AirflowException) as exc: - generator = eks_delete_fargate_profile_trigger.run() - await generator.asend(None) - assert "Delete Fargate Profile failed - max attempts reached: 2" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_delete_fargate_profile_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_creating = WaiterError( - name="test_name", - reason="test_reason", - last_response={"fargateProfile": {"status": "DELETING"}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"fargateProfile": {"status": "DELETE_FAILED"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error_creating, error_creating, error_failed]) - mock_sleep.return_value = True - - eks_delete_fargate_profile_trigger = EksDeleteFargateProfileTrigger( - cluster_name=TEST_CLUSTER_IDENTIFIER, - fargate_profile_name=TEST_FARGATE_PROFILE_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - ) - with pytest.raises(AirflowException) as exc: - generator = eks_delete_fargate_profile_trigger.run() - await generator.asend(None) - assert f"Delete Fargate Profile failed: {error_failed}" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 3 - - -class TestEksNodegroupTrigger: - def test_eks_nodegroup_trigger_serialize(self): - eks_nodegroup_trigger = EksNodegroupTrigger( - waiter_name="test_waiter_name", - cluster_name=TEST_CLUSTER_IDENTIFIER, - nodegroup_name=TEST_NODEGROUP_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - region=TEST_REGION, - ) - - class_path, args = eks_nodegroup_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.eks.EksNodegroupTrigger" - assert args["waiter_name"] == "test_waiter_name" - assert args["cluster_name"] == TEST_CLUSTER_IDENTIFIER - assert args["nodegroup_name"] == TEST_NODEGROUP_NAME - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["waiter_delay"] == str(TEST_WAITER_DELAY) - assert args["waiter_max_attempts"] == str(TEST_WAITER_MAX_ATTEMPTS) - assert args["region"] == TEST_REGION - - @pytest.mark.asyncio - @mock.patch.object(EksHook, "async_conn") - async def test_eks_nodegroup_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock.get_waiter().wait = AsyncMock() - - eks_nodegroup_trigger = EksNodegroupTrigger( - waiter_name="test_waiter_name", - cluster_name=TEST_CLUSTER_IDENTIFIER, - nodegroup_name=TEST_NODEGROUP_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - region=TEST_REGION, - ) - - generator = eks_nodegroup_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent( - { - "status": "success", - "cluster_name": TEST_CLUSTER_IDENTIFIER, - "nodegroup_name": TEST_NODEGROUP_NAME, - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_nodegroup_trigger_run_multiple_attempts(self, mock_async_conn, mock_sleep): - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"nodegroup": {"status": "CREATING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - - eks_nodegroup_trigger = EksNodegroupTrigger( - waiter_name="test_waiter_name", - cluster_name=TEST_CLUSTER_IDENTIFIER, - nodegroup_name=TEST_NODEGROUP_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - region=TEST_REGION, - ) - - generator = eks_nodegroup_trigger.run() - response = await generator.asend(None) - assert a_mock.get_waiter().wait.call_count == 4 - assert response == TriggerEvent( - { - "status": "success", - "cluster_name": TEST_CLUSTER_IDENTIFIER, - "nodegroup_name": TEST_NODEGROUP_NAME, - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_nodegroup_trigger_run_attempts_exceeded(self, mock_async_conn, mock_sleep): - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"nodegroup": {"status": "CREATING"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - - eks_nodegroup_trigger = EksNodegroupTrigger( - waiter_name="test_waiter_name", - cluster_name=TEST_CLUSTER_IDENTIFIER, - nodegroup_name=TEST_NODEGROUP_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=2, - region=TEST_REGION, - ) - - with pytest.raises(AirflowException) as exc: - generator = eks_nodegroup_trigger.run() - await generator.asend(None) - assert "Waiter error: max attempts reached" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EksHook, "async_conn") - async def test_eks_nodegroup_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_creating = WaiterError( - name="test_name", - reason="test_reason", - last_response={"nodegroup": {"status": "CREATING"}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"nodegroup": {"status": "DELETE_FAILED"}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error_creating, error_creating, error_failed]) - mock_sleep.return_value = True - - eks_nodegroup_trigger = EksNodegroupTrigger( - waiter_name="test_waiter_name", - cluster_name=TEST_CLUSTER_IDENTIFIER, - nodegroup_name=TEST_NODEGROUP_NAME, - aws_conn_id=TEST_AWS_CONN_ID, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - region=TEST_REGION, - ) - with pytest.raises(AirflowException) as exc: - generator = eks_nodegroup_trigger.run() - await generator.asend(None) - - assert "Error checking nodegroup" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 3 +class TestEksTriggers: + @pytest.mark.parametrize( + "trigger", + [ + EksCreateFargateProfileTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + fargate_profile_name=TEST_FARGATE_PROFILE_NAME, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + EksDeleteFargateProfileTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + fargate_profile_name=TEST_FARGATE_PROFILE_NAME, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + EksCreateNodegroupTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + nodegroup_name=TEST_NODEGROUP_NAME, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + region_name=TEST_REGION, + ), + EksDeleteNodegroupTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + nodegroup_name=TEST_NODEGROUP_NAME, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + region_name=TEST_REGION, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_emr.py b/tests/providers/amazon/aws/triggers/test_emr.py index ff57eab4a5d1..7066e817e481 100644 --- a/tests/providers/amazon/aws/triggers/test_emr.py +++ b/tests/providers/amazon/aws/triggers/test_emr.py @@ -16,21 +16,14 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - import pytest -from botocore.exceptions import WaiterError -from airflow.exceptions import AirflowException -from airflow.providers.amazon.aws.hooks.emr import EmrHook from airflow.providers.amazon.aws.triggers.emr import ( EmrContainerTrigger, EmrCreateJobFlowTrigger, EmrStepSensorTrigger, EmrTerminateJobFlowTrigger, ) -from airflow.triggers.base import TriggerEvent TEST_JOB_FLOW_ID = "test-job-flow-id" TEST_POLL_INTERVAL = 10 @@ -44,527 +37,44 @@ STEP_ID = "s-1234" -class TestEmrCreateJobFlowTrigger: - def test_emr_create_job_flow_trigger_serialize(self): - """Test serialize method to make sure all parameters are being serialized correctly.""" - emr_create_job_flow_trigger = EmrCreateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - class_path, args = emr_create_job_flow_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.emr.EmrCreateJobFlowTrigger" - assert args["job_flow_id"] == TEST_JOB_FLOW_ID - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempts"] == str(TEST_MAX_ATTEMPTS) - - @pytest.mark.asyncio - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_create_job_flow_trigger_run(self, mock_async_conn, mock_get_waiter): - """ - Test run method, with basic success case to assert TriggerEvent contains the - correct payload. - """ - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_get_waiter().wait = AsyncMock() - - emr_create_job_flow_trigger = EmrCreateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - - generator = emr_create_job_flow_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent( - { - "status": "success", - "message": "JobFlow completed successfully", - "job_flow_id": TEST_JOB_FLOW_ID, - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_create_job_flow_trigger_run_multiple_attempts( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - """ - Test run method with multiple attempts to make sure the waiter retries - are working as expected. - """ - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Cluster": {"Status": {"State": "STARTING", "StateChangeReason": "test-reason"}}}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - emr_create_job_flow_trigger = EmrCreateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - - generator = emr_create_job_flow_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - { - "status": "success", - "message": "JobFlow completed successfully", - "job_flow_id": TEST_JOB_FLOW_ID, - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_create_job_flow_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - """ - Test run method with max_attempts set to 2 to test the Trigger yields - the correct TriggerEvent in the case of max_attempts being exceeded. - """ - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Cluster": {"Status": {"State": "STARTING", "StateChangeReason": "test-reason"}}}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - emr_create_job_flow_trigger = EmrCreateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - ) - - with pytest.raises(AirflowException) as exc: - generator = emr_create_job_flow_trigger.run() - await generator.asend(None) - - assert str(exc.value) == "JobFlow creation failed - max attempts reached: 2" - assert mock_get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_create_job_flow_trigger_run_attempts_failed( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - """ - Test run method with a failure case to test Trigger yields the correct - failure TriggerEvent. - """ - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_starting = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Cluster": {"Status": {"State": "STARTING", "StateChangeReason": "test-reason"}}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={ - "Cluster": {"Status": {"State": "TERMINATED_WITH_ERRORS", "StateChangeReason": "test-reason"}} - }, - ) - mock_get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_starting, error_starting, error_failed] - ) - mock_sleep.return_value = True - - emr_create_job_flow_trigger = EmrCreateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - - with pytest.raises(AirflowException) as exc: - generator = emr_create_job_flow_trigger.run() - await generator.asend(None) - - assert str(exc.value) == f"JobFlow creation failed: {error_failed}" - assert mock_get_waiter().wait.call_count == 3 - - -class TestEmrTerminateJobFlowTrigger: - def test_emr_terminate_job_flow_trigger_serialize(self): - emr_terminate_job_flow_trigger = EmrTerminateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - class_path, args = emr_terminate_job_flow_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.emr.EmrTerminateJobFlowTrigger" - assert args["job_flow_id"] == TEST_JOB_FLOW_ID - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempts"] == str(TEST_MAX_ATTEMPTS) - - @pytest.mark.asyncio - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_terminate_job_flow_trigger_run(self, mock_async_conn, mock_get_waiter): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_get_waiter().wait = AsyncMock() - - emr_terminate_job_flow_trigger = EmrTerminateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - - generator = emr_terminate_job_flow_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent( - { - "status": "success", - "message": "JobFlow terminated successfully", - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_terminate_job_flow_trigger_run_multiple_attempts( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={ - "Cluster": {"Status": {"State": "TERMINATING", "StateChangeReason": "test-reason"}} - }, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - emr_terminate_job_flow_trigger = EmrTerminateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - - generator = emr_terminate_job_flow_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - { - "status": "success", - "message": "JobFlow terminated successfully", - } - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_terminate_job_flow_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={ - "Cluster": {"Status": {"State": "TERMINATING", "StateChangeReason": "test-reason"}} - }, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - emr_terminate_job_flow_trigger = EmrTerminateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - ) - with pytest.raises(AirflowException) as exc: - generator = emr_terminate_job_flow_trigger.run() - await generator.asend(None) - - assert str(exc.value) == "JobFlow termination failed - max attempts reached: 2" - assert mock_get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "get_waiter") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_terminate_job_flow_trigger_run_attempts_failed( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_starting = WaiterError( - name="test_name", - reason="test_reason", - last_response={ - "Cluster": {"Status": {"State": "TERMINATING", "StateChangeReason": "test-reason"}} - }, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={ - "Cluster": {"Status": {"State": "TERMINATED_WITH_ERRORS", "StateChangeReason": "test-reason"}} - }, - ) - mock_get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_starting, error_starting, error_failed] - ) - mock_sleep.return_value = True - - emr_terminate_job_flow_trigger = EmrTerminateJobFlowTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - aws_conn_id=TEST_AWS_CONN_ID, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPTS, - ) - with pytest.raises(AirflowException) as exc: - generator = emr_terminate_job_flow_trigger.run() - await generator.asend(None) - - assert str(exc.value) == f"JobFlow termination failed: {error_failed}" - assert mock_get_waiter().wait.call_count == 3 - - -class TestEmrContainerTrigger: - def test_emr_container_trigger_serialize(self): - emr_trigger = EmrContainerTrigger( - virtual_cluster_id=VIRTUAL_CLUSTER_ID, - job_id=JOB_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - class_path, args = emr_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" - assert args["virtual_cluster_id"] == VIRTUAL_CLUSTER_ID - assert args["job_id"] == JOB_ID - assert args["aws_conn_id"] == AWS_CONN_ID - assert args["poll_interval"] == POLL_INTERVAL - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.async_conn") - async def test_emr_container_trigger_run(self, mock_async_conn, mock_get_waiter): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_get_waiter().wait = AsyncMock() - - emr_trigger = EmrContainerTrigger( - virtual_cluster_id=VIRTUAL_CLUSTER_ID, - job_id=JOB_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - generator = emr_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "job_id": JOB_ID}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.async_conn") - async def test_emr_trigger_run_multiple_attempts(self, mock_async_conn, mock_get_waiter, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"jobRun": {"state": "RUNNING"}}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - emr_trigger = EmrContainerTrigger( - virtual_cluster_id=VIRTUAL_CLUSTER_ID, - job_id=JOB_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - generator = emr_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "job_id": JOB_ID}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.get_waiter") - @mock.patch("airflow.providers.amazon.aws.hooks.emr.EmrContainerHook.async_conn") - async def test_emr_trigger_run_attempts_failed(self, mock_async_conn, mock_get_waiter, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error_available = WaiterError( - name="test_name", - reason="Max attempts exceeded", - last_response={"jobRun": {"state": "FAILED"}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state", - last_response={"jobRun": {"state": "FAILED"}}, - ) - mock_get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_available, error_available, error_failed] - ) - mock_sleep.return_value = True - - emr_trigger = EmrContainerTrigger( - virtual_cluster_id=VIRTUAL_CLUSTER_ID, - job_id=JOB_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - generator = emr_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "failure", "message": f"Job Failed: {error_failed}"}) - - -class TestEmrStepSensorTrigger: - def test_emr_step_trigger_serialize(self): - """Test trigger serialize object and path as expected""" - emr_trigger = EmrStepSensorTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - step_id=STEP_ID, - aws_conn_id=AWS_CONN_ID, - poke_interval=POLL_INTERVAL, - ) - class_path, args = emr_trigger.serialize() - assert class_path == "airflow.providers.amazon.aws.triggers.emr.EmrStepSensorTrigger" - assert args["job_flow_id"] == TEST_JOB_FLOW_ID - assert args["step_id"] == STEP_ID - assert args["aws_conn_id"] == AWS_CONN_ID - assert args["max_attempts"] == 60 - assert args["poke_interval"] == POLL_INTERVAL - - @pytest.mark.asyncio - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_step_trigger_run(self, mock_async_conn): - """Test trigger emit success if condition met""" - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock.get_waiter().wait = AsyncMock() - - emr_trigger = EmrStepSensorTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - step_id=STEP_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - generator = emr_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_trigger_run_multiple_attempts(self, mock_async_conn, mock_sleep): - """Test trigger try max attempt if attempt not exceeded and job still running""" - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Step": {"Status": {"State": "RUNNING"}}}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - - emr_trigger = EmrStepSensorTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - step_id=STEP_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - generator = emr_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 4 - assert response == TriggerEvent({"status": "success"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(EmrHook, "async_conn") - async def test_emr_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - """Test trigger does fail if max attempt exceeded and job still not succeeded""" - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error_running = WaiterError( - name="test_name", - reason="test reason", - last_response={"Step": {"Status": {"State": "RUNNING"}}}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state", - last_response={"Step": {"Status": {"State": "CANCELLED"}}}, - ) - - a_mock.get_waiter().wait = AsyncMock(side_effect=[error_running, error_failed]) - mock_sleep.return_value = True - - emr_trigger = EmrStepSensorTrigger( - job_flow_id=TEST_JOB_FLOW_ID, - step_id=STEP_ID, - aws_conn_id=AWS_CONN_ID, - poll_interval=POLL_INTERVAL, - ) - - with pytest.raises(AirflowException) as exc: - generator = emr_trigger.run() - await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 2 - assert "Error while waiting for step s-1234 to complete" in str(exc.value) +class TestEmrTriggers: + @pytest.mark.parametrize( + "trigger", + [ + EmrCreateJobFlowTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + aws_conn_id=TEST_AWS_CONN_ID, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPTS, + ), + EmrTerminateJobFlowTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + aws_conn_id=TEST_AWS_CONN_ID, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPTS, + ), + EmrContainerTrigger( + virtual_cluster_id=VIRTUAL_CLUSTER_ID, + job_id=JOB_ID, + aws_conn_id=AWS_CONN_ID, + poll_interval=POLL_INTERVAL, + ), + EmrStepSensorTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + step_id=STEP_ID, + aws_conn_id=AWS_CONN_ID, + waiter_delay=POLL_INTERVAL, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_emr_trigger.py b/tests/providers/amazon/aws/triggers/test_emr_trigger.py index 0ec3b5af6eb8..fe28f86edb67 100644 --- a/tests/providers/amazon/aws/triggers/test_emr_trigger.py +++ b/tests/providers/amazon/aws/triggers/test_emr_trigger.py @@ -23,7 +23,13 @@ from botocore.exceptions import WaiterError from airflow.providers.amazon.aws.hooks.emr import EmrHook -from airflow.providers.amazon.aws.triggers.emr import EmrAddStepsTrigger +from airflow.providers.amazon.aws.triggers.emr import ( + EmrAddStepsTrigger, + EmrContainerTrigger, + EmrCreateJobFlowTrigger, + EmrStepSensorTrigger, + EmrTerminateJobFlowTrigger, +) from airflow.triggers.base import TriggerEvent TEST_JOB_FLOW_ID = "test_job_flow_id" @@ -168,3 +174,48 @@ async def test_emr_add_steps_trigger_run_attempts_failed(self, mock_async_conn, assert response == TriggerEvent( {"status": "failure", "message": f"Step {TEST_STEP_IDS[0]} failed: {error_failed}"} ) + + +class TestEmrTriggers: + @pytest.mark.parametrize( + "trigger", + [ + EmrCreateJobFlowTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_POLL_INTERVAL, + waiter_max_attempts=TEST_MAX_ATTEMPTS, + ), + EmrTerminateJobFlowTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_POLL_INTERVAL, + waiter_max_attempts=TEST_MAX_ATTEMPTS, + ), + EmrContainerTrigger( + virtual_cluster_id="my_cluster_id", + job_id=TEST_JOB_FLOW_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_POLL_INTERVAL, + waiter_max_attempts=TEST_MAX_ATTEMPTS, + ), + EmrStepSensorTrigger( + job_flow_id=TEST_JOB_FLOW_ID, + step_id="my_step", + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_POLL_INTERVAL, + waiter_max_attempts=TEST_MAX_ATTEMPTS, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_glue.py b/tests/providers/amazon/aws/triggers/test_glue.py index cc98ecc74831..9c7cd61a712e 100644 --- a/tests/providers/amazon/aws/triggers/test_glue.py +++ b/tests/providers/amazon/aws/triggers/test_glue.py @@ -24,6 +24,7 @@ from airflow import AirflowException from airflow.providers.amazon.aws.hooks.glue import GlueJobHook from airflow.providers.amazon.aws.triggers.glue import GlueJobCompleteTrigger +from airflow.providers.amazon.aws.triggers.glue_crawler import GlueCrawlerCompleteTrigger class TestGlueJobTrigger: @@ -69,3 +70,21 @@ async def test_wait_job_failed(self, get_state_mock: mock.MagicMock): await trigger.run().asend(None) assert get_state_mock.call_count == 3 + + +class TestGlueCrawlerTrigger: + def test_serialize_recreate(self): + trigger = GlueCrawlerCompleteTrigger( + crawler_name="my_crawler", waiter_delay=2, aws_conn_id="my_conn_id" + ) + + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 diff --git a/tests/providers/amazon/aws/triggers/test_redshift_cluster.py b/tests/providers/amazon/aws/triggers/test_redshift_cluster.py index 43a2ee0ae184..af5e4a9da1ad 100644 --- a/tests/providers/amazon/aws/triggers/test_redshift_cluster.py +++ b/tests/providers/amazon/aws/triggers/test_redshift_cluster.py @@ -16,13 +16,8 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - import pytest -from botocore.exceptions import WaiterError -from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftHook from airflow.providers.amazon.aws.triggers.redshift_cluster import ( RedshiftCreateClusterSnapshotTrigger, RedshiftCreateClusterTrigger, @@ -30,7 +25,6 @@ RedshiftPauseClusterTrigger, RedshiftResumeClusterTrigger, ) -from airflow.triggers.base import TriggerEvent TEST_CLUSTER_IDENTIFIER = "test-cluster" TEST_POLL_INTERVAL = 10 @@ -38,598 +32,50 @@ TEST_AWS_CONN_ID = "test-aws-id" -class TestRedshiftCreateClusterTrigger: - def test_redshift_create_cluster_trigger_serialize(self): - redshift_create_cluster_trigger = RedshiftCreateClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempt=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - class_path, args = redshift_create_cluster_trigger.serialize() - assert ( - class_path - == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftCreateClusterTrigger" - ) - assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempt"] == str(TEST_MAX_ATTEMPT) - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn") - async def test_redshift_create_cluster_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - a_mock.get_waiter().wait = AsyncMock() - - redshift_create_cluster_trigger = RedshiftCreateClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempt=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_create_cluster_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Cluster Created"}) - - -class TestRedshiftPauseClusterTrigger: - def test_redshift_pause_cluster_trigger_serialize(self): - redshift_pause_cluster_trigger = RedshiftPauseClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - class_path, args = redshift_pause_cluster_trigger.serialize() - assert ( - class_path == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftPauseClusterTrigger" - ) - assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempts"] == str(TEST_MAX_ATTEMPT) - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - - @pytest.mark.asyncio - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_pause_cluster_trigger_run(self, mock_async_conn, mock_get_waiter): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_get_waiter().wait = AsyncMock() - - redshift_pause_cluster_trigger = RedshiftPauseClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_pause_cluster_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Cluster paused"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_pause_cluster_trigger_run_multiple_attempts( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_pause_cluster_trigger = RedshiftPauseClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_pause_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Cluster paused"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_pause_cluster_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_pause_cluster_trigger = RedshiftPauseClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_pause_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 2 - assert response == TriggerEvent( - {"status": "failure", "message": "Pause Cluster Failed - max attempts reached."} - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_pause_cluster_trigger_run_attempts_failed( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_available = WaiterError( - name="test_name", - reason="Max attempts exceeded", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_available, error_available, error_failed] - ) - mock_sleep.return_value = True - - redshift_pause_cluster_trigger = RedshiftPauseClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_pause_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - {"status": "failure", "message": f"Pause Cluster Failed: {error_failed}"} - ) - - -class TestRedshiftCreateClusterSnapshotTrigger: - def test_redshift_create_cluster_snapshot_trigger_serialize(self): - redshift_create_cluster_trigger = RedshiftCreateClusterSnapshotTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - class_path, args = redshift_create_cluster_trigger.serialize() - assert ( - class_path - == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftCreateClusterSnapshotTrigger" - ) - assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempts"] == str(TEST_MAX_ATTEMPT) - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - - @pytest.mark.asyncio - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_create_cluster_snapshot_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - a_mock.get_waiter().wait = AsyncMock() - - redshift_create_cluster_trigger = RedshiftCreateClusterSnapshotTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_create_cluster_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Cluster Snapshot Created"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_create_cluster_snapshot_trigger_run_multiple_attempts( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Snapshots": [{"Status": "available"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_create_cluster_snapshot_trigger = RedshiftCreateClusterSnapshotTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_create_cluster_snapshot_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Cluster Snapshot Created"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_create_cluster_snapshot_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Snapshots": [{"Status": "available"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_create_cluster_snapshot_trigger = RedshiftCreateClusterSnapshotTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_create_cluster_snapshot_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 2 - assert response == TriggerEvent( - {"status": "failure", "message": "Create Cluster Snapshot Cluster Failed - max attempts reached."} - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_create_cluster_snapshot_trigger_run_attempts_failed( - self, mock_async_conn, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_available = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Snapshots": [{"Status": "available"}]}, - ) - - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"Snapshots": [{"Status": "available"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_available, error_available, error_failed] - ) - mock_sleep.return_value = True - - redshift_create_cluster_snapshot_trigger = RedshiftCreateClusterSnapshotTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_create_cluster_snapshot_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - {"status": "failure", "message": f"Create Cluster Snapshot Failed: {error_failed}"} - ) - - -class TestRedshiftResumeClusterTrigger: - def test_redshift_resume_cluster_trigger_serialize(self): - redshift_resume_cluster_trigger = RedshiftResumeClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - class_path, args = redshift_resume_cluster_trigger.serialize() - assert ( - class_path - == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftResumeClusterTrigger" - ) - assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER - assert args["poll_interval"] == str(TEST_POLL_INTERVAL) - assert args["max_attempts"] == str(TEST_MAX_ATTEMPT) - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - - @pytest.mark.asyncio - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_resume_cluster_trigger_run(self, mock_async_conn, mock_get_waiter): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - mock_get_waiter().wait = AsyncMock() - - redshift_resume_cluster_trigger = RedshiftResumeClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_resume_cluster_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Cluster resumed"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_resume_cluster_trigger_run_multiple_attempts( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_resume_cluster_trigger = RedshiftResumeClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_resume_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Cluster resumed"}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_resume_cluster_trigger_run_attempts_exceeded( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_resume_cluster_trigger = RedshiftResumeClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_resume_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 2 - assert response == TriggerEvent( - {"status": "failure", "message": "Resume Cluster Failed - max attempts reached."} - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "get_waiter") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_resume_cluster_trigger_run_attempts_failed( - self, mock_async_conn, mock_get_waiter, mock_sleep - ): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error_available = WaiterError( - name="test_name", - reason="Max attempts exceeded", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - mock_get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_available, error_available, error_failed] - ) - mock_sleep.return_value = True - - redshift_resume_cluster_trigger = RedshiftResumeClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_resume_cluster_trigger.run() - response = await generator.asend(None) - - assert mock_get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - {"status": "failure", "message": f"Resume Cluster Failed: {error_failed}"} - ) - - -class TestRedshiftDeleteClusterTrigger: - def test_redshift_delete_cluster_trigger_serialize(self): - redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - class_path, args = redshift_delete_cluster_trigger.serialize() - assert ( - class_path - == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger" - ) - assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER - assert args["poll_interval"] == TEST_POLL_INTERVAL - assert args["max_attempts"] == TEST_MAX_ATTEMPT - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - - @pytest.mark.asyncio - @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn") - async def test_redshift_delete_cluster_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - a_mock.get_waiter().wait = AsyncMock() - - redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_delete_cluster_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "message": "Cluster deleted."}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_delete_cluster_trigger_run_multiple_attempts(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_delete_cluster_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent({"status": "success", "message": "Cluster deleted."}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_delete_cluster_trigger_run_attempts_exceeded(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"Clusters": [{"ClusterStatus": "deleting"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock(side_effect=[error, error, True]) - mock_sleep.return_value = True - - redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=2, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_delete_cluster_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 2 - assert response == TriggerEvent( - {"status": "failure", "message": "Delete Cluster Failed - max attempts reached."} - ) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RedshiftHook, "async_conn") - async def test_redshift_delete_cluster_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error_available = WaiterError( - name="test_name", - reason="Max attempts exceeded", - last_response={"Clusters": [{"ClusterStatus": "deleting"}]}, - ) - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"Clusters": [{"ClusterStatus": "available"}]}, - ) - a_mock.get_waiter().wait.side_effect = AsyncMock( - side_effect=[error_available, error_available, error_failed] - ) - mock_sleep.return_value = True - - redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger( - cluster_identifier=TEST_CLUSTER_IDENTIFIER, - poll_interval=TEST_POLL_INTERVAL, - max_attempts=TEST_MAX_ATTEMPT, - aws_conn_id=TEST_AWS_CONN_ID, - ) - - generator = redshift_delete_cluster_trigger.run() - response = await generator.asend(None) - - assert a_mock.get_waiter().wait.call_count == 3 - assert response == TriggerEvent( - {"status": "failure", "message": f"Delete Cluster Failed: {error_failed}"} - ) +class TestRedshiftClusterTriggers: + @pytest.mark.parametrize( + "trigger", + [ + RedshiftCreateClusterTrigger( + cluster_identifier=TEST_CLUSTER_IDENTIFIER, + poll_interval=TEST_POLL_INTERVAL, + max_attempt=TEST_MAX_ATTEMPT, + aws_conn_id=TEST_AWS_CONN_ID, + ), + RedshiftPauseClusterTrigger( + cluster_identifier=TEST_CLUSTER_IDENTIFIER, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPT, + aws_conn_id=TEST_AWS_CONN_ID, + ), + RedshiftCreateClusterSnapshotTrigger( + cluster_identifier=TEST_CLUSTER_IDENTIFIER, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPT, + aws_conn_id=TEST_AWS_CONN_ID, + ), + RedshiftResumeClusterTrigger( + cluster_identifier=TEST_CLUSTER_IDENTIFIER, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPT, + aws_conn_id=TEST_AWS_CONN_ID, + ), + RedshiftDeleteClusterTrigger( + cluster_identifier=TEST_CLUSTER_IDENTIFIER, + poll_interval=TEST_POLL_INTERVAL, + max_attempts=TEST_MAX_ATTEMPT, + aws_conn_id=TEST_AWS_CONN_ID, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 From 869f3a93a8873381a57382f8a0ab88879ca43f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Fri, 7 Jul 2023 11:53:54 -0700 Subject: [PATCH 098/533] Remove ability to specify arbitrary hook params in AWS RDS trigger (#32386) --- airflow/providers/amazon/aws/operators/rds.py | 23 +++++++++++++------ airflow/providers/amazon/aws/triggers/rds.py | 8 +++---- .../providers/amazon/aws/triggers/test_rds.py | 13 ++++++----- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/rds.py b/airflow/providers/amazon/aws/operators/rds.py index c58961db2e8f..2630098bc465 100644 --- a/airflow/providers/amazon/aws/operators/rds.py +++ b/airflow/providers/amazon/aws/operators/rds.py @@ -43,18 +43,27 @@ class RdsBaseOperator(BaseOperator): ui_color = "#eeaa88" ui_fgcolor = "#ffffff" - def __init__(self, *args, aws_conn_id: str = "aws_conn_id", hook_params: dict | None = None, **kwargs): + def __init__( + self, + *args, + aws_conn_id: str = "aws_conn_id", + region_name: str | None = None, + hook_params: dict | None = None, + **kwargs, + ): if hook_params is not None: warnings.warn( "The parameter hook_params is deprecated and will be removed. " - "If you were using it, please get in touch either on airflow slack, " - "or by opening a github issue on the project. " + "Note that it is also incompatible with deferrable mode. " + "You can use the region_name parameter to specify the region. " + "If you were using hook_params for other purposes, please get in touch either on " + "airflow slack, or by opening a github issue on the project. " "You can mention https://github.com/apache/airflow/pull/32352", AirflowProviderDeprecationWarning, stacklevel=3, # 2 is in the operator's init, 3 is in the user code creating the operator ) - self.hook_params = hook_params or {} - self.hook = RdsHook(aws_conn_id=aws_conn_id, **self.hook_params) + self.region_name = region_name + self.hook = RdsHook(aws_conn_id=aws_conn_id, region_name=region_name, **(hook_params or {})) super().__init__(*args, **kwargs) self._await_interval = 60 # seconds @@ -588,7 +597,7 @@ def execute(self, context: Context) -> str: waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, - hook_params=self.hook_params, + region_name=self.region_name, waiter_name="db_instance_available", # ignoring type because create_db_instance is a dict response=create_db_instance, # type: ignore[arg-type] @@ -674,7 +683,7 @@ def execute(self, context: Context) -> str: waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, - hook_params=self.hook_params, + region_name=self.region_name, waiter_name="db_instance_deleted", # ignoring type because delete_db_instance is a dict response=delete_db_instance, # type: ignore[arg-type] diff --git a/airflow/providers/amazon/aws/triggers/rds.py b/airflow/providers/amazon/aws/triggers/rds.py index 0897f764bece..0551d67591df 100644 --- a/airflow/providers/amazon/aws/triggers/rds.py +++ b/airflow/providers/amazon/aws/triggers/rds.py @@ -47,14 +47,14 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - hook_params: dict[str, Any], + region_name: str | None, response: dict[str, Any], ): self.db_instance_identifier = db_instance_identifier self.waiter_delay = waiter_delay self.waiter_max_attempts = waiter_max_attempts self.aws_conn_id = aws_conn_id - self.hook_params = hook_params + self.region_name = region_name self.waiter_name = waiter_name self.response = response @@ -67,14 +67,14 @@ def serialize(self) -> tuple[str, dict[str, Any]]: "waiter_delay": str(self.waiter_delay), "waiter_max_attempts": str(self.waiter_max_attempts), "aws_conn_id": self.aws_conn_id, - "hook_params": self.hook_params, + "region_name": self.region_name, "waiter_name": self.waiter_name, "response": self.response, }, ) async def run(self): - self.hook = RdsHook(aws_conn_id=self.aws_conn_id, **self.hook_params) + self.hook = RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) async with self.hook.async_conn as client: waiter = client.get_waiter(self.waiter_name) await async_wait( diff --git a/tests/providers/amazon/aws/triggers/test_rds.py b/tests/providers/amazon/aws/triggers/test_rds.py index 5ae64b83c3bd..9c518c8eee5f 100644 --- a/tests/providers/amazon/aws/triggers/test_rds.py +++ b/tests/providers/amazon/aws/triggers/test_rds.py @@ -31,6 +31,7 @@ TEST_WAITER_DELAY = 10 TEST_WAITER_MAX_ATTEMPTS = 10 TEST_AWS_CONN_ID = "test-aws-id" +TEST_REGION = "sa-east-1" TEST_RESPONSE = { "DBInstance": { "DBInstanceIdentifier": "test-db-instance-identifier", @@ -47,7 +48,7 @@ def test_rds_db_instance_trigger_serialize(self): waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, aws_conn_id=TEST_AWS_CONN_ID, - hook_params={}, + region_name=TEST_REGION, response=TEST_RESPONSE, ) class_path, args = rds_db_instance_trigger.serialize() @@ -58,7 +59,7 @@ def test_rds_db_instance_trigger_serialize(self): assert args["waiter_delay"] == str(TEST_WAITER_DELAY) assert args["waiter_max_attempts"] == str(TEST_WAITER_MAX_ATTEMPTS) assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["hook_params"] == {} + assert args["region_name"] == TEST_REGION assert args["response"] == TEST_RESPONSE @pytest.mark.asyncio @@ -75,7 +76,7 @@ async def test_rds_db_instance_trigger_run(self, mock_async_conn): waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, aws_conn_id=TEST_AWS_CONN_ID, - hook_params={}, + region_name=TEST_REGION, response=TEST_RESPONSE, ) @@ -104,7 +105,7 @@ async def test_rds_db_instance_trigger_run_multiple_attempts(self, mock_async_co waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, aws_conn_id=TEST_AWS_CONN_ID, - hook_params={}, + region_name=TEST_REGION, response=TEST_RESPONSE, ) @@ -135,7 +136,7 @@ async def test_rds_db_instance_trigger_run_attempts_exceeded(self, mock_async_co waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=2, aws_conn_id=TEST_AWS_CONN_ID, - hook_params={}, + region_name=TEST_REGION, response=TEST_RESPONSE, ) @@ -173,7 +174,7 @@ async def test_rds_db_instance_trigger_run_attempts_failed(self, mock_async_conn waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, aws_conn_id=TEST_AWS_CONN_ID, - hook_params={}, + region_name=TEST_REGION, response=TEST_RESPONSE, ) From 8303ad1b94d3b541a63248f0af0fea2817a15356 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 7 Jul 2023 21:52:18 +0200 Subject: [PATCH 099/533] Ignore error on mssql temp removal when cleaning (#32433) When there is a loack of sudo capability, clean command might return with error. We should ignore it. --- dev/breeze/src/airflow_breeze/commands/ci_commands.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index c35fa3430294..526ef307353a 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -81,9 +81,7 @@ def free_space(): run_command(["docker", "system", "prune", "--all", "--force", "--volumes"]) run_command(["df", "-h"]) run_command(["docker", "logout", "ghcr.io"], check=False) - run_command( - ["sudo", "rm", "-f", os.fspath(Path.home() / MSSQL_TMP_DIR_NAME)], - ) + run_command(["sudo", "rm", "-f", os.fspath(Path.home() / MSSQL_TMP_DIR_NAME)], check=False) @ci_group.command(name="resource-check", help="Check if available docker resources are enough.") From 723eb7d453e50fb82652a8cf1f6a538410be777f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Harabie=C5=84?= Date: Fri, 7 Jul 2023 22:03:16 +0200 Subject: [PATCH 100/533] Give better link to job configuration docs in BigQueryInsertJobOperator (#31736) Link directly to job configuration docs instead of jobs resource in Google REST API docs. Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- airflow/providers/google/cloud/operators/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 970e7813ed9c..a1b53aed78a3 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -2568,7 +2568,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator): :param configuration: The configuration parameter maps directly to BigQuery's configuration field in the job object. For more details see - https://cloud.google.com/bigquery/docs/reference/v2/jobs + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#jobconfiguration :param job_id: The ID of the job. It will be suffixed with hash of job configuration unless ``force_rerun`` is True. The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or From 9cb463e20e4557efb4d1a6320b196c65ae519c23 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 8 Jul 2023 04:37:49 +0800 Subject: [PATCH 101/533] Support Pydantic 2 (#32366) * Remove Pydantic limit * Use base datetime class for Pydantic --- airflow/serialization/pydantic/dag_run.py | 3 +-- setup.cfg | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/airflow/serialization/pydantic/dag_run.py b/airflow/serialization/pydantic/dag_run.py index a5ae6b4a22b5..1e1528eb3564 100644 --- a/airflow/serialization/pydantic/dag_run.py +++ b/airflow/serialization/pydantic/dag_run.py @@ -18,7 +18,6 @@ from datetime import datetime from typing import List, Optional -from pendulum import DateTime from pydantic import BaseModel as BaseModelPydantic from airflow.serialization.pydantic.dataset import DatasetEventPydantic @@ -30,7 +29,7 @@ class DagRunPydantic(BaseModelPydantic): id: int dag_id: str queued_at: Optional[datetime] - execution_date: DateTime + execution_date: datetime start_date: Optional[datetime] end_date: Optional[datetime] state: str diff --git a/setup.cfg b/setup.cfg index 91d3b86331de..d5d91d3d8ff6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,9 +126,7 @@ install_requires = pendulum>=2.0 pluggy>=1.0 psutil>=4.2.0 - # Limit Pydantic to <2.0.0 because it breaks Kubernetes tests and building providers - # This limit should be removed after https://github.com/apache/airflow/issues/32311 is fixed - pydantic>=1.10.0,<2.0.0 + pydantic>=1.10.0 pygments>=2.0.1 pyjwt>=2.0.0 python-daemon>=3.0.0 From 8c6751ff0f2056af1cb08cec03db8a4d6c913ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Fri, 7 Jul 2023 15:39:29 -0700 Subject: [PATCH 102/533] quick fix on RDS operator to prevent parameter collision (#32436) * quick fix on RDS operator to prevent parameter collision without this code, if the user specified a region in the hook params, it'd create an error about the param being specified twice * use initialized value --- airflow/providers/amazon/aws/operators/rds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/rds.py b/airflow/providers/amazon/aws/operators/rds.py index 2630098bc465..c03725164309 100644 --- a/airflow/providers/amazon/aws/operators/rds.py +++ b/airflow/providers/amazon/aws/operators/rds.py @@ -62,8 +62,9 @@ def __init__( AirflowProviderDeprecationWarning, stacklevel=3, # 2 is in the operator's init, 3 is in the user code creating the operator ) - self.region_name = region_name - self.hook = RdsHook(aws_conn_id=aws_conn_id, region_name=region_name, **(hook_params or {})) + hook_params = hook_params or {} + self.region_name = region_name or hook_params.pop("region_name", None) + self.hook = RdsHook(aws_conn_id=aws_conn_id, region_name=self.region_name, **(hook_params)) super().__init__(*args, **kwargs) self._await_interval = 60 # seconds From 566bc1b68b4e1643761b4e8518e5e556b8e6e82c Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:52:14 -0700 Subject: [PATCH 103/533] Fix tests and add logic to handle clearing setup directly (#32430) --- airflow/models/dag.py | 2 ++ tests/models/test_dag.py | 77 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 1ec78b64c379..234db54b1539 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2373,6 +2373,8 @@ def partial_subset( also_include.extend(t.get_upstreams_follow_setups()) else: also_include.extend(t.get_upstreams_only_setups_and_teardowns()) + if t.is_setup and not include_downstream: + also_include.extend(x for x in t.downstream_list if x.is_teardown) direct_upstreams: list[Operator] = [] if include_direct_upstream: diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index bd460e6cc7cb..e4a3290a5d2f 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -3590,12 +3590,23 @@ def cleared_upstream(task): upstream = True return set( task.dag.partial_subset( - task_ids_or_regex=[task.task_id], + task_ids_or_regex=task.task_id, include_downstream=not upstream, include_upstream=upstream, ).tasks ) + @staticmethod + def cleared_neither(task): + """Helper to return tasks that would be cleared if **upstream** selected.""" + return set( + task.dag.partial_subset( + task_ids_or_regex=[task.task_id], + include_downstream=False, + include_upstream=False, + ).tasks + ) + def test_get_flat_relative_ids_with_setup(self): with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: s1, w1, w2, w3, t1 = self.make_tasks(dag, "s1, w1, w2, w3, t1") @@ -3823,18 +3834,70 @@ def test_clear_upstream_not_your_setup(self): """ with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") - s1 >> w1 >> t1 + s1 >> w1 >> t1.as_teardown(setups=s1) s1 >> w2 - self.cleared_upstream(w2) == {s1, w2, t1} + # w2 is downstream of s1, so when clearing upstream, it should clear s1 (since it + # is upstream of w2) and t1 since it's the teardown for s1 even though not downstream of w1 + assert self.cleared_upstream(w2) == {s1, w2, t1} - def clearing_teardown_no_clear_setup(self): + def test_clearing_teardown_no_clear_setup(self): with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: s1, w1, t1 = self.make_tasks(dag, "s1, w1, t1") s1 >> t1 # clearing t1 does not clear s1 - self.cleared_downstream(t1) == {t1} + assert self.cleared_downstream(t1) == {t1} s1 >> w1 >> t1 # that isn't changed with the introduction of w1 - self.cleared_downstream(t1) == {t1} + assert self.cleared_downstream(t1) == {t1} # though, of course, clearing w1 clears them all - self.cleared_downstream(w1) == {s1, w1, t1} + assert self.cleared_downstream(w1) == {s1, w1, t1} + + def test_clearing_setup_clears_teardown(self): + with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: + s1, w1, t1 = self.make_tasks(dag, "s1, w1, t1") + s1 >> t1 + s1 >> w1 >> t1 + # clearing w1 clears all always + assert self.cleared_upstream(w1) == {s1, w1, t1} + assert self.cleared_downstream(w1) == {s1, w1, t1} + assert self.cleared_neither(w1) == {s1, w1, t1} + # clearing s1 clears t1 always + assert self.cleared_upstream(s1) == {s1, t1} + assert self.cleared_downstream(s1) == {s1, w1, t1} + assert self.cleared_neither(s1) == {s1, t1} + + @pytest.mark.parametrize( + "upstream, downstream, expected", + [ + (False, False, {"my_teardown", "my_setup"}), + (False, True, {"my_setup", "my_work", "my_teardown"}), + (True, False, {"my_teardown", "my_setup"}), + (True, True, {"my_setup", "my_work", "my_teardown"}), + ], + ) + def test_clearing_setup_clears_teardown_taskflow(self, upstream, downstream, expected): + with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: + + @setup + def my_setup(): + ... + + @task_decorator + def my_work(): + ... + + @teardown + def my_teardown(): + ... + + s1 = my_setup() + w1 = my_work() + t1 = my_teardown() + s1 >> w1 >> t1 + s1 >> t1 + assert { + x.task_id + for x in dag.partial_subset( + "my_setup", include_upstream=upstream, include_downstream=downstream + ).tasks + } == expected From 257136786c9a3eebbae717738637ab24fd6ab563 Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:31:20 +0530 Subject: [PATCH 104/533] Moves `AzureBlobStorageToGCSOperator` from Azure to Google provider (#32306) * moved AzureBlobStorageToGCSOperator to google provider --- .../cloud/transfers/azure_blob_to_gcs.py | 118 ++++++++++++++++++ airflow/providers/google/provider.yaml | 5 +- .../providers/microsoft/azure/provider.yaml | 5 +- .../azure/transfers/azure_blob_to_gcs.py | 108 +++------------- .../connections/gcp.rst | 2 +- .../operators/transfer/azure_blob_to_gcs.rst | 53 ++++++++ .../redirects.txt | 1 + .../transfer/azure_blob_to_gcs.rst | 44 +------ .../transfers/test_azure_blob_to_gcs.py | 8 +- .../cloud}/azure/example_azure_blob_to_gcs.py | 35 ++---- 10 files changed, 211 insertions(+), 168 deletions(-) create mode 100644 airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py create mode 100644 docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst rename tests/providers/{microsoft/azure => google/cloud}/transfers/test_azure_blob_to_gcs.py (90%) rename tests/system/providers/{microsoft => google/cloud}/azure/example_azure_blob_to_gcs.py (71%) diff --git a/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py b/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py new file mode 100644 index 000000000000..8ba6f2d6eb07 --- /dev/null +++ b/airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import tempfile +from typing import TYPE_CHECKING, Sequence + +from airflow.models import BaseOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class AzureBlobStorageToGCSOperator(BaseOperator): + """ + Operator transfers data from Azure Blob Storage to specified bucket in Google Cloud Storage. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:AzureBlobStorageToGCSOperator` + + :param wasb_conn_id: Reference to the wasb connection. + :param gcp_conn_id: The connection ID to use when fetching connection info. + :param blob_name: Name of the blob + :param container_name: Name of the container + :param bucket_name: The bucket to upload to + :param object_name: The object name to set when uploading the file + :param filename: The local file path to the file to be uploaded + :param gzip: Option to compress local file or file data for upload + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account. + """ + + def __init__( + self, + *, + wasb_conn_id="wasb_default", + gcp_conn_id: str = "google_cloud_default", + blob_name: str, + container_name: str, + bucket_name: str, + object_name: str, + filename: str, + gzip: bool, + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.wasb_conn_id = wasb_conn_id + self.gcp_conn_id = gcp_conn_id + self.blob_name = blob_name + self.container_name = container_name + self.bucket_name = bucket_name + self.object_name = object_name + self.filename = filename + self.gzip = gzip + self.impersonation_chain = impersonation_chain + + template_fields: Sequence[str] = ( + "blob_name", + "container_name", + "bucket_name", + "object_name", + "filename", + ) + + def execute(self, context: Context) -> str: + azure_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) + gcs_hook = GCSHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + with tempfile.NamedTemporaryFile() as temp_file: + self.log.info("Downloading data from blob: %s", self.blob_name) + azure_hook.get_file( + file_path=temp_file.name, + container_name=self.container_name, + blob_name=self.blob_name, + ) + self.log.info( + "Uploading data from blob's: %s into GCP bucket: %s", self.object_name, self.bucket_name + ) + gcs_hook.upload( + bucket_name=self.bucket_name, + object_name=self.object_name, + filename=temp_file.name, + gzip=self.gzip, + ) + self.log.info( + "Resources have been uploaded from blob: %s to GCS bucket:%s", + self.blob_name, + self.bucket_name, + ) + return f"gs://{self.bucket_name}/{self.object_name}" diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 02168a12c953..c240940dc9f9 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -1007,7 +1007,10 @@ transfers: target-integration-name: Google Cloud Storage (GCS) python-module: airflow.providers.google.cloud.transfers.mssql_to_gcs how-to-guide: /docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst - + - source-integration-name: Microsoft Azure Blob Storage + target-integration-name: Google Cloud Storage (GCS) + python-module: airflow.providers.google.cloud.transfers.azure_blob_to_gcs + how-to-guide: /docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst connection-types: - hook-class-name: airflow.providers.google.common.hooks.base_google.GoogleBaseHook diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index a1070177bc11..441f51a5786b 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -242,12 +242,15 @@ transfers: python-module: airflow.providers.microsoft.azure.transfers.local_to_wasb - source-integration-name: Microsoft Azure Blob Storage target-integration-name: Google Cloud Storage (GCS) - how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Microsoft Azure Blob Storage how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb + - source-integration-name: Microsoft Azure Blob Storage + target-integration-name: Google Cloud Storage (GCS) + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst + python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs connection-types: diff --git a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py index 8ba6f2d6eb07..20edde6e0938 100644 --- a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py +++ b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py @@ -17,102 +17,26 @@ # under the License. from __future__ import annotations -import tempfile -from typing import TYPE_CHECKING, Sequence +import warnings -from airflow.models import BaseOperator -from airflow.providers.google.cloud.hooks.gcs import GCSHook -from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.google.cloud.transfers.azure_blob_to_gcs import ( + AzureBlobStorageToGCSOperator as AzureBlobStorageToGCSOperatorFromGoogleProvider, +) -if TYPE_CHECKING: - from airflow.utils.context import Context - -class AzureBlobStorageToGCSOperator(BaseOperator): +class AzureBlobStorageToGCSOperator(AzureBlobStorageToGCSOperatorFromGoogleProvider): """ - Operator transfers data from Azure Blob Storage to specified bucket in Google Cloud Storage. - - .. seealso:: - For more information on how to use this operator, take a look at the guide: - :ref:`howto/operator:AzureBlobStorageToGCSOperator` - - :param wasb_conn_id: Reference to the wasb connection. - :param gcp_conn_id: The connection ID to use when fetching connection info. - :param blob_name: Name of the blob - :param container_name: Name of the container - :param bucket_name: The bucket to upload to - :param object_name: The object name to set when uploading the file - :param filename: The local file path to the file to be uploaded - :param gzip: Option to compress local file or file data for upload - :param impersonation_chain: Optional service account to impersonate using short-term - credentials, or chained list of accounts required to get the access_token - of the last account in the list, which will be impersonated in the request. - If set as a string, the account must grant the originating account - the Service Account Token Creator IAM role. - If set as a sequence, the identities from the list must grant - Service Account Token Creator IAM role to the directly preceding identity, with first - account from the list granting this role to the originating account. + This class is deprecated. + Please use `airflow.providers.google.cloud.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator`. """ - def __init__( - self, - *, - wasb_conn_id="wasb_default", - gcp_conn_id: str = "google_cloud_default", - blob_name: str, - container_name: str, - bucket_name: str, - object_name: str, - filename: str, - gzip: bool, - impersonation_chain: str | Sequence[str] | None = None, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.wasb_conn_id = wasb_conn_id - self.gcp_conn_id = gcp_conn_id - self.blob_name = blob_name - self.container_name = container_name - self.bucket_name = bucket_name - self.object_name = object_name - self.filename = filename - self.gzip = gzip - self.impersonation_chain = impersonation_chain - - template_fields: Sequence[str] = ( - "blob_name", - "container_name", - "bucket_name", - "object_name", - "filename", - ) - - def execute(self, context: Context) -> str: - azure_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) - gcs_hook = GCSHook( - gcp_conn_id=self.gcp_conn_id, - impersonation_chain=self.impersonation_chain, + def __init__(self, *args, **kwargs): + warnings.warn( + """This class is deprecated. + Please use + `airflow.providers.google.cloud.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator`.""", + AirflowProviderDeprecationWarning, + stacklevel=2, ) - - with tempfile.NamedTemporaryFile() as temp_file: - self.log.info("Downloading data from blob: %s", self.blob_name) - azure_hook.get_file( - file_path=temp_file.name, - container_name=self.container_name, - blob_name=self.blob_name, - ) - self.log.info( - "Uploading data from blob's: %s into GCP bucket: %s", self.object_name, self.bucket_name - ) - gcs_hook.upload( - bucket_name=self.bucket_name, - object_name=self.object_name, - filename=temp_file.name, - gzip=self.gzip, - ) - self.log.info( - "Resources have been uploaded from blob: %s to GCS bucket:%s", - self.blob_name, - self.bucket_name, - ) - return f"gs://{self.bucket_name}/{self.object_name}" + super().__init__(*args, **kwargs) diff --git a/docs/apache-airflow-providers-google/connections/gcp.rst b/docs/apache-airflow-providers-google/connections/gcp.rst index c8374d455604..91e54d1b1cd6 100644 --- a/docs/apache-airflow-providers-google/connections/gcp.rst +++ b/docs/apache-airflow-providers-google/connections/gcp.rst @@ -298,4 +298,4 @@ Note that as domain-wide delegation is currently supported by most of the Google * All of Google Cloud operators and hooks. * Firebase hooks. -* All transfer operators that involve Google cloud in different providers, for example: :class:`airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs`. +* All transfer operators that involve Google cloud in different providers, for example: :class:`airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator`. diff --git a/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst new file mode 100644 index 000000000000..1568283ce196 --- /dev/null +++ b/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst @@ -0,0 +1,53 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Azure Blob Storage to Google Cloud Storage (GCS) Transfer Operator +================================================================== +The `Google Cloud Storage `__ (GCS) is used to store large data from various applications. +This is also the same with `Azure Blob Storage `__. +This page shows how to transfer data from Azure Blob Storage to GCS. + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: ../_partials/prerequisite_tasks.rst + +.. _howto/operator:AzureBlobStorageToGCSOperator: + +Transfer Data from Azure Blob Storage to Google Cloud Storage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Operator transfers data from Azure Blob Storage to specified bucket in Google Cloud Storage + +Use the :class:`~airflow.providers.google.cloud.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator` +to transfer data from Azure Blob Storage to Google Cloud Storage. + +Example usage: + +.. exampleinclude:: /../../tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py + :language: python + :start-after: [START how_to_azure_blob_to_gcs] + :end-before: [END how_to_azure_blob_to_gcs] + +Reference +^^^^^^^^^ + +For further information, look at: + +* `GCS Client Library Documentation `__ +* `GCS Product Documentation `__ +* `Azure Blob Storage Client Library Documentation `__ diff --git a/docs/apache-airflow-providers-microsoft-azure/redirects.txt b/docs/apache-airflow-providers-microsoft-azure/redirects.txt index 3ce11766b770..998489b9fbd2 100644 --- a/docs/apache-airflow-providers-microsoft-azure/redirects.txt +++ b/docs/apache-airflow-providers-microsoft-azure/redirects.txt @@ -1,3 +1,4 @@ connections/index.rst connections/azure.rst secrets-backends/index.rst secrets-backends/azure-key-vault-secrets-backend.rst logging.rst logging/index.rst +transfer/azure_blob_to_gcs.rst ../../apache-airflow-providers-google/latest/operators/transfer/azure_blob_to_gcs.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst b/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst index b4c4863ac5f8..df2cd0bcceea 100644 --- a/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst +++ b/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst @@ -1,4 +1,3 @@ - .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -16,45 +15,6 @@ specific language governing permissions and limitations under the License. -Azure Blob Storage to Google Cloud Storage (GCS) Transfer Operator -================================================================== -The Blob service stores text and binary data as objects in the cloud. -The Blob service offers the following three resources: the storage account, containers, and blobs. -Within your storage account, containers provide a way to organize sets of blobs. -For more information about the service visit `Azure Blob Storage API documentation `_. - -Before you begin -^^^^^^^^^^^^^^^^ -Before using Blob Storage within Airflow you need to authenticate your account with Token, Login and Password. -Please follow Azure -`instructions `_ -to do it. - -TOKEN should be added to the Connection in Airflow in JSON format, Login and Password as plain text. -You can check `how to do such connection `_. - -See following example. -Set values for these fields: - -.. code-block:: - - Connection Id: wasb_default - Login: Storage Account Name - Password: KEY1 - Extra: {"sas_token": "TOKEN"} - -.. _howto/operator:AzureBlobStorageToGCSOperator: - -Transfer Data from Blob Storage to Google Cloud Storage -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Operator transfers data from Azure Blob Storage to specified bucket in Google Cloud Storage - -To get information about jobs within a Azure Blob Storage use: -:class:`~airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator` - -Example usage: -.. exampleinclude:: /../../tests/system/providers/microsoft/azure/example_azure_blob_to_gcs.py - :language: python - :start-after: [START how_to_azure_blob_to_gcs] - :end-before: [END how_to_azure_blob_to_gcs] +Upload data from Azure Blob Storage to Google Cloud Storage (Moved to Google Providers) +======================================================================================= diff --git a/tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py b/tests/providers/google/cloud/transfers/test_azure_blob_to_gcs.py similarity index 90% rename from tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py rename to tests/providers/google/cloud/transfers/test_azure_blob_to_gcs.py index 4bc51895f3e4..a0b3eae99d68 100644 --- a/tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_azure_blob_to_gcs.py @@ -18,7 +18,7 @@ from unittest import mock -from airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs import AzureBlobStorageToGCSOperator +from airflow.providers.google.cloud.transfers.azure_blob_to_gcs import AzureBlobStorageToGCSOperator WASB_CONN_ID = "wasb_default" GCP_CONN_ID = "google_cloud_default" @@ -57,9 +57,9 @@ def test_init(self): assert operator.impersonation_chain == IMPERSONATION_CHAIN assert operator.task_id == TASK_ID - @mock.patch("airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs.WasbHook") - @mock.patch("airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs.GCSHook") - @mock.patch("airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs.tempfile") + @mock.patch("airflow.providers.google.cloud.transfers.azure_blob_to_gcs.WasbHook") + @mock.patch("airflow.providers.google.cloud.transfers.azure_blob_to_gcs.GCSHook") + @mock.patch("airflow.providers.google.cloud.transfers.azure_blob_to_gcs.tempfile") def test_execute(self, mock_temp, mock_hook_gcs, mock_hook_wasb): op = AzureBlobStorageToGCSOperator( wasb_conn_id=WASB_CONN_ID, diff --git a/tests/system/providers/microsoft/azure/example_azure_blob_to_gcs.py b/tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py similarity index 71% rename from tests/system/providers/microsoft/azure/example_azure_blob_to_gcs.py rename to tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py index 81fb9993b25d..5d7649479df6 100644 --- a/tests/system/providers/microsoft/azure/example_azure_blob_to_gcs.py +++ b/tests/system/providers/google/cloud/azure/example_azure_blob_to_gcs.py @@ -21,11 +21,10 @@ from datetime import datetime from airflow import DAG +from airflow.providers.google.cloud.transfers.azure_blob_to_gcs import AzureBlobStorageToGCSOperator from airflow.providers.microsoft.azure.sensors.wasb import ( WasbBlobSensor, - WasbPrefixSensor, ) -from airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs import AzureBlobStorageToGCSOperator # Ignore missing args provided by default_args # type: ignore[call-arg] @@ -36,35 +35,23 @@ GCP_BUCKET_FILE_PATH = os.environ.get("GCP_BUCKET_FILE_PATH", "file.txt") GCP_BUCKET_NAME = os.environ.get("GCP_BUCKET_NAME", "INVALID BUCKET NAME") GCP_OBJECT_NAME = os.environ.get("GCP_OBJECT_NAME", "file.txt") -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "example_azure_blob_to_gcs" -PREFIX_NAME = os.environ.get("AZURE_PREFIX_NAME", "20230421") -# [START how_to_azure_blob_to_gcs] with DAG( DAG_ID, schedule=None, start_date=datetime(2021, 1, 1), # Override to match your needs - default_args={ - # azure args - "container_name": AZURE_CONTAINER_NAME, - "blob_name": BLOB_NAME, - "prefix": PREFIX_NAME, - }, ) as dag: - wait_for_blob = WasbBlobSensor(task_id="wait_for_blob") - - wait_for_blob_async = WasbBlobSensor(task_id="wait_for_blob_async", deferrable=True) - - wait_for_blob_prefix = WasbPrefixSensor(task_id="wait_for_blob_prefix") - - wait_for_blob_prefix_async = WasbPrefixSensor( - task_id="wait_for_blob_prefix_async", - deferrable=True, + wait_for_blob = WasbBlobSensor( + task_id="wait_for_blob", container_name=AZURE_CONTAINER_NAME, blob_name=BLOB_NAME ) + # [START how_to_azure_blob_to_gcs] transfer_files_to_gcs = AzureBlobStorageToGCSOperator( task_id="transfer_files_to_gcs", + # azure args + container_name=AZURE_CONTAINER_NAME, + blob_name=BLOB_NAME, # GCP args bucket_name=GCP_BUCKET_NAME, object_name=GCP_OBJECT_NAME, @@ -74,13 +61,7 @@ ) # [END how_to_azure_blob_to_gcs] - ( - wait_for_blob - >> wait_for_blob_async - >> wait_for_blob_prefix - >> wait_for_blob_prefix_async - >> transfer_files_to_gcs - ) + (wait_for_blob >> transfer_files_to_gcs) from tests.system.utils.watcher import watcher From e70bee00cd12ecf1462485a747c0e3296ef7d48c Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Sat, 8 Jul 2023 14:05:23 +0300 Subject: [PATCH 105/533] Fix `cache_control` parameter of upload function in `GCSHook` (#32440) * Fix `cache_control` parameter of upload function in `GCSHook` --------- Co-authored-by: Ying Wu --- airflow/providers/google/cloud/hooks/gcs.py | 2 +- tests/providers/google/cloud/hooks/test_gcs.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py index 42a27e80f898..5350c9f79012 100644 --- a/airflow/providers/google/cloud/hooks/gcs.py +++ b/airflow/providers/google/cloud/hooks/gcs.py @@ -517,7 +517,7 @@ def _call_with_retry(f: Callable[[], None]) -> None: blob.metadata = metadata if cache_control: - blob.cacheControl = cache_control + blob.cache_control = cache_control if filename and data: raise ValueError( diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/tests/providers/google/cloud/hooks/test_gcs.py index 9e7ff971d2ba..61ce7a416271 100644 --- a/tests/providers/google/cloud/hooks/test_gcs.py +++ b/tests/providers/google/cloud/hooks/test_gcs.py @@ -936,6 +936,19 @@ def test_upload_file(self, mock_service, testdata_file): assert metadata == blob_object.return_value.metadata + @mock.patch(GCS_STRING.format("GCSHook.get_conn")) + def test_upload_cache_control(self, mock_service, testdata_file): + test_bucket = "test_bucket" + test_object = "test_object" + cache_control = "public, max-age=3600" + + bucket_mock = mock_service.return_value.bucket + blob_object = bucket_mock.return_value.blob + + self.gcs_hook.upload(test_bucket, test_object, filename=testdata_file, cache_control=cache_control) + + assert cache_control == blob_object.return_value.cache_control + @mock.patch(GCS_STRING.format("GCSHook.get_conn")) def test_upload_file_gzip(self, mock_service, testdata_file): test_bucket = "test_bucket" From 4e30181afd03c66e6d43cc18494fb134cbd2316a Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Sat, 8 Jul 2023 19:55:23 +0300 Subject: [PATCH 106/533] Bump stale bot version to 8 (#32441) --- .github/workflows/recheck-old-bug-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/recheck-old-bug-report.yml b/.github/workflows/recheck-old-bug-report.yml index 0d94764433e8..093e5df4efc4 100644 --- a/.github/workflows/recheck-old-bug-report.yml +++ b/.github/workflows/recheck-old-bug-report.yml @@ -28,7 +28,7 @@ jobs: recheck-old-bug-report: runs-on: ubuntu-22.04 steps: - - uses: actions/stale@v7 + - uses: actions/stale@v8 with: only-issue-labels: 'kind:bug' stale-issue-label: 'Stale Bug Report' From e39362130b8659942672a728a233887f0b02dc8b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 8 Jul 2023 20:30:40 +0200 Subject: [PATCH 107/533] Remove "unknown warning" check in provider verification (#32444) So far we've been checking if there are no "unknown" warnings generated during provider import check, but this has proven to be rather useless other than occasionally annoying everone with failing main when new warnings have been introduced by a dependency. There was likely not a single time when seeing those new warnings have been actionable or useful other than adding them to the exclusion list. Removing the warning check seems to be reasonable thing to do. --- scripts/in_container/verify_providers.py | 166 ++--------------------- 1 file changed, 9 insertions(+), 157 deletions(-) diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index 78cd4c81182d..3f547b008797 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -30,7 +30,6 @@ from inspect import isclass from pathlib import Path from typing import NamedTuple -from warnings import WarningMessage from rich.console import Console @@ -123,94 +122,6 @@ class ProviderPackageDetails(NamedTuple): } -# The set of known deprecation messages that we know about. -# It contains tuples of "message" and the module that generates the warning - so when the -# Same warning is generated by different module, it is not treated as "known" warning. -KNOWN_DEPRECATED_MESSAGES: set[tuple[str, str]] = { - ("This version of Apache Beam has not been sufficiently tested on Python", "apache_beam"), - ("Using or importing the ABCs from 'collections' instead of from 'collections.abc'", "apache_beam"), - ("pyarrow.HadoopFileSystem is deprecated as of", "papermill"), - ("You have an incompatible version of 'pyarrow' installed", "snowflake"), - ("dns.hash module will be removed in future versions.", "dns"), - ("PKCS#7 support in pyOpenSSL is deprecated.", "eventlet"), - ("PKCS#12 support in pyOpenSSL is deprecated.", "eventlet"), - ("the imp module is deprecated in favour of importlib", "hdfs"), - ("SelectableGroups dict interface is deprecated.", "kombu"), - ("The module cloudant is now deprecated.", "cloudant"), - ("'nteract-scrapbook' package has been renamed to `scrapbook`", "scrapbook"), - ("SelectableGroups dict interface is deprecated. Use select.", "markdown"), - ("'_app_ctx_stack' is deprecated and will be", "flask_sqlalchemy"), - ("'_request_ctx_stack' is deprecated and will be", "flask_appbuilder"), - ("'_request_ctx_stack' is deprecated and will be", "flask_jwt_extended"), - ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "azure/datalake/store"), - ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "botocore"), - ("'urllib3.contrib.pyopenssl' module is deprecated and will be", "requests_toolbelt"), - ("zmq.eventloop.ioloop is deprecated in pyzmq 17.", "jupyter_client"), - ("Support for grpcio-gcp is deprecated.", "google"), -} - -KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = { - "distutils Version classes are deprecated. Use packaging.version instead.", - "the imp module is deprecated in favour of importlib", - "Param `schedule_interval` is deprecated and will be removed in a future release.", - "'urllib3.contrib.pyopenssl' module is deprecated", - "Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0", - ( - "Implementing implicit namespace packages (as specified in PEP 420) is " - "preferred to `pkg_resources.declare_namespace`" - ), - "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.operators.pod` instead.", - "urllib3 (1.26.6) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!", - "urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!", - "urllib3 (1.26.12) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't match a supported version!", - "Jupyter is migrating its paths to use standard platformdirs given by the platformdirs library.", - ( - "Importing ErrorTree directly from the jsonschema package is deprecated and will become" - " an ImportError. Import it from jsonschema.exceptions instead." - ), -} - -# The set of warning messages generated by direct importing of some deprecated modules. We should only -# ignore those messages when the warnings are generated directly by importlib - which means that -# we imported it directly during module walk by the importlib library -KNOWN_DEPRECATED_DIRECT_IMPORTS: set[str] = { - "This module is deprecated. Please use `kubernetes.client.models.V1Volume`.", - "This module is deprecated. Please use `kubernetes.client.models.V1VolumeMount`.", - "This module is deprecated. Please use `kubernetes.client.models.V1ResourceRequirements`", - "This module is deprecated. Please use `kubernetes.client.models.V1EnvVar`.", - "numpy.ufunc size changed, may indicate binary incompatibility.", - "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.lambda_function`.", - "S3ToSnowflakeOperator is deprecated.", - "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.operators.pod` instead.", - "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.triggers.pod` instead.", - "This module is deprecated. Please use `airflow.providers.slack.notifications.slack`", -} - - -def filter_known_warnings(warn: warnings.WarningMessage) -> bool: - msg_string = str(warn.message).replace("\n", " ") - for message, origin in KNOWN_DEPRECATED_MESSAGES: - if message in msg_string and warn.filename.find(f"/{origin}/") != -1: - return False - return True - - -def filter_direct_importlib_warning(warn: warnings.WarningMessage) -> bool: - msg_string = str(warn.message).replace("\n", " ") - for m in KNOWN_DEPRECATED_DIRECT_IMPORTS: - if m in msg_string and warn.filename.find("/importlib/") != -1: - return False - return True - - -def filter_known_common_deprecated_messages(warn: warnings.WarningMessage) -> bool: - msg_string = str(warn.message).replace("\n", " ") - for m in KNOWN_COMMON_DEPRECATED_MESSAGES: - if m in msg_string: - return False - return True - - def get_all_providers() -> list[str]: """Returns all providers for regular packages. @@ -227,7 +138,7 @@ def import_all_classes( provider_ids: list[str] | None = None, print_imports: bool = False, print_skips: bool = False, -) -> tuple[list[str], list[WarningMessage], list[str]]: +) -> tuple[list[str], list[str]]: """Imports all classes in providers packages. This method loads and imports all the classes found in providers, so that we @@ -239,8 +150,7 @@ def import_all_classes( :param provider_ids - provider ids that should be loaded. :param print_imports - if imported class should also be printed in output :param print_skips - if skipped classes should also be printed in output - :return: tuple of list of all imported classes and all warnings and all classes - with potential recursion side effects + :return: tuple of list of all imported classes and """ console.print() console.print(f"Walking all package with prefixes in {walkable_paths_and_prefixes}") @@ -269,7 +179,6 @@ def onerror(_): tracebacks.append((package, exception_string)) break - all_warnings: list[WarningMessage] = [] for path, prefix in walkable_paths_and_prefixes.items(): for modinfo in pkgutil.walk_packages(path=[path], prefix=prefix, onerror=onerror): if not any(modinfo.name.startswith(provider_prefix) for provider_prefix in provider_prefixes): @@ -282,7 +191,7 @@ def onerror(_): printed_packages.add(package_to_print) console.print(f"Importing package: {package_to_print}") try: - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True): warnings.filterwarnings("always", category=DeprecationWarning) _module = importlib.import_module(modinfo.name) for attribute_name in dir(_module): @@ -295,8 +204,6 @@ def onerror(_): or issubclass(attribute, BaseSecretsBackend) ): classes_with_potential_circular_import.append(class_name) - if w: - all_warnings.extend(w) except AirflowOptionalProviderFeatureException: # We ignore optional features ... @@ -324,7 +231,7 @@ def onerror(_): console.print("[red]----------------------------------------[/]") sys.exit(1) else: - return imported_classes, all_warnings, classes_with_potential_circular_import + return imported_classes, classes_with_potential_circular_import def is_imported_from_same_module(the_class: str, imported_name: str) -> bool: @@ -688,9 +595,8 @@ def verify_provider_classes_for_single_provider(imported_classes: list[str], pro return total, bad -def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns: list[warnings.WarningMessage]) -> bool: - """Summarises Bad/Good class names for providers and warnings""" - raise_error = False +def summarise_total_vs_bad(total: int, bad: int) -> bool: + """Summarises Bad/Good class names for providers""" if bad == 0: console.print() console.print(f"[green]OK: All {total} entities are properly named[/]") @@ -708,49 +614,6 @@ def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns: list[warnin f"[red]ERROR! There are in total: {bad} entities badly named out of {total} entities[/]" ) console.print() - raise_error = True - if warns: - if os.environ.get("CI") != "" and bad == 0: - console.print("::endgroup::") - console.print() - console.print("[red]Unknown warnings generated:[/]") - console.print() - for w in warns: - one_line_message = str(w.message).replace("\n", " ") - console.print(f"{w.filename}:{w.lineno}:[yellow]{one_line_message}[/]") - console.print() - console.print(f"[red]ERROR! There were {len(warns)} warnings generated during the import[/]") - console.print() - console.print("[yellow]Ideally, fix it, so that no warnings are generated during import.[/]") - console.print("[yellow]There are three cases that are legitimate deprecation warnings though:[/]") - console.print("[yellow] 1) when you deprecate whole module or class and replace it in provider[/]") - console.print("[yellow] 2) when 3rd-party module generates Deprecation and you cannot upgrade it[/]") - console.print( - "[yellow] 3) when many 3rd-party module generates same Deprecation warning that " - "comes from another common library[/]" - ) - console.print() - console.print( - "[yellow]In case 1), add distinctive subset of the deprecation message to " - "the KNOWN_DEPRECATED_DIRECT_IMPORTS in ./scripts/ci/in_container/verify_providers.py[/]" - ) - console.print( - "[yellow]In case 2), add distinctive subset of the deprecation message together with " - "module it is generates from to " - "the KNOWN_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]" - ) - console.print( - "[yellow]In case 3), add distinctive subset of the deprecation message to " - "the KNOWN_COMMON_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]" - ) - console.print() - raise_error = True - else: - console.print() - console.print("[green]OK: No warnings generated[/]") - console.print() - - if raise_error: console.print("[red]Please fix the problems listed above [/]") return False return True @@ -798,7 +661,7 @@ def verify_provider_classes() -> tuple[list[str], list[str]]: for provider_path in get_providers_paths(): walkable_paths_and_prefixes[provider_path] = provider_prefix add_all_namespaced_packages(walkable_paths_and_prefixes, provider_path, provider_prefix) - imported_classes, warns, classes_with_potential_circular_import = import_all_classes( + imported_classes, classes_with_potential_circular_import = import_all_classes( walkable_paths_and_prefixes=walkable_paths_and_prefixes, provider_ids=provider_ids, print_imports=True, @@ -812,25 +675,14 @@ def verify_provider_classes() -> tuple[list[str], list[str]]: ) total += inc_total bad += inc_bad - warns = list(filter(filter_known_warnings, warns)) - warns = list(filter(filter_direct_importlib_warning, warns)) - warns = list(filter(filter_known_common_deprecated_messages, warns)) - if not summarise_total_vs_bad_and_warnings(total, bad, warns): + if not summarise_total_vs_bad(total, bad): sys.exit(1) if len(imported_classes) == 0: console.print("[red]Something is seriously wrong - no classes imported[/]") sys.exit(1) - if warns: - console.print("[yellow]There were warnings generated during the import[/]") - for w in warns: - one_line_message = str(w.message).replace("\n", " ") - print(f"[yellow]{w.filename}:{w.lineno}: {one_line_message}[/]") - console.print() - console.print( - "[green]SUCCESS: All provider packages are importable and no unknown warnings generated![/]\n" - ) + console.print("[green]SUCCESS: All provider packages are importable![/]\n") console.print(f"Imported {len(imported_classes)} classes.") console.print() return imported_classes, classes_with_potential_circular_import From 92497fa727a23ef65478ef56572c7d71427c4a40 Mon Sep 17 00:00:00 2001 From: Aleksandr Musorin Date: Sat, 8 Jul 2023 21:18:12 +0200 Subject: [PATCH 108/533] upgrade flask-appbuilder (#32054) --- setup.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d5d91d3d8ff6..5a975220e9ce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -93,7 +93,7 @@ install_requires = # `airflow/www/fab_security` with their upstream counterparts. In particular, make sure any breaking changes, # for example any new methods, are accounted for. # NOTE! When you change the value here, you also have to update flask-appbuilder[oauth] in setup.py - flask-appbuilder==4.3.1 + flask-appbuilder==4.3.3 flask-caching>=1.5.0 flask-login>=0.6.2 flask-session>=0.4.0 diff --git a/setup.py b/setup.py index 90673ef86dfb..21c0a7300c2d 100644 --- a/setup.py +++ b/setup.py @@ -291,7 +291,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve flask_appbuilder_oauth = [ "authlib>=1.0.0", # The version here should be upgraded at the same time as flask-appbuilder in setup.cfg - "flask-appbuilder[oauth]==4.3.1", + "flask-appbuilder[oauth]==4.3.3", ] kerberos = [ "pykerberos>=1.1.13", From 0f73647bdab79ac6c30961222924f6166f75b55a Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Sat, 8 Jul 2023 12:33:25 -0700 Subject: [PATCH 109/533] D205 Support - Provider: Google (#32356) * D205 Support - Provider: Google * PR Fixes --- .../providers/google/cloud/hooks/automl.py | 22 +++--- .../google/cloud/hooks/bigquery_dts.py | 10 +-- .../providers/google/cloud/hooks/bigtable.py | 8 ++- .../google/cloud/hooks/cloud_build.py | 3 +- .../google/cloud/hooks/cloud_composer.py | 1 + .../providers/google/cloud/hooks/cloud_sql.py | 14 ++-- .../providers/google/cloud/hooks/compute.py | 27 ++++--- .../providers/google/cloud/hooks/dataflow.py | 3 +- .../google/cloud/hooks/datafusion.py | 6 +- .../providers/google/cloud/hooks/dataprep.py | 1 + airflow/providers/google/cloud/hooks/dlp.py | 28 ++++---- airflow/providers/google/cloud/hooks/gcs.py | 36 +++++----- airflow/providers/google/cloud/hooks/gdm.py | 1 + .../google/cloud/hooks/life_sciences.py | 3 +- .../providers/google/cloud/hooks/looker.py | 5 +- .../google/cloud/hooks/natural_language.py | 15 ++-- .../providers/google/cloud/hooks/os_login.py | 6 +- .../providers/google/cloud/hooks/pubsub.py | 3 +- .../providers/google/cloud/hooks/spanner.py | 3 +- .../google/cloud/hooks/stackdriver.py | 44 ++++++------ airflow/providers/google/cloud/hooks/tasks.py | 12 ++-- .../cloud/hooks/vertex_ai/custom_job.py | 12 +++- .../cloud/hooks/vertex_ai/endpoint_service.py | 3 +- .../providers/google/cloud/hooks/vision.py | 44 ++++++++++-- .../providers/google/cloud/hooks/workflows.py | 29 ++++---- .../google/cloud/log/gcs_task_handler.py | 12 ++-- .../google/cloud/operators/automl.py | 5 +- .../google/cloud/operators/bigquery.py | 20 +++--- .../google/cloud/operators/bigquery_dts.py | 8 +-- .../google/cloud/operators/bigtable.py | 5 +- .../google/cloud/operators/cloud_base.py | 6 +- .../google/cloud/operators/cloud_build.py | 4 +- .../cloud/operators/cloud_memorystore.py | 23 +++--- .../google/cloud/operators/cloud_sql.py | 4 +- .../cloud_storage_transfer_service.py | 28 ++++---- .../google/cloud/operators/compute.py | 14 ++-- .../google/cloud/operators/dataflow.py | 30 ++++---- .../google/cloud/operators/datafusion.py | 5 +- .../google/cloud/operators/dataproc.py | 22 +++--- .../providers/google/cloud/operators/dlp.py | 31 ++++---- .../google/cloud/operators/functions.py | 11 +-- .../providers/google/cloud/operators/gcs.py | 25 +++---- .../cloud/operators/kubernetes_engine.py | 6 +- .../google/cloud/operators/mlengine.py | 4 +- .../cloud/operators/natural_language.py | 8 +-- .../google/cloud/operators/pubsub.py | 5 +- .../google/cloud/operators/spanner.py | 9 +-- .../google/cloud/operators/stackdriver.py | 48 +++++++------ .../providers/google/cloud/operators/tasks.py | 7 +- .../cloud/operators/vertex_ai/auto_ml.py | 17 +++-- .../vertex_ai/batch_prediction_job.py | 5 +- .../cloud/operators/vertex_ai/custom_job.py | 18 ++--- .../operators/vertex_ai/endpoint_service.py | 3 +- .../vertex_ai/hyperparameter_tuning_job.py | 5 +- .../google/cloud/operators/workflows.py | 27 ++++--- .../google/cloud/sensors/bigquery.py | 13 ++-- .../google/cloud/sensors/bigtable.py | 1 + .../google/cloud/sensors/cloud_composer.py | 4 +- .../sensors/cloud_storage_transfer_service.py | 3 +- .../cloud/sensors/dataproc_metastore.py | 3 +- airflow/providers/google/cloud/sensors/gcs.py | 31 ++++---- .../providers/google/cloud/sensors/pubsub.py | 15 ++-- .../providers/google/cloud/sensors/tasks.py | 3 +- .../google/cloud/transfers/adls_to_gcs.py | 6 +- .../cloud/transfers/azure_fileshare_to_gcs.py | 5 +- .../google/cloud/transfers/bigquery_to_gcs.py | 4 +- .../cloud/transfers/bigquery_to_mssql.py | 3 +- .../cloud/transfers/bigquery_to_mysql.py | 3 +- .../cloud/transfers/bigquery_to_postgres.py | 3 +- .../google/cloud/transfers/bigquery_to_sql.py | 7 +- .../cloud/transfers/cassandra_to_gcs.py | 28 ++++---- .../google/cloud/transfers/gcs_to_bigquery.py | 5 +- .../google/cloud/transfers/gcs_to_gcs.py | 9 +-- .../google/cloud/transfers/local_to_gcs.py | 3 +- .../google/cloud/transfers/mssql_to_gcs.py | 8 +-- .../google/cloud/transfers/mysql_to_gcs.py | 3 +- .../google/cloud/transfers/oracle_to_gcs.py | 3 +- .../google/cloud/transfers/postgres_to_gcs.py | 4 +- .../google/cloud/transfers/presto_to_gcs.py | 19 +++-- .../google/cloud/transfers/s3_to_gcs.py | 3 +- .../google/cloud/transfers/sql_to_gcs.py | 10 ++- .../google/cloud/transfers/trino_to_gcs.py | 19 +++-- .../google/cloud/triggers/bigquery.py | 3 +- .../google/cloud/triggers/bigquery_dts.py | 23 +++--- .../google/cloud/triggers/cloud_sql.py | 1 + .../google/cloud/triggers/dataproc.py | 1 + .../providers/google/cloud/triggers/gcs.py | 26 +++---- .../providers/google/cloud/utils/bigquery.py | 4 +- .../cloud/utils/credentials_provider.py | 26 +++---- .../providers/google/cloud/utils/dataform.py | 1 + .../google/cloud/utils/field_sanitizer.py | 4 +- .../google/cloud/utils/field_validator.py | 8 +-- .../google/common/hooks/base_google.py | 71 ++++++++----------- .../google/common/hooks/discovery_api.py | 3 +- .../google/firebase/hooks/firestore.py | 3 +- .../google/firebase/operators/firestore.py | 3 +- .../marketing_platform/operators/analytics.py | 5 +- .../google/suite/transfers/gcs_to_gdrive.py | 3 +- 98 files changed, 567 insertions(+), 590 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/automl.py b/airflow/providers/google/cloud/hooks/automl.py index dba14d720b12..cada4eb51129 100644 --- a/airflow/providers/google/cloud/hooks/automl.py +++ b/airflow/providers/google/cloud/hooks/automl.py @@ -122,10 +122,10 @@ def create_model( retry: Retry | _MethodDefault = DEFAULT, ) -> Operation: """ - Creates a model_id. Returns a Model in the `response` field when it - completes. When you create a model, several model evaluations are - created for it: a global evaluation, and one evaluation for each - annotation spec. + Creates a model_id and returns a Model in the `response` field when it completes. + + When you create a model, several model evaluations are created for it: + a global evaluation, and one evaluation for each annotation spec. :param model: The model_id to create. If a dict is provided, it must be of the same form as the protobuf message `google.cloud.automl_v1beta1.types.Model` @@ -163,9 +163,10 @@ def batch_predict( metadata: Sequence[tuple[str, str]] = (), ) -> Operation: """ - Perform a batch prediction. Unlike the online `Predict`, batch - prediction result won't be immediately available in the response. - Instead, a long running operation object is returned. + Perform a batch prediction and returns a long-running operation object. + + Unlike the online `Predict`, batch prediction result won't be immediately + available in the response. Instead, a long-running operation object is returned. :param model_id: Name of the model_id requested to serve the batch prediction. :param input_config: Required. The input configuration for batch prediction. @@ -215,8 +216,7 @@ def predict( metadata: Sequence[tuple[str, str]] = (), ) -> PredictResponse: """ - Perform an online prediction. The prediction result will be directly - returned in the response. + Perform an online prediction and returns the prediction result in the response. :param model_id: Name of the model_id requested to serve the prediction. :param payload: Required. Payload to perform a prediction on. The payload must match the problem type @@ -485,7 +485,9 @@ def deploy_model( metadata: Sequence[tuple[str, str]] = (), ) -> Operation: """ - Deploys a model. If a model is already deployed, deploying it with the same parameters + Deploys a model. + + If a model is already deployed, deploying it with the same parameters has no effect. Deploying with different parameters (as e.g. changing node_number) will reset the deployment state without pausing the model_id's availability. diff --git a/airflow/providers/google/cloud/hooks/bigquery_dts.py b/airflow/providers/google/cloud/hooks/bigquery_dts.py index c2dff601d25e..2d233591fde1 100644 --- a/airflow/providers/google/cloud/hooks/bigquery_dts.py +++ b/airflow/providers/google/cloud/hooks/bigquery_dts.py @@ -75,6 +75,8 @@ def __init__( @staticmethod def _disable_auto_scheduling(config: dict | TransferConfig) -> TransferConfig: """ + Create a transfer config with the automatic scheduling disabled. + In the case of Airflow, the customer needs to create a transfer config with the automatic scheduling disabled (UI, CLI or an Airflow operator) and then trigger a transfer run using a specialized Airflow operator that will @@ -195,10 +197,10 @@ def start_manual_transfer_runs( metadata: Sequence[tuple[str, str]] = (), ) -> StartManualTransferRunsResponse: """ - Start manual transfer runs to be executed now with schedule_time equal - to current time. The transfer runs can be created for a time range where - the run_time is between start_time (inclusive) and end_time - (exclusive), or for a specific run_time. + Start manual transfer runs to be executed now with schedule_time equal to current time. + + The transfer runs can be created for a time range where the run_time is between + start_time (inclusive) and end_time (exclusive), or for a specific run_time. :param transfer_config_id: Id of transfer config to be used. :param requested_time_range: Time range for the transfer runs that should be started. diff --git a/airflow/providers/google/cloud/hooks/bigtable.py b/airflow/providers/google/cloud/hooks/bigtable.py index c1e573a85a73..d01ab2d8ee1a 100644 --- a/airflow/providers/google/cloud/hooks/bigtable.py +++ b/airflow/providers/google/cloud/hooks/bigtable.py @@ -69,8 +69,7 @@ def _get_client(self, project_id: str) -> Client: @GoogleBaseHook.fallback_to_default_project_id def get_instance(self, instance_id: str, project_id: str) -> Instance | None: """ - Retrieves and returns the specified Cloud Bigtable instance if it exists. - Otherwise, returns None. + Retrieves and returns the specified Cloud Bigtable instance if it exists, otherwise returns None. :param instance_id: The ID of the Cloud Bigtable instance. :param project_id: Optional, Google Cloud project ID where the @@ -86,6 +85,7 @@ def get_instance(self, instance_id: str, project_id: str) -> Instance | None: def delete_instance(self, instance_id: str, project_id: str) -> None: """ Deletes the specified Cloud Bigtable instance. + Raises google.api_core.exceptions.NotFound if the Cloud Bigtable instance does not exist. @@ -217,6 +217,7 @@ def create_table( ) -> None: """ Creates the specified Cloud Bigtable table. + Raises ``google.api_core.exceptions.AlreadyExists`` if the table exists. :param instance: The Cloud Bigtable instance that owns the table. @@ -238,6 +239,7 @@ def create_table( def delete_table(self, instance_id: str, table_id: str, project_id: str) -> None: """ Deletes the specified table in Cloud Bigtable. + Raises google.api_core.exceptions.NotFound if the table does not exist. :param instance_id: The ID of the Cloud Bigtable instance. @@ -256,6 +258,7 @@ def delete_table(self, instance_id: str, table_id: str, project_id: str) -> None def update_cluster(instance: Instance, cluster_id: str, nodes: int) -> None: """ Updates number of nodes in the specified Cloud Bigtable cluster. + Raises google.api_core.exceptions.NotFound if the cluster does not exist. :param instance: The Cloud Bigtable instance that owns the cluster. @@ -284,6 +287,7 @@ def get_column_families_for_table(instance: Instance, table_id: str) -> dict[str def get_cluster_states_for_table(instance: Instance, table_id: str) -> dict[str, ClusterState]: """ Fetches Cluster States for the specified table in Cloud Bigtable. + Raises google.api_core.exceptions.NotFound if the table does not exist. :param instance: The Cloud Bigtable instance that owns the table. diff --git a/airflow/providers/google/cloud/hooks/cloud_build.py b/airflow/providers/google/cloud/hooks/cloud_build.py index 0ac929a4ad2f..1b7815d9035a 100644 --- a/airflow/providers/google/cloud/hooks/cloud_build.py +++ b/airflow/providers/google/cloud/hooks/cloud_build.py @@ -502,8 +502,7 @@ def retry_build( location: str = "global", ) -> Build: """ - Creates a new build based on the specified build. This method creates a new build - using the original build request, which may or may not result in an identical build. + Create a new build using the original build request; may or may not result in an identical build. :param id_: Build ID of the original build. :param project_id: Optional, Google Cloud Project project_id where the function belongs. diff --git a/airflow/providers/google/cloud/hooks/cloud_composer.py b/airflow/providers/google/cloud/hooks/cloud_composer.py index 5c994b3a5850..3ad0ba1098ac 100644 --- a/airflow/providers/google/cloud/hooks/cloud_composer.py +++ b/airflow/providers/google/cloud/hooks/cloud_composer.py @@ -156,6 +156,7 @@ def get_environment( ) -> Environment: """ Get an existing environment. + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. :param region: Required. The ID of the Google Cloud region that the service belongs to. :param environment_id: Required. The ID of the Google Cloud environment that the service belongs to. diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py index 304a6f88fa3d..ebd03b69bbb6 100644 --- a/airflow/providers/google/cloud/hooks/cloud_sql.py +++ b/airflow/providers/google/cloud/hooks/cloud_sql.py @@ -304,8 +304,7 @@ def delete_database(self, instance: str, database: str, project_id: str) -> None @GoogleBaseHook.fallback_to_default_project_id def export_instance(self, instance: str, body: dict, project_id: str): """ - Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump - or CSV file. + Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump or CSV file. :param instance: Database instance ID of the Cloud SQL instance. This does not include the project ID. @@ -327,8 +326,7 @@ def export_instance(self, instance: str, body: dict, project_id: str): @GoogleBaseHook.fallback_to_default_project_id def import_instance(self, instance: str, body: dict, project_id: str) -> None: """ - Imports data into a Cloud SQL instance from a SQL dump or CSV file in - Cloud Storage. + Imports data into a Cloud SQL instance from a SQL dump or CSV file in Cloud Storage. :param instance: Database instance ID. This does not include the project ID. @@ -382,8 +380,7 @@ def _wait_for_operation_to_complete( self, project_id: str, operation_name: str, time_to_sleep: int = TIME_TO_SLEEP_IN_SECONDS ) -> None: """ - Waits for the named operation to complete - checks status of the - asynchronous call. + Waits for the named operation to complete - checks status of the asynchronous call. :param project_id: Project ID of the project that contains the instance. :param operation_name: Name of the operation. @@ -721,10 +718,11 @@ def get_socket_path(self) -> str: class CloudSQLDatabaseHook(BaseHook): - """Serves DB connection configuration for Google Cloud SQL (Connections - of *gcpcloudsqldb://* type). + """ + Serves DB connection configuration for Google Cloud SQL (Connections of *gcpcloudsqldb://* type). The hook is a "meta" one. It does not perform an actual connection. + It is there to retrieve all the parameters configured in gcpcloudsql:// connection, start/stop Cloud SQL Proxy if needed, dynamically generate Postgres or MySQL connection in the database and return an actual Postgres or MySQL hook. diff --git a/airflow/providers/google/cloud/hooks/compute.py b/airflow/providers/google/cloud/hooks/compute.py index 064551cc40de..1a95e94c317f 100644 --- a/airflow/providers/google/cloud/hooks/compute.py +++ b/airflow/providers/google/cloud/hooks/compute.py @@ -107,6 +107,7 @@ def insert_instance_template( ) -> None: """ Creates Instance Template using body specified. + Must be called with keyword arguments rather than positional. :param body: Instance Template representation as an object. @@ -158,9 +159,10 @@ def delete_instance_template( ) -> None: """ Deletes Instance Template. - Deleting an Instance Template is permanent and cannot be undone. It - is not possible to delete templates that are already in use by a managed instance group. - Must be called with keyword arguments rather than positional. + + Deleting an Instance Template is permanent and cannot be undone. It is not + possible to delete templates that are already in use by a managed instance + group. Must be called with keyword arguments rather than positional. :param resource_id: Name of the Compute Engine Instance Template resource. :param request_id: Unique request_id that you might add to achieve @@ -210,6 +212,7 @@ def get_instance_template( ) -> InstanceTemplate: """ Retrieves Instance Template by project_id and resource_id. + Must be called with keyword arguments rather than positional. :param resource_id: Name of the Instance Template. @@ -259,6 +262,7 @@ def insert_instance( ) -> None: """ Creates Instance using body specified. + Must be called with keyword arguments rather than positional. :param body: Instance representation as an object. Should at least include 'name', 'machine_type', @@ -332,6 +336,7 @@ def get_instance( ) -> Instance: """ Retrieves Instance by project_id and resource_id. + Must be called with keyword arguments rather than positional. :param resource_id: Name of the Instance @@ -383,8 +388,8 @@ def delete_instance( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Deletes Instance. - Deleting an Instance is permanent and cannot be undone. + Permanently and irrevocably deletes an Instance. + It is not possible to delete Instances that are already in use by a managed instance group. Must be called with keyword arguments rather than positional. @@ -433,6 +438,7 @@ def delete_instance( def start_instance(self, zone: str, resource_id: str, project_id: str) -> None: """ Starts an existing instance defined by project_id, zone and resource_id. + Must be called with keyword arguments rather than positional. :param zone: Google Cloud zone where the instance exists @@ -457,7 +463,8 @@ def start_instance(self, zone: str, resource_id: str, project_id: str) -> None: @GoogleBaseHook.fallback_to_default_project_id def stop_instance(self, zone: str, resource_id: str, project_id: str) -> None: """ - Stops an instance defined by project_id, zone and resource_id + Stops an instance defined by project_id, zone and resource_id. + Must be called with keyword arguments rather than positional. :param zone: Google Cloud zone where the instance exists @@ -483,6 +490,7 @@ def stop_instance(self, zone: str, resource_id: str, project_id: str) -> None: def set_machine_type(self, zone: str, resource_id: str, body: dict, project_id: str) -> None: """ Sets machine type of an instance defined by project_id, zone and resource_id. + Must be called with keyword arguments rather than positional. :param zone: Google Cloud zone where the instance exists. @@ -524,6 +532,7 @@ def insert_instance_group_manager( ) -> None: """ Creates an Instance Group Managers using the body specified. + After the group is created, instances in the group are created using the specified Instance Template. Must be called with keyword arguments rather than positional. @@ -580,6 +589,7 @@ def get_instance_group_manager( ) -> InstanceGroupManager: """ Retrieves Instance Group Manager by project_id, zone and resource_id. + Must be called with keyword arguments rather than positional. :param resource_id: The name of the Managed Instance Group @@ -631,8 +641,8 @@ def delete_instance_group_manager( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Deletes Instance Group Managers. - Deleting an Instance Group Manager is permanent and cannot be undone. + Permanently and irrevocably deletes Instance Group Managers. + Must be called with keyword arguments rather than positional. :param resource_id: Name of the Compute Engine Instance Group Managers resource. @@ -687,6 +697,7 @@ def patch_instance_group_manager( ) -> None: """ Patches Instance Group Manager with the specified body. + Must be called with keyword arguments rather than positional. :param zone: Google Cloud zone where the Instance Group Manager exists diff --git a/airflow/providers/google/cloud/hooks/dataflow.py b/airflow/providers/google/cloud/hooks/dataflow.py index 3e0b07fcb28d..19342cb26b08 100644 --- a/airflow/providers/google/cloud/hooks/dataflow.py +++ b/airflow/providers/google/cloud/hooks/dataflow.py @@ -389,8 +389,7 @@ def _refresh_jobs(self) -> None: def _check_dataflow_job_state(self, job) -> bool: """ - Helper method to check the state of one job in dataflow for this task - if job failed raise exception. + Helper method to check the state of one job in dataflow for this task if job failed raise exception. :return: True if job is done. :raise: Exception diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/airflow/providers/google/cloud/hooks/datafusion.py index a736d11e9784..f74958cc1607 100644 --- a/airflow/providers/google/cloud/hooks/datafusion.py +++ b/airflow/providers/google/cloud/hooks/datafusion.py @@ -103,10 +103,7 @@ def wait_for_pipeline_state( failure_states: list[str] | None = None, timeout: int = 5 * 60, ) -> None: - """ - Polls pipeline state and raises an exception if the state is one of - `failure_states` or the operation timed_out. - """ + """Polls pipeline state and raises an exception if the state fails or times out.""" failure_states = failure_states or FAILURE_STATES success_states = success_states or SUCCESS_STATES start_time = monotonic() @@ -190,6 +187,7 @@ def get_conn(self) -> Resource: def restart_instance(self, instance_name: str, location: str, project_id: str) -> Operation: """ Restart a single Data Fusion instance. + At the end of an operation instance is fully restarted. :param instance_name: The name of the instance to restart. diff --git a/airflow/providers/google/cloud/hooks/dataprep.py b/airflow/providers/google/cloud/hooks/dataprep.py index 7e475ea6a739..c01a48d5ae42 100644 --- a/airflow/providers/google/cloud/hooks/dataprep.py +++ b/airflow/providers/google/cloud/hooks/dataprep.py @@ -105,6 +105,7 @@ def get_jobs_for_job_group(self, job_id: int) -> dict[str, Any]: def get_job_group(self, job_group_id: int, embed: str, include_deleted: bool) -> dict[str, Any]: """ Get the specified job group. + A job group is a job that is executed from a specific node in a flow. :param job_group_id: The ID of the job that will be fetched diff --git a/airflow/providers/google/cloud/hooks/dlp.py b/airflow/providers/google/cloud/hooks/dlp.py index db198de6c555..47950579894d 100644 --- a/airflow/providers/google/cloud/hooks/dlp.py +++ b/airflow/providers/google/cloud/hooks/dlp.py @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This module contains a CloudDLPHook -which allows you to connect to Google Cloud DLP service. +This module contains a CloudDLPHook which allows you to connect to Google Cloud DLP service. .. spelling:word-list:: @@ -65,6 +64,7 @@ class CloudDLPHook(GoogleBaseHook): """ Hook for Google Cloud Data Loss Prevention (DLP) APIs. + Cloud DLP allows clients to detect the presence of Personally Identifiable Information (PII) and other privacy-sensitive data in user-supplied, unstructured data streams, like text blocks or images. The service also @@ -167,8 +167,7 @@ def create_deidentify_template( metadata: Sequence[tuple[str, str]] = (), ) -> DeidentifyTemplate: """ - Creates a deidentify template for re-using frequently used configuration for - de-identifying content, images, and storage. + Create a deidentify template to reuse frequently-used configurations for content, images, and storage. :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. @@ -293,8 +292,7 @@ def create_inspect_template( metadata: Sequence[tuple[str, str]] = (), ) -> InspectTemplate: """ - Creates an inspect template for re-using frequently used configuration for - inspecting content, images, and storage. + Create an inspect template to reuse frequently used configurations for content, images, and storage. :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. @@ -344,8 +342,7 @@ def create_job_trigger( metadata: Sequence[tuple[str, str]] = (), ) -> JobTrigger: """ - Creates a job trigger to run DLP actions such as scanning storage for sensitive - information on a set schedule. + Create a job trigger to run DLP actions such as scanning storage for sensitive info on a set schedule. :param project_id: (Optional) Google Cloud project ID where the DLP Instance exists. If set to None or missing, the default @@ -437,8 +434,7 @@ def deidentify_content( metadata: Sequence[tuple[str, str]] = (), ) -> DeidentifyContentResponse: """ - De-identifies potentially sensitive info from a content item. This method has limits - on input size and output size. + De-identifies potentially sensitive info from a content item; limits input size and output size. :param project_id: (Optional) Google Cloud project ID where the DLP Instance exists. If set to None or missing, the default @@ -531,8 +527,10 @@ def delete_dlp_job( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Deletes a long-running DLP job. This method indicates that the client is no longer - interested in the DLP job result. The job will be cancelled if possible. + Deletes a long-running DLP job. + + This method indicates that the client is no longer interested in the DLP job result. + The job will be cancelled if possible. :param dlp_job_id: The ID of the DLP job resource to be cancelled. :param project_id: (Optional) Google Cloud project ID where the @@ -931,8 +929,7 @@ def inspect_content( metadata: Sequence[tuple[str, str]] = (), ) -> InspectContentResponse: """ - Finds potentially sensitive info in content. This method has limits on input size, - processing time, and output size. + Finds potentially sensitive info in content; limits input size, processing time, and output size. :param project_id: (Optional) Google Cloud project ID where the DLP Instance exists. If set to None or missing, the default @@ -1264,8 +1261,7 @@ def redact_image( metadata: Sequence[tuple[str, str]] = (), ) -> RedactImageResponse: """ - Redacts potentially sensitive info from an image. This method has limits on - input size, processing time, and output size. + Redacts potentially sensitive info from an image; limits input size, processing time, and output size. :param project_id: (Optional) Google Cloud project ID where the DLP Instance exists. If set to None or missing, the default diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py index 5350c9f79012..a42f81c35506 100644 --- a/airflow/providers/google/cloud/hooks/gcs.py +++ b/airflow/providers/google/cloud/hooks/gcs.py @@ -143,10 +143,7 @@ def _inner_wrapper(self, *args, **kwargs) -> RT: class GCSHook(GoogleBaseHook): - """ - Interact with Google Cloud Storage. This hook uses the Google Cloud - connection. - """ + """Use the Google Cloud connection to interact with Google Cloud Storage.""" _conn: storage.Client | None = None @@ -232,9 +229,7 @@ def rewrite( destination_object: str | None = None, ) -> None: """ - Has the same functionality as copy, except that will work on files - over 5 TB, as well as when copying between locations and/or storage - classes. + Similar to copy; supports files over 5 TB, and copying between locations and/or storage classes. destination_object can be omitted, in which case source_object is used. @@ -430,8 +425,7 @@ def provide_file_and_upload( object_url: str | None = None, ) -> Generator[IO[bytes], None, None]: """ - Creates temporary file, returns a file handle and uploads the files content - on close. + Creates temporary file, returns a file handle and uploads the files content on close. You can use this method by passing the bucket_name and object_name parameters or just object_url parameter. @@ -483,7 +477,9 @@ def upload( """ def _call_with_retry(f: Callable[[], None]) -> None: - """Helper functions to upload a file or a string with a retry mechanism and exponential back-off. + """ + Helper functions to upload a file or a string with a retry mechanism and exponential back-off. + :param f: Callable that should be retried. """ num_file_attempts = 0 @@ -834,6 +830,7 @@ def _list_blobs_with_match_glob( ) -> Any: """ List blobs when match_glob param is given. + This method is a patched version of google.cloud.storage Client.list_blobs(). It is used as a temporary workaround to support "match_glob" param, as it isn't officially supported by GCS Python client. @@ -879,8 +876,7 @@ def list_by_timespan( match_glob: str | None = None, ) -> List[str]: """ - List all objects from the bucket with the give string prefix in name that were - updated in the time between ``timespan_start`` and ``timespan_end``. + List all objects from the bucket with the given string prefix that were updated in the time range. :param bucket_name: bucket name :param timespan_start: will return objects that were updated at or after this datetime (UTC) @@ -1002,8 +998,10 @@ def create_bucket( labels: dict | None = None, ) -> str: """ - Creates a new bucket. Google Cloud Storage uses a flat namespace, so - you can't create a bucket with a name that is already in use. + Creates a new bucket. + + Google Cloud Storage uses a flat namespace, so you can't + create a bucket with a name that is already in use. .. seealso:: For more information, see Bucket Naming Guidelines: @@ -1314,10 +1312,7 @@ def _prepare_sync_plan( def gcs_object_is_directory(bucket: str) -> bool: - """ - Return True if given Google Cloud Storage URL (gs:///) - is a directory or an empty bucket. Otherwise return False. - """ + """Return True if given Google Cloud Storage URL (gs:///) is a directory or empty bucket.""" _, blob = _parse_gcs_url(bucket) return len(blob) == 0 or blob.endswith("/") @@ -1355,8 +1350,9 @@ def parse_json_from_gcs(gcp_conn_id: str, file_uri: str) -> Any: def _parse_gcs_url(gsurl: str) -> tuple[str, str]: """ - Given a Google Cloud Storage URL (gs:///), returns a - tuple containing the corresponding bucket and blob. + Given a Google Cloud Storage URL, return a tuple containing the corresponding bucket and blob. + + Expected url format: gs:/// """ parsed_url = urlsplit(gsurl) if not parsed_url.netloc: diff --git a/airflow/providers/google/cloud/hooks/gdm.py b/airflow/providers/google/cloud/hooks/gdm.py index 7d228c8c95ef..203191d1644d 100644 --- a/airflow/providers/google/cloud/hooks/gdm.py +++ b/airflow/providers/google/cloud/hooks/gdm.py @@ -28,6 +28,7 @@ class GoogleDeploymentManagerHook(GoogleBaseHook): """ Interact with Google Cloud Deployment Manager using the Google Cloud connection. + This allows for scheduled and programmatic inspection and deletion of resources managed by GDM. """ diff --git a/airflow/providers/google/cloud/hooks/life_sciences.py b/airflow/providers/google/cloud/hooks/life_sciences.py index 835837ef5fb8..e3cd01c6c3df 100644 --- a/airflow/providers/google/cloud/hooks/life_sciences.py +++ b/airflow/providers/google/cloud/hooks/life_sciences.py @@ -122,8 +122,7 @@ def _location_path(self, project_id: str, location: str) -> str: def _wait_for_operation_to_complete(self, operation_name: str) -> None: """ - Waits for the named operation to complete - checks status of the - asynchronous call. + Waits for the named operation to complete - checks status of the asynchronous call. :param operation_name: The name of the operation. :return: The response returned by the operation. diff --git a/airflow/providers/google/cloud/hooks/looker.py b/airflow/providers/google/cloud/hooks/looker.py index 42477d2fbbba..d1179d707cae 100644 --- a/airflow/providers/google/cloud/hooks/looker.py +++ b/airflow/providers/google/cloud/hooks/looker.py @@ -209,8 +209,9 @@ def __init__( def read_config(self): """ - Overrides the default logic of getting connection settings. Fetches - the connection settings from Airflow's connection object. + Fetches the connection settings from Airflow's connection object. + + Overrides the default logic of getting connection settings. """ config = {} diff --git a/airflow/providers/google/cloud/hooks/natural_language.py b/airflow/providers/google/cloud/hooks/natural_language.py index ee6f25965cf6..673df34b2328 100644 --- a/airflow/providers/google/cloud/hooks/natural_language.py +++ b/airflow/providers/google/cloud/hooks/natural_language.py @@ -90,8 +90,9 @@ def analyze_entities( metadata: Sequence[tuple[str, str]] = (), ) -> AnalyzeEntitiesResponse: """ - Finds named entities in the text along with entity types, - salience, mentions for each entity, and other properties. + Finds named entities in the text along with various properties. + + Examples properties: entity types, salience, mentions for each entity, and others. :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document @@ -120,8 +121,7 @@ def analyze_entity_sentiment( metadata: Sequence[tuple[str, str]] = (), ) -> AnalyzeEntitySentimentResponse: """ - Finds entities, similar to AnalyzeEntities in the text and analyzes sentiment associated with each - entity and its mentions. + Similar to AnalyzeEntities, also analyzes sentiment associated with each entity and its mentions. :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document @@ -179,7 +179,9 @@ def analyze_syntax( metadata: Sequence[tuple[str, str]] = (), ) -> AnalyzeSyntaxResponse: """ - Analyzes the syntax of the text and provides sentence boundaries and tokenization along with part + Analyzes the syntax of the text. + + Provides sentence boundaries and tokenization along with part of speech tags, dependency trees, and other properties. :param document: Input document. @@ -210,8 +212,7 @@ def annotate_text( metadata: Sequence[tuple[str, str]] = (), ) -> AnnotateTextResponse: """ - A convenience method that provides all the features that analyzeSentiment, - analyzeEntities, and analyzeSyntax provide in one call. + Provide all features that analyzeSentiment, analyzeEntities, and analyzeSyntax provide in one call. :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document diff --git a/airflow/providers/google/cloud/hooks/os_login.py b/airflow/providers/google/cloud/hooks/os_login.py index c82772cc821b..9a355d74d9ea 100644 --- a/airflow/providers/google/cloud/hooks/os_login.py +++ b/airflow/providers/google/cloud/hooks/os_login.py @@ -76,9 +76,9 @@ def import_ssh_public_key( metadata: Sequence[tuple[str, str]] = (), ) -> ImportSshPublicKeyResponse: """ - Adds an SSH public key and returns the profile information. Default POSIX - account information is set when no username and UID exist as part of the - login profile. + Adds an SSH public key and returns the profile information. + + Default POSIX account information is set when no username and UID exist as part of the login profile. :param user: The unique ID for the user :param ssh_public_key: The SSH public key and expiration time. diff --git a/airflow/providers/google/cloud/hooks/pubsub.py b/airflow/providers/google/cloud/hooks/pubsub.py index 0e33f22ebf59..6e867c2bdb1f 100644 --- a/airflow/providers/google/cloud/hooks/pubsub.py +++ b/airflow/providers/google/cloud/hooks/pubsub.py @@ -587,7 +587,8 @@ def __init__(self, project_id: str | None = None, **kwargs: Any): async def _get_subscriber_client(self) -> SubscriberAsyncClient: """ - Returns async connection to the Google PubSub + Returns async connection to the Google PubSub. + :return: Google Pub/Sub asynchronous client. """ if not self._client: diff --git a/airflow/providers/google/cloud/hooks/spanner.py b/airflow/providers/google/cloud/hooks/spanner.py index d83cfc598ad8..00fe9a880456 100644 --- a/airflow/providers/google/cloud/hooks/spanner.py +++ b/airflow/providers/google/cloud/hooks/spanner.py @@ -215,8 +215,7 @@ def get_database( project_id: str, ) -> Database | None: """ - Retrieves a database in Cloud Spanner. If the database does not exist - in the specified instance, it returns None. + Retrieves a database in Cloud Spanner; return None if the database does not exist in the instance. :param instance_id: The ID of the Cloud Spanner instance. :param database_id: The ID of the database in Cloud Spanner. diff --git a/airflow/providers/google/cloud/hooks/stackdriver.py b/airflow/providers/google/cloud/hooks/stackdriver.py index 517e211edd0d..28e0b8830357 100644 --- a/airflow/providers/google/cloud/hooks/stackdriver.py +++ b/airflow/providers/google/cloud/hooks/stackdriver.py @@ -77,10 +77,10 @@ def list_alert_policies( metadata: Sequence[tuple[str, str]] = (), ) -> Any: """ - Fetches all the Alert Policies identified by the filter passed as - filter parameter. The desired return type can be specified by the - format parameter, the supported formats are "dict", "json" and None - which returns python dictionary, stringified JSON and protobuf + Fetches all the Alert Policies identified by the filter passed as filter parameter. + + The desired return type can be specified by the format parameter, the supported formats + are "dict", "json" and None which returns python dictionary, stringified JSON and protobuf respectively. :param format_: (Optional) Desired output format of the result. The @@ -158,8 +158,9 @@ def enable_alert_policies( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Enables one or more disabled alerting policies identified by filter - parameter. Inoperative in case the policy is already enabled. + Enables one or more disabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already enabled. :param project_id: The project in which alert needs to be enabled. :param filter_: If provided, this field specifies the criteria that @@ -191,8 +192,9 @@ def disable_alert_policies( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Disables one or more enabled alerting policies identified by filter - parameter. Inoperative in case the policy is already disabled. + Disables one or more enabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already disabled. :param project_id: The project in which alert needs to be disabled. :param filter_: If provided, this field specifies the criteria that @@ -224,8 +226,7 @@ def upsert_alert( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Creates a new alert or updates an existing policy identified - the name field in the alerts parameter. + Creates a new alert or updates an existing policy identified the name field in the alerts parameter. :param project_id: The project in which alert needs to be created/updated. :param alerts: A JSON string or file that specifies all the alerts that needs @@ -352,10 +353,10 @@ def list_notification_channels( metadata: Sequence[tuple[str, str]] = (), ) -> Any: """ - Fetches all the Notification Channels identified by the filter passed as - filter parameter. The desired return type can be specified by the - format parameter, the supported formats are "dict", "json" and None - which returns python dictionary, stringified JSON and protobuf + Fetches all the Notification Channels identified by the filter passed as filter parameter. + + The desired return type can be specified by the format parameter, the supported formats are + "dict", "json" and None which returns python dictionary, stringified JSON and protobuf respectively. :param format_: (Optional) Desired output format of the result. The @@ -435,8 +436,9 @@ def enable_notification_channels( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Enables one or more disabled alerting policies identified by filter - parameter. Inoperative in case the policy is already enabled. + Enables one or more disabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already enabled. :param project_id: The project in which notification channels needs to be enabled. :param filter_: If provided, this field specifies the criteria that @@ -468,8 +470,9 @@ def disable_notification_channels( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Disables one or more enabled notification channels identified by filter - parameter. Inoperative in case the policy is already disabled. + Disables one or more enabled notification channels identified by filter parameter. + + Inoperative in case the policy is already disabled. :param project_id: The project in which notification channels needs to be enabled. :param filter_: If provided, this field specifies the criteria that @@ -501,8 +504,9 @@ def upsert_channel( metadata: Sequence[tuple[str, str]] = (), ) -> dict: """ - Creates a new notification or updates an existing notification channel - identified the name field in the alerts parameter. + Create a new notification or updates an existing notification channel. + + Channel is identified by the name field in the alerts parameter. :param channels: A JSON string or file that specifies all the alerts that needs to be either created or updated. For more details, see diff --git a/airflow/providers/google/cloud/hooks/tasks.py b/airflow/providers/google/cloud/hooks/tasks.py index 1301a3406e76..4807d0fde77c 100644 --- a/airflow/providers/google/cloud/hooks/tasks.py +++ b/airflow/providers/google/cloud/hooks/tasks.py @@ -15,11 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains a CloudTasksHook -which allows you to connect to Google Cloud Tasks service, -performing actions to queues or tasks. -""" +"""This module contains a CloudTasksHook which allows you to connect to Google Cloud Tasks service.""" + from __future__ import annotations from typing import Sequence @@ -37,8 +34,9 @@ class CloudTasksHook(GoogleBaseHook): """ - Hook for Google Cloud Tasks APIs. Cloud Tasks allows developers to manage - the execution of background work in their applications. + Hook for Google Cloud Tasks APIs. + + Cloud Tasks allows developers to manage the execution of background work in their applications. All the methods in the hook where project_id is used must be called with keyword arguments rather than positional. diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py index 368600571367..b5f43d643f1e 100644 --- a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +++ b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py @@ -357,7 +357,9 @@ def cancel_pipeline_job( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Cancels a PipelineJob. Starts asynchronous cancellation on the PipelineJob. The server makes a best + Cancels a PipelineJob. + + Starts asynchronous cancellation on the PipelineJob. The server makes a best effort to cancel the pipeline, but success is not guaranteed. Clients can use [PipelineService.GetPipelineJob][google.cloud.aiplatform.v1.PipelineService.GetPipelineJob] or other methods to check whether the cancellation succeeded or whether the pipeline completed despite @@ -396,7 +398,9 @@ def cancel_training_pipeline( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Cancels a TrainingPipeline. Starts asynchronous cancellation on the TrainingPipeline. The server makes + Cancels a TrainingPipeline. + + Starts asynchronous cancellation on the TrainingPipeline. The server makes a best effort to cancel the pipeline, but success is not guaranteed. Clients can use [PipelineService.GetTrainingPipeline][google.cloud.aiplatform.v1.PipelineService.GetTrainingPipeline] or other methods to check whether the cancellation succeeded or whether the pipeline completed despite @@ -435,7 +439,9 @@ def cancel_custom_job( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ - Cancels a CustomJob. Starts asynchronous cancellation on the CustomJob. The server makes a best effort + Cancels a CustomJob. + + Starts asynchronous cancellation on the CustomJob. The server makes a best effort to cancel the job, but success is not guaranteed. Clients can use [JobService.GetCustomJob][google.cloud.aiplatform.v1.JobService.GetCustomJob] or other methods to check whether the cancellation succeeded or whether the job completed despite cancellation. On diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py b/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py index 2750f70241ad..3c1fb9b3d019 100644 --- a/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +++ b/airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py @@ -316,8 +316,7 @@ def undeploy_model( metadata: Sequence[tuple[str, str]] = (), ) -> Operation: """ - Undeploys a Model from an Endpoint, removing a DeployedModel from it, and freeing all resources it's - using. + Undeploys a Model from an Endpoint, removing a DeployedModel from it, and freeing all used resources. :param project_id: Required. The ID of the Google Cloud project that the service belongs to. :param region: Required. The ID of the Google Cloud region that the service belongs to. diff --git a/airflow/providers/google/cloud/hooks/vision.py b/airflow/providers/google/cloud/hooks/vision.py index e7cc242e4b9b..a6e504ab4fcb 100644 --- a/airflow/providers/google/cloud/hooks/vision.py +++ b/airflow/providers/google/cloud/hooks/vision.py @@ -64,20 +64,16 @@ def get_entity_with_name( self, entity: Any, entity_id: str | None, location: str | None, project_id: str ) -> Any: """ - Check if entity has the `name` attribute set: - * If so, no action is taken. + Check if entity has the `name` attribute set. + * If so, no action is taken. * If not, and the name can be constructed from other parameters provided, it is created and filled in the entity. - * If both the entity's 'name' attribute is set and the name can be constructed from other parameters provided: - * If they are the same - no action is taken - * if they are different - an exception is thrown. - :param entity: Entity :param entity_id: Entity id :param location: Location @@ -178,6 +174,8 @@ def create_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> str: """ + Create product set. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator`. """ @@ -213,6 +211,8 @@ def get_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> dict: """ + Get product set. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionGetProductSetOperator`. """ @@ -237,6 +237,8 @@ def update_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> dict: """ + Update product set. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductSetOperator`. """ @@ -270,6 +272,8 @@ def delete_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ + Delete product set. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductSetOperator`. """ @@ -291,6 +295,8 @@ def create_product( metadata: Sequence[tuple[str, str]] = (), ): """ + Create product. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductOperator`. """ @@ -329,6 +335,8 @@ def get_product( metadata: Sequence[tuple[str, str]] = (), ): """ + Get product. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionGetProductOperator`. """ @@ -353,6 +361,8 @@ def update_product( metadata: Sequence[tuple[str, str]] = (), ): """ + Update product. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductOperator`. """ @@ -384,6 +394,8 @@ def delete_product( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ + Delete product. + For the documentation see: :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductOperator`. """ @@ -406,6 +418,8 @@ def create_reference_image( metadata: Sequence[tuple[str, str]] = (), ) -> str: """ + Create reference image. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateReferenceImageOperator`. """ @@ -448,6 +462,8 @@ def delete_reference_image( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ + Delete reference image. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDeleteReferenceImageOperator`. """ @@ -478,6 +494,8 @@ def add_product_to_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ + Add product to product set. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionAddProductToProductSetOperator`. """ @@ -506,6 +524,8 @@ def remove_product_from_product_set( metadata: Sequence[tuple[str, str]] = (), ) -> None: """ + Remove product from product set. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionRemoveProductFromProductSetOperator`. """ @@ -529,6 +549,8 @@ def annotate_image( timeout: float | None = None, ) -> dict: """ + Annotate image. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator`. """ @@ -550,6 +572,8 @@ def batch_annotate_images( timeout: float | None = None, ) -> dict: """ + Batch annotate images. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator`. """ @@ -574,6 +598,8 @@ def text_detection( additional_properties: dict | None = None, ) -> dict: """ + Text detection. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectTextOperator`. """ @@ -604,6 +630,8 @@ def document_text_detection( additional_properties: dict | None = None, ) -> dict: """ + Document text detection. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionTextDetectOperator`. """ @@ -634,6 +662,8 @@ def label_detection( additional_properties: dict | None = None, ) -> dict: """ + Label detection. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageLabelsOperator`. """ @@ -664,6 +694,8 @@ def safe_search_detection( additional_properties: dict | None = None, ) -> dict: """ + Safe search detection. + For the documentation see: :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageSafeSearchOperator`. """ diff --git a/airflow/providers/google/cloud/hooks/workflows.py b/airflow/providers/google/cloud/hooks/workflows.py index 6e1f6ed1209e..6e3942e3b965 100644 --- a/airflow/providers/google/cloud/hooks/workflows.py +++ b/airflow/providers/google/cloud/hooks/workflows.py @@ -67,10 +67,11 @@ def create_workflow( metadata: Sequence[tuple[str, str]] = (), ) -> Operation: """ - Creates a new workflow. If a workflow with the specified name - already exists in the specified project and location, the long - running operation will return - [ALREADY_EXISTS][google.rpc.Code.ALREADY_EXISTS] error. + Creates a new workflow. + + If a workflow with the specified name already exists in the + specified project and location, the long running operation will + return [ALREADY_EXISTS][google.rpc.Code.ALREADY_EXISTS] error. :param workflow: Required. Workflow to be created. :param workflow_id: Required. The ID of the workflow to be created. @@ -129,6 +130,7 @@ def update_workflow( ) -> Operation: """ Updates an existing workflow. + Running this method has no impact on already running executions of the workflow. A new revision of the workflow may be created as a result of a successful @@ -164,9 +166,7 @@ def delete_workflow( metadata: Sequence[tuple[str, str]] = (), ) -> Operation: """ - Deletes a workflow with the specified name. - This method also cancels and deletes all running - executions of the workflow. + Delete a workflow with the specified name and all running executions of the workflow. :param workflow_id: Required. The ID of the workflow to be created. :param project_id: Required. The ID of the Google Cloud project the cluster belongs to. @@ -194,8 +194,7 @@ def list_workflows( metadata: Sequence[tuple[str, str]] = (), ) -> ListWorkflowsPager: """ - Lists Workflows in a given project and location. - The default order is not specified. + Lists Workflows in a given project and location; the default order is not specified. :param filter_: Filter to restrict results to specific workflows. :param order_by: Comma-separated list of fields that @@ -233,8 +232,7 @@ def create_execution( metadata: Sequence[tuple[str, str]] = (), ) -> Execution: """ - Creates a new execution using the latest revision of - the given workflow. + Creates a new execution using the latest revision of the given workflow. :param execution: Required. Input parameters of the execution represented as a dictionary. :param workflow_id: Required. The ID of the workflow. @@ -328,11 +326,10 @@ def list_executions( metadata: Sequence[tuple[str, str]] = (), ) -> ListExecutionsPager: """ - Returns a list of executions which belong to the - workflow with the given name. The method returns - executions of all workflow revisions. Returned - executions are ordered by their start time (newest - first). + Returns a list of executions which belong to the workflow with the given name. + + The method returns executions of all workflow revisions. Returned + executions are ordered by their start time (newest first). :param workflow_id: Required. The ID of the workflow to be created. :param project_id: Required. The ID of the Google Cloud project the cluster belongs to. diff --git a/airflow/providers/google/cloud/log/gcs_task_handler.py b/airflow/providers/google/cloud/log/gcs_task_handler.py index 0c3f8a27c73c..121a668857b2 100644 --- a/airflow/providers/google/cloud/log/gcs_task_handler.py +++ b/airflow/providers/google/cloud/log/gcs_task_handler.py @@ -59,10 +59,10 @@ def get_default_delete_local_copy(): class GCSTaskHandler(FileTaskHandler, LoggingMixin): """ - GCSTaskHandler is a python log handler that handles and reads - task instance logs. It extends airflow FileTaskHandler and - uploads to and reads from GCS remote storage. Upon log reading - failure, it reads from host machine's local disk. + GCSTaskHandler is a python log handler that handles and reads task instance logs. + + It extends airflow FileTaskHandler and uploads to and reads from GCS remote + storage. Upon log reading failure, it reads from host machine's local disk. :param base_log_folder: Base log folder to place logs. :param gcs_log_folder: Path to a remote location where logs will be saved. It must have the prefix @@ -209,6 +209,7 @@ def _read_remote_logs(self, ti, try_number, metadata=None) -> tuple[list[str], l def _read(self, ti, try_number, metadata=None): """ Read logs of given task instance and try_number from GCS. + If failed, read the log from task instance host machine. todo: when min airflow version >= 2.6, remove this method @@ -231,8 +232,7 @@ def _read(self, ti, try_number, metadata=None): def gcs_write(self, log, remote_log_location) -> bool: """ - Writes the log to the remote_log_location and return `True` when done. Fails silently - and return `False` if no log was created. + Write the log to the remote location and return `True`; fail silently and return `False` on error. :param log: the log to write to the remote_log_location :param remote_log_location: the log's location in remote storage diff --git a/airflow/providers/google/cloud/operators/automl.py b/airflow/providers/google/cloud/operators/automl.py index 0503e6bffefe..e590dc3d0620 100644 --- a/airflow/providers/google/cloud/operators/automl.py +++ b/airflow/providers/google/cloud/operators/automl.py @@ -895,8 +895,9 @@ def execute(self, context: Context): class AutoMLDeployModelOperator(GoogleCloudBaseOperator): """ - Deploys a model. If a model is already deployed, deploying it with the same parameters - has no effect. Deploying with different parameters (as e.g. changing node_number) will + Deploys a model; if a model is already deployed, deploying it with the same parameters has no effect. + + Deploying with different parameters (as e.g. changing node_number) will reset the deployment state without pausing the model_id's availability. Only applicable for Text Classification, Image Object Detection and Tables; all other diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index a1b53aed78a3..70ab30d61a23 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -402,8 +402,7 @@ def execute_complete(self, context: Context, event: dict[str, Any]) -> None: class BigQueryIntervalCheckOperator(_BigQueryDbHookMixin, SQLIntervalCheckOperator): """ - Checks that the values of metrics given as SQL expressions are within - a certain tolerance of the ones from days_back before. + Check that the values of metrics given as SQL expressions are within a tolerance of the older ones. This method constructs a query like so :: @@ -545,9 +544,9 @@ def execute_complete(self, context: Context, event: dict[str, Any]) -> None: class BigQueryColumnCheckOperator(_BigQueryDbHookMixin, SQLColumnCheckOperator): """ - BigQueryColumnCheckOperator subclasses the SQLColumnCheckOperator - in order to provide a job id for OpenLineage to parse. See base class - docstring for usage. + Subclasses the SQLColumnCheckOperator in order to provide a job id for OpenLineage to parse. + + See base class docstring for usage. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -668,9 +667,9 @@ def execute(self, context=None): class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator): """ - BigQueryTableCheckOperator subclasses the SQLTableCheckOperator - in order to provide a job id for OpenLineage to parse. See base class - for usage. + Subclasses the SQLTableCheckOperator in order to provide a job id for OpenLineage to parse. + + See base class for usage. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -770,8 +769,9 @@ def execute(self, context=None): class BigQueryGetDataOperator(GoogleCloudBaseOperator): """ - Fetches the data from a BigQuery table (alternatively fetch data for selected columns) and returns data - in either of the following two formats, based on "as_dict" value: + Fetches the data from a BigQuery table (alternatively fetch data for selected columns) and returns data. + + Data is returned in either of the following two formats, based on "as_dict" value: 1. False (Default) - A Python list of lists, with the number of nested lists equal to the number of rows fetched. Each nested list represents a row, where the elements within it correspond to the column values for that particular row. diff --git a/airflow/providers/google/cloud/operators/bigquery_dts.py b/airflow/providers/google/cloud/operators/bigquery_dts.py index e10618bc39bd..9edae0b65978 100644 --- a/airflow/providers/google/cloud/operators/bigquery_dts.py +++ b/airflow/providers/google/cloud/operators/bigquery_dts.py @@ -219,10 +219,10 @@ def execute(self, context: Context) -> None: class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperator): """ - Start manual transfer runs to be executed now with schedule_time equal - to current time. The transfer runs can be created for a time range where - the run_time is between start_time (inclusive) and end_time - (exclusive), or for a specific run_time. + Start manual transfer runs to be executed now with schedule_time equal to current time. + + The transfer runs can be created for a time range where the run_time is between + start_time (inclusive) and end_time (exclusive), or for a specific run_time. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/bigtable.py b/airflow/providers/google/cloud/operators/bigtable.py index 66d3e46a41dc..c6af84ddd1a4 100644 --- a/airflow/providers/google/cloud/operators/bigtable.py +++ b/airflow/providers/google/cloud/operators/bigtable.py @@ -52,9 +52,10 @@ def _validate_inputs(self): class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidationMixin): """ Creates a new Cloud Bigtable instance. + If the Cloud Bigtable instance with the given ID exists, the operator does not - compare its configuration - and immediately succeeds. No changes are made to the existing instance. + compare its configuration and immediately succeeds. No changes are made to the + existing instance. For more details about instance creation have a look at the reference: https://googleapis.github.io/google-cloud-python/latest/bigtable/instance.html#google.cloud.bigtable.instance.Instance.create diff --git a/airflow/providers/google/cloud/operators/cloud_base.py b/airflow/providers/google/cloud/operators/cloud_base.py index fc7845e1d887..bd1194e6ece6 100644 --- a/airflow/providers/google/cloud/operators/cloud_base.py +++ b/airflow/providers/google/cloud/operators/cloud_base.py @@ -24,14 +24,12 @@ class GoogleCloudBaseOperator(BaseOperator): - """ - Abstract base class that takes care of common specifics of the operators built - on top of Google API client libraries. - """ + """Abstract base class for operators using Google API client libraries.""" def __deepcopy__(self, memo): """ Updating the memo to fix the non-copyable global constant. + This constant can be specified in operator parameters as a retry configuration to indicate a default. See https://github.com/apache/airflow/issues/28751 for details. """ diff --git a/airflow/providers/google/cloud/operators/cloud_build.py b/airflow/providers/google/cloud/operators/cloud_build.py index 14fed55a3af6..db1f44db3715 100644 --- a/airflow/providers/google/cloud/operators/cloud_build.py +++ b/airflow/providers/google/cloud/operators/cloud_build.py @@ -747,8 +747,7 @@ def execute(self, context: Context): class CloudBuildRetryBuildOperator(GoogleCloudBaseOperator): """ - Creates a new build based on the specified build. This method creates a new build - using the original build request, which may or may not result in an identical build. + Creates a new build using the original build request, which may or may not result in an identical build. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -999,6 +998,7 @@ def execute(self, context: Context): class BuildProcessor: """ Processes build configurations to add additional functionality to support the use of operators. + The following improvements are made: * It is required to provide the source and only one type can be given, * It is possible to provide the source as the URL address instead dict. diff --git a/airflow/providers/google/cloud/operators/cloud_memorystore.py b/airflow/providers/google/cloud/operators/cloud_memorystore.py index 250514388fad..4c27b8982f70 100644 --- a/airflow/providers/google/cloud/operators/cloud_memorystore.py +++ b/airflow/providers/google/cloud/operators/cloud_memorystore.py @@ -325,8 +325,9 @@ def execute(self, context: Context) -> None: class CloudMemorystoreFailoverInstanceOperator(GoogleCloudBaseOperator): """ - Initiates a failover of the primary node to current replica node for a specific STANDARD tier Cloud - Memorystore for Redis instance. + Initiate a failover of the primary node for a specific STANDARD tier Cloud Memorystore for Redis instance. + + Uses the current replica node. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -886,8 +887,7 @@ def execute(self, context: Context) -> None: class CloudMemorystoreCreateInstanceAndImportOperator(GoogleCloudBaseOperator): """ - Creates a Redis instance based on the specified tier and memory size and import a Redis RDB snapshot file - from Cloud Storage into a this instance. + Create a Redis instance and import a Redis RDB snapshot file from Cloud Storage into this instance. By default, the instance is accessible from the project's `default network `__. @@ -1007,8 +1007,9 @@ def execute(self, context: Context) -> None: class CloudMemorystoreExportAndDeleteInstanceOperator(GoogleCloudBaseOperator): """ - Export Redis instance data into a Redis RDB format file in Cloud Storage. In next step, deletes a this - instance. + Export Redis instance data into a Redis RDB format file in Cloud Storage. + + In next step, deletes this instance. Redis will continue serving during this operation. @@ -1429,8 +1430,7 @@ def execute(self, context: Context): class CloudMemorystoreMemcachedListInstancesOperator(GoogleCloudBaseOperator): """ - Lists all Memcached instances owned by a project in either the specified location (region) or all - locations. + List all Memcached instances owned by a project in either the specified location/region or all locations. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -1617,9 +1617,10 @@ def execute(self, context: Context): class CloudMemorystoreMemcachedUpdateParametersOperator(GoogleCloudBaseOperator): """ - Updates the defined Memcached Parameters for an existing Instance. This method only stages the - parameters, it must be followed by apply_parameters to apply the parameters to nodes of - the Memcached Instance. + Updates the defined Memcached Parameters for an existing Instance. + + This method only stages the parameters, it must be followed by apply_parameters + to apply the parameters to nodes of the Memcached Instance. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index b1144663c706..c60a5bc09a33 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -1027,8 +1027,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context, event=None) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "success": self.log.info("Operation %s completed successfully", event["operation_name"]) diff --git a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py index dc7bfc65c5b8..8ae31e33f0f9 100644 --- a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +++ b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py @@ -111,8 +111,7 @@ def _reformat_schedule(self) -> None: def process_body(self) -> dict: """ - Injects AWS credentials into body if needed and - reformats schedule information. + Injects AWS credentials into body if needed and reformats schedule information. :return: Preprocessed body """ @@ -162,9 +161,10 @@ def _restrict_aws_credentials(self) -> None: def validate_body(self) -> None: """ - Validates the body. Checks if body specifies `transferSpec` - if yes, then check if AWS credentials are passed correctly and - no more than 1 data source was selected. + Validates the body. + + Checks if body specifies `transferSpec` if yes, then check if AWS credentials + are passed correctly and no more than 1 data source was selected. :raises: AirflowException """ @@ -360,10 +360,11 @@ def execute(self, context: Context) -> dict: class CloudDataTransferServiceDeleteJobOperator(GoogleCloudBaseOperator): """ - Delete a transfer job. This is a soft delete. After a transfer job is - deleted, the job and all the transfer executions are subject to garbage - collection. Transfer jobs become eligible for garbage collection - 30 days after soft delete. + Delete a transfer job. + + This is a soft delete. After a transfer job is deleted, the job and all the transfer + executions are subject to garbage collection. Transfer jobs become eligible for garbage + collection 30 days after soft delete. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -429,8 +430,7 @@ def execute(self, context: Context) -> None: class CloudDataTransferServiceGetOperationOperator(GoogleCloudBaseOperator): """ - Gets the latest state of a long-running operation in Google Storage Transfer - Service. + Gets the latest state of a long-running operation in Google Storage Transfer Service. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -503,8 +503,7 @@ def execute(self, context: Context) -> dict: class CloudDataTransferServiceListOperationsOperator(GoogleCloudBaseOperator): """ - Lists long-running operations in Google Storage Transfer - Service that match the specified filter. + Lists long-running operations in Google Storage Transfer Service that match the specified filter. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -766,8 +765,7 @@ def execute(self, context: Context) -> None: class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator): """ - Synchronizes an S3 bucket with a Google Cloud Storage bucket using the - Google Cloud Storage Transfer Service. + Sync an S3 bucket with a Google Cloud Storage bucket using the Google Cloud Storage Transfer Service. .. warning:: diff --git a/airflow/providers/google/cloud/operators/compute.py b/airflow/providers/google/cloud/operators/compute.py index db8a98e44348..2adb5b8439c4 100644 --- a/airflow/providers/google/cloud/operators/compute.py +++ b/airflow/providers/google/cloud/operators/compute.py @@ -667,8 +667,7 @@ def execute(self, context: Context) -> None: class ComputeEngineSetMachineTypeOperator(ComputeEngineBaseOperator): """ - Changes the machine type for a stopped instance to the machine type specified in - the request. + Changes the machine type for a stopped instance to the machine type specified in the request. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -1261,9 +1260,10 @@ def execute(self, context: Context) -> dict: class ComputeEngineInstanceGroupUpdateManagerTemplateOperator(ComputeEngineBaseOperator): """ - Patches the Instance Group Manager, replacing source template URL with the - destination one. API V1 does not have update/patch operations for Instance - Group Manager, so you must use beta or newer API version. Beta is the default. + Patches the Instance Group Manager, replacing source template URL with the destination one. + + API V1 does not have update/patch operations for Instance Group Manager, + so you must use beta or newer API version. Beta is the default. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -1409,6 +1409,7 @@ def execute(self, context: Context) -> bool | None: class ComputeEngineInsertInstanceGroupManagerOperator(ComputeEngineBaseOperator): """ Creates an Instance Group Managers using the body specified. + After the group is created, instances in the group are created using the specified Instance Template. .. seealso:: @@ -1576,8 +1577,7 @@ def execute(self, context: Context) -> dict: class ComputeEngineDeleteInstanceGroupManagerOperator(ComputeEngineBaseOperator): """ - Deletes an Instance Group Managers. - Deleting an Instance Group Manager is permanent and cannot be undone. + Permanently and irrevocably deletes an Instance Group Managers. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py index a5e958821485..0a225a47b623 100644 --- a/airflow/providers/google/cloud/operators/dataflow.py +++ b/airflow/providers/google/cloud/operators/dataflow.py @@ -61,9 +61,12 @@ class CheckJobRunning(Enum): class DataflowConfiguration: - """Dataflow configuration that can be passed to - :class:`~airflow.providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator` and - :class:`~airflow.providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator`. + """ + Dataflow configuration for BeamRunJavaPipelineOperator and BeamRunPythonPipelineOperator. + + .. seealso:: + :class:`~airflow.providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator` + and :class:`~airflow.providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator`. :param job_name: The 'jobName' to use when executing the Dataflow job (templated). This ends up being set in the pipeline options, so any entry @@ -167,11 +170,10 @@ def __init__( class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator): """ - Start a Java Cloud Dataflow batch job. The parameters of the operation - will be passed to the job. + Start a Java Cloud Dataflow batch job; the parameters of the operation will be passed to the job. This class is deprecated. - Please use `providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator`. + Please use :class:`providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator`. Example usage: @@ -452,8 +454,7 @@ def on_kill(self) -> None: class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator): """ - Start a Templated Cloud Dataflow job. The parameters of the operation - will be passed to the job. + Start a Templated Cloud Dataflow job; the parameters of the operation will be passed to the job. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -1015,14 +1016,14 @@ def on_kill(self) -> None: class DataflowCreatePythonJobOperator(GoogleCloudBaseOperator): """ - Launching Cloud Dataflow jobs written in python. Note that both - dataflow_default_options and options will be merged to specify pipeline - execution parameter, and dataflow_default_options is expected to save - high-level options, for instances, project and zone information, which - apply to all dataflow operators in the DAG. + Launching Cloud Dataflow jobs written in python. + + Note that both dataflow_default_options and options will be merged to specify pipeline + execution parameter, and dataflow_default_options is expected to save high-level options, + for instances, project and zone information, which apply to all dataflow operators in the DAG. This class is deprecated. - Please use `providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator`. + Please use :class:`providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator`. .. seealso:: For more detail on job submission have a look at the reference: @@ -1232,6 +1233,7 @@ def on_kill(self) -> None: class DataflowStopJobOperator(GoogleCloudBaseOperator): """ Stops the job with the specified name prefix or Job ID. + All jobs with provided name prefix will be stopped. Streaming jobs are drained by default. diff --git a/airflow/providers/google/cloud/operators/datafusion.py b/airflow/providers/google/cloud/operators/datafusion.py index 8ae64e1a4b72..e43420d3be7e 100644 --- a/airflow/providers/google/cloud/operators/datafusion.py +++ b/airflow/providers/google/cloud/operators/datafusion.py @@ -51,6 +51,7 @@ def get_project_id(instance): class CloudDataFusionRestartInstanceOperator(GoogleCloudBaseOperator): """ Restart a single Data Fusion instance. + At the end of an operation instance is fully restarted. .. seealso:: @@ -849,8 +850,8 @@ def execute(self, context: Context) -> str: def execute_complete(self, context: Context, event: dict[str, Any]): """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "error": raise AirflowException(event["message"]) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index d14a495bc0cb..bac0c4eb9bf6 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -388,6 +388,7 @@ def _build_cluster_data(self): def make(self): """ Helper method for easier migration. + :return: Dict representing Dataproc cluster. """ return self._build_cluster_data() @@ -655,8 +656,8 @@ def execute(self, context: Context) -> dict: def execute_complete(self, context: Context, event: dict[str, Any]) -> Any: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ cluster_state = event["cluster_state"] cluster_name = event["cluster_name"] @@ -895,8 +896,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> Any: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event and event["status"] == "error": raise AirflowException(event["message"]) @@ -1074,8 +1075,8 @@ def execute(self, context: Context): def execute_complete(self, context, event=None) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ job_state = event["job_state"] job_id = event["job_id"] @@ -1087,10 +1088,7 @@ def execute_complete(self, context, event=None) -> None: return job_id def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.dataproc_job_id: self.hook.cancel_job(project_id=self.project_id, job_id=self.dataproc_job_id, region=self.region) @@ -2201,8 +2199,8 @@ def execute(self, context: Context): def execute_complete(self, context: Context, event: dict[str, Any]) -> Any: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ cluster_state = event["cluster_state"] cluster_name = event["cluster_name"] diff --git a/airflow/providers/google/cloud/operators/dlp.py b/airflow/providers/google/cloud/operators/dlp.py index dde0ce59438a..60b52a4751c8 100644 --- a/airflow/providers/google/cloud/operators/dlp.py +++ b/airflow/providers/google/cloud/operators/dlp.py @@ -15,11 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains various Google Cloud DLP operators -which allow you to perform basic operations using -Cloud DLP. -""" +"""Various Google Cloud DLP operators which allow you to perform basic operations using Cloud DLP.""" + from __future__ import annotations from typing import TYPE_CHECKING, Sequence @@ -152,8 +149,7 @@ def execute(self, context: Context) -> None: class CloudDLPCreateDeidentifyTemplateOperator(GoogleCloudBaseOperator): """ - Creates a DeidentifyTemplate for re-using frequently used configuration for - de-identifying content, images, and storage. + Create a deidentify template to reuse frequently-used configurations for content, images, and storage. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -374,8 +370,7 @@ def execute(self, context: Context): class CloudDLPCreateInspectTemplateOperator(GoogleCloudBaseOperator): """ - Creates an InspectTemplate for re-using frequently used configuration for - inspecting content, images, and storage. + Create an InspectTemplate to reuse frequently-used configurations for content, images, and storage. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -485,8 +480,7 @@ def execute(self, context: Context): class CloudDLPCreateJobTriggerOperator(GoogleCloudBaseOperator): """ - Creates a job trigger to run DLP actions such as scanning storage for sensitive - information on a set schedule. + Create a job trigger to run DLP actions such as scanning storage for sensitive info on a set schedule. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -705,8 +699,7 @@ def execute(self, context: Context): class CloudDLPDeidentifyContentOperator(GoogleCloudBaseOperator): """ - De-identifies potentially sensitive info from a ContentItem. This method has limits - on input size and output size. + De-identifies potentially sensitive info from a content item; limits input size and output size. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -893,8 +886,10 @@ def execute(self, context: Context) -> None: class CloudDLPDeleteDLPJobOperator(GoogleCloudBaseOperator): """ - Deletes a long-running DlpJob. This method indicates that the client is no longer - interested in the DlpJob result. The job will be cancelled if possible. + Deletes a long-running DlpJob. + + This method indicates that the client is no longer interested + in the DlpJob result. The job will be cancelled if possible. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -1676,8 +1671,7 @@ def execute(self, context: Context): class CloudDLPInspectContentOperator(GoogleCloudBaseOperator): """ - Finds potentially sensitive info in content. This method has limits on - input size, processing time, and output size. + Finds potentially sensitive info in content; limits input size, processing time, and output size. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -2317,8 +2311,7 @@ def execute(self, context: Context): class CloudDLPRedactImageOperator(GoogleCloudBaseOperator): """ - Redacts potentially sensitive info from an image. This method has limits on - input size, processing time, and output size. + Redacts potentially sensitive info from an image; limits input size, processing time, and output size. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/functions.py b/airflow/providers/google/cloud/operators/functions.py index 1f01880de4b7..1131790cabdf 100644 --- a/airflow/providers/google/cloud/operators/functions.py +++ b/airflow/providers/google/cloud/operators/functions.py @@ -104,8 +104,7 @@ def _validate_max_instances(value): class CloudFunctionDeployFunctionOperator(GoogleCloudBaseOperator): """ - Creates a function in Google Cloud Functions. - If a function with this name already exists, it will be updated. + Create or update a function in Google Cloud Functions. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -312,10 +311,7 @@ def should_upload_function(self) -> bool: return self.upload_function def preprocess_body(self) -> None: - """ - Modifies sourceUploadUrl body field in special way when zip_path - is not empty. - """ + """Modifies sourceUploadUrl body field in special way when zip_path is not empty.""" self._verify_archive_url_and_zip_path() self._verify_upload_url_and_zip_path() self._verify_upload_url_and_no_zip_path() @@ -412,8 +408,7 @@ def execute(self, context: Context): class CloudFunctionInvokeFunctionOperator(GoogleCloudBaseOperator): """ - Invokes a deployed Cloud Function. To be used for testing - purposes as very limited traffic is allowed. + Invokes a deployed Cloud Function. To be used for testing purposes as very limited traffic is allowed. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py index 1025a5038a59..fd73af42fb66 100644 --- a/airflow/providers/google/cloud/operators/gcs.py +++ b/airflow/providers/google/cloud/operators/gcs.py @@ -43,8 +43,10 @@ class GCSCreateBucketOperator(GoogleCloudBaseOperator): """ - Creates a new bucket. Google Cloud Storage uses a flat namespace, - so you can't create a bucket with a name that is already in use. + Creates a new bucket. + + Google Cloud Storage uses a flat namespace, so you + can't create a bucket with a name that is already in use. .. seealso:: For more information, see Bucket Naming Guidelines: @@ -158,8 +160,7 @@ def execute(self, context: Context) -> None: class GCSListObjectsOperator(GoogleCloudBaseOperator): """ - List all objects from the bucket filtered by given string prefix and delimiter in name, - or match_glob. + List all objects from the bucket filtered by given string prefix and delimiter in name or match_glob. This operator returns a python list with the name of objects which can be used by XCom in the downstream task. @@ -265,9 +266,7 @@ def execute(self, context: Context) -> list: class GCSDeleteObjectsOperator(GoogleCloudBaseOperator): """ - Deletes objects from a Google Cloud Storage bucket, either - from an explicit list of object names or all objects - matching a prefix. + Deletes objects from a list or all objects matching a prefix from a Google Cloud Storage bucket. :param bucket_name: The GCS bucket to delete from :param objects: List of objects to delete. These should be the names @@ -495,11 +494,11 @@ def execute(self, context: Context) -> None: class GCSFileTransformOperator(GoogleCloudBaseOperator): """ - Copies data from a source GCS location to a temporary location on the - local filesystem. Runs a transformation on this file as specified by - the transformation script and uploads the output to a destination bucket. - If the output bucket is not specified the original file will be - overwritten. + Copies data from a source GCS location to a temporary location on the local filesystem. + + Runs a transformation on this file as specified by the transformation script + and uploads the output to a destination bucket. If the output bucket is not + specified the original file will be overwritten. The locations of the source and the destination files in the local filesystem is provided as an first and second arguments to the @@ -601,6 +600,8 @@ def execute(self, context: Context) -> None: class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator): """ + Copy objects that were modified during a time span, run a transform, and upload results to a bucket. + Determines a list of objects that were added or modified at a GCS source location during a specific time-span, copies them to a temporary location on the local file system, runs a transform on this file as specified by diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 086a7d99b7e6..3f730d5bda34 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -186,8 +186,7 @@ def _get_hook(self) -> GKEHook: class GKECreateClusterOperator(GoogleCloudBaseOperator): """ - Create a Google Kubernetes Engine Cluster of specified dimensions - The operator will wait until the cluster is created. + Create a Google Kubernetes Engine Cluster of specified dimensions and wait until the cluster is created. The **minimum** required to define a cluster to create is: @@ -394,8 +393,7 @@ def _get_hook(self) -> GKEHook: class GKEStartPodOperator(KubernetesPodOperator): """ - Executes a task in a Kubernetes pod in the specified Google Kubernetes - Engine cluster. + Executes a task in a Kubernetes pod in the specified Google Kubernetes Engine cluster. This Operator assumes that the system has gcloud installed and has configured a connection id with a service account. diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/airflow/providers/google/cloud/operators/mlengine.py index 20d9d1988623..554167662e17 100644 --- a/airflow/providers/google/cloud/operators/mlengine.py +++ b/airflow/providers/google/cloud/operators/mlengine.py @@ -1296,8 +1296,8 @@ def _wait_for_job_done(self, project_id: str, job_id: str, interval: int = 30): def execute_complete(self, context: Context, event: dict[str, Any]): """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "error": raise AirflowException(event["message"]) diff --git a/airflow/providers/google/cloud/operators/natural_language.py b/airflow/providers/google/cloud/operators/natural_language.py index 2665f9ced60a..21ddb01f8a89 100644 --- a/airflow/providers/google/cloud/operators/natural_language.py +++ b/airflow/providers/google/cloud/operators/natural_language.py @@ -37,8 +37,9 @@ class CloudNaturalLanguageAnalyzeEntitiesOperator(GoogleCloudBaseOperator): """ - Finds named entities in the text along with entity types, - salience, mentions for each entity, and other properties. + Finds named entities in the text along with various properties. + + Examples properties: entity types, salience, mentions for each entity, and others. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -109,8 +110,7 @@ def execute(self, context: Context): class CloudNaturalLanguageAnalyzeEntitySentimentOperator(GoogleCloudBaseOperator): """ - Finds entities, similar to AnalyzeEntities in the text and analyzes sentiment associated with each - entity and its mentions. + Similar to AnalyzeEntities, also analyzes sentiment associated with each entity and its mentions. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/pubsub.py b/airflow/providers/google/cloud/operators/pubsub.py index 2eb6882ceb99..8e0f7a12d99a 100644 --- a/airflow/providers/google/cloud/operators/pubsub.py +++ b/airflow/providers/google/cloud/operators/pubsub.py @@ -678,7 +678,9 @@ def execute(self, context: Context) -> None: class PubSubPullOperator(GoogleCloudBaseOperator): - """Pulls messages from a PubSub subscription and passes them through XCom. + """ + Pulls messages from a PubSub subscription and passes them through XCom. + If the queue is empty, returns empty list - never waits for messages. If you do need to wait, please use :class:`airflow.providers.google.cloud.sensors.PubSubPullSensor` instead. @@ -781,6 +783,7 @@ def _default_message_callback( ) -> list: """ This method can be overridden by subclasses or by `messages_callback` constructor argument. + This default implementation converts `ReceivedMessage` objects into JSON-serializable dicts. :param pulled_messages: messages received from the topic. diff --git a/airflow/providers/google/cloud/operators/spanner.py b/airflow/providers/google/cloud/operators/spanner.py index 72a2e7000d8e..d235c146b5b4 100644 --- a/airflow/providers/google/cloud/operators/spanner.py +++ b/airflow/providers/google/cloud/operators/spanner.py @@ -31,8 +31,7 @@ class SpannerDeployInstanceOperator(GoogleCloudBaseOperator): """ - Creates a new Cloud Spanner instance, or if an instance with the same instance_id - exists in the specified project, updates the Cloud Spanner instance. + Create or update a Cloud Spanner instance. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -128,8 +127,7 @@ def execute(self, context: Context) -> None: class SpannerDeleteInstanceOperator(GoogleCloudBaseOperator): """ - Deletes a Cloud Spanner instance. If an instance does not exist, - no action is taken and the operator succeeds. + Delete a Cloud Spanner instance; if an instance does not exist, no action is taken and the task succeeds. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -309,8 +307,7 @@ def sanitize_queries(queries: list[str]) -> None: class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator): """ - Creates a new Cloud Spanner database, or if database exists, - the operator does nothing. + Creates a new Cloud Spanner database; if database exists, the operator does nothing. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/stackdriver.py b/airflow/providers/google/cloud/operators/stackdriver.py index 33a082355e0d..aa2b3d368c67 100644 --- a/airflow/providers/google/cloud/operators/stackdriver.py +++ b/airflow/providers/google/cloud/operators/stackdriver.py @@ -36,11 +36,11 @@ class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator): """ - Fetches all the Alert Policies identified by the filter passed as - filter parameter. The desired return type can be specified by the - format parameter, the supported formats are "dict", "json" and None - which returns python dictionary, stringified JSON and protobuf - respectively. + Fetches all the Alert Policies identified by the filter passed as filter parameter. + + The desired return type can be specified by the format parameter, the supported + formats are "dict", "json" and None which returns python dictionary, stringified + JSON and protobuf respectively. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -150,8 +150,9 @@ def execute(self, context: Context): class StackdriverEnableAlertPoliciesOperator(GoogleCloudBaseOperator): """ - Enables one or more disabled alerting policies identified by filter - parameter. Inoperative in case the policy is already enabled. + Enables one or more disabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already enabled. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -232,8 +233,9 @@ def execute(self, context: Context): # Disable Alert Operator class StackdriverDisableAlertPoliciesOperator(GoogleCloudBaseOperator): """ - Disables one or more enabled alerting policies identified by filter - parameter. Inoperative in case the policy is already disabled. + Disables one or more enabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already disabled. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -313,8 +315,7 @@ def execute(self, context: Context): class StackdriverUpsertAlertOperator(GoogleCloudBaseOperator): """ - Creates a new alert or updates an existing policy identified - the name field in the alerts parameter. + Creates a new alert or updates an existing policy identified the name field in the alerts parameter. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -470,11 +471,11 @@ def execute(self, context: Context): class StackdriverListNotificationChannelsOperator(GoogleCloudBaseOperator): """ - Fetches all the Notification Channels identified by the filter passed as - filter parameter. The desired return type can be specified by the - format parameter, the supported formats are "dict", "json" and None - which returns python dictionary, stringified JSON and protobuf - respectively. + Fetches all the Notification Channels identified by the filter passed as filter parameter. + + The desired return type can be specified by the format parameter, the + supported formats are "dict", "json" and None which returns python + dictionary, stringified JSON and protobuf respectively. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -584,8 +585,9 @@ def execute(self, context: Context): class StackdriverEnableNotificationChannelsOperator(GoogleCloudBaseOperator): """ - Enables one or more disabled alerting policies identified by filter - parameter. Inoperative in case the policy is already enabled. + Enables one or more disabled alerting policies identified by filter parameter. + + Inoperative in case the policy is already enabled. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -668,8 +670,9 @@ def execute(self, context: Context): class StackdriverDisableNotificationChannelsOperator(GoogleCloudBaseOperator): """ - Disables one or more enabled notification channels identified by filter - parameter. Inoperative in case the policy is already disabled. + Disables one or more enabled notification channels identified by filter parameter. + + Inoperative in case the policy is already disabled. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -752,8 +755,9 @@ def execute(self, context: Context): class StackdriverUpsertNotificationChannelOperator(GoogleCloudBaseOperator): """ - Creates a new notification or updates an existing notification channel - identified the name field in the alerts parameter. + Create a new notification or updates an existing notification channel. + + Channel is identified by the name field in the alerts parameter. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/operators/tasks.py b/airflow/providers/google/cloud/operators/tasks.py index f0964214e2c8..f78399d76673 100644 --- a/airflow/providers/google/cloud/operators/tasks.py +++ b/airflow/providers/google/cloud/operators/tasks.py @@ -15,11 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains various Google Cloud Tasks operators -which allow you to perform basic operations using -Cloud Tasks queues/tasks. -""" +"""Google Cloud Tasks operators which allow you to perform basic operations using Cloud Tasks queues/tasks.""" + from __future__ import annotations from typing import TYPE_CHECKING, Sequence, Tuple diff --git a/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py b/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py index a3bcb2158d34..88a8560e85dc 100644 --- a/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +++ b/airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py @@ -80,10 +80,7 @@ def __init__( self.hook: AutoMLHook | None = None def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_auto_ml_job() @@ -537,8 +534,11 @@ def execute(self, context: Context): class DeleteAutoMLTrainingJobOperator(GoogleCloudBaseOperator): - """Deletes an AutoMLForecastingTrainingJob, AutoMLImageTrainingJob, AutoMLTabularTrainingJob, - AutoMLTextTrainingJob, or AutoMLVideoTrainingJob. + """ + Delete an AutoML training job. + + Can be used with AutoMLForecastingTrainingJob, AutoMLImageTrainingJob, + AutoMLTabularTrainingJob, AutoMLTextTrainingJob, or AutoMLVideoTrainingJob. """ template_fields = ("training_pipeline", "region", "project_id", "impersonation_chain") @@ -588,7 +588,10 @@ def execute(self, context: Context): class ListAutoMLTrainingJobOperator(GoogleCloudBaseOperator): - """Lists AutoMLForecastingTrainingJob, AutoMLImageTrainingJob, AutoMLTabularTrainingJob, + """ + List an AutoML training job. + + Can be used with AutoMLForecastingTrainingJob, AutoMLImageTrainingJob, AutoMLTabularTrainingJob, AutoMLTextTrainingJob, or AutoMLVideoTrainingJob in a Location. """ diff --git a/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py b/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py index 0247d33f8440..684cfdd8f2e8 100644 --- a/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +++ b/airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py @@ -268,10 +268,7 @@ def execute(self, context: Context): return batch_prediction_job def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_batch_prediction_job() diff --git a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py index 7eb7866ff367..9896cdcccd69 100644 --- a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +++ b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py @@ -489,10 +489,7 @@ def execute(self, context: Context): return result def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_job() @@ -839,10 +836,7 @@ def execute(self, context: Context): return result def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_job() @@ -1191,16 +1185,14 @@ def execute(self, context: Context): return result def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_job() class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator): - """Deletes a CustomTrainingJob, CustomPythonTrainingJob, or CustomContainerTrainingJob. + """ + Deletes a CustomTrainingJob, CustomPythonTrainingJob, or CustomContainerTrainingJob. :param training_pipeline_id: Required. The name of the TrainingPipeline resource to be deleted. :param custom_job_id: Required. The name of the CustomJob to delete. diff --git a/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py b/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py index 10c722b2a0e2..b2bf7c421339 100644 --- a/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +++ b/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py @@ -458,8 +458,7 @@ def execute(self, context: Context): class UndeployModelOperator(GoogleCloudBaseOperator): """ - Undeploys a Model from an Endpoint, removing a DeployedModel from it, and freeing all resources it's - using. + Undeploys a Model from an Endpoint, removing a DeployedModel from it, and freeing all used resources. :param project_id: Required. The ID of the Google Cloud project that the service belongs to. :param region: Required. The ID of the Google Cloud region that the service belongs to. diff --git a/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py b/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py index c3d6c82b86d3..69c7e76016d3 100644 --- a/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +++ b/airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py @@ -264,10 +264,7 @@ def execute(self, context: Context): return hyperparameter_tuning_job def on_kill(self) -> None: - """ - Callback called when the operator is killed. - Cancel any running job. - """ + """Callback called when the operator is killed; cancel any running job.""" if self.hook: self.hook.cancel_hyperparameter_tuning_job() diff --git a/airflow/providers/google/cloud/operators/workflows.py b/airflow/providers/google/cloud/operators/workflows.py index cff6e15ef0b2..4f2517f5da01 100644 --- a/airflow/providers/google/cloud/operators/workflows.py +++ b/airflow/providers/google/cloud/operators/workflows.py @@ -49,9 +49,10 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator): """ - Creates a new workflow. If a workflow with the specified name - already exists in the specified project and location, the long - running operation will return + Creates a new workflow. + + If a workflow with the specified name already exists in the specified + project and location, the long running operation will return [ALREADY_EXISTS][google.rpc.Code.ALREADY_EXISTS] error. .. seealso:: @@ -159,6 +160,7 @@ def execute(self, context: Context): class WorkflowsUpdateWorkflowOperator(GoogleCloudBaseOperator): """ Updates an existing workflow. + Running this method has no impact on already running executions of the workflow. A new revision of the workflow may be created as a result of a successful @@ -245,9 +247,7 @@ def execute(self, context: Context): class WorkflowsDeleteWorkflowOperator(GoogleCloudBaseOperator): """ - Deletes a workflow with the specified name. - This method also cancels and deletes all running - executions of the workflow. + Delete a workflow with the specified name and all running executions of the workflow. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -305,8 +305,7 @@ def execute(self, context: Context): class WorkflowsListWorkflowsOperator(GoogleCloudBaseOperator): """ - Lists Workflows in a given project and location. - The default order is not specified. + Lists Workflows in a given project and location; the default order is not specified. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -447,8 +446,7 @@ def execute(self, context: Context): class WorkflowsCreateExecutionOperator(GoogleCloudBaseOperator): """ - Creates a new execution using the latest revision of - the given workflow. + Creates a new execution using the latest revision of the given workflow. .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -597,11 +595,10 @@ def execute(self, context: Context): class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator): """ - Returns a list of executions which belong to the - workflow with the given name. The method returns - executions of all workflow revisions. Returned - executions are ordered by their start time (newest - first). + Returns a list of executions which belong to the workflow with the given name. + + The method returns executions of all workflow revisions. Returned + executions are ordered by their start time (newest first). .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/sensors/bigquery.py b/airflow/providers/google/cloud/sensors/bigquery.py index e4e1819ef134..d13748da1807 100644 --- a/airflow/providers/google/cloud/sensors/bigquery.py +++ b/airflow/providers/google/cloud/sensors/bigquery.py @@ -133,8 +133,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: dict[str, Any], event: dict[str, str] | None = None) -> str: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ table_uri = f"{self.project_id}:{self.dataset_id}.{self.table_id}" self.log.info("Sensor checks existence of table: %s", table_uri) @@ -216,10 +216,7 @@ def poke(self, context: Context) -> bool: ) def execute(self, context: Context) -> None: - """ - Airflow runs this method on the worker and defers using the triggers - if deferrable is set to True. - """ + """Airflow runs this method on the worker and defers using the triggers if deferrable is True.""" if not self.deferrable: super().execute(context) else: @@ -243,8 +240,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: dict[str, Any], event: dict[str, str] | None = None) -> str: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ table_uri = f"{self.project_id}:{self.dataset_id}.{self.table_id}" self.log.info('Sensor checks existence of partition: "%s" in table: %s', self.partition_id, table_uri) diff --git a/airflow/providers/google/cloud/sensors/bigtable.py b/airflow/providers/google/cloud/sensors/bigtable.py index c69d4099cad1..0b823f0a1a07 100644 --- a/airflow/providers/google/cloud/sensors/bigtable.py +++ b/airflow/providers/google/cloud/sensors/bigtable.py @@ -36,6 +36,7 @@ class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValidationMixin): """ Sensor that waits for Cloud Bigtable table to be fully replicated to its clusters. + No exception will be raised if the instance or the table does not exist. For more details about cluster states for a table, have a look at the reference: diff --git a/airflow/providers/google/cloud/sensors/cloud_composer.py b/airflow/providers/google/cloud/sensors/cloud_composer.py index 238c9533051c..ecd717aa5a15 100644 --- a/airflow/providers/google/cloud/sensors/cloud_composer.py +++ b/airflow/providers/google/cloud/sensors/cloud_composer.py @@ -84,8 +84,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: dict[str, Any], event: dict[str, str] | None = None) -> str: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event.get("operation_done"): diff --git a/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py b/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py index cccc628d3c60..60edca6337d1 100644 --- a/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +++ b/airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py @@ -35,8 +35,7 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator): """ - Waits for at least one operation belonging to the job to have the - expected status. + Waits for at least one operation belonging to the job to have the expected status. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/sensors/dataproc_metastore.py b/airflow/providers/google/cloud/sensors/dataproc_metastore.py index 50ba4ddf55ee..fb895abba33d 100644 --- a/airflow/providers/google/cloud/sensors/dataproc_metastore.py +++ b/airflow/providers/google/cloud/sensors/dataproc_metastore.py @@ -33,7 +33,8 @@ class MetastoreHivePartitionSensor(BaseSensorOperator): """ Waits for partitions to show up in Hive. - This sensor uses Google Cloud SDK and passes requests via gRPC. + + This sensor uses Google Cloud SDK and passes requests via gRPC. :param service_id: Required. Dataproc Metastore service id. :param region: Required. The ID of the Google Cloud region that the service belongs to. diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/airflow/providers/google/cloud/sensors/gcs.py index 7048789601e4..34df5f52570f 100644 --- a/airflow/providers/google/cloud/sensors/gcs.py +++ b/airflow/providers/google/cloud/sensors/gcs.py @@ -120,8 +120,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, str]) -> str: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "error": raise AirflowException(event["message"]) @@ -160,10 +160,10 @@ def __init__(self, **kwargs: Any) -> None: def ts_function(context): """ - Default callback for the GoogleCloudStorageObjectUpdatedSensor. The default - behaviour is check for the object being updated after the data interval's - end, or execution_date + interval on Airflow versions prior to 2.2 (before - AIP-39 implementation). + Default callback for the GoogleCloudStorageObjectUpdatedSensor. + + The default behaviour is check for the object being updated after the data interval's end, + or execution_date + interval on Airflow versions prior to 2.2 (before AIP-39 implementation). """ try: return context["data_interval_end"] @@ -340,10 +340,7 @@ def execute(self, context: Context): ) def execute_complete(self, context: dict[str, Any], event: dict[str, str | list[str]]) -> str | list[str]: - """ - Callback for when the trigger fires; returns immediately. - Relies on trigger to throw a success event. - """ + """Callback for the trigger; returns immediately and relies on trigger to throw a success event.""" self.log.info("Resuming from trigger and checking status") if event["status"] == "success": return event["matches"] @@ -351,16 +348,15 @@ def execute_complete(self, context: dict[str, Any], event: dict[str, str | list[ def get_time(): - """ - This is just a wrapper of datetime.datetime.now to simplify mocking in the - unittests. - """ + """This is just a wrapper of datetime.datetime.now to simplify mocking in the unittests.""" return datetime.now() @poke_mode_only class GCSUploadSessionCompleteSensor(BaseSensorOperator): """ + Return True if the inactivity period has passed with no increase in the number of objects in the bucket. + Checks for changes in the number of objects at prefix in Google Cloud Storage bucket and returns True if the inactivity period has passed with no increase in the number of objects. Note, this sensor will not behave correctly @@ -441,8 +437,7 @@ def _get_gcs_hook(self) -> GCSHook | None: def is_bucket_updated(self, current_objects: set[str]) -> bool: """ - Checks whether new objects have been uploaded and the inactivity_period - has passed and updates the state of the sensor accordingly. + Check whether new objects have been added and the inactivity_period has passed, and update the state. :param current_objects: set of object ids in bucket during last poke. """ @@ -545,8 +540,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: dict[str, Any], event: dict[str, str] | None = None) -> str: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event: if event["status"] == "success": diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/airflow/providers/google/cloud/sensors/pubsub.py index db9f39b19b0b..c818f7168e76 100644 --- a/airflow/providers/google/cloud/sensors/pubsub.py +++ b/airflow/providers/google/cloud/sensors/pubsub.py @@ -34,7 +34,9 @@ class PubSubPullSensor(BaseSensorOperator): - """Pulls messages from a PubSub subscription and passes them through XCom. + """ + Pulls messages from a PubSub subscription and passes them through XCom. + Always waits for at least one message to be returned from the subscription. .. seealso:: @@ -146,10 +148,7 @@ def poke(self, context: Context) -> bool: return bool(pulled_messages) def execute(self, context: Context) -> None: - """ - Airflow runs this method on the worker and defers using the triggers - if deferrable is set to True. - """ + """Airflow runs this method on the worker and defers using the triggers if deferrable is True.""" if not self.deferrable: super().execute(context) return self._return_value @@ -170,10 +169,7 @@ def execute(self, context: Context) -> None: ) def execute_complete(self, context: dict[str, Any], event: dict[str, str | list[str]]) -> str | list[str]: - """ - Callback for when the trigger fires; returns immediately. - Relies on trigger to throw a success event. - """ + """Callback for the trigger; returns immediately and relies on trigger to throw a success event.""" if event["status"] == "success": self.log.info("Sensor pulls messages: %s", event["message"]) return event["message"] @@ -187,6 +183,7 @@ def _default_message_callback( ): """ This method can be overridden by subclasses or by `messages_callback` constructor argument. + This default implementation converts `ReceivedMessage` objects into JSON-serializable dicts. :param pulled_messages: messages received from the topic. diff --git a/airflow/providers/google/cloud/sensors/tasks.py b/airflow/providers/google/cloud/sensors/tasks.py index 714fcd18d630..f90154e7feac 100644 --- a/airflow/providers/google/cloud/sensors/tasks.py +++ b/airflow/providers/google/cloud/sensors/tasks.py @@ -29,8 +29,7 @@ class TaskQueueEmptySensor(BaseSensorOperator): """ - Pulls tasks count from a cloud task queue. - Always waits for queue returning tasks count as 0. + Pulls tasks count from a cloud task queue; waits for queue to return task count as 0. :param project_id: the Google Cloud project ID for the subscription (templated) :param gcp_conn_id: The connection ID to use connecting to Google Cloud. diff --git a/airflow/providers/google/cloud/transfers/adls_to_gcs.py b/airflow/providers/google/cloud/transfers/adls_to_gcs.py index 4810ab41af20..75b691ba41d2 100644 --- a/airflow/providers/google/cloud/transfers/adls_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/adls_to_gcs.py @@ -15,10 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains Azure Data Lake Storage to -Google Cloud Storage operator. -""" +"""This module contains Azure Data Lake Storage to Google Cloud Storage operator.""" + from __future__ import annotations import os diff --git a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py b/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py index 39d14bd04814..e8410cb5daa4 100644 --- a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py @@ -31,8 +31,9 @@ class AzureFileShareToGCSOperator(BaseOperator): """ - Synchronizes a Azure FileShare directory content (excluding subdirectories), - possibly filtered by a prefix, with a Google Cloud Storage destination path. + Sync an Azure FileShare directory with a Google Cloud Storage destination path. + + Does not include subdirectories. May be filtered by prefix. :param share_name: The Azure FileShare share where to find the objects. (templated) :param directory_name: (Optional) Path to Azure FileShare directory which content is to be transferred. diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py index 467751115151..a1ec7a1b5141 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_gcs.py @@ -265,8 +265,8 @@ def execute(self, context: Context): def execute_complete(self, context: Context, event: dict[str, Any]): """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "error": raise AirflowException(event["message"]) diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py index 0c7e58a458c0..d46a001c28a2 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py @@ -32,8 +32,7 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator): """ - Fetches the data from a BigQuery table (alternatively fetch data for selected columns) - and insert that data into a MSSQL table. + Fetch data from a BigQuery table (alternatively fetch selected columns) and insert it into a MSSQL table. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py index 99b01fe9e86b..c8aec07b0e40 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py @@ -28,8 +28,7 @@ class BigQueryToMySqlOperator(BigQueryToSqlBaseOperator): """ - Fetches the data from a BigQuery table (alternatively fetch data for selected columns) - and insert that data into a MySQL table. + Fetch data from a BigQuery table (alternatively fetch selected columns) and insert it into a MySQL table. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py b/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py index 34d639962047..3d1415687c68 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py @@ -26,8 +26,7 @@ class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator): """ - Fetches the data from a BigQuery table (alternatively fetch data for selected columns) - and insert that data into a PostgreSQL table. + Fetch data from a BigQuery table (alternatively fetch selected columns) and insert into PostgreSQL table. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_sql.py b/airflow/providers/google/cloud/transfers/bigquery_to_sql.py index 0e3d7a2a6997..36acb9363811 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_sql.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_sql.py @@ -32,9 +32,10 @@ class BigQueryToSqlBaseOperator(BaseOperator): """ - Fetches the data from a BigQuery table (alternatively fetch data for selected columns) - and insert that data into a SQL table. This is a BaseOperator; an abstract class. Refer - to children classes which are related to specific SQL databases (MySQL, MsSQL, Postgres...). + Fetch data from a BigQuery table (alternatively fetch selected columns) and insert it into an SQL table. + + This is a BaseOperator; an abstract class. Refer to children classes + which are related to specific SQL databases (MySQL, MsSQL, Postgres...). .. note:: If you pass fields to ``selected_fields`` which are in different order than the diff --git a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py b/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py index ffd3e3aa7837..408977d7ccfa 100644 --- a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py @@ -15,10 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains operator for copying -data from Cassandra to Google Cloud Storage in JSON format. -""" +"""This module contains operator for copying data from Cassandra to Google Cloud Storage in JSON format.""" + from __future__ import annotations import json @@ -220,8 +218,7 @@ def _write_local_data_files(self, cursor): def _write_local_schema_file(self, cursor): """ - Takes a cursor, and writes the BigQuery schema for the results to a - local file system. + Takes a cursor, and writes the BigQuery schema for the results to a local file system. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that @@ -295,9 +292,9 @@ def convert_array_types(self, value: list[Any] | SortedSet) -> list[Any]: def convert_user_type(self, value: Any) -> dict[str, Any]: """ - Converts a user type to RECORD that contains n fields, where n is the - number of attributes. Each element in the user type class will be converted to its - corresponding data type in BQ. + Converts a user type to RECORD that contains n fields, where n is the number of attributes. + + Each element in the user type class will be converted to its corresponding data type in BQ. """ names = value._fields values = [self.convert_value(getattr(value, name)) for name in names] @@ -305,17 +302,20 @@ def convert_user_type(self, value: Any) -> dict[str, Any]: def convert_tuple_type(self, values: tuple[Any]) -> dict[str, Any]: """ - Converts a tuple to RECORD that contains n fields, each will be converted - to its corresponding data type in bq and will be named 'field_', where - index is determined by the order of the tuple elements defined in cassandra. + Converts a tuple to RECORD that contains n fields. + + Each field will be converted to its corresponding data type in bq and + will be named 'field_', where index is determined by the order + of the tuple elements defined in cassandra. """ names = ["field_" + str(i) for i in range(len(values))] return self.generate_data_dict(names, values) def convert_map_type(self, value: OrderedMapSerializedKey) -> list[dict[str, Any]]: """ - Converts a map to a repeated RECORD that contains two fields: 'key' and 'value', - each will be converted to its corresponding data type in BQ. + Converts a map to a repeated RECORD that contains two fields: 'key' and 'value'. + + Each will be converted to its corresponding data type in BQ. """ converted_map = [] for k, v in zip(value.keys(), value.values()): diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 46c5adc95b88..7b8b99e8b4b2 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -443,8 +443,8 @@ def execute(self, context: Context): def execute_complete(self, context: Context, event: dict[str, Any]): """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event["status"] == "error": raise AirflowException(event["message"]) @@ -692,6 +692,7 @@ def _validate_src_fmt_configs( ) -> dict: """ Validates the given src_fmt_configs against a valid configuration for the source format. + Adds the backward compatibility config to the src_fmt_configs. :param source_format: File format to export. diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py index 222a1ecad7f8..f9433d7434c6 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py @@ -312,10 +312,11 @@ def _ignore_existing_files(self, hook, prefix, **kwargs): def _copy_source_without_wildcard(self, hook, prefix): """ - For source_objects with no wildcard, this operator would first list - all files in source_objects, using provided delimiter if any. Then copy - files from source_objects to destination_object and rename each source - file. + List all files in source_objects, copy files to destination_object, and rename each source file. + + For source_objects with no wildcard, this operator would first list all + files in source_objects, using provided delimiter if any. Then copy files + from source_objects to destination_object and rename each source file. Example 1: diff --git a/airflow/providers/google/cloud/transfers/local_to_gcs.py b/airflow/providers/google/cloud/transfers/local_to_gcs.py index 26e70109bc74..6a287fa95a79 100644 --- a/airflow/providers/google/cloud/transfers/local_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/local_to_gcs.py @@ -31,8 +31,7 @@ class LocalFilesystemToGCSOperator(BaseOperator): """ - Uploads a file or list of files to Google Cloud Storage. - Optionally can compress the file for upload. + Uploads a file or list of files to Google Cloud Storage; optionally can compress the file for upload. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py b/airflow/providers/google/cloud/transfers/mssql_to_gcs.py index 152145a8cbd3..d9b728295545 100644 --- a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/mssql_to_gcs.py @@ -27,8 +27,8 @@ class MSSQLToGCSOperator(BaseSQLToGCSOperator): - """Copy data from Microsoft SQL Server to Google Cloud Storage - in JSON, CSV or Parquet format. + """ + Copy data from Microsoft SQL Server to Google Cloud Storage in JSON, CSV or Parquet format. :param bit_fields: Sequence of fields names of MSSQL "BIT" data type, to be interpreted in the schema as "BOOLEAN". "BIT" fields that won't @@ -99,8 +99,8 @@ def field_to_bigquery(self, field) -> dict[str, str]: @classmethod def convert_type(cls, value, schema_type, **kwargs): """ - Takes a value from MSSQL, and converts it to a value that's safe for - JSON/Google Cloud Storage/BigQuery. + Take a value from MSSQL and convert it to a value safe for JSON/Google Cloud Storage/BigQuery. + Datetime, Date and Time are converted to ISO formatted strings. """ if isinstance(value, decimal.Decimal): diff --git a/airflow/providers/google/cloud/transfers/mysql_to_gcs.py b/airflow/providers/google/cloud/transfers/mysql_to_gcs.py index 8f9e4dad4f84..cefde7c4ab07 100644 --- a/airflow/providers/google/cloud/transfers/mysql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/mysql_to_gcs.py @@ -94,8 +94,7 @@ def field_to_bigquery(self, field) -> dict[str, str]: def convert_type(self, value, schema_type: str, **kwargs): """ - Takes a value from MySQLdb, and converts it to a value that's safe for - JSON/Google Cloud Storage/BigQuery. + Take a value from MySQLdb and convert it to a value safe for JSON/Google Cloud Storage/BigQuery. * Datetimes are converted to `str(value)` (`datetime.isoformat(' ')`) strings. diff --git a/airflow/providers/google/cloud/transfers/oracle_to_gcs.py b/airflow/providers/google/cloud/transfers/oracle_to_gcs.py index fcf8458ef9ec..fc96357d5172 100644 --- a/airflow/providers/google/cloud/transfers/oracle_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/oracle_to_gcs.py @@ -87,8 +87,7 @@ def field_to_bigquery(self, field) -> dict[str, str]: def convert_type(self, value, schema_type, **kwargs): """ - Takes a value from Oracle db, and converts it to a value that's safe for - JSON/Google Cloud Storage/BigQuery. + Take a value from Oracle db and convert it to a value safe for JSON/Google Cloud Storage/BigQuery. * Datetimes are converted to UTC seconds. * Decimals are converted to floats. diff --git a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py index b7637db0a918..f83b47ea325c 100644 --- a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py @@ -130,8 +130,8 @@ def field_to_bigquery(self, field) -> dict[str, str]: def convert_type(self, value, schema_type, stringify_dict=True): """ - Takes a value from Postgres, and converts it to a value that's safe for - JSON/Google Cloud Storage/BigQuery. + Take a value from Postgres and convert it to a value safe for JSON/Google Cloud Storage/BigQuery. + Timezone aware Datetime are converted to UTC seconds. Unaware Datetime, Date and Time are converted to ISO formatted strings. Decimals are converted to floats. diff --git a/airflow/providers/google/cloud/transfers/presto_to_gcs.py b/airflow/providers/google/cloud/transfers/presto_to_gcs.py index 1433435c7529..b9385876de6e 100644 --- a/airflow/providers/google/cloud/transfers/presto_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/presto_to_gcs.py @@ -86,8 +86,10 @@ def execute(self, *args, **kwargs) -> PrestoResult: def executemany(self, *args, **kwargs): """ - Prepare a database operation (query or command) and then execute it against all parameter - sequences or mappings found in the sequence seq_of_parameters. + Prepare and execute a database operation. + + Prepare a database operation (query or command) and then execute it against + all parameter sequences or mappings found in the sequence seq_of_parameters. """ self.initialized = False self.rows = [] @@ -101,18 +103,16 @@ def peekone(self) -> Any: return element def fetchone(self) -> Any: - """ - Fetch the next row of a query result set, returning a single sequence, or - ``None`` when no more data is available. - """ + """Fetch the next row of a query result set, returning a single sequence, or ``None``.""" if self.rows: return self.rows.pop(0) return self.cursor.fetchone() def fetchmany(self, size=None) -> list: """ - Fetch the next set of rows of a query result, returning a sequence of sequences - (e.g. a list of tuples). An empty sequence is returned when no more rows are available. + Fetch the next set of rows of a query result, returning a sequence of sequences. + + An empty sequence is returned when no more rows are available. """ if size is None: size = self.cursor.arraysize @@ -128,8 +128,7 @@ def fetchmany(self, size=None) -> list: def __next__(self) -> Any: """ - Return the next row from the currently executing SQL statement using the same semantics as - ``.fetchone()``. + Return the next row from the current SQL statement using the same semantics as ``.fetchone()``. A ``StopIteration`` exception is raised when the result set is exhausted. """ diff --git a/airflow/providers/google/cloud/transfers/s3_to_gcs.py b/airflow/providers/google/cloud/transfers/s3_to_gcs.py index 39a2f2a68f56..b0dbdf90daae 100644 --- a/airflow/providers/google/cloud/transfers/s3_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/s3_to_gcs.py @@ -35,8 +35,7 @@ class S3ToGCSOperator(S3ListOperator): """ - Synchronizes an S3 key, possibly a prefix, with a Google Cloud Storage - destination path. + Synchronizes an S3 key, possibly a prefix, with a Google Cloud Storage destination path. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/cloud/transfers/sql_to_gcs.py b/airflow/providers/google/cloud/transfers/sql_to_gcs.py index ce283ff86617..e696487f65f0 100644 --- a/airflow/providers/google/cloud/transfers/sql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/sql_to_gcs.py @@ -371,9 +371,7 @@ def _get_file_mime_type(self): return file_mime_type def _configure_csv_file(self, file_handle, schema): - """Configure a csv writer with the file_handle and write schema - as headers for the new file. - """ + """Configure a csv writer with the file_handle and write schema as headers for the new file.""" csv_writer = csv.writer(file_handle, delimiter=self.field_delimiter) csv_writer.writerow(schema) return csv_writer @@ -438,9 +436,9 @@ def _get_col_type_dict(self): def _write_local_schema_file(self, cursor): """ - Takes a cursor, and writes the BigQuery schema for the results to a - local file system. Schema for database will be read from cursor if - not specified. + Takes a cursor, and writes the BigQuery schema for the results to a local file system. + + Schema for database will be read from cursor if not specified. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that diff --git a/airflow/providers/google/cloud/transfers/trino_to_gcs.py b/airflow/providers/google/cloud/transfers/trino_to_gcs.py index 60dbecb4b73b..138f69acf1c0 100644 --- a/airflow/providers/google/cloud/transfers/trino_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/trino_to_gcs.py @@ -86,8 +86,10 @@ def execute(self, *args, **kwargs) -> TrinoResult: def executemany(self, *args, **kwargs): """ - Prepare a database operation (query or command) and then execute it against all parameter - sequences or mappings found in the sequence seq_of_parameters. + Prepare and execute a database query. + + Prepare a database operation (query or command) and then execute it against + all parameter sequences or mappings found in the sequence seq_of_parameters. """ self.initialized = False self.rows = [] @@ -101,18 +103,16 @@ def peekone(self) -> Any: return element def fetchone(self) -> Any: - """ - Fetch the next row of a query result set, returning a single sequence, or - ``None`` when no more data is available. - """ + """Fetch the next row of a query result set, returning a single sequence, or ``None``.""" if self.rows: return self.rows.pop(0) return self.cursor.fetchone() def fetchmany(self, size=None) -> list: """ - Fetch the next set of rows of a query result, returning a sequence of sequences - (e.g. a list of tuples). An empty sequence is returned when no more rows are available. + Fetch the next set of rows of a query result, returning a sequence of sequences. + + An empty sequence is returned when no more rows are available. """ if size is None: size = self.cursor.arraysize @@ -128,8 +128,7 @@ def fetchmany(self, size=None) -> list: def __next__(self) -> Any: """ - Return the next row from the currently executing SQL statement using the same semantics as - ``.fetchone()``. + Return the next row from the current SQL statement using the same semantics as ``.fetchone()``. A ``StopIteration`` exception is raised when the result set is exhausted. """ diff --git a/airflow/providers/google/cloud/triggers/bigquery.py b/airflow/providers/google/cloud/triggers/bigquery.py index bb3973c338e8..b1c659a845a1 100644 --- a/airflow/providers/google/cloud/triggers/bigquery.py +++ b/airflow/providers/google/cloud/triggers/bigquery.py @@ -521,8 +521,7 @@ async def _table_exists( self, hook: BigQueryTableAsyncHook, dataset: str, table_id: str, project_id: str ) -> bool: """ - Create client session and make call to BigQueryTableAsyncHook and check for the table in - Google Big Query. + Create session, make call to BigQueryTableAsyncHook, and check for the table in Google Big Query. :param hook: BigQueryTableAsyncHook Hook class :param dataset: The name of the dataset in which to look for the table storage bucket. diff --git a/airflow/providers/google/cloud/triggers/bigquery_dts.py b/airflow/providers/google/cloud/triggers/bigquery_dts.py index 95e6aed39711..def8b90b66a3 100644 --- a/airflow/providers/google/cloud/triggers/bigquery_dts.py +++ b/airflow/providers/google/cloud/triggers/bigquery_dts.py @@ -27,7 +27,9 @@ class BigQueryDataTransferRunTrigger(BaseTrigger): - """Triggers class to watch the Transfer Run state to define when the job is done. + """ + Triggers class to watch the Transfer Run state to define when the job is done. + :param project_id: The BigQuery project id where the transfer configuration should be :param config_id: ID of the config of the Transfer Run which should be watched. :param run_id: ID of the Transfer Run which should be watched. @@ -35,13 +37,13 @@ class BigQueryDataTransferRunTrigger(BaseTrigger): :param gcp_conn_id: The connection ID used to connect to Google Cloud. :param location: BigQuery Transfer Service location for regional transfers. :param impersonation_chain: Optional service account to impersonate using short-term - credentials, or chained list of accounts required to get the access_token - of the last account in the list, which will be impersonated in the request. - If set as a string, the account must grant the originating account - the Service Account Token Creator IAM role. - If set as a sequence, the identities from the list must grant - Service Account Token Creator IAM role to the directly preceding identity, with first - account from the list granting this role to the originating account (templated). + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). """ def __init__( @@ -79,10 +81,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """ - Get Transfer Run status and if it one of the statuses which mean end of the job - then yield TriggerEvent object. - """ + """If the Transfer Run is in a terminal state, then yield TriggerEvent object.""" hook = self._get_async_hook() while True: try: diff --git a/airflow/providers/google/cloud/triggers/cloud_sql.py b/airflow/providers/google/cloud/triggers/cloud_sql.py index 7d2cd5a323c6..e04ada9277fc 100644 --- a/airflow/providers/google/cloud/triggers/cloud_sql.py +++ b/airflow/providers/google/cloud/triggers/cloud_sql.py @@ -28,6 +28,7 @@ class CloudSQLExportTrigger(BaseTrigger): """ Trigger that periodically polls information from Cloud SQL API to verify job status. + Implementation leverages asynchronous transport. """ diff --git a/airflow/providers/google/cloud/triggers/dataproc.py b/airflow/providers/google/cloud/triggers/dataproc.py index 6f2be8512469..3f94c4996506 100644 --- a/airflow/providers/google/cloud/triggers/dataproc.py +++ b/airflow/providers/google/cloud/triggers/dataproc.py @@ -289,6 +289,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: class DataprocWorkflowTrigger(DataprocBaseTrigger): """ Trigger that periodically polls information from Dataproc API to verify status. + Implementation leverages asynchronous transport. """ diff --git a/airflow/providers/google/cloud/triggers/gcs.py b/airflow/providers/google/cloud/triggers/gcs.py index 9595cc6ca89d..8b7f3abe783a 100644 --- a/airflow/providers/google/cloud/triggers/gcs.py +++ b/airflow/providers/google/cloud/triggers/gcs.py @@ -209,6 +209,7 @@ async def _is_blob_updated_after( class GCSPrefixBlobTrigger(GCSBlobTrigger): """ Looks for objects in bucket matching a prefix. + If none found, sleep for interval and check again. Otherwise, return matches. :param bucket: the bucket in the google cloud storage where the objects are residing. @@ -287,14 +288,10 @@ async def _list_blobs_with_prefix(self, hook: GCSAsyncHook, bucket_name: str, pr class GCSUploadSessionTrigger(GCSPrefixBlobTrigger): """ - Checks for changes in the number of objects at prefix in Google Cloud Storage - bucket and returns Trigger Event if the inactivity period has passed with no - increase in the number of objects. - - :param bucket: The Google Cloud Storage bucket where the objects are. - expected. - :param prefix: The name of the prefix to check in the Google cloud - storage bucket. + Return Trigger Event if the inactivity period has passed with no increase in the number of objects. + + :param bucket: The Google Cloud Storage bucket where the objects are expected. + :param prefix: The name of the prefix to check in the Google cloud storage bucket. :param poke_interval: polling period in seconds to check :param inactivity_period: The total seconds of inactivity to designate an upload session is over. Note, this mechanism is not real time and @@ -354,10 +351,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """ - Simple loop until no change in any new files or deleted in list blob is - found for the inactivity_period. - """ + """Loop until no new files or deleted files in list blob for the inactivity_period.""" try: hook = self._get_async_hook() while True: @@ -373,16 +367,12 @@ async def run(self) -> AsyncIterator[TriggerEvent]: yield TriggerEvent({"status": "error", "message": str(e)}) def _get_time(self) -> datetime: - """ - This is just a wrapper of datetime.datetime.now to simplify mocking in the - unittests. - """ + """This is just a wrapper of datetime.datetime.now to simplify mocking in the unittests.""" return datetime.now() def _is_bucket_updated(self, current_objects: set[str]) -> dict[str, str]: """ - Checks whether new objects have been uploaded and the inactivity_period - has passed and updates the state of the sensor accordingly. + Check whether new objects have been uploaded and the inactivity_period has passed; update the state. :param current_objects: set of object ids in bucket during last check. """ diff --git a/airflow/providers/google/cloud/utils/bigquery.py b/airflow/providers/google/cloud/utils/bigquery.py index e96e10a77030..76dc9637bef6 100644 --- a/airflow/providers/google/cloud/utils/bigquery.py +++ b/airflow/providers/google/cloud/utils/bigquery.py @@ -22,6 +22,7 @@ def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str: """ Helper method that casts a BigQuery row to the appropriate data types. + This is useful because BigQuery returns all fields as strings. """ if string_field is None: @@ -40,7 +41,8 @@ def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str: def convert_job_id(job_id: str | list[str], project_id: str, location: str | None) -> Any: """ - Helper method that converts to path: project_id:location:job_id + Helper method that converts to path: project_id:location:job_id. + :param project_id: Required. The ID of the Google Cloud project where workspace located. :param location: Optional. The ID of the Google Cloud region where workspace located. :param job_id: Required. The ID of the job. diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/airflow/providers/google/cloud/utils/credentials_provider.py index a5063cd738dc..b6d7bcc385d8 100644 --- a/airflow/providers/google/cloud/utils/credentials_provider.py +++ b/airflow/providers/google/cloud/utils/credentials_provider.py @@ -15,10 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains a mechanism for providing temporary -Google Cloud authentication. -""" +"""This module contains a mechanism for providing temporary Google Cloud authentication.""" + from __future__ import annotations import json @@ -52,8 +50,7 @@ def build_gcp_conn( project_id: str | None = None, ) -> str: """ - Builds a uri that can be used as :envvar:`AIRFLOW_CONN_{CONN_ID}` with provided service key, - scopes and project id. + Build a uri that can be used as :envvar:`AIRFLOW_CONN_{CONN_ID}` with provided values. :param key_file_path: Path to service key. :param scopes: Required OAuth scopes. @@ -80,8 +77,10 @@ def provide_gcp_credentials( key_file_dict: dict | None = None, ) -> Generator[None, None, None]: """ - Context manager that provides a Google Cloud credentials for application supporting - `Application Default Credentials (ADC) strategy`__. + Context manager that provides Google Cloud credentials for Application Default Credentials (ADC). + + .. seealso:: + `Application Default Credentials (ADC) strategy`__. It can be used to provide credentials for external programs (e.g. gcloud) that expect authorization file in ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable. @@ -117,9 +116,9 @@ def provide_gcp_connection( project_id: str | None = None, ) -> Generator[None, None, None]: """ - Context manager that provides a temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` - connection. It build a new connection that includes path to provided service json, - required scopes and project id. + Context manager that provides a temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` connection. + + It builds a new connection that includes path to provided service json, required scopes and project id. :param key_file_path: Path to file with Google Cloud Service Account .json file. :param scopes: OAuth scopes for the connection @@ -365,8 +364,7 @@ def get_credentials_and_project_id(*args, **kwargs) -> tuple[google.auth.credent def _get_scopes(scopes: str | None = None) -> Sequence[str]: """ - Parse a comma-separated string containing OAuth2 scopes if `scopes` is provided. - Otherwise, default scope will be returned. + Parse a comma-separated string containing OAuth2 scopes if `scopes` is provided; otherwise return default. :param scopes: A comma-separated string containing OAuth2 scopes :return: Returns the scope defined in the connection configuration, or the default scope @@ -378,6 +376,8 @@ def _get_target_principal_and_delegates( impersonation_chain: str | Sequence[str] | None = None, ) -> tuple[str | None, Sequence[str] | None]: """ + Get the target_principal and optional list of delegates from impersonation_chain. + Analyze contents of impersonation_chain and return target_principal (the service account to directly impersonate using short-term credentials, if any) and optional list of delegates required to get the access_token of target_principal. diff --git a/airflow/providers/google/cloud/utils/dataform.py b/airflow/providers/google/cloud/utils/dataform.py index d47b8228a1a0..e183e29a1a75 100644 --- a/airflow/providers/google/cloud/utils/dataform.py +++ b/airflow/providers/google/cloud/utils/dataform.py @@ -45,6 +45,7 @@ def make_initialization_workspace_flow( ) -> tuple: """ Creates flow which simulates the initialization of the default project. + :param project_id: Required. The ID of the Google Cloud project where workspace located. :param region: Required. The ID of the Google Cloud region where workspace located. :param repository_id: Required. The ID of the Dataform repository where workspace located. diff --git a/airflow/providers/google/cloud/utils/field_sanitizer.py b/airflow/providers/google/cloud/utils/field_sanitizer.py index de43e8c89498..ef2150824c18 100644 --- a/airflow/providers/google/cloud/utils/field_sanitizer.py +++ b/airflow/providers/google/cloud/utils/field_sanitizer.py @@ -103,9 +103,7 @@ class GcpFieldSanitizerException(AirflowException): - """Thrown when sanitizer finds unexpected field type in the path - (other than dict or array). - """ + """Thrown when sanitizer finds unexpected field type in the path (other than dict or array).""" class GcpBodyFieldSanitizer(LoggingMixin): diff --git a/airflow/providers/google/cloud/utils/field_validator.py b/airflow/providers/google/cloud/utils/field_validator.py index 499cb0f7e33c..1e853d44ed88 100644 --- a/airflow/providers/google/cloud/utils/field_validator.py +++ b/airflow/providers/google/cloud/utils/field_validator.py @@ -413,10 +413,10 @@ def _validate_field(self, validation_spec, dictionary_to_validate, parent=None, def validate(self, body_to_validate: dict) -> None: """ - Validates if the body (dictionary) follows specification that the validator was - instantiated with. Raises ValidationSpecificationException or - ValidationFieldException in case of problems with specification or the - body not conforming to the specification respectively. + Validates if the body (dictionary) follows specification that the validator was instantiated with. + + Raises ValidationSpecificationException or ValidationFieldException in case of problems + with specification or the body not conforming to the specification respectively. :param body_to_validate: body that must follow the specification :return: None diff --git a/airflow/providers/google/common/hooks/base_google.py b/airflow/providers/google/common/hooks/base_google.py index 3ee344a03a88..30287c27049f 100644 --- a/airflow/providers/google/common/hooks/base_google.py +++ b/airflow/providers/google/common/hooks/base_google.py @@ -93,8 +93,7 @@ def is_soft_quota_exception(exception: Exception): def is_operation_in_progress_exception(exception: Exception) -> bool: """ - Some of the calls return 429 (too many requests!) or 409 errors (Conflict) - in case of operation in progress. + Some calls return 429 (too many requests!) or 409 errors (Conflict) in case of operation in progress. * Google Cloud SQL """ @@ -142,11 +141,12 @@ def get_field(extras: dict, field_name: str): class GoogleBaseHook(BaseHook): """ - A base hook for Google cloud-related hooks. Google cloud has a shared REST - API client that is built in the same way no matter which service you use. - This class helps construct and authorize the credentials needed to then - call googleapiclient.discovery.build() to actually discover and build a client - for a Google cloud service. + A base hook for Google cloud-related hooks. + + Google cloud has a shared REST API client that is built in the same way no matter + which service you use. This class helps construct and authorize the credentials + needed to then call googleapiclient.discovery.build() to actually discover and + build a client for a Google cloud service. The class also contains some miscellaneous helper functions. @@ -326,10 +326,7 @@ def _get_credentials_email(self) -> str: return oauth2_client.tokeninfo().execute()["email"] def _authorize(self) -> google_auth_httplib2.AuthorizedHttp: - """ - Returns an authorized HTTP object to be used to build a Google cloud - service hook connection. - """ + """Returns an authorized HTTP object to be used to build a Google cloud service hook connection.""" credentials = self.get_credentials() http = build_http() http = set_user_agent(http, "airflow/" + version.version) @@ -338,10 +335,11 @@ def _authorize(self) -> google_auth_httplib2.AuthorizedHttp: def _get_field(self, f: str, default: Any = None) -> Any: """ - Fetches a field from extras, and returns it. This is some Airflow - magic. The google_cloud_platform hook type adds custom UI elements - to the hook page, which allow admins to specify service_account, - key_path, etc. They get formatted as shown below. + Fetches a field from extras, and returns it. + + This is some Airflow magic. The google_cloud_platform hook type adds + custom UI elements to the hook page, which allow admins to specify + service_account, key_path, etc. They get formatted as shown below. """ return hasattr(self, "extras") and get_field(self.extras, f) or default @@ -407,10 +405,7 @@ def scopes(self) -> Sequence[str]: @staticmethod def quota_retry(*args, **kwargs) -> Callable: - """ - A decorator that provides a mechanism to repeat requests in response to exceeding a temporary quote - limit. - """ + """Provides a mechanism to repeat requests in response to exceeding a temporary quota limit.""" def decorator(fun: Callable): default_kwargs = { @@ -426,11 +421,7 @@ def decorator(fun: Callable): @staticmethod def operation_in_progress_retry(*args, **kwargs) -> Callable[[T], T]: - """ - A decorator that provides a mechanism to repeat requests in response to - operation in progress (HTTP 409) - limit. - """ + """Provides a mechanism to repeat requests in response to operation in progress (HTTP 409) limit.""" def decorator(fun: T): default_kwargs = { @@ -447,8 +438,9 @@ def decorator(fun: T): @staticmethod def fallback_to_default_project_id(func: Callable[..., RT]) -> Callable[..., RT]: """ - Decorator that provides fallback for Google Cloud project id. If - the project is None it will be replaced with the project_id from the + Decorator that provides fallback for Google Cloud project id. + + If the project is None it will be replaced with the project_id from the service account the Hook is authenticated with. Project id can be specified either via project_id kwarg or via first parameter in positional args. @@ -479,12 +471,11 @@ def inner_wrapper(self: GoogleBaseHook, *args, **kwargs) -> RT: @staticmethod def provide_gcp_credential_file(func: T) -> T: """ - Function decorator that provides a Google Cloud credentials for application supporting Application - Default Credentials (ADC) strategy. + Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support. - It is recommended to use ``provide_gcp_credential_file_as_context`` context manager to limit the - scope when authorization data is available. Using context manager also - makes it easier to use multiple connection in one function. + It is recommended to use ``provide_gcp_credential_file_as_context`` context + manager to limit the scope when authorization data is available. Using context + manager also makes it easier to use multiple connection in one function. """ @functools.wraps(func) @@ -497,8 +488,11 @@ def wrapper(self: GoogleBaseHook, *args, **kwargs): @contextmanager def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]: """ - Context manager that provides a Google Cloud credentials for application supporting `Application - Default Credentials (ADC) strategy `__. + Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support. + + See: + `Application Default Credentials (ADC) + strategy `__. It can be used to provide credentials for external programs (e.g. gcloud) that expect authorization file in ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable. @@ -588,10 +582,10 @@ def provide_authorized_gcloud(self) -> Generator[None, None, None]: def download_content_from_request(file_handle, request: dict, chunk_size: int) -> None: """ Download media resources. - Note that the Python file object is compatible with io.Base and can be used with this class also. - :param file_handle: io.Base or file object. The stream in which to write the downloaded - bytes. + Note that the Python file object is compatible with io.Base and can be used with this class also. + + :param file_handle: io.Base or file object. The stream in which to write the downloaded bytes. :param request: googleapiclient.http.HttpRequest, the media request to perform in chunks. :param chunk_size: int, File will be downloaded in chunks of this many bytes. """ @@ -628,10 +622,7 @@ def __init__(self, **kwargs: Any): self._sync_hook = None async def get_sync_hook(self) -> Any: - """ - Sync version of the Google Cloud Hooks makes blocking calls in ``__init__`` so we don't inherit - from it. - """ + """Sync version of the Google Cloud Hook makes blocking calls in ``__init__``; don't inherit it.""" if not self._sync_hook: self._sync_hook = await sync_to_async(self.sync_hook_class)(**self._hook_kwargs) return self._sync_hook diff --git a/airflow/providers/google/common/hooks/discovery_api.py b/airflow/providers/google/common/hooks/discovery_api.py index 13daf7ccf7ce..da0b9f9ce494 100644 --- a/airflow/providers/google/common/hooks/discovery_api.py +++ b/airflow/providers/google/common/hooks/discovery_api.py @@ -82,8 +82,7 @@ def get_conn(self) -> Resource: def query(self, endpoint: str, data: dict, paginate: bool = False, num_retries: int = 0) -> dict: """ - Creates a dynamic API call to any Google API registered in Google's API Client Library - and queries it. + Creates a dynamic API call to any Google API registered in Google's API Client Library and queries it. :param endpoint: The client libraries path to the api call's executing method. For example: 'analyticsreporting.reports.batchGet' diff --git a/airflow/providers/google/firebase/hooks/firestore.py b/airflow/providers/google/firebase/hooks/firestore.py index 93fc154e997a..58c981182973 100644 --- a/airflow/providers/google/firebase/hooks/firestore.py +++ b/airflow/providers/google/firebase/hooks/firestore.py @@ -110,8 +110,7 @@ def export_documents( def _wait_for_operation_to_complete(self, operation_name: str) -> None: """ - Waits for the named operation to complete - checks status of the - asynchronous call. + Waits for the named operation to complete - checks status of the asynchronous call. :param operation_name: The name of the operation. :return: The response returned by the operation. diff --git a/airflow/providers/google/firebase/operators/firestore.py b/airflow/providers/google/firebase/operators/firestore.py index af7ec700bc31..d14242e6b046 100644 --- a/airflow/providers/google/firebase/operators/firestore.py +++ b/airflow/providers/google/firebase/operators/firestore.py @@ -28,8 +28,7 @@ class CloudFirestoreExportDatabaseOperator(BaseOperator): """ - Exports a copy of all or a subset of documents from Google Cloud Firestore to another storage system, - such as Google Cloud Storage. + Export documents from Google Cloud Firestore to another storage system, such as Google Cloud Storage. .. seealso:: For more information on how to use this operator, take a look at the guide: diff --git a/airflow/providers/google/marketing_platform/operators/analytics.py b/airflow/providers/google/marketing_platform/operators/analytics.py index 2996838f12c2..0098980e9bee 100644 --- a/airflow/providers/google/marketing_platform/operators/analytics.py +++ b/airflow/providers/google/marketing_platform/operators/analytics.py @@ -385,8 +385,9 @@ def execute(self, context: Context) -> None: class GoogleAnalyticsModifyFileHeadersDataImportOperator(BaseOperator): """ - GA has a very particular naming convention for Data Import. Ability to - prefix "ga:" to all column headers and also a dict to rename columns to + GA has a very particular naming convention for Data Import. + + Ability to prefix "ga:" to all column headers and also a dict to rename columns to match the custom dimension ID in GA i.e clientId : dimensionX. :param storage_bucket: The Google cloud storage bucket where the file is stored. diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py index c1e796258f53..9220b6b3b59d 100644 --- a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py +++ b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py @@ -35,8 +35,7 @@ class GCSToGoogleDriveOperator(BaseOperator): """ - Copies objects from a Google Cloud Storage service to a Google Drive service, with renaming - if requested. + Copies objects from a Google Cloud Storage service to a Google Drive service, with renaming if requested. Using this operator requires the following OAuth 2.0 scope: From c99c03135ca7e7c41e1c6d338de9e41422ea84f0 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Sat, 8 Jul 2023 14:14:36 -0700 Subject: [PATCH 110/533] D205 Support - Providers: Stragglers and new additions (#32447) This completes the updates to all provider packages. --- airflow/providers/amazon/aws/hooks/s3.py | 3 +-- airflow/providers/amazon/aws/sensors/s3.py | 4 ++-- airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py | 3 ++- airflow/providers/cncf/kubernetes/utils/pod_manager.py | 6 ++++-- .../microsoft/azure/transfers/azure_blob_to_gcs.py | 4 +++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index 364aeafb81e6..ae007d38f38b 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -646,8 +646,7 @@ async def is_keys_unchanged_async( last_activity_time: datetime | None = None, ) -> dict[str, Any]: """ - Checks whether new objects have been uploaded and the inactivity_period - has passed and updates the state of the sensor accordingly. + Check if new objects have been uploaded and the period has passed; update sensor state accordingly. :param client: aiobotocore client :param bucket_name: the name of the bucket diff --git a/airflow/providers/amazon/aws/sensors/s3.py b/airflow/providers/amazon/aws/sensors/s3.py index 4d15cdc2125d..c245ab374447 100644 --- a/airflow/providers/amazon/aws/sensors/s3.py +++ b/airflow/providers/amazon/aws/sensors/s3.py @@ -356,8 +356,8 @@ def execute(self, context: Context) -> None: def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: """ Callback for when the trigger fires - returns immediately. - Relies on trigger to throw an exception, otherwise it assumes execution was - successful. + + Relies on trigger to throw an exception, otherwise it assumes execution was successful. """ if event and event["status"] == "error": raise AirflowException(event["message"]) diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py index e2d2c6ed7e69..a41ccc2dae10 100644 --- a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py @@ -84,7 +84,8 @@ class DynamoDBToS3Operator(AwsToAwsBaseOperator): :param dynamodb_table_name: Dynamodb table to replicate data from :param s3_bucket_name: S3 bucket to replicate data to :param file_size: Flush file to s3 if file size >= file_size - :param dynamodb_scan_kwargs: kwargs pass to + :param dynamodb_scan_kwargs: kwargs pass to + :param s3_key_prefix: Prefix of s3 object key :param process_func: How we transform a dynamodb item to bytes. By default, we dump the json :param export_time: Time in the past from which to export table data, counted in seconds from the start of diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 979180f1d92f..acd40d4b1755 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -129,6 +129,7 @@ def container_is_running(pod: V1Pod, container_name: str) -> bool: def container_is_completed(pod: V1Pod, container_name: str) -> bool: """ Examines V1Pod ``pod`` to determine whether ``container_name`` is completed. + If that container is present and completed, returns True. Returns False otherwise. """ container_status = get_container_status(pod, container_name) @@ -430,8 +431,9 @@ def fetch_requested_container_logs( self, pod: V1Pod, container_logs: Iterable[str] | str | Literal[True], follow_logs=False ) -> list[PodLoggingStatus]: """ - Follow the logs of containers in the pod specified by input parameter and publish - it to airflow logging. Returns when all the containers exit. + Follow the logs of containers in the specified pod and publish it to airflow logging. + + Returns when all the containers exit. """ pod_logging_statuses = [] all_containers = self.get_container_names(pod) diff --git a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py index 20edde6e0938..dd05ec90f5d1 100644 --- a/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py +++ b/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py @@ -28,7 +28,9 @@ class AzureBlobStorageToGCSOperator(AzureBlobStorageToGCSOperatorFromGoogleProvider): """ This class is deprecated. - Please use `airflow.providers.google.cloud.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator`. + + Please use + :class:`airflow.providers.google.cloud.transfers.azure_blob_to_gcs.AzureBlobStorageToGCSOperator`. """ def __init__(self, *args, **kwargs): From f12233c123d70f4c59c490bed556ca40c8fcd4c8 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sun, 9 Jul 2023 03:45:39 +0530 Subject: [PATCH 111/533] Add Dag Processor status to cluster activity page. (#32446) * Add Dag Processor status to cluster activity page. * Update tests. --- .../www/static/js/cluster-activity/index.test.tsx | 14 ++++++++++++-- .../js/cluster-activity/live-metrics/Health.tsx | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/airflow/www/static/js/cluster-activity/index.test.tsx b/airflow/www/static/js/cluster-activity/index.test.tsx index 13f6d4234434..c35a8bb41b60 100644 --- a/airflow/www/static/js/cluster-activity/index.test.tsx +++ b/airflow/www/static/js/cluster-activity/index.test.tsx @@ -59,12 +59,21 @@ const mockHistoricalMetricsData = { }, }; +const heartbeat = "2023-05-19T12:00:36.109924+00:00"; const mockHealthData = { metadatabase: { status: "healthy", }, scheduler: { - latest_scheduler_heartbeat: "2023-05-19T12:00:36.109924+00:00", + latestSchedulerHeartbeat: heartbeat, + status: "healthy", + }, + triggerer: { + latestTriggererHeartbeat: heartbeat, + status: "healthy", + }, + dagProcessor: { + latestDagProcessorHeartbeat: heartbeat, status: "healthy", }, }; @@ -148,7 +157,8 @@ describe("Test ToggleGroups", () => { expect(getAllByTestId("echart-container")).toHaveLength(4); - expect(getAllByText("healthy")).toHaveLength(2); + expect(getAllByText("healthy")).toHaveLength(4); + expect(getAllByText("last heartbeat:")).toHaveLength(3); expect(getByText("Unpaused DAGs")).toBeInTheDocument(); expect(getByText("No dag running")).toBeInTheDocument(); }); diff --git a/airflow/www/static/js/cluster-activity/live-metrics/Health.tsx b/airflow/www/static/js/cluster-activity/live-metrics/Health.tsx index fb59439c4b75..49bafaeac042 100644 --- a/airflow/www/static/js/cluster-activity/live-metrics/Health.tsx +++ b/airflow/www/static/js/cluster-activity/live-metrics/Health.tsx @@ -106,6 +106,11 @@ const Health = (props: CenterProps) => { status={data?.triggerer?.status} latestHeartbeat={data?.triggerer?.latestTriggererHeartbeat} /> + From d97bb4117fbbb5b9090711481988225ff6521ac2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Jul 2023 00:17:32 +0200 Subject: [PATCH 112/533] Bump tough-cookie from 4.0.0 to 4.1.3 in /airflow/www (#32443) Bumps [tough-cookie](https://github.com/salesforce/tough-cookie) from 4.0.0 to 4.1.3. - [Release notes](https://github.com/salesforce/tough-cookie/releases) - [Changelog](https://github.com/salesforce/tough-cookie/blob/master/CHANGELOG.md) - [Commits](https://github.com/salesforce/tough-cookie/compare/v4.0.0...v4.1.3) --- updated-dependencies: - dependency-name: tough-cookie dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- airflow/www/yarn.lock | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 1916d9c9001a..a202168a77e1 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -9964,6 +9964,11 @@ pure-color@^1.2.0: resolved "https://registry.yarnpkg.com/pure-color/-/pure-color-1.3.0.tgz#1fe064fb0ac851f0de61320a8bf796836422f33e" integrity sha512-QFADYnsVoBMw1srW7OVKEYjG+MbIa49s54w1MA1EDY6r2r/sTcKKYqRX1f4GYvnXP7eN/Pe9HFcX+hwzmrXRHA== +querystringify@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" + integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -10444,6 +10449,11 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -11399,13 +11409,14 @@ toggle-selection@^1.0.6: integrity sha1-bkWxJj8gF/oKzH2J14sVuL932jI= tough-cookie@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" - integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + version "4.1.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" + integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== dependencies: psl "^1.1.33" punycode "^2.1.1" - universalify "^0.1.2" + universalify "^0.2.0" + url-parse "^1.5.3" tr46@^2.1.0: version "2.1.0" @@ -11712,10 +11723,10 @@ unist-util-visit@^4.0.0: unist-util-is "^5.0.0" unist-util-visit-parents "^5.1.1" -universalify@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== +universalify@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" + integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== unload@2.2.0: version "2.2.0" @@ -11741,6 +11752,14 @@ url-loader@4.1.0: mime-types "^2.1.26" schema-utils "^2.6.5" +url-parse@^1.5.3: + version "1.5.10" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + url-search-params-polyfill@^8.1.0: version "8.1.1" resolved "https://registry.yarnpkg.com/url-search-params-polyfill/-/url-search-params-polyfill-8.1.1.tgz#9e69e4dba300a71ae7ad3cead62c7717fd99329f" From 8690d3fe98865c5385cb43393c049f85306df614 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Jul 2023 00:30:20 +0200 Subject: [PATCH 113/533] Bump stylelint from 13.13.1 to 15.10.1 in /airflow/www (#32435) Bumps [stylelint](https://github.com/stylelint/stylelint) from 13.13.1 to 15.10.1. - [Release notes](https://github.com/stylelint/stylelint/releases) - [Changelog](https://github.com/stylelint/stylelint/blob/main/CHANGELOG.md) - [Commits](https://github.com/stylelint/stylelint/compare/13.13.1...15.10.1) --- updated-dependencies: - dependency-name: stylelint dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- airflow/www/package.json | 2 +- airflow/www/yarn.lock | 915 ++++++++++++--------------------------- 2 files changed, 285 insertions(+), 632 deletions(-) diff --git a/airflow/www/package.json b/airflow/www/package.json index c5a9c69e6c1a..6ee89d5c83bd 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -74,7 +74,7 @@ "openapi-typescript": "^5.4.1", "prettier": "^2.8.4", "style-loader": "^1.2.1", - "stylelint": "^13.6.1", + "stylelint": "^15.10.1", "stylelint-config-prettier": "^9.0.5", "stylelint-config-standard": "^20.0.0", "terser-webpack-plugin": "<5.0.0", diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index a202168a77e1..1d154dfc6231 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -74,7 +74,7 @@ semver "^6.3.0" source-map "^0.5.0" -"@babel/core@^7.17.9", "@babel/core@^7.18.5": +"@babel/core@^7.18.5": version "7.18.5" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.18.5.tgz#c597fa680e58d571c28dda9827669c78cdd7f000" integrity sha512-MGY8vg3DxMnctw0LdvSEojOsumc70g0t18gNyUdAZqB1Rpd1Bqo/svHGvt+UJ6JcGX+DIekGFDxxIWofBxLCnQ== @@ -2241,6 +2241,26 @@ resolved "https://registry.yarnpkg.com/@chakra-ui/visually-hidden/-/visually-hidden-2.0.13.tgz#6553467d93f206d17716bcbe6e895a84eef87472" integrity sha512-sDEeeEjLfID333EC46NdCbhK2HyMXlpl5HzcJjuwWIpyVz4E1gKQ9hlwpq6grijvmzeSywQ5D3tTwUrvZck4KQ== +"@csstools/css-parser-algorithms@^2.3.0": + version "2.3.0" + resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-2.3.0.tgz#0cc3a656dc2d638370ecf6f98358973bfbd00141" + integrity sha512-dTKSIHHWc0zPvcS5cqGP+/TPFUJB0ekJ9dGKvMAFoNuBFhDPBt9OMGNZiIA5vTiNdGHHBeScYPXIGBMnVOahsA== + +"@csstools/css-tokenizer@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-2.1.1.tgz#07ae11a0a06365d7ec686549db7b729bc036528e" + integrity sha512-GbrTj2Z8MCTUv+52GE0RbFGM527xuXZ0Xa5g0Z+YN573uveS4G0qi6WNOMyz3yrFM/jaILTTwJ0+umx81EzqfA== + +"@csstools/media-query-list-parser@^2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@csstools/media-query-list-parser/-/media-query-list-parser-2.1.2.tgz#6ef642b728d30c1009bfbba3211c7e4c11302728" + integrity sha512-M8cFGGwl866o6++vIY7j1AKuq9v57cf+dGepScwCcbut9ypJNr4Cj+LLTWligYUZ0uyhEoJDKt5lvyBfh2L3ZQ== + +"@csstools/selector-specificity@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@csstools/selector-specificity/-/selector-specificity-3.0.0.tgz#798622546b63847e82389e473fd67f2707d82247" + integrity sha512-hBI9tfBtuPIi885ZsZ32IMEU/5nlZH/KOVYJCOh7gyMxaVLGmLedYqFN6Ui1LXkI8JlC8IsuC0rF0btcRZKd5g== + "@discoveryjs/json-ext@^0.5.0": version "0.5.7" resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" @@ -2840,21 +2860,6 @@ dependencies: "@sinonjs/commons" "^1.7.0" -"@stylelint/postcss-css-in-js@^0.37.2": - version "0.37.3" - resolved "https://registry.yarnpkg.com/@stylelint/postcss-css-in-js/-/postcss-css-in-js-0.37.3.tgz#d149a385e07ae365b0107314c084cb6c11adbf49" - integrity sha512-scLk3cSH1H9KggSniseb2KNAU5D9FWc3H7BxCSAIdtU9OWIyw0zkEZ9qEKHryRM+SExYXRKNb7tOOVNAsQ3iwg== - dependencies: - "@babel/core" "^7.17.9" - -"@stylelint/postcss-markdown@^0.36.2": - version "0.36.2" - resolved "https://registry.yarnpkg.com/@stylelint/postcss-markdown/-/postcss-markdown-0.36.2.tgz#0a540c4692f8dcdfc13c8e352c17e7bfee2bb391" - integrity sha512-2kGbqUVJUGE8dM+bMzXG/PYUWKkjLIkRLWNh39OaADkiabDRdw8ATFCgbMz5xdIcvwspPAluSL7uY+ZiTWdWmQ== - dependencies: - remark "^13.0.0" - unist-util-find-all-after "^3.0.2" - "@testing-library/dom@^8.5.0": version "8.13.0" resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-8.13.0.tgz#bc00bdd64c7d8b40841e27a70211399ad3af46f5" @@ -3332,10 +3337,10 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.4.tgz#f0ec25dbf2f0e4b18647313ac031134ca5b24b21" integrity sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA== -"@types/minimist@^1.2.0": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.1.tgz#283f669ff76d7b8260df8ab7a4262cc83d988256" - integrity sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg== +"@types/minimist@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" + integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== "@types/ms@*": version "0.7.31" @@ -3457,7 +3462,7 @@ dependencies: source-map "^0.6.1" -"@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2": +"@types/unist@*", "@types/unist@^2.0.0": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== @@ -3889,7 +3894,7 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.21.3" -ansi-regex@^5.0.0, ansi-regex@^5.0.1: +ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== @@ -4071,19 +4076,6 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== -autoprefixer@^9.8.6: - version "9.8.8" - resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.8.tgz#fd4bd4595385fa6f06599de749a4d5f7a474957a" - integrity sha512-eM9d/swFopRt5gdJ7jrpCwgvEMIayITpojhkkSMRsFHYuH5bkSQ4p/9qTEHtmNudUZh22Tehu7I6CxAW0IXTKA== - dependencies: - browserslist "^4.12.0" - caniuse-lite "^1.0.30001109" - normalize-range "^0.1.2" - num2fraction "^1.2.2" - picocolors "^0.2.1" - postcss "^7.0.32" - postcss-value-parser "^4.1.0" - available-typed-arrays@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" @@ -4247,11 +4239,6 @@ babel-preset-jest@^27.5.1: babel-plugin-jest-hoist "^27.5.1" babel-preset-current-node-syntax "^1.0.0" -bail@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.5.tgz#b6fa133404a392cbc1f8c4bf63f5953351e7a776" - integrity sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ== - bail@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/bail/-/bail-2.0.2.tgz#d26f5cd8fe5d6f832a31517b9f7c356040ba6d5d" @@ -4312,7 +4299,7 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.1: +braces@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -4338,7 +4325,7 @@ browser-process-hrtime@^1.0.0: resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== -browserslist@^4.0.0, browserslist@^4.12.0, browserslist@^4.14.5, browserslist@^4.16.6, browserslist@^4.17.5, browserslist@^4.17.6, browserslist@^4.20.2, browserslist@^4.20.3: +browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.6, browserslist@^4.17.5, browserslist@^4.17.6, browserslist@^4.20.2, browserslist@^4.20.3: version "4.20.4" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.20.4.tgz#98096c9042af689ee1e0271333dbc564b8ce4477" integrity sha512-ok1d+1WpnU24XYN7oC3QWgTyMhY/avPJ/r9T00xxvUOIparA/gc+UPUMaod3i+G6s+nI2nUb9xZ5k794uIwShw== @@ -4409,15 +4396,6 @@ callsites@^3.0.0: resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -camelcase-keys@^6.2.2: - version "6.2.2" - resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" - integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg== - dependencies: - camelcase "^5.3.1" - map-obj "^4.0.0" - quick-lru "^4.0.1" - camelcase-keys@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-7.0.0.tgz#40fcbe171f7432888369d0c871df7cfa5ce4f788" @@ -4448,7 +4426,7 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001349: +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001349: version "1.0.30001439" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001439.tgz" integrity sha512-1MgUzEkoMO6gKfXflStpYgZDlFM7M/ck/bgfVCACO5vnAf0fXoNVHdWtqGU+MYca+4bL9Z5bpOVmR33cWW9G2A== @@ -4490,14 +4468,6 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.1.1: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - chalk@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.0.1.tgz#ca57d71e82bb534a296df63bbacc4a1c22b2a4b6" @@ -4597,13 +4567,6 @@ clone-deep@^4.0.1: kind-of "^6.0.2" shallow-clone "^3.0.0" -clone-regexp@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clone-regexp/-/clone-regexp-2.2.0.tgz#7d65e00885cd8796405c35a737e7a86b7429e36f" - integrity sha512-beMpP7BOtTipFuW8hrJvREQ2DrRu3BE7by0ZpibtfBA+qfHYvMGTc2Yb1JMYPKg/JUw0CHYvpg796aNTSW9z7Q== - dependencies: - is-regexp "^2.0.0" - clsx@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" @@ -4669,10 +4632,10 @@ color@^4.2.3: color-convert "^2.0.1" color-string "^1.9.0" -colord@^2.9.1: - version "2.9.2" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.2.tgz#25e2bacbbaa65991422c07ea209e2089428effb1" - integrity sha512-Uqbg+J445nc1TKn4FoDPS6ZZqAvEDnwrH42yo8B40JSOgSLxMZ/gt3h4nmCtPLQeXhjJJkqBx7SCY35WnIixaQ== +colord@^2.9.1, colord@^2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.3.tgz#4f8ce919de456f1d5c1c368c307fe20f3e59fb43" + integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw== colorette@^1.2.0: version "1.2.2" @@ -4805,16 +4768,15 @@ cosmiconfig@^6.0.0: path-type "^4.0.0" yaml "^1.7.2" -cosmiconfig@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" - integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA== +cosmiconfig@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.2.0.tgz#f7d17c56a590856cd1e7cee98734dca272b0d8fd" + integrity sha512-3rTMnFJA1tCOPwRxtgF4wd7Ab2qvDbL8jX+3smjIbS4HlZBagTlpERbdN7iAbWlrfxE3M8c27kTwTawQ7st+OQ== dependencies: - "@types/parse-json" "^4.0.0" import-fresh "^3.2.1" + js-yaml "^4.1.0" parse-json "^5.0.0" path-type "^4.0.0" - yaml "^1.10.0" cross-fetch@^3.1.5: version "3.1.5" @@ -4844,6 +4806,11 @@ css-declaration-sorter@^6.2.2: resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.3.0.tgz#72ebd995c8f4532ff0036631f7365cce9759df14" integrity sha512-OGT677UGHJTAVMRhPO+HJ4oKln3wkBTwtDFH0ojbqm+MJm6xuDMHp2nkhh/ThaBqq20IbraBQSWKfSLNHQO9Og== +css-functions-list@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/css-functions-list/-/css-functions-list-3.1.0.tgz#cf5b09f835ad91a00e5959bcfc627cd498e1321b" + integrity sha512-/9lCvYZaUbBGvYUgYGFJ4dcYiyqdhSjG7IPVluoV8A1ILjkF7ilmhp1OGUz8n+nmBcu0RNrQAzgD8B6FJbrt2w== + css-loader@5.2.7: version "5.2.7" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-5.2.7.tgz#9b9f111edf6fb2be5dc62525644cbc9c232064ae" @@ -4891,6 +4858,14 @@ css-tree@^1.1.2, css-tree@^1.1.3: mdn-data "2.0.14" source-map "^0.6.1" +css-tree@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-2.3.1.tgz#10264ce1e5442e8572fc82fbe490644ff54b5c20" + integrity sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw== + dependencies: + mdn-data "2.0.30" + source-map-js "^1.0.1" + css-what@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" @@ -5429,10 +5404,10 @@ date-now@^0.1.4: resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" integrity sha1-6vQ5/U1ISK105cx9vvIAZyueNFs= -debug@4: - version "4.3.2" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" - integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== +debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" @@ -5443,20 +5418,6 @@ debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.0.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - -debug@^4.1.0, debug@^4.1.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== - dependencies: - ms "2.1.2" - decamelize-keys@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9" @@ -5465,11 +5426,16 @@ decamelize-keys@^1.1.0: decamelize "^1.1.0" map-obj "^1.0.0" -decamelize@^1.1.0, decamelize@^1.2.0: +decamelize@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= +decamelize@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-5.0.1.tgz#db11a92e58c741ef339fb0a2868d8a06a9a7b1e9" + integrity sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA== + decimal.js@^10.2.1: version "10.3.1" resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" @@ -5626,7 +5592,7 @@ dom-serializer@^1.0.1: domhandler "^4.2.0" entities "^2.0.0" -domelementtype@1, domelementtype@^1.3.1: +domelementtype@1: version "1.3.1" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== @@ -5650,13 +5616,6 @@ domhandler@2.3: dependencies: domelementtype "1" -domhandler@^2.3.0: - version "2.4.2" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803" - integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA== - dependencies: - domelementtype "1" - domhandler@^4.0.0, domhandler@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.0.tgz#f9768a5f034be60a89a27c2e4d0f74eba0d8b059" @@ -5684,14 +5643,6 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -domutils@^1.5.1: - version "1.7.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" - integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg== - dependencies: - dom-serializer "0" - domelementtype "1" - domutils@^2.5.2: version "2.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" @@ -5761,11 +5712,6 @@ entities@1.0: resolved "https://registry.yarnpkg.com/entities/-/entities-1.0.0.tgz#b2987aa3821347fcde642b24fdfc9e4fb712bf26" integrity sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY= -entities@^1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" - integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w== - entities@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" @@ -6263,13 +6209,6 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" -execall@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/execall/-/execall-2.0.0.tgz#16a06b5fe5099df7d00be5d9c06eecded1663b45" - integrity sha512-0FU2hZ5Hh6iQnarpRtQurM/aAvp3RIbfvgLHrcqJYzhXyV2KFruhuChf9NC6waAhiUR7FFtlugkI4p7f2Fqlow== - dependencies: - clone-regexp "^2.1.0" - exit@0.1.2, exit@0.1.x, exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -6295,10 +6234,10 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-glob@^3.2.4, fast-glob@^3.2.5, fast-glob@^3.2.9: - version "3.2.11" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9" - integrity sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew== +fast-glob@^3.2.4, fast-glob@^3.2.9, fast-glob@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.0.tgz#7c40cb491e1e2ed5664749e87bfb516dbe8727c0" + integrity sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" @@ -6321,10 +6260,10 @@ fast-safe-stringify@^2.0.7: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== -fastest-levenshtein@^1.0.12: - version "1.0.12" - resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz#9990f7d3a88cc5a9ffd1f1745745251700d497e2" - integrity sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow== +fastest-levenshtein@^1.0.12, fastest-levenshtein@^1.0.16: + version "1.0.16" + resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" + integrity sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg== fastq@^1.6.0: version "1.11.0" @@ -6425,6 +6364,14 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + flat-cache@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" @@ -6598,11 +6545,6 @@ get-package-type@^0.1.0: resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== -get-stdin@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-8.0.0.tgz#cbad6a73feb75f6eeb22ba9e01f89aa28aa97a53" - integrity sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg== - get-stream@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" @@ -6699,7 +6641,7 @@ globalyzer@0.1.0: resolved "https://registry.yarnpkg.com/globalyzer/-/globalyzer-0.1.0.tgz#cb76da79555669a1519d5a8edf093afaa0bf1465" integrity sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q== -globby@^11.0.1, globby@^11.0.3, globby@^11.1.0: +globby@^11.0.1, globby@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== @@ -6732,13 +6674,6 @@ globrex@^0.1.2: resolved "https://registry.yarnpkg.com/globrex/-/globrex-0.1.2.tgz#dd5d9ec826232730cd6793a5e33a9302985e6098" integrity sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg== -gonzales-pe@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/gonzales-pe/-/gonzales-pe-4.3.0.tgz#fe9dec5f3c557eead09ff868c65826be54d067b3" - integrity sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ== - dependencies: - minimist "^1.2.5" - gopd@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" @@ -6864,11 +6799,6 @@ hoist-non-react-statics@^3.3.1: dependencies: react-is "^16.7.0" -hosted-git-info@^2.1.4: - version "2.8.9" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" - integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== - hosted-git-info@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.0.2.tgz#5e425507eede4fea846b7262f0838456c4209961" @@ -6888,10 +6818,10 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== -html-tags@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.2.0.tgz#dbb3518d20b726524e4dd43de397eb0a95726961" - integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== +html-tags@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.3.1.tgz#a04026a18c882e4bba8a01a3d39cfe465d40b5ce" + integrity sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ== htmlparser2@3.8.x: version "3.8.3" @@ -6904,18 +6834,6 @@ htmlparser2@3.8.x: entities "1.0" readable-stream "1.1" -htmlparser2@^3.10.0: - version "3.10.1" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" - integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ== - dependencies: - domelementtype "^1.3.1" - domhandler "^2.3.0" - domutils "^1.5.1" - entities "^1.1.1" - inherits "^2.0.1" - readable-stream "^3.1.1" - htmlparser2@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7" @@ -6965,15 +6883,10 @@ icss-utils@^5.0.0, icss-utils@^5.1.0: resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== -ignore@^5.1.1: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== - -ignore@^5.1.8, ignore@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" - integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== +ignore@^5.1.1, ignore@^5.2.0, ignore@^5.2.4: + version "5.2.4" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" + integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== import-fresh@^3.0.0, import-fresh@^3.1.0, import-fresh@^3.2.1: version "3.3.0" @@ -7016,6 +6929,11 @@ indent-string@^4.0.0: resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== +indent-string@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5" + integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg== + infer-owner@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" @@ -7029,7 +6947,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1: +inherits@2, inherits@^2.0.4, inherits@~2.0.1: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -7157,6 +7075,13 @@ is-core-module@^2.2.0: dependencies: has "^1.0.3" +is-core-module@^2.5.0: + version "2.12.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.12.1.tgz#0c0b6885b6f80011c71541ce15c8d66cf5a4f9fd" + integrity sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg== + dependencies: + has "^1.0.3" + is-core-module@^2.8.1: version "2.9.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69" @@ -7252,11 +7177,6 @@ is-plain-obj@^1.1.0: resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4= -is-plain-obj@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" - integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== - is-plain-obj@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-4.1.0.tgz#d65025edec3657ce032fd7db63c97883eaed71f0" @@ -7269,6 +7189,11 @@ is-plain-object@^2.0.4: dependencies: isobject "^3.0.1" +is-plain-object@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" + integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== + is-potential-custom-element-name@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" @@ -7290,11 +7215,6 @@ is-regex@^1.1.4: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-regexp@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-2.1.0.tgz#cd734a56864e23b956bf4e7c66c396a4c0b22c2d" - integrity sha512-OZ4IlER3zmRIoB9AqNhEggVxqIH4ofDns5nRrPS6yQxXE1TPCUpFznBfRQmQa8uC+pXqjMnukiJBxCisIxiLGA== - is-shared-array-buffer@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" @@ -7342,11 +7262,6 @@ is-typedarray@^1.0.0: resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= -is-unicode-supported@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" - integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== - is-weakref@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2" @@ -8068,10 +7983,10 @@ kleur@^4.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780" integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ== -known-css-properties@^0.21.0: - version "0.21.0" - resolved "https://registry.yarnpkg.com/known-css-properties/-/known-css-properties-0.21.0.tgz#15fbd0bbb83447f3ce09d8af247ed47c68ede80d" - integrity sha512-sZLUnTqimCkvkgRS+kbPlYW5o8q5w1cu+uIisKpEWkj31I8mx8kNG162DwRav8Zirkva6N5uoFsm9kzK4mUXjw== +known-css-properties@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/known-css-properties/-/known-css-properties-0.27.0.tgz#82a9358dda5fe7f7bd12b5e7142c0a205393c0c5" + integrity sha512-uMCj6+hZYDoffuvAJjFAPz56E9uoowFHmTkqRtRq5WyC5Q6Cu/fTZKNQpX/RbzChBYLLl3lo8CjFZBAZXq9qFg== language-subtag-registry@~0.3.2: version "0.3.21" @@ -8137,6 +8052,13 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + lodash.curry@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.curry/-/lodash.curry-4.1.1.tgz#248e36072ede906501d75966200a86dab8b23170" @@ -8197,19 +8119,6 @@ lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0, lodash@~4.17.2 resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-symbols@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" - integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== - dependencies: - chalk "^4.1.0" - is-unicode-supported "^0.1.0" - -longest-streak@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-2.0.4.tgz#b8599957da5b5dab64dee3fe316fa774597d90e4" - integrity sha512-vM6rUVCVUJJt33bnmHiZEvr7wPT78ztX7rojL+LW51bHtLh6HTjx84LA5W4+oa6aKEJA7jJu5LR6vQRBpA5DVg== - longest-streak@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4" @@ -8266,11 +8175,6 @@ map-obj@^1.0.0: resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= -map-obj@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.2.1.tgz#e4ea399dbc979ae735c83c863dd31bdf364277b7" - integrity sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ== - map-obj@^4.1.0: version "4.3.0" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a" @@ -8322,17 +8226,6 @@ mdast-util-find-and-replace@^2.0.0: unist-util-is "^5.0.0" unist-util-visit-parents "^5.0.0" -mdast-util-from-markdown@^0.8.0: - version "0.8.5" - resolved "https://registry.yarnpkg.com/mdast-util-from-markdown/-/mdast-util-from-markdown-0.8.5.tgz#d1ef2ca42bc377ecb0463a987910dae89bd9a28c" - integrity sha512-2hkTXtYYnr+NubD/g6KGBS/0mFmBcifAsI0yIWRiRo0PjVs6SSOSOdtzbp6kSGnShDN6G5aWZpKQ2lWRy27mWQ== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-to-string "^2.0.0" - micromark "~2.11.0" - parse-entities "^2.0.0" - unist-util-stringify-position "^2.0.0" - mdast-util-from-markdown@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mdast-util-from-markdown/-/mdast-util-from-markdown-1.2.0.tgz#84df2924ccc6c995dec1e2368b2b208ad0a76268" @@ -8424,18 +8317,6 @@ mdast-util-to-hast@^12.1.0: unist-util-position "^4.0.0" unist-util-visit "^4.0.0" -mdast-util-to-markdown@^0.6.0: - version "0.6.5" - resolved "https://registry.yarnpkg.com/mdast-util-to-markdown/-/mdast-util-to-markdown-0.6.5.tgz#b33f67ca820d69e6cc527a93d4039249b504bebe" - integrity sha512-XeV9sDE7ZlOQvs45C9UKMtfTcctcaj/pGwH8YLbMHoMOXNNCn2LsqVQOqrF1+/NU8lKDAqozme9SCXWyo9oAcQ== - dependencies: - "@types/unist" "^2.0.0" - longest-streak "^2.0.0" - mdast-util-to-string "^2.0.0" - parse-entities "^2.0.0" - repeat-string "^1.0.0" - zwitch "^1.0.0" - mdast-util-to-markdown@^1.0.0, mdast-util-to-markdown@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/mdast-util-to-markdown/-/mdast-util-to-markdown-1.3.0.tgz#38b6cdc8dc417de642a469c4fc2abdf8c931bd1e" @@ -8449,11 +8330,6 @@ mdast-util-to-markdown@^1.0.0, mdast-util-to-markdown@^1.3.0: unist-util-visit "^4.0.0" zwitch "^2.0.0" -mdast-util-to-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz#b8cfe6a713e1091cb5b728fc48885a4767f8b97b" - integrity sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w== - mdast-util-to-string@^3.0.0, mdast-util-to-string@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-3.1.0.tgz#56c506d065fbf769515235e577b5a261552d56e9" @@ -8464,28 +8340,33 @@ mdn-data@2.0.14: resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== +mdn-data@2.0.30: + version "2.0.30" + resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.30.tgz#ce4df6f80af6cfbe218ecd5c552ba13c4dfa08cc" + integrity sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA== + memoize-one@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== -meow@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364" - integrity sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ== +meow@^10.1.5: + version "10.1.5" + resolved "https://registry.yarnpkg.com/meow/-/meow-10.1.5.tgz#be52a1d87b5f5698602b0f32875ee5940904aa7f" + integrity sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw== dependencies: - "@types/minimist" "^1.2.0" - camelcase-keys "^6.2.2" - decamelize "^1.2.0" + "@types/minimist" "^1.2.2" + camelcase-keys "^7.0.0" + decamelize "^5.0.0" decamelize-keys "^1.1.0" hard-rejection "^2.1.0" minimist-options "4.1.0" - normalize-package-data "^3.0.0" - read-pkg-up "^7.0.1" - redent "^3.0.0" - trim-newlines "^3.0.0" - type-fest "^0.18.0" - yargs-parser "^20.2.3" + normalize-package-data "^3.0.2" + read-pkg-up "^8.0.0" + redent "^4.0.0" + trim-newlines "^4.0.2" + type-fest "^1.2.2" + yargs-parser "^20.2.9" merge-stream@^2.0.0: version "2.0.0" @@ -8772,21 +8653,13 @@ micromark@^3.0.0: micromark-util-types "^1.0.1" uvu "^0.5.0" -micromark@~2.11.0: - version "2.11.4" - resolved "https://registry.yarnpkg.com/micromark/-/micromark-2.11.4.tgz#d13436138eea826383e822449c9a5c50ee44665a" - integrity sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA== - dependencies: - debug "^4.0.0" - parse-entities "^2.0.0" - -micromatch@^4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" - integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== +micromatch@^4.0.4, micromatch@^4.0.5: + version "4.0.5" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" + integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== dependencies: - braces "^3.0.1" - picomatch "^2.2.3" + braces "^3.0.2" + picomatch "^2.3.1" microseconds@0.2.0: version "0.2.0" @@ -8815,7 +8688,7 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -min-indent@^1.0.0: +min-indent@^1.0.0, min-indent@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== @@ -8859,7 +8732,7 @@ minimist-options@4.1.0: is-plain-obj "^1.1.0" kind-of "^6.0.3" -minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.6: version "1.2.7" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== @@ -8972,10 +8845,10 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" -nanoid@^3.3.4: - version "3.3.4" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== +nanoid@^3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" + integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== natural-compare@^1.4.0: version "1.4.0" @@ -9042,23 +8915,13 @@ node-releases@^2.0.5: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.5.tgz#280ed5bc3eba0d96ce44897d8aee478bfb3d9666" integrity sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q== -normalize-package-data@^2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" - integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== - dependencies: - hosted-git-info "^2.1.4" - resolve "^1.10.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -normalize-package-data@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.2.tgz#cae5c410ae2434f9a6c1baa65d5bc3b9366c8699" - integrity sha512-6CdZocmfGaKnIHPVFhJJZ3GuR8SsLKvDANFp47Jmy51aKIr8akjAWTSxtpI+MBgBFdSMRyo4hMpDlT6dTffgZg== +normalize-package-data@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz#dbcc3e2da59509a0983422884cd172eefdfa525e" + integrity sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA== dependencies: hosted-git-info "^4.0.1" - resolve "^1.20.0" + is-core-module "^2.5.0" semver "^7.3.4" validate-npm-package-license "^3.0.1" @@ -9067,21 +8930,11 @@ normalize-path@^3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -normalize-range@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942" - integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== - normalize-registry-url@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/normalize-registry-url/-/normalize-registry-url-1.0.0.tgz#f75d2c48373da780c76f1f0eeb6382c06e784d13" integrity sha512-0v6T4851b72ykk5zEtFoN4QX/Fqyk7pouIj9xZyAvAe9jlDhAwT4z6FlwsoQCHjeuK2EGUoAwy/F4y4B1uZq9A== -normalize-selector@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/normalize-selector/-/normalize-selector-0.2.0.tgz#d0b145eb691189c63a78d201dc4fdb1293ef0c03" - integrity sha512-dxvWdI8gw6eAvk9BlPffgEoGfM7AdijoCwOEJge3e3ulT2XLgmU7KvvxprOaCu05Q1uGRHmOhHe1r6emZoKyFw== - normalize-url@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" @@ -9101,11 +8954,6 @@ nth-check@^2.0.1: dependencies: boolbase "^1.0.0" -num2fraction@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/num2fraction/-/num2fraction-1.2.2.tgz#6f682b6a027a4e9ddfa4564cd2589d1d4e669ede" - integrity sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg== - nvd3@^1.8.6: version "1.8.6" resolved "https://registry.yarnpkg.com/nvd3/-/nvd3-1.8.6.tgz#2d3eba74bf33363b5101ebf1d093c59a53ae73c4" @@ -9340,6 +9188,13 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" @@ -9431,11 +9286,6 @@ perfect-scrollbar@^1.5.1: resolved "https://registry.yarnpkg.com/perfect-scrollbar/-/perfect-scrollbar-1.5.1.tgz#8ee5b3ca06ce9c3f7338fd4ab67a55248a6cf3be" integrity sha512-MrSImINnIh3Tm1hdPT6bji6fmIeRorVEegQvyUnhqko2hDGTHhmjPefHXfxG/Jb8xVbfCwgmUIlIajERGXjVXQ== -picocolors@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" - integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== - picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -9446,6 +9296,11 @@ picomatch@^2.0.4, picomatch@^2.2.3: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== +picomatch@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + pify@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" @@ -9555,25 +9410,6 @@ postcss-discard-overridden@^5.1.0: resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz#7e8c5b53325747e9d90131bb88635282fb4a276e" integrity sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw== -postcss-html@^0.36.0: - version "0.36.0" - resolved "https://registry.yarnpkg.com/postcss-html/-/postcss-html-0.36.0.tgz#b40913f94eaacc2453fd30a1327ad6ee1f88b204" - integrity sha512-HeiOxGcuwID0AFsNAL0ox3mW6MHH5cstWN1Z3Y+n6H+g12ih7LHdYxWwEA/QmrebctLjo79xz9ouK3MroHwOJw== - dependencies: - htmlparser2 "^3.10.0" - -postcss-less@^3.1.4: - version "3.1.4" - resolved "https://registry.yarnpkg.com/postcss-less/-/postcss-less-3.1.4.tgz#369f58642b5928ef898ffbc1a6e93c958304c5ad" - integrity sha512-7TvleQWNM2QLcHqvudt3VYjULVB49uiW6XzEUFmvwHzvsOEF5MwBrIXZDJQvJNFGjJQTzSzZnDoCJ8h/ljyGXA== - dependencies: - postcss "^7.0.14" - -postcss-media-query-parser@^0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/postcss-media-query-parser/-/postcss-media-query-parser-0.2.3.tgz#27b39c6f4d94f81b1a73b8f76351c609e5cef244" - integrity sha1-J7Ocb02U+Bsac7j3Y1HGCeXO8kQ= - postcss-merge-longhand@^5.1.5: version "5.1.5" resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.1.5.tgz#b0e03bee3b964336f5f33c4fc8eacae608e91c05" @@ -9743,48 +9579,15 @@ postcss-resolve-nested-selector@^0.1.1: resolved "https://registry.yarnpkg.com/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.1.tgz#29ccbc7c37dedfac304e9fff0bf1596b3f6a0e4e" integrity sha1-Kcy8fDfe36wwTp//C/FZaz9qDk4= -postcss-safe-parser@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/postcss-safe-parser/-/postcss-safe-parser-4.0.2.tgz#a6d4e48f0f37d9f7c11b2a581bf00f8ba4870b96" - integrity sha512-Uw6ekxSWNLCPesSv/cmqf2bY/77z11O7jZGPax3ycZMFU/oi2DMH9i89AdHc1tRwFg/arFoEwX0IS3LCUxJh1g== - dependencies: - postcss "^7.0.26" - -postcss-sass@^0.4.4: - version "0.4.4" - resolved "https://registry.yarnpkg.com/postcss-sass/-/postcss-sass-0.4.4.tgz#91f0f3447b45ce373227a98b61f8d8f0785285a3" - integrity sha512-BYxnVYx4mQooOhr+zer0qWbSPYnarAy8ZT7hAQtbxtgVf8gy+LSLT/hHGe35h14/pZDTw1DsxdbrwxBN++H+fg== - dependencies: - gonzales-pe "^4.3.0" - postcss "^7.0.21" - -postcss-scss@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/postcss-scss/-/postcss-scss-2.1.1.tgz#ec3a75fa29a55e016b90bf3269026c53c1d2b383" - integrity sha512-jQmGnj0hSGLd9RscFw9LyuSVAa5Bl1/KBPqG1NQw9w8ND55nY4ZEsdlVuYJvLPpV+y0nwTV5v/4rHPzZRihQbA== - dependencies: - postcss "^7.0.6" - -postcss-selector-parser@^6.0.2: - version "6.0.11" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz#2e41dc39b7ad74046e1615185185cd0b17d0c8dc" - integrity sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5: - version "6.0.6" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.6.tgz#2c5bba8174ac2f6981ab631a42ab0ee54af332ea" - integrity sha512-9LXrvaaX3+mcv5xkg5kFwqSzSH1JIObIx51PrndZwlmznwXRfxMddDvo9gve3gVR8ZTKgoFDdWkbRFmEhT4PMg== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" +postcss-safe-parser@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/postcss-safe-parser/-/postcss-safe-parser-6.0.0.tgz#bb4c29894171a94bc5c996b9a30317ef402adaa1" + integrity sha512-FARHN8pwH+WiS2OPCxJI8FuRJpTVnn6ZNFiqAM2aeW2LwTHWWmWgIyKC6cUo0L8aeKiF/14MNvnpls6R2PBeMQ== -postcss-selector-parser@^6.0.9: - version "6.0.10" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz#79b61e2c0d1bfc2602d549e11d0876256f8df88d" - integrity sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w== +postcss-selector-parser@^6.0.13, postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.9: + version "6.0.13" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz#d05d8d76b1e8e173257ef9d60b706a8e5e99bf1b" + integrity sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ== dependencies: cssesc "^3.0.0" util-deprecate "^1.0.2" @@ -9797,11 +9600,6 @@ postcss-svgo@^5.1.0: postcss-value-parser "^4.2.0" svgo "^2.7.0" -postcss-syntax@^0.36.2: - version "0.36.2" - resolved "https://registry.yarnpkg.com/postcss-syntax/-/postcss-syntax-0.36.2.tgz#f08578c7d95834574e5593a82dfbfa8afae3b51c" - integrity sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w== - postcss-unique-selectors@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz#a9f273d1eacd09e9aa6088f4b0507b18b1b541b6" @@ -9809,39 +9607,17 @@ postcss-unique-selectors@^5.1.1: dependencies: postcss-selector-parser "^6.0.5" -postcss-value-parser@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb" - integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ== - -postcss-value-parser@^4.2.0: +postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514" integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== -postcss@^7.0.14, postcss@^7.0.2, postcss@^7.0.21, postcss@^7.0.26, postcss@^7.0.32, postcss@^7.0.35, postcss@^7.0.6: - version "7.0.39" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309" - integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== - dependencies: - picocolors "^0.2.1" - source-map "^0.6.1" - -postcss@^8.2.15: - version "8.4.20" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.20.tgz#64c52f509644cecad8567e949f4081d98349dc56" - integrity sha512-6Q04AXR1212bXr5fh03u8aAwbLxAQNGQ/Q1LNa0VfOI06ZAlhPHtQvE4OIdpj4kLThXilalPnmDSOD65DcHt+g== - dependencies: - nanoid "^3.3.4" - picocolors "^1.0.0" - source-map-js "^1.0.2" - -postcss@^8.4.13: - version "8.4.14" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.14.tgz#ee9274d5622b4858c1007a74d76e42e56fd21caf" - integrity sha512-E398TUmfAYFPBSdzgeieK2Y1+1cpdxJx8yXbK/m57nRhKSmk1GB2tO4lbLBtlkfPQTDKfe4Xqv1ASWPpayPEig== +postcss@^8.2.15, postcss@^8.4.13, postcss@^8.4.24: + version "8.4.25" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.25.tgz#4a133f5e379eda7f61e906c3b1aaa9b81292726f" + integrity sha512-7taJ/8t2av0Z+sQEvNzCkpDynl0tX3uJMCODi6nT3PfASC7dYCWV9aQ+uiCf+KBD4SEFcu+GvJdGdwzQ6OSjCw== dependencies: - nanoid "^3.3.4" + nanoid "^3.3.6" picocolors "^1.0.0" source-map-js "^1.0.2" @@ -9974,11 +9750,6 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== -quick-lru@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" - integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== - quick-lru@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" @@ -10215,24 +9986,24 @@ reactflow@^11.4.0: "@reactflow/minimap" "11.3.0" "@reactflow/node-toolbar" "1.1.0" -read-pkg-up@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507" - integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg== +read-pkg-up@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-8.0.0.tgz#72f595b65e66110f43b052dd9af4de6b10534670" + integrity sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ== dependencies: - find-up "^4.1.0" - read-pkg "^5.2.0" - type-fest "^0.8.1" + find-up "^5.0.0" + read-pkg "^6.0.0" + type-fest "^1.0.1" -read-pkg@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc" - integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg== +read-pkg@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-6.0.0.tgz#a67a7d6a1c2b0c3cd6aa2ea521f40c458a4a504c" + integrity sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q== dependencies: "@types/normalize-package-data" "^2.4.0" - normalize-package-data "^2.5.0" - parse-json "^5.0.0" - type-fest "^0.6.0" + normalize-package-data "^3.0.2" + parse-json "^5.2.0" + type-fest "^1.0.1" readable-stream@1.1: version "1.1.13" @@ -10244,15 +10015,6 @@ readable-stream@1.1: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^3.1.1: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - rechoir@^0.7.0: version "0.7.1" resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.1.tgz#9478a96a1ca135b5e88fc027f03ee92d6c645686" @@ -10268,6 +10030,14 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +redent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-4.0.0.tgz#0c0ba7caabb24257ab3bb7a4fd95dd1d5c5681f9" + integrity sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag== + dependencies: + indent-string "^5.0.0" + strip-indent "^4.0.0" + redoc@^2.0.0-rc.72: version "2.0.0-rc.72" resolved "https://registry.yarnpkg.com/redoc/-/redoc-2.0.0-rc.72.tgz#9eee22104d652b4a90e19ca50009b0b623a7b5b3" @@ -10396,13 +10166,6 @@ remark-parse@^10.0.0: mdast-util-from-markdown "^1.0.0" unified "^10.0.0" -remark-parse@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-9.0.0.tgz#4d20a299665880e4f4af5d90b7c7b8a935853640" - integrity sha512-geKatMwSzEXKHuzBNU1z676sGcDcFoChMK38TgdHJNAYfFtsfHDQG7MoJAjs6sgYMqyLduCYWDIWZIxiPeafEw== - dependencies: - mdast-util-from-markdown "^0.8.0" - remark-rehype@^10.0.0: version "10.1.0" resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-10.1.0.tgz#32dc99d2034c27ecaf2e0150d22a6dcccd9a6279" @@ -10413,32 +10176,11 @@ remark-rehype@^10.0.0: mdast-util-to-hast "^12.1.0" unified "^10.0.0" -remark-stringify@^9.0.0: - version "9.0.1" - resolved "https://registry.yarnpkg.com/remark-stringify/-/remark-stringify-9.0.1.tgz#576d06e910548b0a7191a71f27b33f1218862894" - integrity sha512-mWmNg3ZtESvZS8fv5PTvaPckdL4iNlCHTt8/e/8oN08nArHRHjNZMKzA/YW3+p7/lYqIw4nx1XsjCBo/AxNChg== - dependencies: - mdast-util-to-markdown "^0.6.0" - -remark@^13.0.0: - version "13.0.0" - resolved "https://registry.yarnpkg.com/remark/-/remark-13.0.0.tgz#d15d9bf71a402f40287ebe36067b66d54868e425" - integrity sha512-HDz1+IKGtOyWN+QgBiAT0kn+2s6ovOxHyPAFGKVE81VSzJ+mq7RwHFledEvB5F1p4iJvOah/LOKdFuzvRnNLCA== - dependencies: - remark-parse "^9.0.0" - remark-stringify "^9.0.0" - unified "^9.1.0" - remove-accents@0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5" integrity sha1-CkPTqq4egNuRngeuJUsoXZ4ce7U= -repeat-string@^1.0.0: - version "1.6.1" - resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== - require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -10476,7 +10218,7 @@ resolve.exports@^1.1.0: resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-1.1.0.tgz#5ce842b94b05146c0e03076985d1d0e7e48c90c9" integrity sha512-J1l+Zxxp4XK3LUDZ9m60LRJF/mAe4z6a4xyabPHk7pvK5t35dACV32iIjJDFeWZFfZlO29w6SZ67knR0tHzJtQ== -resolve@^1.10.0, resolve@^1.10.1, resolve@^1.12.0, resolve@^1.14.2, resolve@^1.20.0: +resolve@^1.10.1, resolve@^1.12.0, resolve@^1.14.2, resolve@^1.20.0: version "1.20.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== @@ -10548,7 +10290,7 @@ sade@^1.7.3: dependencies: mri "^1.1.0" -safe-buffer@^5.1.0, safe-buffer@~5.2.0: +safe-buffer@^5.1.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -10619,11 +10361,6 @@ schema-utils@^4.0.0: ajv-formats "^2.1.1" ajv-keywords "^5.0.0" -"semver@2 || 3 || 4 || 5": - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== - semver@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" @@ -10756,6 +10493,11 @@ signal-exit@^3.0.3: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.5.tgz#9e3e8cc0c75a99472b44321033a7702e7738252f" integrity sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ== +signal-exit@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.0.2.tgz#ff55bb1d9ff2114c13b400688fa544ac63c36967" + integrity sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q== + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -10792,7 +10534,7 @@ source-list-map@^2.0.0, source-list-map@^2.0.1: resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== -source-map-js@^1.0.2: +source-map-js@^1.0.1, source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== @@ -10871,11 +10613,6 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.9.tgz#8a595135def9592bda69709474f1cbeea7c2467f" integrity sha512-Ki212dKK4ogX+xDo4CtOZBVIwhsKBEfsEEcwmJfLQzirgc2jIWdzg40Unxz/HzEUqM1WFzVlQSMF9kZZ2HboLQ== -specificity@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/specificity/-/specificity-0.4.1.tgz#aab5e645012db08ba182e151165738d00887b019" - integrity sha512-1klA3Gi5PD1Wv9Q0wUoOQN1IWAuPu0D1U03ThXTr0cJ20+/iq2tHSDnK7Kk/0LXJ1ztUB2/1Os0wKmfyNgUQfg== - sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -10918,16 +10655,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -string-width@^4.1.0, string-width@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" - integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" - -string-width@^4.2.2, string-width@^4.2.3: +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -11011,26 +10739,12 @@ string.prototype.trimstart@^1.0.6: define-properties "^1.1.4" es-abstract "^1.20.4" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== - dependencies: - ansi-regex "^5.0.0" - -strip-ansi@^6.0.1: +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -11064,6 +10778,13 @@ strip-indent@^3.0.0: dependencies: min-indent "^1.0.0" +strip-indent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-4.0.0.tgz#b41379433dd06f5eae805e21d631e07ee670d853" + integrity sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA== + dependencies: + min-indent "^1.0.1" + strip-json-comments@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" @@ -11132,59 +10853,51 @@ stylelint-config-standard@^20.0.0: dependencies: stylelint-config-recommended "^3.0.0" -stylelint@^13.6.1: - version "13.13.1" - resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-13.13.1.tgz#fca9c9f5de7990ab26a00f167b8978f083a18f3c" - integrity sha512-Mv+BQr5XTUrKqAXmpqm6Ddli6Ief+AiPZkRsIrAoUKFuq/ElkUh9ZMYxXD0iQNZ5ADghZKLOWz1h7hTClB7zgQ== +stylelint@^15.10.1: + version "15.10.1" + resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-15.10.1.tgz#93f189958687e330c106b010cbec0c41dcae506d" + integrity sha512-CYkzYrCFfA/gnOR+u9kJ1PpzwG10WLVnoxHDuBA/JiwGqdM9+yx9+ou6SE/y9YHtfv1mcLo06fdadHTOx4gBZQ== dependencies: - "@stylelint/postcss-css-in-js" "^0.37.2" - "@stylelint/postcss-markdown" "^0.36.2" - autoprefixer "^9.8.6" + "@csstools/css-parser-algorithms" "^2.3.0" + "@csstools/css-tokenizer" "^2.1.1" + "@csstools/media-query-list-parser" "^2.1.2" + "@csstools/selector-specificity" "^3.0.0" balanced-match "^2.0.0" - chalk "^4.1.1" - cosmiconfig "^7.0.0" - debug "^4.3.1" - execall "^2.0.0" - fast-glob "^3.2.5" - fastest-levenshtein "^1.0.12" + colord "^2.9.3" + cosmiconfig "^8.2.0" + css-functions-list "^3.1.0" + css-tree "^2.3.1" + debug "^4.3.4" + fast-glob "^3.3.0" + fastest-levenshtein "^1.0.16" file-entry-cache "^6.0.1" - get-stdin "^8.0.0" global-modules "^2.0.0" - globby "^11.0.3" + globby "^11.1.0" globjoin "^0.1.4" - html-tags "^3.1.0" - ignore "^5.1.8" + html-tags "^3.3.1" + ignore "^5.2.4" import-lazy "^4.0.0" imurmurhash "^0.1.4" - known-css-properties "^0.21.0" - lodash "^4.17.21" - log-symbols "^4.1.0" + is-plain-object "^5.0.0" + known-css-properties "^0.27.0" mathml-tag-names "^2.1.3" - meow "^9.0.0" - micromatch "^4.0.4" - normalize-selector "^0.2.0" - postcss "^7.0.35" - postcss-html "^0.36.0" - postcss-less "^3.1.4" - postcss-media-query-parser "^0.2.3" + meow "^10.1.5" + micromatch "^4.0.5" + normalize-path "^3.0.0" + picocolors "^1.0.0" + postcss "^8.4.24" postcss-resolve-nested-selector "^0.1.1" - postcss-safe-parser "^4.0.2" - postcss-sass "^0.4.4" - postcss-scss "^2.1.1" - postcss-selector-parser "^6.0.5" - postcss-syntax "^0.36.2" - postcss-value-parser "^4.1.0" + postcss-safe-parser "^6.0.0" + postcss-selector-parser "^6.0.13" + postcss-value-parser "^4.2.0" resolve-from "^5.0.0" - slash "^3.0.0" - specificity "^0.4.1" - string-width "^4.2.2" - strip-ansi "^6.0.0" + string-width "^4.2.3" + strip-ansi "^6.0.1" style-search "^0.1.0" - sugarss "^2.0.0" + supports-hyperlinks "^3.0.0" svg-tags "^1.0.0" - table "^6.6.0" - v8-compile-cache "^2.3.0" - write-file-atomic "^3.0.3" + table "^6.8.1" + write-file-atomic "^5.0.1" stylis@4.0.13: version "4.0.13" @@ -11196,13 +10909,6 @@ stylis@^4.0.3: resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.0.10.tgz#446512d1097197ab3f02fb3c258358c3f7a14240" integrity sha512-m3k+dk7QeJw660eIKRRn3xPF6uuvHs/FFzjX3HQ5ove0qYsiygoAhwn5a3IYKaZPo5LrYD0rfVmtv1gNY1uYwg== -sugarss@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/sugarss/-/sugarss-2.0.0.tgz#ddd76e0124b297d40bf3cca31c8b22ecb43bc61d" - integrity sha512-WfxjozUk0UVA4jm+U1d736AUpzSrNsQcIbyOkoE364GrtWmIrFdk5lksEupgWMD4VaT/0kVx1dobpiDumSgmJQ== - dependencies: - postcss "^7.0.2" - supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" @@ -11232,6 +10938,14 @@ supports-hyperlinks@^2.0.0: has-flag "^4.0.0" supports-color "^7.0.0" +supports-hyperlinks@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-3.0.0.tgz#c711352a5c89070779b4dad54c05a2f14b15c94b" + integrity sha512-QBDPHyPQDRTy9ku4URNGY5Lah8PAaXs6tAAwp55sL5WCsSW7GIfdf6W5ixfziW+t7wh3GVvHyHHyQ1ESsoRvaA== + dependencies: + has-flag "^4.0.0" + supports-color "^7.0.0" + supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" @@ -11282,10 +10996,10 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -table@^6.6.0: - version "6.8.0" - resolved "https://registry.yarnpkg.com/table/-/table-6.8.0.tgz#87e28f14fa4321c3377ba286f07b79b281a3b3ca" - integrity sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA== +table@^6.8.1: + version "6.8.1" + resolved "https://registry.yarnpkg.com/table/-/table-6.8.1.tgz#ea2b71359fe03b017a5fbc296204471158080bdf" + integrity sha512-Y4X9zqrCftUhMeH2EptSSERdVKt/nEdijTOacGD/97EKjhQ/Qs8RTlEGABSJNNN8lac9kheH+af7yAkEWlgneA== dependencies: ajv "^8.0.1" lodash.truncate "^4.4.2" @@ -11435,15 +11149,10 @@ trim-lines@^3.0.0: resolved "https://registry.yarnpkg.com/trim-lines/-/trim-lines-3.0.1.tgz#d802e332a07df861c48802c04321017b1bd87338" integrity sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg== -trim-newlines@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" - integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== - -trough@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" - integrity sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA== +trim-newlines@^4.0.2: + version "4.1.1" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-4.1.1.tgz#28c88deb50ed10c7ba6dc2474421904a00139125" + integrity sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ== trough@^2.0.0: version "2.1.0" @@ -11506,11 +11215,6 @@ type-detect@4.0.8: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== -type-fest@^0.18.0: - version "0.18.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.18.1.tgz#db4bc151a4a2cf4eebf9add5db75508db6cc841f" - integrity sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw== - type-fest@^0.20.2: version "0.20.2" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" @@ -11521,17 +11225,7 @@ type-fest@^0.21.3: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== -type-fest@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" - integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== - -type-fest@^0.8.1: - version "0.8.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" - integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== - -type-fest@^1.2.1: +type-fest@^1.0.1, type-fest@^1.2.1, type-fest@^1.2.2: version "1.4.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1" integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA== @@ -11630,18 +11324,6 @@ unified@^10.0.0: trough "^2.0.0" vfile "^5.0.0" -unified@^9.1.0: - version "9.2.2" - resolved "https://registry.yarnpkg.com/unified/-/unified-9.2.2.tgz#67649a1abfc3ab85d2969502902775eb03146975" - integrity sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ== - dependencies: - bail "^1.0.0" - extend "^3.0.0" - is-buffer "^2.0.0" - is-plain-obj "^2.0.0" - trough "^1.0.0" - vfile "^4.0.0" - unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -11663,23 +11345,11 @@ unist-builder@^3.0.0: dependencies: "@types/unist" "^2.0.0" -unist-util-find-all-after@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/unist-util-find-all-after/-/unist-util-find-all-after-3.0.2.tgz#fdfecd14c5b7aea5e9ef38d5e0d5f774eeb561f6" - integrity sha512-xaTC/AGZ0rIM2gM28YVRAFPIZpzbpDtU3dRmp7EXlNVA8ziQc4hY3H7BHXM1J49nEmiqc3svnqMReW+PGqbZKQ== - dependencies: - unist-util-is "^4.0.0" - unist-util-generated@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/unist-util-generated/-/unist-util-generated-2.0.0.tgz#86fafb77eb6ce9bfa6b663c3f5ad4f8e56a60113" integrity sha512-TiWE6DVtVe7Ye2QxOVW9kqybs6cZexNwTwSMVgkfjEReqy/xwGpAXb99OxktoWwmL+Z+Epb0Dn8/GNDYP1wnUw== -unist-util-is@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-4.1.0.tgz#976e5f462a7a5de73d94b706bac1b90671b57797" - integrity sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg== - unist-util-is@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-5.1.1.tgz#e8aece0b102fa9bc097b0fef8f870c496d4a6236" @@ -11692,13 +11362,6 @@ unist-util-position@^4.0.0: dependencies: "@types/unist" "^2.0.0" -unist-util-stringify-position@^2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz#cce3bfa1cdf85ba7375d1d5b17bdc4cada9bd9da" - integrity sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g== - dependencies: - "@types/unist" "^2.0.2" - unist-util-stringify-position@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-3.0.2.tgz#5c6aa07c90b1deffd9153be170dce628a869a447" @@ -11807,7 +11470,7 @@ use-sync-external-store@1.2.0: resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== -util-deprecate@^1.0.1, util-deprecate@^1.0.2: +util-deprecate@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= @@ -11822,7 +11485,7 @@ uvu@^0.5.0: kleur "^4.0.3" sade "^1.7.3" -v8-compile-cache@^2.0.3, v8-compile-cache@^2.3.0: +v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== @@ -11849,14 +11512,6 @@ validator@^13.9.0: resolved "https://registry.yarnpkg.com/validator/-/validator-13.9.0.tgz#33e7b85b604f3bbce9bb1a05d5c3e22e1c2ff855" integrity sha512-B+dGG8U3fdtM0/aNK4/X8CXq/EcxU2WPrPEkJGslb47qyHsxmbggTWK0yEA4qnYVNF+nxNlN88o14hIcPmSIEA== -vfile-message@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-2.0.4.tgz#5b43b88171d409eae58477d13f23dd41d52c371a" - integrity sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ== - dependencies: - "@types/unist" "^2.0.0" - unist-util-stringify-position "^2.0.0" - vfile-message@^3.0.0: version "3.1.3" resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-3.1.3.tgz#1360c27a99234bebf7bddbbbca67807115e6b0dd" @@ -11865,16 +11520,6 @@ vfile-message@^3.0.0: "@types/unist" "^2.0.0" unist-util-stringify-position "^3.0.0" -vfile@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/vfile/-/vfile-4.2.1.tgz#03f1dce28fc625c625bc6514350fbdb00fa9e624" - integrity sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA== - dependencies: - "@types/unist" "^2.0.0" - is-buffer "^2.0.0" - unist-util-stringify-position "^2.0.0" - vfile-message "^2.0.0" - vfile@^5.0.0: version "5.3.6" resolved "https://registry.yarnpkg.com/vfile/-/vfile-5.3.6.tgz#61b2e70690cc835a5d0d0fd135beae74e5a39546" @@ -12121,7 +11766,7 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= -write-file-atomic@^3.0.0, write-file-atomic@^3.0.3: +write-file-atomic@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== @@ -12131,6 +11776,14 @@ write-file-atomic@^3.0.0, write-file-atomic@^3.0.3: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" +write-file-atomic@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-5.0.1.tgz#68df4717c55c6fa4281a7860b4c2ba0a6d2b11e7" + integrity sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw== + dependencies: + imurmurhash "^0.1.4" + signal-exit "^4.0.1" + ws@^7.4.6: version "7.5.5" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" @@ -12171,11 +11824,16 @@ yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== -yargs-parser@^20.2.2, yargs-parser@^20.2.3: +yargs-parser@^20.2.2: version "20.2.7" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.7.tgz#61df85c113edfb5a7a4e36eb8aa60ef423cbc90a" integrity sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw== +yargs-parser@^20.2.9: + version "20.2.9" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== + yargs-parser@^21.0.1: version "21.0.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.0.1.tgz#0267f286c877a4f0f728fceb6f8a3e4cb95c6e35" @@ -12226,11 +11884,6 @@ zustand@^4.1.1: dependencies: use-sync-external-store "1.2.0" -zwitch@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920" - integrity sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw== - zwitch@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7" From c6f151fa8f569687f3d889bce04bc270f114d208 Mon Sep 17 00:00:00 2001 From: Bartosz Jankiewicz Date: Sun, 9 Jul 2023 00:32:13 +0200 Subject: [PATCH 114/533] Fix an issue that crashes Airflow Webserver when passed invalid private key path to Snowflake (#32016) * Validate private key file path and size in Snowflake hook * Test update for snowflake hook * Code review fixes * Fix import Co-authored-by: xrmr <17521278+xrmr@users.noreply.github.com> * Fixes * Fix error message. --------- Co-authored-by: Tzu-ping Chung Co-authored-by: xrmr <17521278+xrmr@users.noreply.github.com> Co-authored-by: bjankiewicz --- airflow/providers/snowflake/hooks/snowflake.py | 7 ++++++- .../snowflake/hooks/test_snowflake.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index 2a202441a76c..59199cf8cd2e 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -248,7 +248,12 @@ def _get_conn_params(self) -> dict[str, str | None]: "Please remove one." ) elif private_key_file: - private_key_pem = Path(private_key_file).read_bytes() + private_key_file_path = Path(private_key_file) + if not private_key_file_path.is_file() or private_key_file_path.stat().st_size == 0: + raise ValueError("The private_key_file path points to an empty or invalid file.") + if private_key_file_path.stat().st_size > 4096: + raise ValueError("The private_key_file size is too big. Please keep it less than 4 KB.") + private_key_pem = Path(private_key_file_path).read_bytes() elif private_key_content: private_key_pem = private_key_content.encode() diff --git a/tests/providers/snowflake/hooks/test_snowflake.py b/tests/providers/snowflake/hooks/test_snowflake.py index e1105fbb5948..6a738952d90f 100644 --- a/tests/providers/snowflake/hooks/test_snowflake.py +++ b/tests/providers/snowflake/hooks/test_snowflake.py @@ -393,6 +393,24 @@ def test_get_conn_params_should_support_private_auth_with_unencrypted_key( ), pytest.raises(TypeError, match="Password was given but private key is not encrypted."): SnowflakeHook(snowflake_conn_id="test_conn")._get_conn_params() + def test_get_conn_params_should_fail_on_invalid_key(self): + connection_kwargs = { + **BASE_CONNECTION_KWARGS, + "password": None, + "extra": { + "database": "db", + "account": "airflow", + "warehouse": "af_wh", + "region": "af_region", + "role": "af_role", + "private_key_file": "/dev/urandom", + }, + } + with mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri() + ), pytest.raises(ValueError, match="The private_key_file path points to an empty or invalid file."): + SnowflakeHook(snowflake_conn_id="test_conn").get_conn() + def test_should_add_partner_info(self): with mock.patch.dict( "os.environ", From 156b85ad85ded6131e0902a4e00c27506eee62f3 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Sat, 8 Jul 2023 16:49:59 -0700 Subject: [PATCH 115/533] D205 Support - Jobs (#32450) Part of https://github.com/apache/airflow/issues/10742 D205 asserts that all docstrings must have a one-line summary ending in a period. If there is more than one sentence then there must be a blank line before the rest of the docstring. Meeting these requirements could be as simple as adding a newline, or might require some rephrasing. There are almost a thousand violations in the repo so we're going to have to take this in bites. ### PLEASE NOTE There should be zero logic changes in this PR, only changes to docstrings and whitespace. If you see otherwise, please call it out. ### Included in this chunk All files in the `airflow/jobs` module. ### To test If you comment out [this line](https://github.com/apache/airflow/blob/main/pyproject.toml#L68) and run pre-commit in main you will get 284 errors. After these changes, "only" 269 remain and no files in the dag_processing folder should be on the list. After uncommenting that line and rerunning pre-commits, there should be zero regressions. --- airflow/jobs/scheduler_job_runner.py | 24 ++++------- airflow/jobs/triggerer_job_runner.py | 64 +++++++++------------------- 2 files changed, 30 insertions(+), 58 deletions(-) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index a85a0925fb23..8e9db65647ed 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -1138,10 +1138,7 @@ def _create_dagruns_for_dags(self, guard: CommitProhibitorGuard, session: Sessio # END: create dagruns def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -> None: - """ - Unconditionally create a DAG run for the given DAG, and update the dag_model's fields to control - if/when the next DAGRun should be created. - """ + """Create a DAG run and update the dag_model to control if/when the next DAGRun should be created.""" # Bulk Fetch DagRuns with dag_id and execution_date same # as DagModel.dag_id and DagModel.next_dagrun # This list is used to verify if the DagRun already exist so that we don't attempt to create @@ -1571,8 +1568,7 @@ def _emit_pool_metrics(self, session: Session = NEW_SESSION) -> None: @provide_session def adopt_or_reset_orphaned_tasks(self, session: Session = NEW_SESSION) -> int: """ - Reset any TaskInstance still in QUEUED or SCHEDULED states that were - enqueued by a SchedulerJob that is no longer running. + Reset any TaskInstance in QUEUED or SCHEDULED state if its SchedulerJob is no longer running. :return: the number of TIs reset """ @@ -1658,10 +1654,7 @@ def adopt_or_reset_orphaned_tasks(self, session: Session = NEW_SESSION) -> int: @provide_session def check_trigger_timeouts(self, session: Session = NEW_SESSION) -> None: - """ - Looks at all tasks that are in the "deferred" state and whose trigger - or execution timeout has passed, so they can be marked as failed. - """ + """Mark any "deferred" task as failed if the trigger or execution timeout has passed.""" num_timed_out_tasks = session.execute( update(TI) .where( @@ -1680,9 +1673,9 @@ def check_trigger_timeouts(self, session: Session = NEW_SESSION) -> None: def _find_zombies(self) -> None: """ - Find zombie task instances, which are tasks haven't heartbeated for too long - or have a no-longer-running LocalTaskJob, and create a TaskCallbackRequest - to be handled by the DAG processor. + Find zombie task instances and create a TaskCallbackRequest to be handled by the DAG processor. + + Zombie instances are tasks haven't heartbeated for too long or have a no-longer-running LocalTaskJob. """ from airflow.jobs.job import Job @@ -1775,8 +1768,9 @@ def _set_orphaned(self, dataset: DatasetModel) -> int: @provide_session def _orphan_unreferenced_datasets(self, session: Session = NEW_SESSION) -> None: """ - Detects datasets that are no longer referenced in any DAG schedule parameters or task outlets and - sets the dataset is_orphaned flag to True. + Detect orphaned datasets and set is_orphaned flag to True. + + An orphaned dataset is no longer referenced in any DAG schedule parameters or task outlets. """ orphaned_dataset_query = session.scalars( select(DatasetModel) diff --git a/airflow/jobs/triggerer_job_runner.py b/airflow/jobs/triggerer_job_runner.py index 633e90944cd7..d92c281cf002 100644 --- a/airflow/jobs/triggerer_job_runner.py +++ b/airflow/jobs/triggerer_job_runner.py @@ -83,14 +83,12 @@ def configure_trigger_log_handler(): """ - Configure logging such that each trigger logs to its own file and - can be exposed through the airflow webserver. + Configure logging where each trigger logs to its own file and can be exposed via the airflow webserver. Generally speaking, we take the log handler configured for logger ``airflow.task``, wrap it with TriggerHandlerWrapper, and set it as the handler for root logger. - If there already is a handler configured for the root logger - and it supports triggers, we wrap it instead. + If there already is a handler configured for the root logger and it supports triggers, we wrap it instead. :meta private: """ @@ -237,9 +235,7 @@ def setup_queue_listener(): class TriggererJobRunner(BaseJobRunner["Job | JobPydantic"], LoggingMixin): """ - TriggererJobRunner continuously runs active triggers in asyncio, watching - for them to fire off their events and then dispatching that information - to their dependent tasks/DAGs. + Run active triggers in asyncio and update their dependent tests/DAGs once their events have fired. It runs as two threads: - The main thread does DB calls/checkins @@ -291,17 +287,14 @@ def register_signals(self) -> None: @provide_session def is_needed(cls, session) -> bool: """ - Tests if the triggerer job needs to be run (i.e., if there are triggers - in the trigger table). + Tests if the triggerer job needs to be run (i.e., if there are triggers in the trigger table). + This is used for the warning boxes in the UI. """ return session.query(func.count(Trigger.id)).scalar() > 0 def on_kill(self): - """ - Called when there is an external kill command (via the heartbeat - mechanism, for example). - """ + """Called when there is an external kill command (via the heartbeat mechanism, for example).""" self.trigger_runner.stop = True def _kill_listener(self): @@ -369,20 +362,13 @@ def _run_trigger_loop(self) -> None: time.sleep(1) def load_triggers(self): - """ - Queries the database to get the triggers we're supposed to be running, - adds them to our runner, and then removes ones from it we no longer - need. - """ + """Query the database for the triggers we're supposed to be running and update the runner.""" Trigger.assign_unassigned(self.job.id, self.capacity, self.job.heartrate) ids = Trigger.ids_for_triggerer(self.job.id) self.trigger_runner.update_triggers(set(ids)) def handle_events(self): - """ - Handles outbound events from triggers - dispatching them into the Trigger - model where they are then pushed into the relevant task instances. - """ + """Dispatch outbound events to the Trigger model which pushes them to the relevant task instances.""" while self.trigger_runner.events: # Get the event and its trigger ID trigger_id, event = self.trigger_runner.events.popleft() @@ -393,8 +379,9 @@ def handle_events(self): def handle_failed_triggers(self): """ - Handles "failed" triggers - ones that errored or exited before they - sent an event. Task Instances that depend on them need failing. + Handles "failed" triggers. - ones that errored or exited before they sent an event. + + Task Instances that depend on them need failing. """ while self.trigger_runner.failed_triggers: # Tell the model to fail this trigger's deps @@ -492,10 +479,7 @@ async def arun(self): await watchdog async def create_triggers(self): - """ - Drain the to_create queue and create all triggers that have been - requested in the DB that we don't yet have. - """ + """Drain the to_create queue and create all new triggers that have been requested in the DB.""" while self.to_create: trigger_id, trigger_instance = self.to_create.popleft() if trigger_id not in self.triggers: @@ -516,8 +500,9 @@ async def create_triggers(self): async def cancel_triggers(self): """ - Drain the to_cancel queue and ensure all triggers that are not in the - DB are cancelled, so the cleanup job deletes them. + Drain the to_cancel queue and ensure all triggers that are not in the DB are cancelled. + + This allows the the cleanup job to delete them. """ while self.to_cancel: trigger_id = self.to_cancel.popleft() @@ -528,9 +513,9 @@ async def cancel_triggers(self): async def cleanup_finished_triggers(self): """ - Go through all trigger tasks (coroutines) and clean up entries for - ones that have exited, optionally warning users if the exit was - not normal. + Go through all trigger tasks (coroutines) and clean up entries for ones that have exited. + + Optionally warn users if the exit was not normal. """ for trigger_id, details in list(self.triggers.items()): if details["task"].done(): @@ -594,10 +579,7 @@ async def block_watchdog(self): @staticmethod def set_individual_trigger_logging(trigger): - """ - Setting these context vars allows log messages for individual triggers - to be routed to distinct files and filtered from triggerer stdout. - """ + """Configure trigger logging to allow individual files and stdout filtering.""" # set logging context vars for routing to appropriate handler ctx_task_instance.set(trigger.task_instance) ctx_trigger_id.set(trigger.trigger_id) @@ -607,10 +589,7 @@ def set_individual_trigger_logging(trigger): ctx_indiv_trigger.set(True) async def run_trigger(self, trigger_id, trigger): - """ - Wrapper which runs an actual trigger (they are async generators) - and pushes their events into our outbound event deque. - """ + """Run a trigger (they are async generators) and push their events into our outbound event deque.""" name = self.triggers[trigger_id]["name"] self.log.info("trigger %s starting", name) try: @@ -653,8 +632,7 @@ def mark_trigger_end(trigger): def update_triggers(self, requested_trigger_ids: set[int]): """ - Called from the main thread to request that we update what - triggers we're running. + Called from the main thread to request that we update what triggers we're running. Works out the differences - ones to add, and ones to remove - then adds them to the deques so the subthread can actually mutate the running From ea420eee2a6e37977ba16e1a6bdaeb92c0f86937 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Sat, 8 Jul 2023 23:01:34 -0700 Subject: [PATCH 116/533] D205 Support - Dag Processing (#32449) --- airflow/dag_processing/manager.py | 25 +++++++++++++++---------- airflow/dag_processing/processor.py | 3 +++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py index da024161b89c..7e44e89816f8 100644 --- a/airflow/dag_processing/manager.py +++ b/airflow/dag_processing/manager.py @@ -167,11 +167,10 @@ def start(self) -> None: def run_single_parsing_loop(self) -> None: """ Should only be used when launched DAG file processor manager in sync mode. - Send agent heartbeat signal to the manager, requesting that it runs one - processing "loop". - Call wait_until_finished to ensure that any launched processors have - finished before continuing + Send agent heartbeat signal to the manager, requesting that it runs one processing "loop". + + Call wait_until_finished to ensure that any launched processors have finished before continuing. """ if not self._parent_signal_conn or not self._process: raise ValueError("Process not started.") @@ -477,6 +476,7 @@ def _exit_gracefully(self, signum, frame): def start(self): """ Use multiple processes to parse and generate tasks for the DAGs in parallel. + By processing them in separate processes, we can get parallelism and isolation from potentially harmful user code. """ @@ -519,6 +519,7 @@ def deactivate_stale_dags( ): """ Detects DAGs which are no longer present in files. + Deactivate them and remove them in the serialized_dag table. """ to_deactivate = set() @@ -888,6 +889,7 @@ def _log_file_processing_stats(self, known_file_paths): def get_pid(self, file_path) -> int | None: """ Retrieve the PID of the process processing the given file or None if the file is not being processed. + :param file_path: the path to the file that's being processed. """ if file_path in self._processors: @@ -905,6 +907,7 @@ def get_all_pids(self) -> list[int]: def get_last_runtime(self, file_path) -> float | None: """ Retrieve the last processing time of a specific path. + :param file_path: the path to the file that was processed :return: the runtime (in seconds) of the process of the last run, or None if the file was never processed. @@ -915,9 +918,9 @@ def get_last_runtime(self, file_path) -> float | None: def get_last_dag_count(self, file_path) -> int | None: """ Retrieve the total DAG count at a specific path. + :param file_path: the path to the file that was processed - :return: the number of dags loaded from that file, or None if the file - was never processed. + :return: the number of dags loaded from that file, or None if the file was never processed. """ stat = self._file_stats.get(file_path) return stat.num_dags if stat else None @@ -925,9 +928,9 @@ def get_last_dag_count(self, file_path) -> int | None: def get_last_error_count(self, file_path) -> int | None: """ Retrieve the total number of errors from processing a specific path. + :param file_path: the path to the file that was processed - :return: the number of import errors from processing, or None if the file - was never processed. + :return: the number of import errors from processing, or None if the file was never processed. """ stat = self._file_stats.get(file_path) return stat.import_errors if stat else None @@ -935,9 +938,9 @@ def get_last_error_count(self, file_path) -> int | None: def get_last_finish_time(self, file_path) -> datetime | None: """ Retrieve the last completion time for processing a specific path. + :param file_path: the path to the file that was processed - :return: the finish time of the process of the last run, or None if the - file was never processed. + :return: the finish time of the process of the last run, or None if the file was never processed. """ stat = self._file_stats.get(file_path) return stat.last_finish_time if stat else None @@ -945,6 +948,7 @@ def get_last_finish_time(self, file_path) -> datetime | None: def get_start_time(self, file_path) -> datetime | None: """ Retrieve the last start time for processing a specific path. + :param file_path: the path to the file that's being processed :return: the start time of the process that's processing the specified file or None if the file is not currently being processed. @@ -956,6 +960,7 @@ def get_start_time(self, file_path) -> datetime | None: def get_run_count(self, file_path) -> int: """ The number of times the given file has been parsed. + :param file_path: the path to the file that's being processed. """ stat = self._file_stats.get(file_path) diff --git a/airflow/dag_processing/processor.py b/airflow/dag_processing/processor.py index 9ee865bbf861..5c175571c0ca 100644 --- a/airflow/dag_processing/processor.py +++ b/airflow/dag_processing/processor.py @@ -598,6 +598,7 @@ def update_import_errors( ) -> None: """ Update any import errors to be displayed in the UI. + For the DAGs in the given DagBag, record any associated import errors and clears errors for files that no longer have them. These are usually displayed through the Airflow UI so that users know that there are issues parsing DAGs. @@ -664,6 +665,7 @@ def check_pools(dag): def update_dag_warnings(self, *, session: Session, dagbag: DagBag) -> None: """ Update any import warnings to be displayed in the UI. + For the DAGs in the given DagBag, record any associated configuration warnings and clear warnings for files that no longer have them. These are usually displayed through the Airflow UI so that users know that there are issues parsing DAGs. @@ -691,6 +693,7 @@ def execute_callbacks( ) -> None: """ Execute on failure callbacks. + These objects can come from SchedulerJobRunner or from DagProcessorJobRunner. :param dagbag: Dag Bag of dags From d27532b21abd3db9695a66d1df783babe872d856 Mon Sep 17 00:00:00 2001 From: Detlev V Date: Sun, 9 Jul 2023 08:25:09 +0200 Subject: [PATCH 117/533] fix(helm): dagprocessor livenessProbe include --local and --job-type args (#32426) --------- Co-authored-by: Detlev Vandaele --- chart/templates/_helpers.yaml | 13 +++++++++++-- helm_tests/airflow_core/test_dag_processor.py | 18 ++++++++++++++++++ helm_tests/airflow_core/test_scheduler.py | 18 ++++++++++++++++++ helm_tests/airflow_core/test_triggerer.py | 17 +++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 651dfa6914b5..fc805e56b515 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -690,16 +690,25 @@ server_tls_key_file = /etc/pgbouncer/server.key - -c - | CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ - airflow jobs check --job-type TriggererJob --hostname $(hostname) + airflow jobs check --job-type TriggererJob --hostname $(hostname) {{- end }} {{- end }} {{- define "dag_processor_liveness_check_command" }} + {{- $commandArgs := (list) -}} + {{- if semverCompare ">=2.5.0" .Values.airflowVersion }} + {{- $commandArgs = append $commandArgs "--local" -}} + {{- if semverCompare ">=2.5.2" .Values.airflowVersion }} + {{- $commandArgs = concat $commandArgs (list "--job-type" "DagProcessorJob") -}} + {{- end }} + {{- else }} + {{- $commandArgs = concat $commandArgs (list "--hostname" "$(hostname)") -}} + {{- end }} - sh - -c - | CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \ - airflow jobs check --hostname $(hostname) + airflow jobs check {{ join " " $commandArgs }} {{- end }} {{- define "registry_docker_config" }} diff --git a/helm_tests/airflow_core/test_dag_processor.py b/helm_tests/airflow_core/test_dag_processor.py index dbde3f7b33de..c25fea08ab0c 100644 --- a/helm_tests/airflow_core/test_dag_processor.py +++ b/helm_tests/airflow_core/test_dag_processor.py @@ -338,6 +338,24 @@ def test_livenessprobe_values_are_configurable(self): "spec.template.spec.containers[0].livenessProbe.exec.command", docs[0] ) + @pytest.mark.parametrize( + "airflow_version, probe_command", + [ + ("2.4.9", "airflow jobs check --hostname $(hostname)"), + ("2.5.0", "airflow jobs check --local"), + ("2.5.2", "airflow jobs check --local --job-type DagProcessorJob"), + ], + ) + def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command): + docs = render_chart( + values={"airflowVersion": f"{airflow_version}", "dagProcessor": {"enabled": True}}, + show_only=["templates/dag-processor/dag-processor-deployment.yaml"], + ) + assert ( + probe_command + in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1] + ) + @pytest.mark.parametrize( "log_persistence_values, expected_volume", [ diff --git a/helm_tests/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py index faf666854b55..698ba4f6eb11 100644 --- a/helm_tests/airflow_core/test_scheduler.py +++ b/helm_tests/airflow_core/test_scheduler.py @@ -357,6 +357,24 @@ def test_livenessprobe_values_are_configurable(self): "spec.template.spec.containers[0].livenessProbe.exec.command", docs[0] ) + @pytest.mark.parametrize( + "airflow_version, probe_command", + [ + ("1.9.0", "from airflow.jobs.scheduler_job import SchedulerJob"), + ("2.1.0", "airflow jobs check --job-type SchedulerJob --hostname $(hostname)"), + ("2.5.0", "airflow jobs check --job-type SchedulerJob --local"), + ], + ) + def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command): + docs = render_chart( + values={"airflowVersion": f"{airflow_version}"}, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + assert ( + probe_command + in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1] + ) + @pytest.mark.parametrize( "log_persistence_values, expected_volume", [ diff --git a/helm_tests/airflow_core/test_triggerer.py b/helm_tests/airflow_core/test_triggerer.py index 0154c8f8c0c9..22a8ad7f5219 100644 --- a/helm_tests/airflow_core/test_triggerer.py +++ b/helm_tests/airflow_core/test_triggerer.py @@ -359,6 +359,23 @@ def test_livenessprobe_values_are_configurable(self): "spec.template.spec.containers[0].livenessProbe.exec.command", docs[0] ) + @pytest.mark.parametrize( + "airflow_version, probe_command", + [ + ("2.4.9", "airflow jobs check --job-type TriggererJob --hostname $(hostname)"), + ("2.5.0", "airflow jobs check --job-type TriggererJob --local"), + ], + ) + def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version, probe_command): + docs = render_chart( + values={"airflowVersion": f"{airflow_version}"}, + show_only=["templates/triggerer/triggerer-deployment.yaml"], + ) + assert ( + probe_command + in jmespath.search("spec.template.spec.containers[0].livenessProbe.exec.command", docs[0])[-1] + ) + @pytest.mark.parametrize( "log_persistence_values, expected_volume", [ From ccbf1b125b583ab930a128b1bb8315c6ae5544ea Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Sun, 9 Jul 2023 09:29:02 +0300 Subject: [PATCH 118/533] Add elasticsearch to boring-cyborg.yml (#32451) --- .github/boring-cyborg.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index e364b37984a5..5aa495022b3f 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -148,6 +148,12 @@ labelPRBasedOnFilePath: - docs/apache-airflow-providers-openlineage/**/* - tests/providers/openlineage/**/* + provider:elasticsearch: + - airflow/providers/elasticsearch/**/* + - docs/apache-airflow-providers-elasticsearch/**/* + - tests/providers/elasticsearch/**/* + - tests/system/providers/elasticsearch/**/* + area:providers: - airflow/providers/**/* - docs/apache-airflow-providers-*/**/* From 5d8c54ec52723d9595d1c300e0ae57b2fc10cb2d Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Sun, 9 Jul 2023 15:03:29 +0800 Subject: [PATCH 119/533] Add type annotation to code examples (#32422) --------- Co-authored-by: Syed Hussain <103602455+syedahsn@users.noreply.github.com> --- .../authoring-and-scheduling/deferring.rst | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/deferring.rst b/docs/apache-airflow/authoring-and-scheduling/deferring.rst index 81526737e8d1..5d4e6e2634be 100644 --- a/docs/apache-airflow/authoring-and-scheduling/deferring.rst +++ b/docs/apache-airflow/authoring-and-scheduling/deferring.rst @@ -56,38 +56,44 @@ Writing a deferrable operator takes a bit more work. There are some main points * You can defer multiple times, and you can defer before/after your Operator does significant work, or only defer if certain conditions are met (e.g. a system does not have an immediate answer). Deferral is entirely under your control. * Any Operator can defer; no special marking on its class is needed, and it's not limited to Sensors. * In order for any changes to a Trigger to be reflected, the *triggerer* needs to be restarted whenever the Trigger is modified. -* If you want add an operator or sensor that supports both deferrable and non-deferrable modes. It's suggested to add ``deferable: bool = conf.getboolean("operators", "default_deferrable", fallback=False)`` to the ``__init__`` method of the operator and use it to decide whether to run the operator in deferrable mode. You'll be able to configure the default value of ``deferrable`` of all the operators and sensors that supports switch between deferrable and non-deferrable mode through ``default_deferrable`` in the ``operator`` section. Here's an example of a sensor that supports both modes.:: +* If you want to add an operator or sensor that supports both deferrable and non-deferrable modes, it's suggested to add ``deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False)`` to the ``__init__`` method of the operator and use it to decide whether to run the operator in deferrable mode. You'll be able to configure the default value of ``deferrable`` of all the operators and sensors that support switching between deferrable and non-deferrable mode through ``default_deferrable`` in the ``operator`` section. Here's an example of a sensor that supports both modes. + +.. code-block:: python import time from datetime import timedelta + from typing import Any + from airflow.configuration import conf from airflow.sensors.base import BaseSensorOperator from airflow.triggers.temporal import TimeDeltaTrigger + from airflow.utils.context import Context class WaitOneHourSensor(BaseSensorOperator): def __init__( - self, - deferable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), - **kwargs - ): + self, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs + ) -> None: super().__init__(**kwargs) self.deferrable = deferable - def execute(self, context): - if deferrable: + def execute(self, context: Context) -> None: + if self.deferrable: self.defer( trigger=TimeDeltaTrigger(timedelta(hours=1)), - method_name="execute_complete" + method_name="execute_complete", ) else: time.sleep(3600) - def execute_complete(self, context, event=None): + def execute_complete( + self, + context: Context, + event: dict[str, Any] | None = None, + ) -> None: # We have no more work to do here. Mark as complete. return - Triggering Deferral ~~~~~~~~~~~~~~~~~~~ @@ -106,19 +112,23 @@ If your Operator returns from either its first ``execute()`` method when it's ne You are free to set ``method_name`` to ``execute`` if you want your Operator to have one entrypoint, but it, too, will have to accept ``event`` as an optional keyword argument. -Here's a basic example of how a sensor might trigger deferral:: +Here's a basic example of how a sensor might trigger deferral + +.. code-block:: python from datetime import timedelta + from typing import Any from airflow.sensors.base import BaseSensorOperator from airflow.triggers.temporal import TimeDeltaTrigger + from airflow.utils.context import Context class WaitOneHourSensor(BaseSensorOperator): - def execute(self, context): + def execute(self, context: Context) -> None: self.defer(trigger=TimeDeltaTrigger(timedelta(hours=1)), method_name="execute_complete") - def execute_complete(self, context, event=None): + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: # We have no more work to do here. Mark as complete. return @@ -151,8 +161,9 @@ There's also some design constraints to be aware of: Currently Triggers are only used up to their first event, as they are only used for resuming deferred tasks (which happens on the first event fired). However, we plan to allow DAGs to be launched from triggers in future, which is where multi-event triggers will be more useful. -Here's the structure of a basic Trigger:: +Here's the structure of a basic Trigger +.. code-block:: python import asyncio @@ -161,7 +172,6 @@ Here's the structure of a basic Trigger:: class DateTimeTrigger(BaseTrigger): - def __init__(self, moment): super().__init__() self.moment = moment @@ -174,6 +184,7 @@ Here's the structure of a basic Trigger:: await asyncio.sleep(1) yield TriggerEvent(self.moment) + This is a very simplified version of Airflow's ``DateTimeTrigger``, and you can see several things here: * ``__init__`` and ``serialize`` are written as a pair; the Trigger is instantiated once when it is submitted by the Operator as part of its deferral request, then serialized and re-instantiated on any *triggerer* process that runs the trigger. From afcb63e806755f07cb1d435ef050bfaf96f99c4a Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 9 Jul 2023 15:49:27 +0200 Subject: [PATCH 120/533] Fix triggerer HA doc (#32454) Signed-off-by: Hussein Awala --- docs/apache-airflow/authoring-and-scheduling/deferring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/deferring.rst b/docs/apache-airflow/authoring-and-scheduling/deferring.rst index 5d4e6e2634be..6b5f124e8f62 100644 --- a/docs/apache-airflow/authoring-and-scheduling/deferring.rst +++ b/docs/apache-airflow/authoring-and-scheduling/deferring.rst @@ -203,7 +203,7 @@ Triggers are designed from the ground-up to be highly-available; if you want to Depending on how much work the triggers are doing, you can fit from hundreds to tens of thousands of triggers on a single ``triggerer`` host. By default, every ``triggerer`` will have a capacity of 1000 triggers it will try to run at once; you can change this with the ``--capacity`` argument. If you have more triggers trying to run than you have capacity across all of your ``triggerer`` processes, some triggers will be delayed from running until others have completed. -Airflow tries to only run triggers in one place at once, and maintains a heartbeat to all ``triggerers`` that are currently running. If a ``triggerer`` dies, or becomes partitioned from the network where Airflow's database is running, Airflow will automatically re-schedule triggers that were on that host to run elsewhere (after waiting 30 seconds for the machine to re-appear). +Airflow tries to only run triggers in one place at once, and maintains a heartbeat to all ``triggerers`` that are currently running. If a ``triggerer`` dies, or becomes partitioned from the network where Airflow's database is running, Airflow will automatically re-schedule triggers that were on that host to run elsewhere (after waiting (2.1 * ``triggerer.job_heartbeat_sec``) seconds for the machine to re-appear). This means it's possible, but unlikely, for triggers to run in multiple places at once; this is designed into the Trigger contract, however, and entirely expected. Airflow will de-duplicate events fired when a trigger is running in multiple places simultaneously, so this process should be transparent to your Operators. From cd1ac927a7c8451fb27547ee9dc94950214df393 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Sun, 9 Jul 2023 18:07:57 +0300 Subject: [PATCH 121/533] Prepare docs for July 2023 RC3 amazon provider (#32455) --- airflow/providers/amazon/CHANGELOG.rst | 6 ++++++ docs/apache-airflow-providers-amazon/commits.rst | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index e8227e109e18..f0a03a85c001 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -47,6 +47,7 @@ Features * ``Deferrable mode for ECS operators (#31881)`` * ``feature: AWS - GlueJobOperator - job_poll_interval (#32147)`` * ``Added 'AzureBlobStorageToS3Operator' transfer operator (#32270)`` +* ``Introduce a base class for aws triggers (#32274)`` Bug Fixes ~~~~~~~~~ @@ -67,6 +68,9 @@ Misc * ``Add 'on_finish_action' to 'KubernetesPodOperator' (#30718)`` * ``Add default_deferrable config (#31712)`` * ``deprecate arbitrary parameter passing to RDS hook (#32352)`` +* ``quick fix on RDS operator to prevent parameter collision (#32436)`` +* ``Remove ability to specify arbitrary hook params in AWS RDS trigger (#32386)`` +* ``Only update crawler tags if present in config dict (#32331)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): @@ -80,6 +84,8 @@ Misc * ``cleanup Amazon CHANGELOG.rst (#32031)`` * ``Remove spurious headers for provider changelogs (#32373)`` * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: Stragglers and new additions (#32447)`` + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` 8.2.0 ..... diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index 58362aebb88a..538c13b75604 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -31,11 +31,17 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-08 ``D205 Support - Providers: Stragglers and new additions (#32447)`` +`8c6751ff0f `_ 2023-07-07 ``quick fix on RDS operator to prevent parameter collision (#32436)`` +`869f3a93a8 `_ 2023-07-07 ``Remove ability to specify arbitrary hook params in AWS RDS trigger (#32386)`` +`05f1acfcb7 `_ 2023-07-07 ``Introduce a base class for aws triggers (#32274)`` +`7a3bc8d7c8 `_ 2023-07-06 ``Only update crawler tags if present in config dict (#32331)`` +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` From 59c49044c60d2b6222cd8502c236c0ae92be6044 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 9 Jul 2023 19:36:09 +0200 Subject: [PATCH 122/533] Fix installation from local not remote airflow for docs building (#32464) --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 27aacafcf8c7..4219caf9b4d4 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -390,7 +390,7 @@ You will need to change it manually to see the docs **NOTE** In order to run the publish documentation you need to activate virtualenv where you installed apache-airflow with doc extra: -* `pip install 'apache-airflow[doc_gen]'` +* `pip install '.[doc_gen]'` If you don't have virtual env set you can do: From 63a9591a4c09135b25e358ab8040b9d1f3ca0d81 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Sun, 9 Jul 2023 20:54:10 +0300 Subject: [PATCH 123/533] fix vote email template in provider release doc (#32461) --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 4219caf9b4d4..025889b4f34a 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -488,7 +488,8 @@ cat < Date: Sun, 9 Jul 2023 20:56:09 +0200 Subject: [PATCH 124/533] Auto-detect docker context for breeze commands (#32463) Recently Docker Desktop - particularly on Mac OS disabled creation of /var/run/docker.sock by default, leaving it only as advanced option, thus deprecating the "default" context for docker. So far we recommended to enable the option for Breeze, but this change allows to autodetect if there is a "desktop-linux" context available (which uses a user-home-dir socket instead of the global /var/run one) and if it gets detected, "desktop-linux" context is used. This is slightly more complicated, because we need to also leave the ``--builder`` available for cases where our auto-detection fails, so we need to expose the option in all commands that might attempt to build image automatically, we also need to find and forward the right socket to inside breeze - as we cannot unconditionally forward the /var/run/docker.sock. Alongside the chage "static-checks" command gets the same capability of rebuilding the CI image automatically. --- BREEZE.rst | 11 +- .../commands/ci_image_commands.py | 5 +- .../commands/developer_commands.py | 26 +++- .../commands/developer_commands_config.py | 14 +- .../airflow_breeze/commands/main_command.py | 2 + .../params/common_build_params.py | 2 +- .../src/airflow_breeze/params/shell_params.py | 1 + .../airflow_breeze/utils/common_options.py | 3 +- .../utils/docker_command_utils.py | 90 +++++++----- dev/breeze/tests/test_docker_command_utils.py | 124 +++++++--------- images/breeze/output-commands-hash.txt | 18 +-- images/breeze/output-commands.svg | 82 ++++++----- images/breeze/output_build-docs.svg | 22 +-- images/breeze/output_ci-image_build.svg | 92 ++++++------ images/breeze/output_prod-image_build.svg | 132 +++++++++--------- images/breeze/output_shell.svg | 74 +++++----- images/breeze/output_start-airflow.svg | 8 +- images/breeze/output_static-checks.svg | 44 ++++-- scripts/ci/docker-compose/base.yml | 3 + 19 files changed, 425 insertions(+), 328 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 13796254b856..dd1993505f27 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -56,6 +56,12 @@ Docker Desktop - **Docker problems**: Sometimes it is not obvious that space is an issue when you run into a problem with Docker. If you see a weird behaviour, try ``breeze cleanup`` command. Also see `pruning `_ instructions from Docker. +- **Docker context**: In recent versions Docker Desktop is by default configured to use ``desktop-linux`` + docker context that uses docker socket created in user home directory. Older versions (and plain docker) + uses ``/var/run/docker.sock`` socket and ``default`` context. Breeze will attempt to detect if you have + ``desktop-linux`` context configured and will use it if it is available, but you can force the + context by adding ``--builder`` flag to the commands that build image or run the container and forward + the socket to inside the image. Here is an example configuration with more than 200GB disk space for Docker: @@ -84,7 +90,8 @@ Here is an example configuration with more than 200GB disk space for Docker: - ``newgrp docker`` - 4. Check if docker can be run without root - ``docker run hello-world`` -- 5. Make sure that "Allow the default Docker socket to be used" in "Advanced" tab of "Docker Desktop" settings is checked +- 5. In some cases you might make sure that "Allow the default Docker socket to + be used" in "Advanced" tab of "Docker Desktop" settings is checked .. raw:: html @@ -852,7 +859,7 @@ blocked by it, or your proxy setting has not been done properly. Advanced commands ================= -Airflow Breeze is a bash script serving as a "swiss-army-knife" of Airflow testing. Under the +Airflow Breeze is a Python script serving as a "swiss-army-knife" of Airflow testing. Under the hood it uses other scripts that you can also run manually if you have problem with running the Breeze environment. Breeze script allows performing the following tasks: diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 78a7f3f84aee..5b6b9ee3b8d6 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -575,6 +575,7 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara ) ci_image_params = BuildCiParams( python=command_params.python, + builder=command_params.builder, github_repository=command_params.github_repository, upgrade_to_newer_dependencies=False, image_tag=command_params.image_tag, @@ -597,8 +598,8 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara get_console().print(f"[info]{command_params.image_type} image already built locally.[/]") else: get_console().print( - f"[warning]{command_params.image_type} image was never built locally or deleted. " - "Forcing build.[/]" + f"[warning]{command_params.image_type} image for Python {command_params.python} " + f"was never built locally or was deleted. Forcing build.[/]" ) ci_image_params.force_build = True if check_if_image_building_is_needed( diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 4586e4cd5066..52e1e6955cbc 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -45,6 +45,7 @@ option_airflow_extras, option_answer, option_backend, + option_builder, option_celery_broker, option_celery_flower, option_db_reset, @@ -119,6 +120,7 @@ def run(self): @option_python @option_platform_single @option_backend +@option_builder @option_postgres_version @option_mysql_version @option_mssql_version @@ -147,6 +149,7 @@ def run(self): def shell( python: str, backend: str, + builder: str, integration: tuple[str], postgres_version: str, mysql_version: str, @@ -182,6 +185,7 @@ def shell( python=python, github_repository=github_repository, backend=backend, + builder=builder, integration=integration, postgres_version=postgres_version, mysql_version=mysql_version, @@ -211,6 +215,7 @@ def shell( @option_python @option_platform_single @option_backend +@option_builder @option_postgres_version @option_load_example_dags @option_load_default_connection @@ -250,6 +255,7 @@ def shell( def start_airflow( python: str, backend: str, + builder: str, integration: tuple[str], postgres_version: str, load_example_dags: bool, @@ -290,6 +296,7 @@ def start_airflow( python=python, github_repository=github_repository, backend=backend, + builder=builder, integration=integration, postgres_version=postgres_version, load_default_connections=load_default_connections, @@ -320,6 +327,7 @@ def start_airflow( @main.command(name="build-docs") @click.option("-d", "--docs-only", help="Only build documentation.", is_flag=True) @click.option("-s", "--spellcheck-only", help="Only run spell checking.", is_flag=True) +@option_builder @click.option( "--package-filter", help="List of packages to consider.", @@ -350,6 +358,7 @@ def build_docs( docs_only: bool, spellcheck_only: bool, for_production: bool, + builder: str, clean_build: bool, one_pass_only: bool, package_filter: tuple[str], @@ -361,7 +370,9 @@ def build_docs( clean_build = True perform_environment_checks() cleanup_python_generated_files() - params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) + params = BuildCiParams( + github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION, builder=builder + ) rebuild_or_pull_ci_image_if_needed(command_params=params) if clean_build: docs_dir = AIRFLOW_SOURCES_ROOT / "docs" @@ -446,6 +457,9 @@ def build_docs( type=click.IntRange(1, 10), default=3, ) +@option_image_tag_for_running +@option_force_build +@option_builder @option_github_repository @option_verbose @option_dry_run @@ -461,10 +475,20 @@ def static_checks( precommit_args: tuple, initialize_environment: bool, max_initialization_attempts: int, + image_tag: str, + force_build: bool, + builder: str, github_repository: str, ): assert_pre_commit_installed() perform_environment_checks() + build_params = BuildCiParams( + builder=builder, + force_build=force_build, + image_tag=image_tag, + github_repository=github_repository, + ) + rebuild_or_pull_ci_image_if_needed(command_params=build_params) if initialize_environment: get_console().print("[info]Make sure that pre-commit is installed and environment initialized[/]") diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index 5fb6cb2b2228..eb129950331c 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -44,6 +44,7 @@ "--db-reset", "--max-time", "--github-repository", + "--builder", ], }, ], @@ -77,6 +78,7 @@ "--platform", "--image-tag", "--github-repository", + "--builder", ], }, { @@ -144,6 +146,7 @@ "--force-build", "--image-tag", "--github-repository", + "--builder", ], }, { @@ -186,6 +189,7 @@ "--one-pass-only", "--package-filter", "--github-repository", + "--builder", ], }, ], @@ -197,7 +201,6 @@ "--show-diff-on-failure", "--initialize-environment", "--max-initialization-attempts", - "--github-repository", ], }, { @@ -210,6 +213,15 @@ "--only-my-changes", ], }, + { + "name": "Building image before running checks", + "options": [ + "--force-build", + "--image-tag", + "--github-repository", + "--builder", + ], + }, ], "breeze cleanup": [ { diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index f4e4185b9c83..cf29cb345007 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -32,6 +32,7 @@ from airflow_breeze.utils.common_options import ( option_answer, option_backend, + option_builder, option_db_reset, option_dry_run, option_forward_credentials, @@ -100,6 +101,7 @@ def get_command(self, ctx: Context, cmd_name: str): ) @option_python @option_backend +@option_builder @option_postgres_version @option_mysql_version @option_mssql_version diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 4275cfbb83cb..bbc0da9d9a61 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -45,7 +45,7 @@ class CommonBuildParams: ) airflow_constraints_location: str = "" build_id: int = 0 - builder: str = "default" + builder: str = "autodetect" constraints_github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY dev_apt_command: str = "" dev_apt_deps: str = "" diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 22c3b9057fd5..c8dfd9cd84ca 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -78,6 +78,7 @@ class ShellParams: airflow_extras: str = "" backend: str = ALLOWED_BACKENDS[0] base_branch: str = "main" + builder: str = "autodetect" ci: bool = False collect_only: bool = False db_reset: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index e8c056d12170..06fff97b6974 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -502,7 +502,8 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, "--builder", help="Buildx builder used to perform `docker buildx build` commands.", envvar="BUILDER", - default="default", + show_default=True, + default="autodetect", ) option_include_success_outputs = click.option( "--include-success-outputs", diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 8a85c6b4d6d3..ee52138c5136 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -367,38 +367,6 @@ def check_docker_compose_version(): ) -def check_docker_context(): - """Checks whether Docker is using the expected context.""" - expected_docker_context = "default" - response = run_command( - ["docker", "info", "--format", "{{json .ClientInfo.Context}}"], - no_output_dump_on_exception=False, - text=True, - capture_output=True, - dry_run_override=False, - ) - if response.returncode != 0: - get_console().print( - "[warning]Could not check for Docker context.[/]\n" - '[warning]Please make sure that Docker is using the right context by running "docker info" and ' - "checking the active Context.[/]" - ) - return - - used_docker_context = response.stdout.strip().replace('"', "") - - if used_docker_context == expected_docker_context: - get_console().print(f"[success]Good Docker context used: {used_docker_context}.[/]") - else: - get_console().print( - f"[error]Docker is not using the default context, used context is: {used_docker_context}[/]\n" - f"[warning]Please make sure Docker is using the {expected_docker_context} context.[/]\n" - f'[warning]You can try switching contexts by running: "docker context use ' - f'{expected_docker_context}"[/]' - ) - sys.exit(1) - - def get_env_variable_value(arg_name: str, params: CommonBuildParams | ShellParams): raw_value = getattr(params, arg_name, None) value = str(raw_value) if raw_value is not None else "" @@ -452,7 +420,9 @@ def prepare_docker_build_cache_command( build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) - final_command.extend(["buildx", "build", "--builder", image_params.builder, "--progress=auto"]) + final_command.extend( + ["buildx", "build", "--builder", get_docker_context(image_params.builder), "--progress=auto"] + ) final_command.extend(build_flags) final_command.extend(["--pull"]) final_command.extend(arguments) @@ -488,7 +458,7 @@ def prepare_base_build_command(image_params: CommonBuildParams) -> list[str]: "buildx", "build", "--builder", - image_params.builder, + get_docker_context(image_params.builder), "--progress=auto", "--push" if image_params.push else "--load", ] @@ -569,7 +539,7 @@ def build_cache(image_params: CommonBuildParams, output: Output | None) -> RunCo def make_sure_builder_configured(params: CommonBuildParams): - if params.builder != "default": + if params.builder != "autodetect": cmd = ["docker", "buildx", "inspect", params.builder] buildx_command_result = run_command(cmd, text=True, check=False, dry_run_override=False) if buildx_command_result and buildx_command_result.returncode != 0: @@ -747,7 +717,6 @@ def perform_environment_checks(): check_docker_is_running() check_docker_version() check_docker_compose_version() - check_docker_context() def get_docker_syntax_version() -> str: @@ -759,10 +728,11 @@ def get_docker_syntax_version() -> str: def warm_up_docker_builder(image_params: CommonBuildParams): from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT - if image_params.builder == "default": + docker_context = get_docker_context(image_params.builder) + if docker_context == "default": return docker_syntax = get_docker_syntax_version() - get_console().print(f"[info]Warming up the {image_params.builder} builder for syntax: {docker_syntax}") + get_console().print(f"[info]Warming up the {docker_context} builder for syntax: {docker_syntax}") warm_up_image_param = deepcopy(image_params) warm_up_image_param.image_tag = "warmup" warm_up_image_param.push = False @@ -829,3 +799,47 @@ def remove_docker_networks(networks: list[str] | None = None) -> None: check=False, stderr=DEVNULL, ) + + +# When you are using Docker Desktop (specifically on MacOS). the preferred context is "desktop-linux" +# because the docker socket to use is created in the .docker/ directory in the user's home directory +# and it does not require the user to belong to the "docker" group. +# The "default" context is the traditional one that requires "/var/run/docker.sock" to be writeable by the +# user running the docker command. +PREFERRED_CONTEXTS = ["desktop-linux", "default"] + + +def autodetect_docker_context(): + """ + Auto-detects which docker context to use. + + :return: name of the docker context to use + """ + output = run_command(["docker", "context", "ls", "-q"], capture_output=True, check=False, text=True) + if output.returncode != 0: + get_console().print("[warning]Could not detect docker builder. Using default.[/]") + return "default" + context_list = output.stdout.splitlines() + if len(context_list) == 0: + get_console().print("[warning]Could not detect docker builder. Using default.[/]") + return "default" + if len(context_list) == 1: + get_console().print(f"[info]Using {context_list[0]} as context.[/]") + return context_list[0] + if len(context_list) > 1: + for preferred_context in PREFERRED_CONTEXTS: + if preferred_context in context_list: + get_console().print(f"[info]Using {preferred_context} as context.[/]") + return preferred_context + fallback_context = context_list[0] + get_console().print( + f"[warning]Could not use any of the preferred docker contexts {PREFERRED_CONTEXTS}.\n" + f"Using {fallback_context} as context.[/]" + ) + return fallback_context + + +def get_docker_context(context: str): + if context != "autodetect": + return context + return autodetect_docker_context() diff --git a/dev/breeze/tests/test_docker_command_utils.py b/dev/breeze/tests/test_docker_command_utils.py index 89f522ad0dd4..83081ad77066 100644 --- a/dev/breeze/tests/test_docker_command_utils.py +++ b/dev/breeze/tests/test_docker_command_utils.py @@ -16,14 +16,15 @@ # under the License. from __future__ import annotations +import json from unittest import mock from unittest.mock import call import pytest from airflow_breeze.utils.docker_command_utils import ( + autodetect_docker_context, check_docker_compose_version, - check_docker_context, check_docker_version, ) @@ -190,75 +191,54 @@ def test_check_docker_compose_version_ok(mock_get_console, mock_run_command): ) -@mock.patch("airflow_breeze.utils.docker_command_utils.run_command") -@mock.patch("airflow_breeze.utils.docker_command_utils.get_console") -def test_check_docker_compose_version_higher(mock_get_console, mock_run_command): - mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "1.29.2" - check_docker_compose_version() - mock_run_command.assert_called_with( - ["docker", "compose", "version"], - no_output_dump_on_exception=True, - capture_output=True, - text=True, - dry_run_override=False, - ) - mock_get_console.return_value.print.assert_called_with( - "[success]Good version of docker-compose: 1.29.2[/]" - ) - - -@mock.patch("airflow_breeze.utils.docker_command_utils.run_command") -@mock.patch("airflow_breeze.utils.docker_command_utils.get_console") -def test_check_docker_context_default(mock_get_console, mock_run_command): - mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "default" - check_docker_context() - mock_run_command.assert_called_with( - ["docker", "info", "--format", "{{json .ClientInfo.Context}}"], - no_output_dump_on_exception=False, - text=True, - capture_output=True, - dry_run_override=False, - ) - mock_get_console.return_value.print.assert_called_with("[success]Good Docker context used: default.[/]") - - -@mock.patch("airflow_breeze.utils.docker_command_utils.run_command") -@mock.patch("airflow_breeze.utils.docker_command_utils.get_console") -def test_check_docker_context_other(mock_get_console, mock_run_command): - mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "other" - with pytest.raises(SystemExit): - check_docker_context() - mock_run_command.assert_called_with( - ["docker", "info", "--format", "{{json .ClientInfo.Context}}"], - no_output_dump_on_exception=False, - text=True, - capture_output=True, - dry_run_override=False, - ) - mock_get_console.return_value.print.assert_called_with( - "[error]Docker is not using the default context, used context is: other[/]\n" - "[warning]Please make sure Docker is using the default context.[/]\n" - '[warning]You can try switching contexts by running: "docker context use default"[/]' - ) - +@pytest.mark.parametrize( + "context_output, selected_context, console_output", + [ + ( + "default", + "default", + "[info]Using default as context", + ), + ("", "default", "[warning]Could not detect docker builder"), + ("a\nb", "a", "[warning]Could not use any of the preferred docker contexts"), + ("a\ndesktop-linux", "desktop-linux", "[info]Using desktop-linux as context"), + ("a\ndefault", "default", "[info]Using default as context"), + ("a\ndefault\ndesktop-linux", "desktop-linux", "[info]Using desktop-linux as context"), + ], +) +def test_autodetect_docker_context(context_output: str, selected_context: str, console_output: str): + with mock.patch("airflow_breeze.utils.docker_command_utils.run_command") as mock_run_command: + mock_run_command.return_value.returncode = 0 + mock_run_command.return_value.stdout = context_output + with mock.patch("airflow_breeze.utils.docker_command_utils.get_console") as mock_get_console: + mock_get_console.return_value.input.return_value = selected_context + assert autodetect_docker_context() == selected_context + mock_get_console.return_value.print.assert_called_once() + assert console_output in mock_get_console.return_value.print.call_args[0][0] + + +SOCKET_INFO = json.dumps( + [ + { + "Name": "default", + "Metadata": {}, + "Endpoints": {"docker": {"Host": "unix:///not-standard/docker.sock", "SkipTLSVerify": False}}, + "TLSMaterial": {}, + "Storage": {"MetadataPath": "\u003cIN MEMORY\u003e", "TLSPath": "\u003cIN MEMORY\u003e"}, + } + ] +) -@mock.patch("airflow_breeze.utils.docker_command_utils.run_command") -@mock.patch("airflow_breeze.utils.docker_command_utils.get_console") -def test_check_docker_context_command_failed(mock_get_console, mock_run_command): - mock_run_command.return_value.returncode = 1 - check_docker_context() - mock_run_command.assert_called_with( - ["docker", "info", "--format", "{{json .ClientInfo.Context}}"], - no_output_dump_on_exception=False, - text=True, - capture_output=True, - dry_run_override=False, - ) - mock_get_console.return_value.print.assert_called_with( - "[warning]Could not check for Docker context.[/]\n" - '[warning]Please make sure that Docker is using the right context by running "docker info" and ' - "checking the active Context.[/]" - ) +SOCKET_INFO_DESKTOP_LINUX = json.dumps( + [ + { + "Name": "desktop-linux", + "Metadata": {}, + "Endpoints": { + "docker": {"Host": "unix:///VERY_NON_STANDARD/docker.sock", "SkipTLSVerify": False} + }, + "TLSMaterial": {}, + "Storage": {"MetadataPath": "\u003cIN MEMORY\u003e", "TLSPath": "\u003cIN MEMORY\u003e"}, + } + ] +) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 88363d44e93c..f0748105966b 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,18 +1,18 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:b27f70a44bf5697c3852fa932120774d -build-docs:7222d86cd61be577a54cf169979133f4 +main:344261ca3aa7ff31e098b1d88280566a +build-docs:891c3f3a4e5558e96083204ed3512c44 ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd ci:free-space:47234aa0a60b0efd84972e6e797379f8 ci:get-workflow-info:01ee34c33ad62fa5dc33e0ac8773223f ci:resource-check:1d4fe47dff9fc64ac1648ec4beb2d85c ci:selective-check:f2f76d192723dfdf6cf0ada6aba05fee ci:2f985ab2973ce0bbea558d6fa9111dd1 -ci-image:build:0633b20a81c2d3ba6b35f73923b0e0d2 +ci-image:build:5587bfb880f2779e9e7757ee33e9769c ci-image:pull:3b0789fe9ec4cf6131bbec69903334d2 ci-image:verify:49fdafafa8cc3c42e0a70a4cc3fbc14b -ci-image:7700033d8ee564b70c82f40cf8a3785e +ci-image:1af2ced25364bf2ce8985a735ae88b0a cleanup:231de69d5f47ba29c883164e4575e310 compile-www-assets:c8a8c4f002f7246d0541897fc7c70313 down:99efb2055475cd40656d5498aee74ef6 @@ -31,10 +31,10 @@ k8s:status:cab572023393734e3a76a3d678f48f79 k8s:tests:789bded9b3cf35e386d96f9896844e76 k8s:upload-k8s-image:32a6d0f5bea8901f1330a0a181209e40 k8s:c1f106fa973fdd8bc1ccf71a025aa51b -prod-image:build:23ae4a22a4090d91a3411b4a55021057 +prod-image:build:fd443e6dd0aa416f505f7f3179d53eab prod-image:pull:451622b1399f3b181ec2b8620b856e3f prod-image:verify:5afd065f9eab823456371882fd2d4331 -prod-image:d289cb4e47374cac2e90d6a26f867f71 +prod-image:2da480c21d7eb120075489dc1da0f55b release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c release-management:generate-constraints:876bbc97000b5146ed29d031ec2b2fa1 release-management:generate-issue-content-providers:07c55519fff305338a0b6e753201fd39 @@ -58,9 +58,9 @@ setup:regenerate-command-images:91a65c8225fcf11c0110606fd74d8151 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 setup:201c30ea237ea013a9a209a77092a2e8 -shell:13f90c5749811e2f00e24d95e44e946d -start-airflow:22c118d58b13a9d190e966bed5bb8ed8 -static-checks:521edd8e7669b666b288ad0243b45be7 +shell:48db0babd831d704b97118d053ee639a +start-airflow:52ad9113f74b39c1ffbd37464e1415e6 +static-checks:d1cfa5b96a62490ba7f5775e1d4ffc8b testing:docker-compose-tests:70167e67853cacd9ca784695d65a7846 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b testing:integration-tests:35f0ac57157bf8fe227fd080cf216622 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index e5b2130124d5..708e4d6758c5 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - +
    ' ) - # This function is called after changing the state from State.RUNNING, + # This function is called after changing the state from RUNNING, # so we need to subtract 1 from self.try_number here. current_try_number = self.try_number - 1 additional_context: dict[str, Any] = { @@ -2572,7 +2572,7 @@ def get_num_running_task_instances(self, session: Session, same_dagrun=False) -> num_running_task_instances_query = session.query(func.count()).filter( TaskInstance.dag_id == self.dag_id, TaskInstance.task_id == self.task_id, - TaskInstance.state == State.RUNNING, + TaskInstance.state == TaskInstanceState.RUNNING, ) if same_dagrun: num_running_task_instances_query = num_running_task_instances_query.filter( @@ -2894,7 +2894,7 @@ def _is_further_mapped_inside(operator: Operator, container: TaskGroup) -> bool: # State of the task instance. # Stores string version of the task state. -TaskInstanceStateType = Tuple[TaskInstanceKey, str] +TaskInstanceStateType = Tuple[TaskInstanceKey, TaskInstanceState] class SimpleTaskInstance: diff --git a/airflow/operators/subdag.py b/airflow/operators/subdag.py index 30daf638bd9c..0f242c09f374 100644 --- a/airflow/operators/subdag.py +++ b/airflow/operators/subdag.py @@ -37,7 +37,7 @@ from airflow.sensors.base import BaseSensorOperator from airflow.utils.context import Context from airflow.utils.session import NEW_SESSION, create_session, provide_session -from airflow.utils.state import State +from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType @@ -137,17 +137,17 @@ def _reset_dag_run_and_task_instances(self, dag_run: DagRun, execution_date: dat :param execution_date: Execution date to select task instances. """ with create_session() as session: - dag_run.state = State.RUNNING + dag_run.state = DagRunState.RUNNING session.merge(dag_run) failed_task_instances = ( session.query(TaskInstance) .filter(TaskInstance.dag_id == self.subdag.dag_id) .filter(TaskInstance.execution_date == execution_date) - .filter(TaskInstance.state.in_([State.FAILED, State.UPSTREAM_FAILED])) + .filter(TaskInstance.state.in_((TaskInstanceState.FAILED, TaskInstanceState.UPSTREAM_FAILED))) ) for task_instance in failed_task_instances: - task_instance.state = State.NONE + task_instance.state = None session.merge(task_instance) session.commit() @@ -164,7 +164,7 @@ def pre_execute(self, context): dag_run = self.subdag.create_dagrun( run_type=DagRunType.SCHEDULED, execution_date=execution_date, - state=State.RUNNING, + state=DagRunState.RUNNING, conf=self.conf, external_trigger=True, data_interval=data_interval, @@ -172,13 +172,13 @@ def pre_execute(self, context): self.log.info("Created DagRun: %s", dag_run.run_id) else: self.log.info("Found existing DagRun: %s", dag_run.run_id) - if dag_run.state == State.FAILED: + if dag_run.state == DagRunState.FAILED: self._reset_dag_run_and_task_instances(dag_run, execution_date) def poke(self, context: Context): execution_date = context["execution_date"] dag_run = self._get_dagrun(execution_date=execution_date) - return dag_run.state != State.RUNNING + return dag_run.state != DagRunState.RUNNING def post_execute(self, context, result=None): super().post_execute(context) @@ -186,7 +186,7 @@ def post_execute(self, context, result=None): dag_run = self._get_dagrun(execution_date=execution_date) self.log.info("Execution finished. State is %s", dag_run.state) - if dag_run.state != State.SUCCESS: + if dag_run.state != DagRunState.SUCCESS: raise AirflowException(f"Expected state: SUCCESS. Actual state: {dag_run.state}") if self.propagate_skipped_state and self._check_skipped_states(context): @@ -196,9 +196,9 @@ def _check_skipped_states(self, context): leaves_tis = self._get_leaves_tis(context["execution_date"]) if self.propagate_skipped_state == SkippedStatePropagationOptions.ANY_LEAF: - return any(ti.state == State.SKIPPED for ti in leaves_tis) + return any(ti.state == TaskInstanceState.SKIPPED for ti in leaves_tis) if self.propagate_skipped_state == SkippedStatePropagationOptions.ALL_LEAVES: - return all(ti.state == State.SKIPPED for ti in leaves_tis) + return all(ti.state == TaskInstanceState.SKIPPED for ti in leaves_tis) raise AirflowException( f"Unimplemented SkippedStatePropagationOptions {self.propagate_skipped_state} used." ) diff --git a/airflow/sentry.py b/airflow/sentry.py index 3e222405c47a..443063af8a62 100644 --- a/airflow/sentry.py +++ b/airflow/sentry.py @@ -25,27 +25,26 @@ from airflow.configuration import conf from airflow.executors.executor_loader import ExecutorLoader from airflow.utils.session import find_session_idx, provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState if TYPE_CHECKING: from sqlalchemy.orm import Session + from airflow.models.taskinstance import TaskInstance + log = logging.getLogger(__name__) class DummySentry: """Blank class for Sentry.""" - @classmethod - def add_tagging(cls, task_instance): + def add_tagging(self, task_instance): """Blank function for tagging.""" - @classmethod - def add_breadcrumbs(cls, task_instance, session: Session | None = None): + def add_breadcrumbs(self, task_instance, session: Session | None = None): """Blank function for breadcrumbs.""" - @classmethod - def enrich_errors(cls, run): + def enrich_errors(self, run): """Blank function for formatting a TaskInstance._run_raw_task.""" return run @@ -137,13 +136,17 @@ def add_tagging(self, task_instance): scope.set_tag("operator", task.__class__.__name__) @provide_session - def add_breadcrumbs(self, task_instance, session=None): + def add_breadcrumbs( + self, + task_instance: TaskInstance, + session: Session | None = None, + ) -> None: """Function to add breadcrumbs inside of a task_instance.""" if session is None: return dr = task_instance.get_dagrun(session) task_instances = dr.get_task_instances( - state={State.SUCCESS, State.FAILED}, + state={TaskInstanceState.SUCCESS, TaskInstanceState.FAILED}, session=session, ) diff --git a/airflow/ti_deps/deps/dagrun_exists_dep.py b/airflow/ti_deps/deps/dagrun_exists_dep.py index 781ab0ebaf02..0a364628c7f4 100644 --- a/airflow/ti_deps/deps/dagrun_exists_dep.py +++ b/airflow/ti_deps/deps/dagrun_exists_dep.py @@ -19,7 +19,7 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import DagRunState class DagrunRunningDep(BaseTIDep): @@ -31,7 +31,7 @@ class DagrunRunningDep(BaseTIDep): @provide_session def _get_dep_statuses(self, ti, session, dep_context): dr = ti.get_dagrun(session) - if dr.state != State.RUNNING: + if dr.state != DagRunState.RUNNING: yield self._failing_status( reason=f"Task instance's dagrun was not in the 'running' state but in the state '{dr.state}'." ) diff --git a/airflow/ti_deps/deps/not_in_retry_period_dep.py b/airflow/ti_deps/deps/not_in_retry_period_dep.py index b3b5d4ec568f..90954f29f2f5 100644 --- a/airflow/ti_deps/deps/not_in_retry_period_dep.py +++ b/airflow/ti_deps/deps/not_in_retry_period_dep.py @@ -20,7 +20,7 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils import timezone from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState class NotInRetryPeriodDep(BaseTIDep): @@ -38,7 +38,7 @@ def _get_dep_statuses(self, ti, session, dep_context): ) return - if ti.state != State.UP_FOR_RETRY: + if ti.state != TaskInstanceState.UP_FOR_RETRY: yield self._passing_status(reason="The task instance was not marked for retrying.") return diff --git a/airflow/ti_deps/deps/not_previously_skipped_dep.py b/airflow/ti_deps/deps/not_previously_skipped_dep.py index fdc8274d9046..855f04af533d 100644 --- a/airflow/ti_deps/deps/not_previously_skipped_dep.py +++ b/airflow/ti_deps/deps/not_previously_skipped_dep.py @@ -40,7 +40,7 @@ def _get_dep_statuses(self, ti, session, dep_context): XCOM_SKIPMIXIN_SKIPPED, SkipMixin, ) - from airflow.utils.state import State + from airflow.utils.state import TaskInstanceState upstream = ti.task.get_direct_relatives(upstream=True) @@ -87,7 +87,7 @@ def _get_dep_statuses(self, ti, session, dep_context): reason=("Task should be skipped but the the past depends are not met") ) return - ti.set_state(State.SKIPPED, session) + ti.set_state(TaskInstanceState.SKIPPED, session) yield self._failing_status( reason=f"Skipping because of previous XCom result from parent task {parent.task_id}" ) diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py b/airflow/ti_deps/deps/prev_dagrun_dep.py index b3165eb7f21a..62acdbca331b 100644 --- a/airflow/ti_deps/deps/prev_dagrun_dep.py +++ b/airflow/ti_deps/deps/prev_dagrun_dep.py @@ -22,7 +22,7 @@ from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState class PrevDagrunDep(BaseTIDep): @@ -107,7 +107,7 @@ def _get_dep_statuses(self, ti: TI, session, dep_context): ) return - if previous_ti.state not in {State.SKIPPED, State.SUCCESS}: + if previous_ti.state not in {TaskInstanceState.SKIPPED, TaskInstanceState.SUCCESS}: yield self._failing_status( reason=( f"depends_on_past is true for this task, but the previous task instance {previous_ti} " diff --git a/airflow/ti_deps/deps/ready_to_reschedule.py b/airflow/ti_deps/deps/ready_to_reschedule.py index c20ef98a08cc..4fca6f5538e8 100644 --- a/airflow/ti_deps/deps/ready_to_reschedule.py +++ b/airflow/ti_deps/deps/ready_to_reschedule.py @@ -22,7 +22,7 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils import timezone from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState class ReadyToRescheduleDep(BaseTIDep): @@ -31,7 +31,7 @@ class ReadyToRescheduleDep(BaseTIDep): NAME = "Ready To Reschedule" IGNORABLE = True IS_TASK_DEP = True - RESCHEDULEABLE_STATES = {State.UP_FOR_RESCHEDULE, State.NONE} + RESCHEDULEABLE_STATES = {TaskInstanceState.UP_FOR_RESCHEDULE, None} @provide_session def _get_dep_statuses(self, ti, session, dep_context): diff --git a/airflow/ti_deps/deps/task_not_running_dep.py b/airflow/ti_deps/deps/task_not_running_dep.py index fd76873466d3..7299a4f3e2b0 100644 --- a/airflow/ti_deps/deps/task_not_running_dep.py +++ b/airflow/ti_deps/deps/task_not_running_dep.py @@ -20,7 +20,7 @@ from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState class TaskNotRunningDep(BaseTIDep): @@ -37,7 +37,7 @@ def __hash__(self): @provide_session def _get_dep_statuses(self, ti, session, dep_context=None): - if ti.state != State.RUNNING: + if ti.state != TaskInstanceState.RUNNING: yield self._passing_status(reason="Task is not in running state.") return diff --git a/airflow/utils/dot_renderer.py b/airflow/utils/dot_renderer.py index 3f35d1b21df6..d3b329cb5453 100644 --- a/airflow/utils/dot_renderer.py +++ b/airflow/utils/dot_renderer.py @@ -58,7 +58,7 @@ def _draw_task( ) -> None: """Draw a single task on the given parent_graph.""" if states_by_task_id: - state = states_by_task_id.get(task.task_id, State.NONE) + state = states_by_task_id.get(task.task_id) color = State.color_fg(state) fill_color = State.color(state) else: diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 0c98d2550378..9184a2042091 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -341,7 +341,10 @@ def _read( ) log_pos = len(logs) messages = "".join([f"*** {x}\n" for x in messages_list]) - end_of_log = ti.try_number != try_number or ti.state not in [State.RUNNING, State.DEFERRED] + end_of_log = ti.try_number != try_number or ti.state not in ( + TaskInstanceState.RUNNING, + TaskInstanceState.DEFERRED, + ) if metadata and "log_pos" in metadata: previous_chars = metadata["log_pos"] logs = logs[previous_chars:] # Cut off previously passed log test as new tail diff --git a/airflow/utils/log/log_reader.py b/airflow/utils/log/log_reader.py index a4589ebca0af..d93f15bb1a98 100644 --- a/airflow/utils/log/log_reader.py +++ b/airflow/utils/log/log_reader.py @@ -28,7 +28,7 @@ from airflow.utils.helpers import render_log_filename from airflow.utils.log.logging_mixin import ExternalLoggingMixin from airflow.utils.session import NEW_SESSION, provide_session -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState class TaskLogReader: @@ -86,7 +86,8 @@ def read_log_stream(self, ti: TaskInstance, try_number: int | None, metadata: di for host, log in logs[0]: yield "\n".join([host or "", log]) + "\n" if "end_of_log" not in metadata or ( - not metadata["end_of_log"] and ti.state not in [State.RUNNING, State.DEFERRED] + not metadata["end_of_log"] + and ti.state not in (TaskInstanceState.RUNNING, TaskInstanceState.DEFERRED) ): if not logs[0]: # we did not receive any logs in this loop diff --git a/airflow/utils/state.py b/airflow/utils/state.py index f4a8dc1a0a3d..fc74732accab 100644 --- a/airflow/utils/state.py +++ b/airflow/utils/state.py @@ -21,8 +21,7 @@ class TaskInstanceState(str, Enum): - """ - Enum that represents all possible states that a Task Instance can be in. + """All possible states that a Task Instance can be in. Note that None is also allowed, so always use this in a type hint with Optional. """ @@ -53,8 +52,7 @@ def __str__(self) -> str: class DagRunState(str, Enum): - """ - Enum that represents all possible states that a DagRun can be in. + """All possible states that a DagRun can be in. These are "shared" with TaskInstanceState in some parts of the code, so please ensure that their values always match the ones with the diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 1ca628912249..bb48f81ccc47 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -87,7 +87,9 @@ def get_instance_with_map(task_instance, session): def get_try_count(try_number: int, state: State): - return try_number + 1 if state in [State.DEFERRED, State.UP_FOR_RESCHEDULE] else try_number + if state in (TaskInstanceState.DEFERRED, TaskInstanceState.UP_FOR_RESCHEDULE): + return try_number + 1 + return try_number priority: list[None | TaskInstanceState] = [ diff --git a/airflow/www/views.py b/airflow/www/views.py index 370e15ee007d..e198e390ace7 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -767,7 +767,7 @@ def index(self): # find DAGs which have a RUNNING DagRun running_dags = dags_query.join(DagRun, DagModel.dag_id == DagRun.dag_id).where( - DagRun.state == State.RUNNING + DagRun.state == DagRunState.RUNNING ) # find DAGs for which the latest DagRun is FAILED @@ -778,7 +778,7 @@ def index(self): ) subq_failed = ( select(DagRun.dag_id, func.max(DagRun.start_date).label("start_date")) - .where(DagRun.state == State.FAILED) + .where(DagRun.state == DagRunState.FAILED) .group_by(DagRun.dag_id) .subquery() ) @@ -1127,7 +1127,7 @@ def task_stats(self, session: Session = NEW_SESSION): running_dag_run_query_result = ( select(DagRun.dag_id, DagRun.run_id) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .where(DagRun.state == State.RUNNING, DagModel.is_active) + .where(DagRun.state == DagRunState.RUNNING, DagModel.is_active) ) running_dag_run_query_result = running_dag_run_query_result.where(DagRun.dag_id.in_(filter_dag_ids)) @@ -1151,7 +1151,7 @@ def task_stats(self, session: Session = NEW_SESSION): last_dag_run = ( select(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) .join(DagModel, DagModel.dag_id == DagRun.dag_id) - .where(DagRun.state != State.RUNNING, DagModel.is_active) + .where(DagRun.state != DagRunState.RUNNING, DagModel.is_active) .group_by(DagRun.dag_id) ) @@ -1856,7 +1856,7 @@ def include_task_attrs(attr_name): "Airflow administrator for assistance.".format( "- This task instance already ran and had it's state changed manually " "(e.g. cleared in the UI)
    " - if ti and ti.state == State.NONE + if ti and ti.state is None else "" ), ) @@ -2169,7 +2169,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): run_type=DagRunType.MANUAL, execution_date=execution_date, data_interval=dag.timetable.infer_manual_data_interval(run_after=execution_date), - state=State.QUEUED, + state=DagRunState.QUEUED, conf=run_conf, external_trigger=True, dag_hash=get_airflow_app().dag_bag.dags_hash.get(dag_id), @@ -5426,23 +5426,28 @@ def action_muldelete(self, items: list[DagRun]): @action_logging def action_set_queued(self, drs: list[DagRun]): """Set state to queued.""" - return self._set_dag_runs_to_active_state(drs, State.QUEUED) + return self._set_dag_runs_to_active_state(drs, DagRunState.QUEUED) @action("set_running", "Set state to 'running'", "", single=False) @action_has_dag_edit_access @action_logging def action_set_running(self, drs: list[DagRun]): """Set state to running.""" - return self._set_dag_runs_to_active_state(drs, State.RUNNING) + return self._set_dag_runs_to_active_state(drs, DagRunState.RUNNING) @provide_session - def _set_dag_runs_to_active_state(self, drs: list[DagRun], state: str, session: Session = NEW_SESSION): + def _set_dag_runs_to_active_state( + self, + drs: list[DagRun], + state: DagRunState, + session: Session = NEW_SESSION, + ): """This routine only supports Running and Queued state.""" try: count = 0 for dr in session.scalars(select(DagRun).where(DagRun.id.in_(dagrun.id for dagrun in drs))): count += 1 - if state == State.RUNNING: + if state == DagRunState.RUNNING: dr.start_date = timezone.utcnow() dr.state = state session.commit() @@ -5863,7 +5868,12 @@ def action_muldelete(self, items): return redirect(self.get_redirect()) @provide_session - def set_task_instance_state(self, tis, target_state, session: Session = NEW_SESSION): + def set_task_instance_state( + self, + tis: Collection[TaskInstance], + target_state: TaskInstanceState, + session: Session = NEW_SESSION, + ) -> None: """Set task instance state.""" try: count = len(tis) @@ -5879,7 +5889,7 @@ def set_task_instance_state(self, tis, target_state, session: Session = NEW_SESS @action_logging def action_set_running(self, tis): """Set state to 'running'.""" - self.set_task_instance_state(tis, State.RUNNING) + self.set_task_instance_state(tis, TaskInstanceState.RUNNING) self.update_redirect() return redirect(self.get_redirect()) @@ -5888,7 +5898,7 @@ def action_set_running(self, tis): @action_logging def action_set_failed(self, tis): """Set state to 'failed'.""" - self.set_task_instance_state(tis, State.FAILED) + self.set_task_instance_state(tis, TaskInstanceState.FAILED) self.update_redirect() return redirect(self.get_redirect()) @@ -5897,7 +5907,7 @@ def action_set_failed(self, tis): @action_logging def action_set_success(self, tis): """Set state to 'success'.""" - self.set_task_instance_state(tis, State.SUCCESS) + self.set_task_instance_state(tis, TaskInstanceState.SUCCESS) self.update_redirect() return redirect(self.get_redirect()) @@ -5906,7 +5916,7 @@ def action_set_success(self, tis): @action_logging def action_set_retry(self, tis): """Set state to 'up_for_retry'.""" - self.set_task_instance_state(tis, State.UP_FOR_RETRY) + self.set_task_instance_state(tis, TaskInstanceState.UP_FOR_RETRY) self.update_redirect() return redirect(self.get_redirect()) diff --git a/dev/perf/scheduler_dag_execution_timing.py b/dev/perf/scheduler_dag_execution_timing.py index 613a929e9e40..db7f4f2e8de3 100755 --- a/dev/perf/scheduler_dag_execution_timing.py +++ b/dev/perf/scheduler_dag_execution_timing.py @@ -63,7 +63,7 @@ def change_state(self, key, state, info=None): Change the state of scheduler by waiting till the tasks is complete and then shut down the scheduler after the task is complete """ - from airflow.utils.state import State + from airflow.utils.state import TaskInstanceState super().change_state(key, state, info=info) @@ -83,7 +83,7 @@ def change_state(self, key, state, info=None): run = list(airflow.models.DagRun.find(dag_id=dag_id, execution_date=execution_date))[0] self.dags_to_watch[dag_id].runs[execution_date] = run - if run and all(t.state == State.SUCCESS for t in run.get_task_instances()): + if run and all(t.state == TaskInstanceState.SUCCESS for t in run.get_task_instances()): self.dags_to_watch[dag_id].runs.pop(execution_date) self.dags_to_watch[dag_id].waiting_for -= 1 @@ -156,7 +156,7 @@ def create_dag_runs(dag, num_runs, session): Create `num_runs` of dag runs for sub-sequent schedules """ from airflow.utils import timezone - from airflow.utils.state import State + from airflow.utils.state import DagRunState try: from airflow.utils.types import DagRunType @@ -175,7 +175,7 @@ def create_dag_runs(dag, num_runs, session): run_id=f"{id_prefix}{logical_date.isoformat()}", execution_date=logical_date, start_date=timezone.utcnow(), - state=State.RUNNING, + state=DagRunState.RUNNING, external_trigger=False, session=session, ) From 50a68c6c4ecb0a45272be7df7939ded6f28cf2f9 Mon Sep 17 00:00:00 2001 From: Richard Goodman Date: Tue, 11 Jul 2023 23:06:25 +0100 Subject: [PATCH 159/533] Append region info to S3ToRedshitOperator if present (#32328) * Append region info to S3ToRedshitOperator if present It's possible to copy from S3 into Redshift across different regions, however, currently you are unable to do so with the S3ToRedshiftOperator. This PR simply makes this possible, by checking the aws connection passed has the region set in the extras part of the connection config. If this is set, it'll use this in line with the syntax defined [here](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) * Update tests to make assertion checking valid Following on from discussion in PR, currently the way assertion is done is kind of redundant, as it's asserting static variables == other static variables. Instead, this now gets compared to what gets generated from the `_build_copy_query` function, this has been reflected for all applicable test cases in this file. --- .../amazon/aws/transfers/s3_to_redshift.py | 13 +- .../aws/transfers/test_s3_to_redshift.py | 116 +++++++++++++----- 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py index 0d2a059f6e72..b42b2d8cbb7f 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py @@ -119,13 +119,16 @@ def __init__( if arg in self.redshift_data_api_kwargs.keys(): raise AirflowException(f"Cannot include param '{arg}' in Redshift Data API kwargs") - def _build_copy_query(self, copy_destination: str, credentials_block: str, copy_options: str) -> str: + def _build_copy_query( + self, copy_destination: str, credentials_block: str, region_info: str, copy_options: str + ) -> str: column_names = "(" + ", ".join(self.column_list) + ")" if self.column_list else "" return f""" COPY {copy_destination} {column_names} FROM 's3://{self.s3_bucket}/{self.s3_key}' credentials '{credentials_block}' + {region_info} {copy_options}; """ @@ -139,7 +142,9 @@ def execute(self, context: Context) -> None: else: redshift_hook = RedshiftSQLHook(redshift_conn_id=self.redshift_conn_id) conn = S3Hook.get_connection(conn_id=self.aws_conn_id) - + region_info = "" + if conn.extra_dejson.get("region", False): + region_info = f"region '{conn.extra_dejson['region']}'" if conn.extra_dejson.get("role_arn", False): credentials_block = f"aws_iam_role={conn.extra_dejson['role_arn']}" else: @@ -151,7 +156,9 @@ def execute(self, context: Context) -> None: destination = f"{self.schema}.{self.table}" copy_destination = f"#{self.table}" if self.method == "UPSERT" else destination - copy_statement = self._build_copy_query(copy_destination, credentials_block, copy_options) + copy_statement = self._build_copy_query( + copy_destination, credentials_block, region_info, copy_options + ) sql: str | Iterable[str] diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py b/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py index 33d38b94e745..f73acf661e69 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py @@ -63,17 +63,19 @@ def test_execute(self, mock_run, mock_session, mock_connection, mock_hook): dag=None, ) op.execute(None) - copy_query = """ + expected_copy_query = """ COPY schema.table FROM 's3://bucket/key' credentials 'aws_access_key_id=aws_access_key_id;aws_secret_access_key=aws_secret_access_key' ; """ + actual_copy_query = mock_run.call_args.args[0] + assert mock_run.call_count == 1 - assert access_key in copy_query - assert secret_key in copy_query - assert_equal_ignore_multiple_spaces(mock_run.call_args.args[0], copy_query) + assert access_key in actual_copy_query + assert secret_key in actual_copy_query + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") @mock.patch("airflow.models.connection.Connection") @@ -110,17 +112,19 @@ def test_execute_with_column_list(self, mock_run, mock_session, mock_connection, dag=None, ) op.execute(None) - copy_query = """ + expected_copy_query = """ COPY schema.table (column_1, column_2) FROM 's3://bucket/key' credentials 'aws_access_key_id=aws_access_key_id;aws_secret_access_key=aws_secret_access_key' ; """ + actual_copy_query = mock_run.call_args.args[0] + assert mock_run.call_count == 1 - assert access_key in copy_query - assert secret_key in copy_query - assert_equal_ignore_multiple_spaces(mock_run.call_args.args[0], copy_query) + assert access_key in actual_copy_query + assert secret_key in actual_copy_query + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") @mock.patch("airflow.models.connection.Connection") @@ -263,18 +267,20 @@ def test_execute_sts_token(self, mock_run, mock_session, mock_connection, mock_h dag=None, ) op.execute(None) - copy_statement = """ + expected_copy_query = """ COPY schema.table FROM 's3://bucket/key' credentials 'aws_access_key_id=ASIA_aws_access_key_id;aws_secret_access_key=aws_secret_access_key;token=aws_secret_token' ; """ - assert access_key in copy_statement - assert secret_key in copy_statement - assert token in copy_statement + actual_copy_query = mock_run.call_args.args[0] + + assert access_key in actual_copy_query + assert secret_key in actual_copy_query + assert token in actual_copy_query assert mock_run.call_count == 1 - assert_equal_ignore_multiple_spaces(mock_run.call_args.args[0], copy_statement) + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") @mock.patch("airflow.models.connection.Connection") @@ -312,17 +318,68 @@ def test_execute_role_arn(self, mock_run, mock_session, mock_connection, mock_ho dag=None, ) op.execute(None) - copy_statement = """ + expected_copy_query = """ COPY schema.table FROM 's3://bucket/key' credentials 'aws_iam_role=arn:aws:iam::112233445566:role/myRole' ; """ + actual_copy_query = mock_run.call_args.args[0] + + assert extra["role_arn"] in actual_copy_query + assert mock_run.call_count == 1 + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) + + @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_connection") + @mock.patch("airflow.models.connection.Connection") + @mock.patch("boto3.session.Session") + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook.run") + def test_different_region(self, mock_run, mock_session, mock_connection, mock_hook): + access_key = "aws_access_key_id" + secret_key = "aws_secret_access_key" + extra = {"region": "eu-central-1"} + mock_session.return_value = Session(access_key, secret_key) + mock_session.return_value.access_key = access_key + mock_session.return_value.secret_key = secret_key + mock_session.return_value.token = None + + mock_connection.return_value = Connection(extra=extra) + mock_hook.return_value = Connection(extra=extra) + + schema = "schema" + table = "table" + s3_bucket = "bucket" + s3_key = "key" + copy_options = "" + + op = S3ToRedshiftOperator( + schema=schema, + table=table, + s3_bucket=s3_bucket, + s3_key=s3_key, + copy_options=copy_options, + redshift_conn_id="redshift_conn_id", + aws_conn_id="aws_conn_id", + task_id="task_id", + dag=None, + ) + op.execute(None) + expected_copy_query = """ + COPY schema.table + FROM 's3://bucket/key' + credentials + 'aws_access_key_id=aws_access_key_id;aws_secret_access_key=aws_secret_access_key' + region 'eu-central-1' + ; + """ + actual_copy_query = mock_run.call_args.args[0] - assert extra["role_arn"] in copy_statement + assert access_key in actual_copy_query + assert secret_key in actual_copy_query + assert extra["region"] in actual_copy_query assert mock_run.call_count == 1 - assert_equal_ignore_multiple_spaces(mock_run.call_args.args[0], copy_statement) + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) def test_template_fields_overrides(self): assert S3ToRedshiftOperator.template_fields == ( @@ -420,19 +477,10 @@ def test_using_redshift_data_api(self, mock_rs, mock_run, mock_session, mock_con ), ) op.execute(None) - copy_query = """ - COPY schema.table - FROM 's3://bucket/key' - credentials - 'aws_access_key_id=aws_access_key_id;aws_secret_access_key=aws_secret_access_key' - ; - """ + mock_run.assert_not_called() - assert access_key in copy_query - assert secret_key in copy_query mock_rs.execute_statement.assert_called_once() - # test with all args besides sql _call = deepcopy(mock_rs.execute_statement.call_args.kwargs) _call.pop("Sql") assert _call == dict( @@ -443,8 +491,20 @@ def test_using_redshift_data_api(self, mock_rs, mock_run, mock_session, mock_con StatementName=statement_name, WithEvent=False, ) + + expected_copy_query = """ + COPY schema.table + FROM 's3://bucket/key' + credentials + 'aws_access_key_id=aws_access_key_id;aws_secret_access_key=aws_secret_access_key' + ; + """ + actual_copy_query = mock_rs.execute_statement.call_args.kwargs["Sql"] + mock_rs.describe_statement.assert_called_once_with( Id="STATEMENT_ID", ) - # test sql arg - assert_equal_ignore_multiple_spaces(mock_rs.execute_statement.call_args.kwargs["Sql"], copy_query) + + assert access_key in actual_copy_query + assert secret_key in actual_copy_query + assert_equal_ignore_multiple_spaces(actual_copy_query, expected_copy_query) From 977b9aee2b5a32182bb2dce348d962069693bb0c Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 12 Jul 2023 01:00:59 +0200 Subject: [PATCH 160/533] remove ferruzzi from collaborators list (#32531) Signed-off-by: Hussein Awala --- .asf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index ea22101f1cd9..a5611163f369 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -72,7 +72,6 @@ github: collaborators: - mhenc - - ferruzzi - pankajkoti - utkarsharma2 - Lee-W From f7d4d98f2b531378613a180de86620a2b0e5d925 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Tue, 11 Jul 2023 16:21:17 -0700 Subject: [PATCH 161/533] Kubernetes Executor Load Time Optimizations (#30727) Import optimizations to decrease the amount of time it takes to load/import the kubernetes executor. Move some expensive typing related imports to be under TYPE_CHECKING. Refactor expensive classes (other than the KubernetesExecutor) out of the kubernetes_executor.py module into a utils modules so they are only loaded at runtime. Also move some imports to runtime imports, closer to their usage. --- airflow/cli/commands/kubernetes_command.py | 3 +- airflow/executors/kubernetes_executor.py | 525 ++---------------- .../executors/kubernetes_executor_types.py | 35 ++ .../executors/kubernetes_executor_utils.py | 477 ++++++++++++++++ .../kubernetes/kubernetes_helper_functions.py | 15 +- tests/executors/test_kubernetes_executor.py | 94 ++-- 6 files changed, 628 insertions(+), 521 deletions(-) create mode 100644 airflow/executors/kubernetes_executor_types.py create mode 100644 airflow/executors/kubernetes_executor_utils.py diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py index 25cfecc2b47b..c367d4be87b6 100644 --- a/airflow/cli/commands/kubernetes_command.py +++ b/airflow/cli/commands/kubernetes_command.py @@ -25,9 +25,10 @@ from kubernetes.client.api_client import ApiClient from kubernetes.client.rest import ApiException -from airflow.executors.kubernetes_executor import KubeConfig, create_pod_id +from airflow.executors.kubernetes_executor import KubeConfig from airflow.kubernetes import pod_generator from airflow.kubernetes.kube_client import get_kube_client +from airflow.kubernetes.kubernetes_helper_functions import create_pod_id from airflow.kubernetes.pod_generator import PodGenerator from airflow.models import DagRun, TaskInstance from airflow.utils import cli as cli_utils, yaml diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py index ea84ad6dab64..c1f8861f895f 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/executors/kubernetes_executor.py @@ -29,477 +29,36 @@ import time from collections import defaultdict from contextlib import suppress +from datetime import datetime from queue import Empty, Queue -from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Any, Sequence -from kubernetes import client, watch -from kubernetes.client import Configuration, models as k8s -from kubernetes.client.rest import ApiException from sqlalchemy.orm import Session -from urllib3.exceptions import ReadTimeoutError from airflow.configuration import conf -from airflow.exceptions import AirflowException, PodMutationHookException, PodReconciliationError +from airflow.exceptions import PodMutationHookException, PodReconciliationError from airflow.executors.base_executor import BaseExecutor -from airflow.kubernetes import pod_generator -from airflow.kubernetes.kube_client import get_kube_client +from airflow.executors.kubernetes_executor_types import POD_EXECUTOR_DONE_KEY from airflow.kubernetes.kube_config import KubeConfig -from airflow.kubernetes.kubernetes_helper_functions import ( - annotations_for_logging_task_metadata, - annotations_to_key, - create_pod_id, -) -from airflow.kubernetes.pod_generator import PodGenerator +from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key from airflow.utils.event_scheduler import EventScheduler -from airflow.utils.log.logging_mixin import LoggingMixin, remove_escape_codes +from airflow.utils.log.logging_mixin import remove_escape_codes from airflow.utils.session import NEW_SESSION, provide_session -from airflow.utils.singleton import Singleton from airflow.utils.state import State, TaskInstanceState if TYPE_CHECKING: + from kubernetes import client + from kubernetes.client import models as k8s + from airflow.executors.base_executor import CommandType + from airflow.executors.kubernetes_executor_types import ( + KubernetesJobType, + KubernetesResultsType, + ) + from airflow.executors.kubernetes_executor_utils import AirflowKubernetesScheduler from airflow.models.taskinstance import TaskInstance from airflow.models.taskinstancekey import TaskInstanceKey - # TaskInstance key, command, configuration, pod_template_file - KubernetesJobType = Tuple[TaskInstanceKey, CommandType, Any, Optional[str]] - - # key, pod state, pod_name, namespace, resource_version - KubernetesResultsType = Tuple[TaskInstanceKey, Optional[str], str, str, str] - - # pod_name, namespace, pod state, annotations, resource_version - KubernetesWatchType = Tuple[str, str, Optional[str], Dict[str, str], str] - -ALL_NAMESPACES = "ALL_NAMESPACES" -POD_EXECUTOR_DONE_KEY = "airflow_executor_done" - - -class ResourceVersion(metaclass=Singleton): - """Singleton for tracking resourceVersion from Kubernetes.""" - - resource_version: dict[str, str] = {} - - -class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin): - """Watches for Kubernetes jobs.""" - - def __init__( - self, - namespace: str, - watcher_queue: Queue[KubernetesWatchType], - resource_version: str | None, - scheduler_job_id: str, - kube_config: Configuration, - ): - super().__init__() - self.namespace = namespace - self.scheduler_job_id = scheduler_job_id - self.watcher_queue = watcher_queue - self.resource_version = resource_version - self.kube_config = kube_config - - def run(self) -> None: - """Performs watching.""" - if TYPE_CHECKING: - assert self.scheduler_job_id - - kube_client: client.CoreV1Api = get_kube_client() - while True: - try: - self.resource_version = self._run( - kube_client, self.resource_version, self.scheduler_job_id, self.kube_config - ) - except ReadTimeoutError: - self.log.warning( - "There was a timeout error accessing the Kube API. Retrying request.", exc_info=True - ) - time.sleep(1) - except Exception: - self.log.exception("Unknown error in KubernetesJobWatcher. Failing") - self.resource_version = "0" - ResourceVersion().resource_version[self.namespace] = "0" - raise - else: - self.log.warning( - "Watch died gracefully, starting back up with: last resource_version: %s", - self.resource_version, - ) - - def _pod_events(self, kube_client: client.CoreV1Api, query_kwargs: dict): - watcher = watch.Watch() - try: - if self.namespace == ALL_NAMESPACES: - return watcher.stream(kube_client.list_pod_for_all_namespaces, **query_kwargs) - else: - return watcher.stream(kube_client.list_namespaced_pod, self.namespace, **query_kwargs) - except ApiException as e: - if e.status == 410: # Resource version is too old - if self.namespace == ALL_NAMESPACES: - pods = kube_client.list_pod_for_all_namespaces(watch=False) - else: - pods = kube_client.list_namespaced_pod(namespace=self.namespace, watch=False) - resource_version = pods.metadata.resource_version - query_kwargs["resource_version"] = resource_version - return self._pod_events(kube_client=kube_client, query_kwargs=query_kwargs) - else: - raise - - def _run( - self, - kube_client: client.CoreV1Api, - resource_version: str | None, - scheduler_job_id: str, - kube_config: Any, - ) -> str | None: - self.log.info("Event: and now my watch begins starting at resource_version: %s", resource_version) - - kwargs = {"label_selector": f"airflow-worker={scheduler_job_id}"} - if resource_version: - kwargs["resource_version"] = resource_version - if kube_config.kube_client_request_args: - for key, value in kube_config.kube_client_request_args.items(): - kwargs[key] = value - - last_resource_version: str | None = None - - for event in self._pod_events(kube_client=kube_client, query_kwargs=kwargs): - task = event["object"] - self.log.debug("Event: %s had an event of type %s", task.metadata.name, event["type"]) - if event["type"] == "ERROR": - return self.process_error(event) - annotations = task.metadata.annotations - task_instance_related_annotations = { - "dag_id": annotations["dag_id"], - "task_id": annotations["task_id"], - "execution_date": annotations.get("execution_date"), - "run_id": annotations.get("run_id"), - "try_number": annotations["try_number"], - } - map_index = annotations.get("map_index") - if map_index is not None: - task_instance_related_annotations["map_index"] = map_index - - self.process_status( - pod_name=task.metadata.name, - namespace=task.metadata.namespace, - status=task.status.phase, - annotations=task_instance_related_annotations, - resource_version=task.metadata.resource_version, - event=event, - ) - last_resource_version = task.metadata.resource_version - - return last_resource_version - - def process_error(self, event: Any) -> str: - """Process error response.""" - self.log.error("Encountered Error response from k8s list namespaced pod stream => %s", event) - raw_object = event["raw_object"] - if raw_object["code"] == 410: - self.log.info( - "Kubernetes resource version is too old, must reset to 0 => %s", (raw_object["message"],) - ) - # Return resource version 0 - return "0" - raise AirflowException( - f"Kubernetes failure for {raw_object['reason']} with code {raw_object['code']} and message: " - f"{raw_object['message']}" - ) - - def process_status( - self, - pod_name: str, - namespace: str, - status: str, - annotations: dict[str, str], - resource_version: str, - event: Any, - ) -> None: - pod = event["object"] - annotations_string = annotations_for_logging_task_metadata(annotations) - """Process status response.""" - if status == "Pending": - # deletion_timestamp is set by kube server when a graceful deletion is requested. - # since kube server have received request to delete pod set TI state failed - if event["type"] == "DELETED" and pod.metadata.deletion_timestamp: - self.log.info("Event: Failed to start pod %s, annotations: %s", pod_name, annotations_string) - self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) - else: - self.log.debug("Event: %s Pending, annotations: %s", pod_name, annotations_string) - elif status == "Failed": - self.log.error("Event: %s Failed, annotations: %s", pod_name, annotations_string) - self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) - elif status == "Succeeded": - # We get multiple events once the pod hits a terminal state, and we only want to - # send it along to the scheduler once. - # If our event type is DELETED, we have the POD_EXECUTOR_DONE_KEY, or the pod has - # a deletion timestamp, we've already seen the initial Succeeded event and sent it - # along to the scheduler. - if ( - event["type"] == "DELETED" - or POD_EXECUTOR_DONE_KEY in pod.metadata.labels - or pod.metadata.deletion_timestamp - ): - self.log.info( - "Skipping event for Succeeded pod %s - event for this pod already sent to executor", - pod_name, - ) - return - self.log.info("Event: %s Succeeded, annotations: %s", pod_name, annotations_string) - self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version)) - elif status == "Running": - # deletion_timestamp is set by kube server when a graceful deletion is requested. - # since kube server have received request to delete pod set TI state failed - if event["type"] == "DELETED" and pod.metadata.deletion_timestamp: - self.log.info( - "Event: Pod %s deleted before it could complete, annotations: %s", - pod_name, - annotations_string, - ) - self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) - else: - self.log.info("Event: %s is Running, annotations: %s", pod_name, annotations_string) - else: - self.log.warning( - "Event: Invalid state: %s on pod: %s in namespace %s with annotations: %s with " - "resource_version: %s", - status, - pod_name, - namespace, - annotations, - resource_version, - ) - - -class AirflowKubernetesScheduler(LoggingMixin): - """Airflow Scheduler for Kubernetes.""" - - def __init__( - self, - kube_config: Any, - result_queue: Queue[KubernetesResultsType], - kube_client: client.CoreV1Api, - scheduler_job_id: str, - ): - super().__init__() - self.log.debug("Creating Kubernetes executor") - self.kube_config = kube_config - self.result_queue = result_queue - self.namespace = self.kube_config.kube_namespace - self.log.debug("Kubernetes using namespace %s", self.namespace) - self.kube_client = kube_client - self._manager = multiprocessing.Manager() - self.watcher_queue = self._manager.Queue() - self.scheduler_job_id = scheduler_job_id - self.kube_watchers = self._make_kube_watchers() - - def run_pod_async(self, pod: k8s.V1Pod, **kwargs): - """Runs POD asynchronously.""" - sanitized_pod = self.kube_client.api_client.sanitize_for_serialization(pod) - json_pod = json.dumps(sanitized_pod, indent=2) - - self.log.debug("Pod Creation Request: \n%s", json_pod) - try: - resp = self.kube_client.create_namespaced_pod( - body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs - ) - self.log.debug("Pod Creation Response: %s", resp) - except Exception as e: - self.log.exception("Exception when attempting to create Namespaced Pod: %s", json_pod) - raise e - return resp - - def _make_kube_watcher(self, namespace) -> KubernetesJobWatcher: - resource_version = ResourceVersion().resource_version.get(namespace, "0") - watcher = KubernetesJobWatcher( - watcher_queue=self.watcher_queue, - namespace=namespace, - resource_version=resource_version, - scheduler_job_id=self.scheduler_job_id, - kube_config=self.kube_config, - ) - watcher.start() - return watcher - - def _make_kube_watchers(self) -> dict[str, KubernetesJobWatcher]: - watchers = {} - if self.kube_config.multi_namespace_mode: - namespaces_to_watch = ( - self.kube_config.multi_namespace_mode_namespace_list - if self.kube_config.multi_namespace_mode_namespace_list - else [ALL_NAMESPACES] - ) - else: - namespaces_to_watch = [self.kube_config.kube_namespace] - - for namespace in namespaces_to_watch: - watchers[namespace] = self._make_kube_watcher(namespace) - return watchers - - def _health_check_kube_watchers(self): - for namespace, kube_watcher in self.kube_watchers.items(): - if kube_watcher.is_alive(): - self.log.debug("KubeJobWatcher for namespace %s alive, continuing", namespace) - else: - self.log.error( - ( - "Error while health checking kube watcher process for namespace %s. " - "Process died for unknown reasons" - ), - namespace, - ) - ResourceVersion().resource_version[namespace] = "0" - self.kube_watchers[namespace] = self._make_kube_watcher(namespace) - - def run_next(self, next_job: KubernetesJobType) -> None: - """Receives the next job to run, builds the pod, and creates it.""" - key, command, kube_executor_config, pod_template_file = next_job - - dag_id, task_id, run_id, try_number, map_index = key - - if command[0:3] != ["airflow", "tasks", "run"]: - raise ValueError('The command must start with ["airflow", "tasks", "run"].') - - base_worker_pod = get_base_pod_from_template(pod_template_file, self.kube_config) - - if not base_worker_pod: - raise AirflowException( - f"could not find a valid worker template yaml at {self.kube_config.pod_template_file}" - ) - - pod = PodGenerator.construct_pod( - namespace=self.namespace, - scheduler_job_id=self.scheduler_job_id, - pod_id=create_pod_id(dag_id, task_id), - dag_id=dag_id, - task_id=task_id, - kube_image=self.kube_config.kube_image, - try_number=try_number, - map_index=map_index, - date=None, - run_id=run_id, - args=command, - pod_override_object=kube_executor_config, - base_worker_pod=base_worker_pod, - with_mutation_hook=True, - ) - # Reconcile the pod generated by the Operator and the Pod - # generated by the .cfg file - self.log.info( - "Creating kubernetes pod for job is %s, with pod name %s, annotations: %s", - key, - pod.metadata.name, - annotations_for_logging_task_metadata(pod.metadata.annotations), - ) - self.log.debug("Kubernetes running for command %s", command) - self.log.debug("Kubernetes launching image %s", pod.spec.containers[0].image) - - # the watcher will monitor pods, so we do not block. - self.run_pod_async(pod, **self.kube_config.kube_client_request_args) - self.log.debug("Kubernetes Job created!") - - def delete_pod(self, pod_name: str, namespace: str) -> None: - """Deletes Pod from a namespace. Does not raise if it does not exist.""" - try: - self.log.debug("Deleting pod %s in namespace %s", pod_name, namespace) - self.kube_client.delete_namespaced_pod( - pod_name, - namespace, - body=client.V1DeleteOptions(**self.kube_config.delete_option_kwargs), - **self.kube_config.kube_client_request_args, - ) - except ApiException as e: - # If the pod is already deleted - if e.status != 404: - raise - - def patch_pod_executor_done(self, *, pod_name: str, namespace: str): - """Add a "done" annotation to ensure we don't continually adopt pods.""" - self.log.debug("Patching pod %s in namespace %s to mark it as done", pod_name, namespace) - try: - self.kube_client.patch_namespaced_pod( - name=pod_name, - namespace=namespace, - body={"metadata": {"labels": {POD_EXECUTOR_DONE_KEY: "True"}}}, - ) - except ApiException as e: - self.log.info("Failed to patch pod %s with done annotation. Reason: %s", pod_name, e) - - def sync(self) -> None: - """ - Checks the status of all currently running kubernetes jobs. - - If a job is completed, its status is placed in the result queue to be sent back to the scheduler. - """ - self.log.debug("Syncing KubernetesExecutor") - self._health_check_kube_watchers() - while True: - try: - task = self.watcher_queue.get_nowait() - try: - self.log.debug("Processing task %s", task) - self.process_watcher_task(task) - finally: - self.watcher_queue.task_done() - except Empty: - break - - def process_watcher_task(self, task: KubernetesWatchType) -> None: - """Process the task by watcher.""" - pod_name, namespace, state, annotations, resource_version = task - self.log.debug( - "Attempting to finish pod; pod_name: %s; state: %s; annotations: %s", - pod_name, - state, - annotations_for_logging_task_metadata(annotations), - ) - key = annotations_to_key(annotations=annotations) - if key: - self.log.debug("finishing job %s - %s (%s)", key, state, pod_name) - self.result_queue.put((key, state, pod_name, namespace, resource_version)) - - def _flush_watcher_queue(self) -> None: - self.log.debug("Executor shutting down, watcher_queue approx. size=%d", self.watcher_queue.qsize()) - while True: - try: - task = self.watcher_queue.get_nowait() - # Ignoring it since it can only have either FAILED or SUCCEEDED pods - self.log.warning("Executor shutting down, IGNORING watcher task=%s", task) - self.watcher_queue.task_done() - except Empty: - break - - def terminate(self) -> None: - """Terminates the watcher.""" - self.log.debug("Terminating kube_watchers...") - for namespace, kube_watcher in self.kube_watchers.items(): - kube_watcher.terminate() - kube_watcher.join() - self.log.debug("kube_watcher=%s", kube_watcher) - self.log.debug("Flushing watcher_queue...") - self._flush_watcher_queue() - # Queue should be empty... - self.watcher_queue.join() - self.log.debug("Shutting down manager...") - self._manager.shutdown() - - -def get_base_pod_from_template(pod_template_file: str | None, kube_config: Any) -> k8s.V1Pod: - """ - Get base pod from template. - - Reads either the pod_template_file set in the executor_config or the base pod_template_file - set in the airflow.cfg to craft a "base pod" that will be used by the KubernetesExecutor - - :param pod_template_file: absolute path to a pod_template_file.yaml or None - :param kube_config: The KubeConfig class generated by airflow that contains all kube metadata - :return: a V1Pod that can be used as the base pod for k8s tasks - """ - if pod_template_file: - return PodGenerator.deserialize_model_file(pod_template_file) - else: - return PodGenerator.deserialize_model_file(kube_config.pod_template_file) - class KubernetesExecutor(BaseExecutor): """Executor for Kubernetes.""" @@ -536,6 +95,19 @@ def _list_pods(self, query_kwargs): return pods + def _make_safe_label_value(self, input_value: str | datetime) -> str: + """ + Normalize a provided label to be of valid length and characters. + See airflow.kubernetes.pod_generator.make_safe_label_value for more details. + """ + # airflow.kubernetes is an expensive import, locally import it here to + # speed up load times of the kubernetes_executor module. + from airflow.kubernetes import pod_generator + + if isinstance(input_value, datetime): + return pod_generator.datetime_to_label_safe_datestring(input_value) + return pod_generator.make_safe_label_value(input_value) + @provide_session def clear_not_launched_queued_tasks(self, session: Session = NEW_SESSION) -> None: """ @@ -580,9 +152,9 @@ def clear_not_launched_queued_tasks(self, session: Session = NEW_SESSION) -> Non # Build the pod selector base_label_selector = ( - f"dag_id={pod_generator.make_safe_label_value(ti.dag_id)}," - f"task_id={pod_generator.make_safe_label_value(ti.task_id)}," - f"airflow-worker={pod_generator.make_safe_label_value(str(ti.queued_by_job_id))}" + f"dag_id={self._make_safe_label_value(ti.dag_id)}," + f"task_id={self._make_safe_label_value(ti.task_id)}," + f"airflow-worker={self._make_safe_label_value(str(ti.queued_by_job_id))}" ) if ti.map_index >= 0: # Old tasks _couldn't_ be mapped, so we don't have to worry about compat @@ -592,15 +164,14 @@ def clear_not_launched_queued_tasks(self, session: Session = NEW_SESSION) -> Non kwargs.update(**self.kube_config.kube_client_request_args) # Try run_id first - kwargs["label_selector"] += ",run_id=" + pod_generator.make_safe_label_value(ti.run_id) + kwargs["label_selector"] += ",run_id=" + self._make_safe_label_value(ti.run_id) pod_list = self._list_pods(kwargs) if pod_list: continue # Fallback to old style of using execution_date - kwargs["label_selector"] = ( - f"{base_label_selector}," - f"execution_date={pod_generator.datetime_to_label_safe_datestring(ti.execution_date)}" - ) + kwargs[ + "label_selector" + ] = f"{base_label_selector},execution_date={self._make_safe_label_value(ti.execution_date)}" pod_list = self._list_pods(kwargs) if pod_list: continue @@ -617,6 +188,9 @@ def start(self) -> None: self.log.info("Start Kubernetes executor") self.scheduler_job_id = str(self.job_id) self.log.debug("Start with scheduler_job_id: %s", self.scheduler_job_id) + from airflow.executors.kubernetes_executor_utils import AirflowKubernetesScheduler + from airflow.kubernetes.kube_client import get_kube_client + self.kube_client = get_kube_client() self.kube_scheduler = AirflowKubernetesScheduler( kube_config=self.kube_config, @@ -650,6 +224,8 @@ def execute_async( else: self.log.info("Add task %s with command %s", key, command) + from airflow.kubernetes.pod_generator import PodGenerator + try: kube_executor_config = PodGenerator.from_obj(executor_config) except Exception: @@ -706,15 +282,20 @@ def sync(self) -> None: except Empty: break + from airflow.executors.kubernetes_executor_utils import ResourceVersion + resource_instance = ResourceVersion() for ns in resource_instance.resource_version.keys(): resource_instance.resource_version[ns] = ( last_resource_version[ns] or resource_instance.resource_version[ns] ) + from kubernetes.client.rest import ApiException + for _ in range(self.kube_config.worker_pods_creation_batch_size): try: task = self.task_queue.get_nowait() + try: self.kube_scheduler.run_next(task) except PodReconciliationError as e: @@ -766,7 +347,6 @@ def _change_state( ) -> None: if TYPE_CHECKING: assert self.kube_scheduler - from airflow.models.taskinstance import TaskInstance if state == State.RUNNING: self.event_buffer[key] = state, None @@ -787,6 +367,8 @@ def _change_state( # If we don't have a TI state, look it up from the db. event_buffer expects the TI state if state is None: + from airflow.models.taskinstance import TaskInstance + state = session.query(TaskInstance.state).filter(TaskInstance.filter_for_tis([key])).scalar() self.event_buffer[key] = state, None @@ -803,6 +385,7 @@ def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], li messages = [] log = [] try: + from airflow.kubernetes.kube_client import get_kube_client from airflow.kubernetes.pod_generator import PodGenerator @@ -849,7 +432,7 @@ def try_adopt_task_instances(self, tis: Sequence[TaskInstance]) -> Sequence[Task tis_to_flush_by_key = {ti.key: ti for ti in tis if ti.queued_by_job_id} kube_client: client.CoreV1Api = self.kube_client for scheduler_job_id in scheduler_job_ids: - scheduler_job_id = pod_generator.make_safe_label_value(str(scheduler_job_id)) + scheduler_job_id = self._make_safe_label_value(str(scheduler_job_id)) # We will look for any pods owned by the no-longer-running scheduler, # but will exclude only successful pods, as those TIs will have a terminal state # and not be up for adoption! @@ -880,6 +463,8 @@ def cleanup_stuck_queued_tasks(self, tis: list[TaskInstance]) -> list[str]: :param tis: List of Task Instances to clean up :return: List of readable task instances for a warning message """ + from airflow.kubernetes.pod_generator import PodGenerator + if TYPE_CHECKING: assert self.kube_client assert self.kube_scheduler @@ -930,8 +515,11 @@ def adopt_launched_task( self.log.error("attempting to adopt taskinstance which was not specified by database: %s", ti_key) return - new_worker_id_label = pod_generator.make_safe_label_value(self.scheduler_job_id) + new_worker_id_label = self._make_safe_label_value(self.scheduler_job_id) + from kubernetes.client.rest import ApiException + try: + kube_client.patch_namespaced_pod( name=pod.metadata.name, namespace=pod.metadata.namespace, @@ -953,7 +541,7 @@ def _adopt_completed_pods(self, kube_client: client.CoreV1Api) -> None: if TYPE_CHECKING: assert self.scheduler_job_id - new_worker_id_label = pod_generator.make_safe_label_value(self.scheduler_job_id) + new_worker_id_label = self._make_safe_label_value(self.scheduler_job_id) query_kwargs = { "field_selector": "status.phase=Succeeded", "label_selector": ( @@ -964,7 +552,10 @@ def _adopt_completed_pods(self, kube_client: client.CoreV1Api) -> None: pod_list = self._list_pods(query_kwargs) for pod in pod_list: self.log.info("Attempting to adopt pod %s", pod.metadata.name) + from kubernetes.client.rest import ApiException + try: + kube_client.patch_namespaced_pod( name=pod.metadata.name, namespace=pod.metadata.namespace, diff --git a/airflow/executors/kubernetes_executor_types.py b/airflow/executors/kubernetes_executor_types.py new file mode 100644 index 000000000000..a13cd35f8d76 --- /dev/null +++ b/airflow/executors/kubernetes_executor_types.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple + +if TYPE_CHECKING: + from airflow.executors.base_executor import CommandType + from airflow.models.taskinstance import TaskInstanceKey + + # TaskInstance key, command, configuration, pod_template_file + KubernetesJobType = Tuple[TaskInstanceKey, CommandType, Any, Optional[str]] + + # key, pod state, pod_name, namespace, resource_version + KubernetesResultsType = Tuple[TaskInstanceKey, Optional[str], str, str, str] + + # pod_name, namespace, pod state, annotations, resource_version + KubernetesWatchType = Tuple[str, str, Optional[str], Dict[str, str], str] + +ALL_NAMESPACES = "ALL_NAMESPACES" +POD_EXECUTOR_DONE_KEY = "airflow_executor_done" diff --git a/airflow/executors/kubernetes_executor_utils.py b/airflow/executors/kubernetes_executor_utils.py new file mode 100644 index 000000000000..b1ee49cb695e --- /dev/null +++ b/airflow/executors/kubernetes_executor_utils.py @@ -0,0 +1,477 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +import multiprocessing +import time +from queue import Empty, Queue +from typing import TYPE_CHECKING, Any + +from kubernetes import client, watch +from kubernetes.client import Configuration, models as k8s +from kubernetes.client.rest import ApiException +from urllib3.exceptions import ReadTimeoutError + +from airflow.exceptions import AirflowException +from airflow.kubernetes.kube_client import get_kube_client +from airflow.kubernetes.kubernetes_helper_functions import ( + annotations_for_logging_task_metadata, + annotations_to_key, + create_pod_id, +) +from airflow.kubernetes.pod_generator import PodGenerator +from airflow.utils.log.logging_mixin import LoggingMixin +from airflow.utils.singleton import Singleton +from airflow.utils.state import State + +if TYPE_CHECKING: + from airflow.executors.kubernetes_executor_types import ( + KubernetesJobType, + KubernetesResultsType, + KubernetesWatchType, + ) + + +from airflow.executors.kubernetes_executor_types import ALL_NAMESPACES, POD_EXECUTOR_DONE_KEY + + +class ResourceVersion(metaclass=Singleton): + """Singleton for tracking resourceVersion from Kubernetes.""" + + resource_version: dict[str, str] = {} + + +class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin): + """Watches for Kubernetes jobs.""" + + def __init__( + self, + namespace: str, + watcher_queue: Queue[KubernetesWatchType], + resource_version: str | None, + scheduler_job_id: str, + kube_config: Configuration, + ): + super().__init__() + self.namespace = namespace + self.scheduler_job_id = scheduler_job_id + self.watcher_queue = watcher_queue + self.resource_version = resource_version + self.kube_config = kube_config + + def run(self) -> None: + """Performs watching.""" + if TYPE_CHECKING: + assert self.scheduler_job_id + + kube_client: client.CoreV1Api = get_kube_client() + while True: + try: + self.resource_version = self._run( + kube_client, self.resource_version, self.scheduler_job_id, self.kube_config + ) + except ReadTimeoutError: + self.log.warning( + "There was a timeout error accessing the Kube API. Retrying request.", exc_info=True + ) + time.sleep(1) + except Exception: + self.log.exception("Unknown error in KubernetesJobWatcher. Failing") + self.resource_version = "0" + ResourceVersion().resource_version[self.namespace] = "0" + raise + else: + self.log.warning( + "Watch died gracefully, starting back up with: last resource_version: %s", + self.resource_version, + ) + + def _pod_events(self, kube_client: client.CoreV1Api, query_kwargs: dict): + watcher = watch.Watch() + try: + if self.namespace == ALL_NAMESPACES: + return watcher.stream(kube_client.list_pod_for_all_namespaces, **query_kwargs) + else: + return watcher.stream(kube_client.list_namespaced_pod, self.namespace, **query_kwargs) + except ApiException as e: + if e.status == 410: # Resource version is too old + if self.namespace == ALL_NAMESPACES: + pods = kube_client.list_pod_for_all_namespaces(watch=False) + else: + pods = kube_client.list_namespaced_pod(namespace=self.namespace, watch=False) + resource_version = pods.metadata.resource_version + query_kwargs["resource_version"] = resource_version + return self._pod_events(kube_client=kube_client, query_kwargs=query_kwargs) + else: + raise + + def _run( + self, + kube_client: client.CoreV1Api, + resource_version: str | None, + scheduler_job_id: str, + kube_config: Any, + ) -> str | None: + self.log.info("Event: and now my watch begins starting at resource_version: %s", resource_version) + + kwargs = {"label_selector": f"airflow-worker={scheduler_job_id}"} + if resource_version: + kwargs["resource_version"] = resource_version + if kube_config.kube_client_request_args: + for key, value in kube_config.kube_client_request_args.items(): + kwargs[key] = value + + last_resource_version: str | None = None + + for event in self._pod_events(kube_client=kube_client, query_kwargs=kwargs): + task = event["object"] + self.log.debug("Event: %s had an event of type %s", task.metadata.name, event["type"]) + if event["type"] == "ERROR": + return self.process_error(event) + annotations = task.metadata.annotations + task_instance_related_annotations = { + "dag_id": annotations["dag_id"], + "task_id": annotations["task_id"], + "execution_date": annotations.get("execution_date"), + "run_id": annotations.get("run_id"), + "try_number": annotations["try_number"], + } + map_index = annotations.get("map_index") + if map_index is not None: + task_instance_related_annotations["map_index"] = map_index + + self.process_status( + pod_name=task.metadata.name, + namespace=task.metadata.namespace, + status=task.status.phase, + annotations=task_instance_related_annotations, + resource_version=task.metadata.resource_version, + event=event, + ) + last_resource_version = task.metadata.resource_version + + return last_resource_version + + def process_error(self, event: Any) -> str: + """Process error response.""" + self.log.error("Encountered Error response from k8s list namespaced pod stream => %s", event) + raw_object = event["raw_object"] + if raw_object["code"] == 410: + self.log.info( + "Kubernetes resource version is too old, must reset to 0 => %s", (raw_object["message"],) + ) + # Return resource version 0 + return "0" + raise AirflowException( + f"Kubernetes failure for {raw_object['reason']} with code {raw_object['code']} and message: " + f"{raw_object['message']}" + ) + + def process_status( + self, + pod_name: str, + namespace: str, + status: str, + annotations: dict[str, str], + resource_version: str, + event: Any, + ) -> None: + pod = event["object"] + annotations_string = annotations_for_logging_task_metadata(annotations) + """Process status response.""" + if status == "Pending": + # deletion_timestamp is set by kube server when a graceful deletion is requested. + # since kube server have received request to delete pod set TI state failed + if event["type"] == "DELETED" and pod.metadata.deletion_timestamp: + self.log.info("Event: Failed to start pod %s, annotations: %s", pod_name, annotations_string) + self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) + else: + self.log.debug("Event: %s Pending, annotations: %s", pod_name, annotations_string) + elif status == "Failed": + self.log.error("Event: %s Failed, annotations: %s", pod_name, annotations_string) + self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) + elif status == "Succeeded": + # We get multiple events once the pod hits a terminal state, and we only want to + # send it along to the scheduler once. + # If our event type is DELETED, we have the POD_EXECUTOR_DONE_KEY, or the pod has + # a deletion timestamp, we've already seen the initial Succeeded event and sent it + # along to the scheduler. + if ( + event["type"] == "DELETED" + or POD_EXECUTOR_DONE_KEY in pod.metadata.labels + or pod.metadata.deletion_timestamp + ): + self.log.info( + "Skipping event for Succeeded pod %s - event for this pod already sent to executor", + pod_name, + ) + return + self.log.info("Event: %s Succeeded, annotations: %s", pod_name, annotations_string) + self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version)) + elif status == "Running": + # deletion_timestamp is set by kube server when a graceful deletion is requested. + # since kube server have received request to delete pod set TI state failed + if event["type"] == "DELETED" and pod.metadata.deletion_timestamp: + self.log.info( + "Event: Pod %s deleted before it could complete, annotations: %s", + pod_name, + annotations_string, + ) + self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version)) + else: + self.log.info("Event: %s is Running, annotations: %s", pod_name, annotations_string) + else: + self.log.warning( + "Event: Invalid state: %s on pod: %s in namespace %s with annotations: %s with " + "resource_version: %s", + status, + pod_name, + namespace, + annotations, + resource_version, + ) + + +class AirflowKubernetesScheduler(LoggingMixin): + """Airflow Scheduler for Kubernetes.""" + + def __init__( + self, + kube_config: Any, + result_queue: Queue[KubernetesResultsType], + kube_client: client.CoreV1Api, + scheduler_job_id: str, + ): + super().__init__() + self.log.debug("Creating Kubernetes executor") + self.kube_config = kube_config + self.result_queue = result_queue + self.namespace = self.kube_config.kube_namespace + self.log.debug("Kubernetes using namespace %s", self.namespace) + self.kube_client = kube_client + self._manager = multiprocessing.Manager() + self.watcher_queue = self._manager.Queue() + self.scheduler_job_id = scheduler_job_id + self.kube_watchers = self._make_kube_watchers() + + def run_pod_async(self, pod: k8s.V1Pod, **kwargs): + """Runs POD asynchronously.""" + sanitized_pod = self.kube_client.api_client.sanitize_for_serialization(pod) + json_pod = json.dumps(sanitized_pod, indent=2) + + self.log.debug("Pod Creation Request: \n%s", json_pod) + try: + resp = self.kube_client.create_namespaced_pod( + body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs + ) + self.log.debug("Pod Creation Response: %s", resp) + except Exception as e: + self.log.exception("Exception when attempting to create Namespaced Pod: %s", json_pod) + raise e + return resp + + def _make_kube_watcher(self, namespace) -> KubernetesJobWatcher: + resource_version = ResourceVersion().resource_version.get(namespace, "0") + watcher = KubernetesJobWatcher( + watcher_queue=self.watcher_queue, + namespace=namespace, + resource_version=resource_version, + scheduler_job_id=self.scheduler_job_id, + kube_config=self.kube_config, + ) + watcher.start() + return watcher + + def _make_kube_watchers(self) -> dict[str, KubernetesJobWatcher]: + watchers = {} + if self.kube_config.multi_namespace_mode: + namespaces_to_watch = ( + self.kube_config.multi_namespace_mode_namespace_list + if self.kube_config.multi_namespace_mode_namespace_list + else [ALL_NAMESPACES] + ) + else: + namespaces_to_watch = [self.kube_config.kube_namespace] + + for namespace in namespaces_to_watch: + watchers[namespace] = self._make_kube_watcher(namespace) + return watchers + + def _health_check_kube_watchers(self): + for namespace, kube_watcher in self.kube_watchers.items(): + if kube_watcher.is_alive(): + self.log.debug("KubeJobWatcher for namespace %s alive, continuing", namespace) + else: + self.log.error( + ( + "Error while health checking kube watcher process for namespace %s. " + "Process died for unknown reasons" + ), + namespace, + ) + ResourceVersion().resource_version[namespace] = "0" + self.kube_watchers[namespace] = self._make_kube_watcher(namespace) + + def run_next(self, next_job: KubernetesJobType) -> None: + """Receives the next job to run, builds the pod, and creates it.""" + key, command, kube_executor_config, pod_template_file = next_job + + dag_id, task_id, run_id, try_number, map_index = key + + if command[0:3] != ["airflow", "tasks", "run"]: + raise ValueError('The command must start with ["airflow", "tasks", "run"].') + + base_worker_pod = get_base_pod_from_template(pod_template_file, self.kube_config) + + if not base_worker_pod: + raise AirflowException( + f"could not find a valid worker template yaml at {self.kube_config.pod_template_file}" + ) + + pod = PodGenerator.construct_pod( + namespace=self.namespace, + scheduler_job_id=self.scheduler_job_id, + pod_id=create_pod_id(dag_id, task_id), + dag_id=dag_id, + task_id=task_id, + kube_image=self.kube_config.kube_image, + try_number=try_number, + map_index=map_index, + date=None, + run_id=run_id, + args=command, + pod_override_object=kube_executor_config, + base_worker_pod=base_worker_pod, + with_mutation_hook=True, + ) + # Reconcile the pod generated by the Operator and the Pod + # generated by the .cfg file + self.log.info( + "Creating kubernetes pod for job is %s, with pod name %s, annotations: %s", + key, + pod.metadata.name, + annotations_for_logging_task_metadata(pod.metadata.annotations), + ) + self.log.debug("Kubernetes running for command %s", command) + self.log.debug("Kubernetes launching image %s", pod.spec.containers[0].image) + + # the watcher will monitor pods, so we do not block. + self.run_pod_async(pod, **self.kube_config.kube_client_request_args) + self.log.debug("Kubernetes Job created!") + + def delete_pod(self, pod_name: str, namespace: str) -> None: + """Deletes Pod from a namespace. Does not raise if it does not exist.""" + try: + self.log.debug("Deleting pod %s in namespace %s", pod_name, namespace) + self.kube_client.delete_namespaced_pod( + pod_name, + namespace, + body=client.V1DeleteOptions(**self.kube_config.delete_option_kwargs), + **self.kube_config.kube_client_request_args, + ) + except ApiException as e: + # If the pod is already deleted + if e.status != 404: + raise + + def patch_pod_executor_done(self, *, pod_name: str, namespace: str): + """Add a "done" annotation to ensure we don't continually adopt pods.""" + self.log.debug("Patching pod %s in namespace %s to mark it as done", pod_name, namespace) + try: + self.kube_client.patch_namespaced_pod( + name=pod_name, + namespace=namespace, + body={"metadata": {"labels": {POD_EXECUTOR_DONE_KEY: "True"}}}, + ) + except ApiException as e: + self.log.info("Failed to patch pod %s with done annotation. Reason: %s", pod_name, e) + + def sync(self) -> None: + """ + Checks the status of all currently running kubernetes jobs. + + If a job is completed, its status is placed in the result queue to be sent back to the scheduler. + """ + self.log.debug("Syncing KubernetesExecutor") + self._health_check_kube_watchers() + while True: + try: + task = self.watcher_queue.get_nowait() + try: + self.log.debug("Processing task %s", task) + self.process_watcher_task(task) + finally: + self.watcher_queue.task_done() + except Empty: + break + + def process_watcher_task(self, task: KubernetesWatchType) -> None: + """Process the task by watcher.""" + pod_name, namespace, state, annotations, resource_version = task + self.log.debug( + "Attempting to finish pod; pod_name: %s; state: %s; annotations: %s", + pod_name, + state, + annotations_for_logging_task_metadata(annotations), + ) + key = annotations_to_key(annotations=annotations) + if key: + self.log.debug("finishing job %s - %s (%s)", key, state, pod_name) + self.result_queue.put((key, state, pod_name, namespace, resource_version)) + + def _flush_watcher_queue(self) -> None: + self.log.debug("Executor shutting down, watcher_queue approx. size=%d", self.watcher_queue.qsize()) + while True: + try: + task = self.watcher_queue.get_nowait() + # Ignoring it since it can only have either FAILED or SUCCEEDED pods + self.log.warning("Executor shutting down, IGNORING watcher task=%s", task) + self.watcher_queue.task_done() + except Empty: + break + + def terminate(self) -> None: + """Terminates the watcher.""" + self.log.debug("Terminating kube_watchers...") + for namespace, kube_watcher in self.kube_watchers.items(): + kube_watcher.terminate() + kube_watcher.join() + self.log.debug("kube_watcher=%s", kube_watcher) + self.log.debug("Flushing watcher_queue...") + self._flush_watcher_queue() + # Queue should be empty... + self.watcher_queue.join() + self.log.debug("Shutting down manager...") + self._manager.shutdown() + + +def get_base_pod_from_template(pod_template_file: str | None, kube_config: Any) -> k8s.V1Pod: + """ + Get base pod from template. + + Reads either the pod_template_file set in the executor_config or the base pod_template_file + set in the airflow.cfg to craft a "base pod" that will be used by the KubernetesExecutor + + :param pod_template_file: absolute path to a pod_template_file.yaml or None + :param kube_config: The KubeConfig class generated by airflow that contains all kube metadata + :return: a V1Pod that can be used as the base pod for k8s tasks + """ + if pod_template_file: + return PodGenerator.deserialize_model_file(pod_template_file) + else: + return PodGenerator.deserialize_model_file(kube_config.pod_template_file) diff --git a/airflow/kubernetes/kubernetes_helper_functions.py b/airflow/kubernetes/kubernetes_helper_functions.py index fdb76b0aa85f..4cd3422cb646 100644 --- a/airflow/kubernetes/kubernetes_helper_functions.py +++ b/airflow/kubernetes/kubernetes_helper_functions.py @@ -19,13 +19,16 @@ import logging import secrets import string +from typing import TYPE_CHECKING import pendulum from slugify import slugify from airflow.compat.functools import cache from airflow.configuration import conf -from airflow.models.taskinstancekey import TaskInstanceKey + +if TYPE_CHECKING: + from airflow.models.taskinstancekey import TaskInstanceKey log = logging.getLogger(__name__) @@ -91,12 +94,12 @@ def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey: annotation_run_id = annotations.get("run_id") map_index = int(annotations.get("map_index", -1)) - if not annotation_run_id and "execution_date" in annotations: - # Compat: Look up the run_id from the TI table! - from airflow.models.dagrun import DagRun - from airflow.models.taskinstance import TaskInstance - from airflow.settings import Session + # Compat: Look up the run_id from the TI table! + from airflow.models.dagrun import DagRun + from airflow.models.taskinstance import TaskInstance, TaskInstanceKey + from airflow.settings import Session + if not annotation_run_id and "execution_date" in annotations: execution_date = pendulum.parse(annotations["execution_date"]) # Do _not_ use create-session, we don't want to expunge session = Session() diff --git a/tests/executors/test_kubernetes_executor.py b/tests/executors/test_kubernetes_executor.py index ba9c33a36e42..4b4685e079e6 100644 --- a/tests/executors/test_kubernetes_executor.py +++ b/tests/executors/test_kubernetes_executor.py @@ -40,10 +40,10 @@ from tests.test_utils.config import conf_vars try: - from airflow.executors.kubernetes_executor import ( - POD_EXECUTOR_DONE_KEY, + from airflow.executors.kubernetes_executor import KubernetesExecutor + from airflow.executors.kubernetes_executor_types import POD_EXECUTOR_DONE_KEY + from airflow.executors.kubernetes_executor_utils import ( AirflowKubernetesScheduler, - KubernetesExecutor, KubernetesJobWatcher, ResourceVersion, create_pod_id, @@ -160,9 +160,9 @@ def test_execution_date_serialize_deserialize(self): @pytest.mark.skipif( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.client") - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") def test_delete_pod_successfully(self, mock_watcher, mock_client, mock_kube_client): pod_name = "my-pod-1" namespace = "my-namespace-1" @@ -182,9 +182,9 @@ def test_delete_pod_successfully(self, mock_watcher, mock_client, mock_kube_clie @pytest.mark.skipif( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.client") - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") def test_delete_pod_raises_404(self, mock_watcher, mock_client, mock_kube_client): pod_name = "my-pod-1" namespace = "my-namespace-2" @@ -205,9 +205,9 @@ def test_delete_pod_raises_404(self, mock_watcher, mock_client, mock_kube_client @pytest.mark.skipif( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.client") - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") def test_delete_pod_404_not_raised(self, mock_watcher, mock_client, mock_kube_client): pod_name = "my-pod-1" namespace = "my-namespace-3" @@ -249,8 +249,8 @@ def setup_method(self) -> None: pytest.param(400, False, id="400 BadRequest"), ], ) - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_run_next_exception_requeue( self, mock_get_kube_client, mock_kubernetes_job_watcher, status, should_requeue ): @@ -319,7 +319,7 @@ def test_run_next_exception_requeue( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) @mock.patch("airflow.settings.pod_mutation_hook") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_run_next_pmh_error(self, mock_get_kube_client, mock_pmh): """ Exception during Pod Mutation Hook execution should be handled gracefully. @@ -357,8 +357,8 @@ def test_run_next_pmh_error(self, mock_get_kube_client, mock_pmh): @pytest.mark.skipif( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_run_next_pod_reconciliation_error(self, mock_get_kube_client, mock_kubernetes_job_watcher): """ When construct_pod raises PodReconciliationError, we should fail the task. @@ -417,8 +417,8 @@ def test_gauge_executor_metrics(self, mock_stats_gauge, mock_trigger_tasks, mock ] mock_stats_gauge.assert_has_calls(calls) - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_invalid_executor_config(self, mock_get_kube_client, mock_kubernetes_job_watcher): executor = self.kubernetes_executor executor.start() @@ -443,8 +443,8 @@ def test_invalid_executor_config(self, mock_get_kube_client, mock_kubernetes_job @pytest.mark.skipif( AirflowKubernetesScheduler is None, reason="kubernetes python package is not installed" ) - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_pod_async") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler.run_pod_async") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_pod_template_file_override_in_executor_config(self, mock_get_kube_client, mock_run_pod_async): current_folder = pathlib.Path(__file__).parent.resolve() template_file = str( @@ -528,8 +528,8 @@ def test_pod_template_file_override_in_executor_config(self, mock_get_kube_clien finally: executor.end() - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_change_state_running(self, mock_get_kube_client, mock_kubernetes_job_watcher): executor = self.kubernetes_executor executor.start() @@ -542,9 +542,9 @@ def test_change_state_running(self, mock_get_kube_client, mock_kubernetes_job_wa finally: executor.end() - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler.delete_pod") def test_change_state_success(self, mock_delete_pod, mock_get_kube_client, mock_kubernetes_job_watcher): executor = self.kubernetes_executor executor.start() @@ -558,9 +558,9 @@ def test_change_state_success(self, mock_delete_pod, mock_get_kube_client, mock_ finally: executor.end() - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler") def test_change_state_failed_no_deletion( self, mock_kubescheduler, mock_get_kube_client, mock_kubernetes_job_watcher ): @@ -584,9 +584,9 @@ def test_change_state_failed_no_deletion( @pytest.mark.parametrize( "ti_state", [TaskInstanceState.SUCCESS, TaskInstanceState.FAILED, TaskInstanceState.DEFERRED] ) - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler.delete_pod") def test_change_state_none( self, mock_delete_pod, @@ -616,7 +616,7 @@ def test_change_state_none( pytest.param(None, ["ALL_NAMESPACES"]), ], ) - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_watchers_under_multi_namespace_mode( self, mock_get_kube_client, multi_namespace_mode_namespace_list, watchers_keys ): @@ -632,9 +632,9 @@ def test_watchers_under_multi_namespace_mode( finally: executor.end() - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler") def test_change_state_skip_pod_deletion( self, mock_kubescheduler, mock_get_kube_client, mock_kubernetes_job_watcher ): @@ -656,9 +656,9 @@ def test_change_state_skip_pod_deletion( finally: executor.end() - @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher") - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler") + @mock.patch("airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler") def test_change_state_failed_pod_deletion( self, mock_kubescheduler, mock_get_kube_client, mock_kubernetes_job_watcher ): @@ -778,7 +778,7 @@ def test_try_adopt_task_instances_no_matching_pods( mock_adopt_launched_task.assert_not_called() mock_adopt_completed_pods.assert_called_once() - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_adopt_launched_task(self, mock_kube_client): executor = self.kubernetes_executor executor.scheduler_job_id = "modified" @@ -803,7 +803,7 @@ def test_adopt_launched_task(self, mock_kube_client): assert tis_to_flush_by_key == {} assert executor.running == {ti_key} - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_adopt_launched_task_api_exception(self, mock_kube_client): """We shouldn't think we are running the task if aren't able to patch the pod""" executor = self.kubernetes_executor @@ -828,7 +828,7 @@ def test_adopt_launched_task_api_exception(self, mock_kube_client): assert tis_to_flush_by_key == {ti_key: {}} assert executor.running == set() - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_adopt_completed_pods(self, mock_kube_client): """We should adopt all completed pods from other schedulers""" executor = self.kubernetes_executor @@ -878,7 +878,7 @@ def get_annotations(pod_name): ) assert executor.running == expected_running_ti_keys - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") def test_not_adopt_unassigned_task(self, mock_kube_client): """ We should not adopt any tasks that were not assigned by the scheduler. @@ -904,8 +904,8 @@ def test_not_adopt_unassigned_task(self, mock_kube_client): assert not mock_kube_client.patch_namespaced_pod.called assert tis_to_flush_by_key == {"foobar": {}} - @mock.patch("airflow.executors.kubernetes_executor.get_kube_client") - @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod") + @mock.patch("airflow.kubernetes.kube_client.get_kube_client") + @mock.patch("airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler.delete_pod") def test_cleanup_stuck_queued_tasks(self, mock_delete_pod, mock_kube_client, dag_maker, session): """Delete any pods associated with a task stuck in queued.""" executor = KubernetesExecutor() @@ -1232,7 +1232,7 @@ def setup_method(self): self.events = [] def _run(self): - with mock.patch("airflow.executors.kubernetes_executor.watch") as mock_watch: + with mock.patch("airflow.executors.kubernetes_executor_utils.watch") as mock_watch: mock_watch.Watch.return_value.stream.return_value = self.events latest_resource_version = self.watcher._run( self.kube_client, @@ -1362,7 +1362,7 @@ def effect(): self.watcher._run = mock_underscore_run - with mock.patch("airflow.executors.kubernetes_executor.get_kube_client"): + with mock.patch("airflow.executors.kubernetes_executor_utils.get_kube_client"): try: # self.watcher._run() is mocked and return "500" as last resource_version self.watcher.run() From 46715164caf2fadf698f28f1a54443f58b9eb5e8 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 12 Jul 2023 02:02:22 +0200 Subject: [PATCH 162/533] Improve the readability of the conn view tests (#32529) --- tests/www/views/test_views_connection.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1897db364bb1..c7698e3b7c5d 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -344,16 +344,17 @@ def test_duplicate_connection(admin_client): data = {"action": "mulduplicate", "rowid": [conn1.id, conn3.id]} resp = admin_client.post("/connection/action_post", data=data, follow_redirects=True) - expected_result = { + assert resp.status_code == 200 + + expected_connections_ids = { "test_duplicate_gcp_connection", "test_duplicate_gcp_connection_copy1", "test_duplicate_mysql_connection", "test_duplicate_postgres_connection_copy1", "test_duplicate_postgres_connection_copy2", } - response = {conn[0] for conn in session.query(Connection.conn_id).all()} - assert resp.status_code == 200 - assert expected_result == response + connections_ids = {conn.conn_id for conn in session.query(Connection.conn_id)} + assert expected_connections_ids == connections_ids def test_duplicate_connection_error(admin_client): @@ -380,12 +381,11 @@ def test_duplicate_connection_error(admin_client): data = {"action": "mulduplicate", "rowid": [connections[0].id]} resp = admin_client.post("/connection/action_post", data=data, follow_redirects=True) - - expected_result = {f"test_duplicate_postgres_connection_copy{i}" for i in range(1, 11)} - assert resp.status_code == 200 - response = {conn[0] for conn in session.query(Connection.conn_id).all()} - assert expected_result == response + + expected_connections_ids = {f"test_duplicate_postgres_connection_copy{i}" for i in range(1, 11)} + connections_ids = {conn.conn_id for conn in session.query(Connection.conn_id)} + assert expected_connections_ids == connections_ids @pytest.fixture() From 2f44ea5996b9664865d6d891537f770caed19a0d Mon Sep 17 00:00:00 2001 From: Phani Kumar <94376113+phanikumv@users.noreply.github.com> Date: Wed, 12 Jul 2023 11:00:24 +0530 Subject: [PATCH 163/533] Refactor Sqlalchemy queries to 2.0 style (Part 5) (#32474) --- airflow/cli/commands/dag_command.py | 22 ++++--- airflow/cli/commands/jobs_command.py | 11 ++-- .../cli/commands/rotate_fernet_key_command.py | 7 +- airflow/cli/commands/task_command.py | 21 +++--- airflow/cli/commands/variable_command.py | 6 +- airflow/models/renderedtifields.py | 15 ++--- airflow/utils/sqlalchemy.py | 64 +++++++++++++++++-- tests/cli/commands/test_task_command.py | 8 +-- 8 files changed, 103 insertions(+), 51 deletions(-) diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index f57e26b38c9f..66decad78b16 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -28,7 +28,7 @@ import warnings from graphviz.dot import Dot -from sqlalchemy import delete +from sqlalchemy import delete, select from sqlalchemy.orm import Session from airflow import settings @@ -287,7 +287,7 @@ def dag_state(args, session: Session = NEW_SESSION) -> None: if not dag: raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table") - dr = session.query(DagRun).filter_by(dag_id=args.dag_id, execution_date=args.execution_date).one_or_none() + dr = session.scalar(select(DagRun).filter_by(dag_id=args.dag_id, execution_date=args.execution_date)) out = dr.state if dr else None conf_out = "" if out and dr.conf: @@ -309,7 +309,9 @@ def dag_next_execution(args) -> None: print("[INFO] Please be reminded this DAG is PAUSED now.", file=sys.stderr) with create_session() as session: - last_parsed_dag: DagModel = session.query(DagModel).filter(DagModel.dag_id == dag.dag_id).one() + last_parsed_dag: DagModel = session.scalars( + select(DagModel).where(DagModel.dag_id == dag.dag_id) + ).one() def print_execution_interval(interval: DataInterval | None): if interval is None: @@ -428,8 +430,10 @@ def dag_list_jobs(args, dag: DAG | None = None, session: Session = NEW_SESSION) queries.append(Job.state == args.state) fields = ["dag_id", "state", "job_type", "start_date", "end_date"] - all_jobs = session.query(Job).filter(*queries).order_by(Job.start_date.desc()).limit(args.limit).all() - all_jobs = [{f: str(job.__getattribute__(f)) for f in fields} for job in all_jobs] + all_jobs_iter = session.scalars( + select(Job).where(*queries).order_by(Job.start_date.desc()).limit(args.limit) + ) + all_jobs = [{f: str(job.__getattribute__(f)) for f in fields} for job in all_jobs_iter] AirflowConsole().print_as( data=all_jobs, @@ -492,14 +496,12 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No imgcat = args.imgcat_dagrun filename = args.save_dagrun if show_dagrun or imgcat or filename: - tis = ( - session.query(TaskInstance) - .filter( + tis = session.scalars( + select(TaskInstance).where( TaskInstance.dag_id == args.dag_id, TaskInstance.execution_date == execution_date, ) - .all() - ) + ).all() dot_graph = render_dag(dag, tis=tis) print() diff --git a/airflow/cli/commands/jobs_command.py b/airflow/cli/commands/jobs_command.py index 79e0aad79fde..339896d877f2 100644 --- a/airflow/cli/commands/jobs_command.py +++ b/airflow/cli/commands/jobs_command.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +from sqlalchemy import select from sqlalchemy.orm import Session from airflow.jobs.job import Job @@ -32,17 +33,17 @@ def check(args, session: Session = NEW_SESSION) -> None: if args.hostname and args.local: raise SystemExit("You can't use --hostname and --local at the same time") - query = session.query(Job).filter(Job.state == State.RUNNING).order_by(Job.latest_heartbeat.desc()) + query = select(Job).where(Job.state == State.RUNNING).order_by(Job.latest_heartbeat.desc()) if args.job_type: - query = query.filter(Job.job_type == args.job_type) + query = query.where(Job.job_type == args.job_type) if args.hostname: - query = query.filter(Job.hostname == args.hostname) + query = query.where(Job.hostname == args.hostname) if args.local: - query = query.filter(Job.hostname == get_hostname()) + query = query.where(Job.hostname == get_hostname()) if args.limit > 0: query = query.limit(args.limit) - alive_jobs: list[Job] = [job for job in query.all() if job.is_alive()] + alive_jobs: list[Job] = [job for job in session.scalars(query) if job.is_alive()] count_alive_jobs = len(alive_jobs) if count_alive_jobs == 0: diff --git a/airflow/cli/commands/rotate_fernet_key_command.py b/airflow/cli/commands/rotate_fernet_key_command.py index f9e18735976c..e9973978e0a1 100644 --- a/airflow/cli/commands/rotate_fernet_key_command.py +++ b/airflow/cli/commands/rotate_fernet_key_command.py @@ -17,6 +17,8 @@ """Rotate Fernet key command.""" from __future__ import annotations +from sqlalchemy import select + from airflow.models import Connection, Variable from airflow.utils import cli as cli_utils from airflow.utils.session import create_session @@ -26,7 +28,8 @@ def rotate_fernet_key(args): """Rotates all encrypted connection credentials and variables.""" with create_session() as session: - for conn in session.query(Connection).filter(Connection.is_encrypted | Connection.is_extra_encrypted): + conns_query = select(Connection).where(Connection.is_encrypted | Connection.is_extra_encrypted) + for conn in session.scalars(conns_query): conn.rotate_fernet_key() - for var in session.query(Variable).filter(Variable.is_encrypted): + for var in session.scalars(select(Variable).where(Variable.is_encrypted)): var.rotate_fernet_key() diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index a5e20e87518b..796d3a8cf120 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -29,6 +29,7 @@ import pendulum from pendulum.parsing.exceptions import ParserError +from sqlalchemy import select from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm.session import Session @@ -111,11 +112,9 @@ def _get_dag_run( with suppress(ParserError, TypeError): execution_date = timezone.parse(exec_date_or_run_id) try: - dag_run = ( - session.query(DagRun) - .filter(DagRun.dag_id == dag.dag_id, DagRun.execution_date == execution_date) - .one() - ) + dag_run = session.scalars( + select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date == execution_date) + ).one() except NoResultFound: if not create_if_necessary: raise DagRunNotFound( @@ -534,18 +533,14 @@ def _guess_debugger() -> _SupportedDebugger: @provide_session def task_states_for_dag_run(args, session: Session = NEW_SESSION) -> None: """Get the status of all task instances in a DagRun.""" - dag_run = ( - session.query(DagRun) - .filter(DagRun.run_id == args.execution_date_or_run_id, DagRun.dag_id == args.dag_id) - .one_or_none() + dag_run = session.scalar( + select(DagRun).where(DagRun.run_id == args.execution_date_or_run_id, DagRun.dag_id == args.dag_id) ) if not dag_run: try: execution_date = timezone.parse(args.execution_date_or_run_id) - dag_run = ( - session.query(DagRun) - .filter(DagRun.execution_date == execution_date, DagRun.dag_id == args.dag_id) - .one_or_none() + dag_run = session.scalar( + select(DagRun).where(DagRun.execution_date == execution_date, DagRun.dag_id == args.dag_id) ) except (ParserError, TypeError) as err: raise AirflowException(f"Error parsing the supplied execution_date. Error: {str(err)}") diff --git a/airflow/cli/commands/variable_command.py b/airflow/cli/commands/variable_command.py index 009b4704aaae..32f6b0c1987c 100644 --- a/airflow/cli/commands/variable_command.py +++ b/airflow/cli/commands/variable_command.py @@ -22,6 +22,8 @@ import os from json import JSONDecodeError +from sqlalchemy import select + from airflow.cli.simple_table import AirflowConsole from airflow.models import Variable from airflow.utils import cli as cli_utils @@ -33,7 +35,7 @@ def variables_list(args): """Displays all the variables.""" with create_session() as session: - variables = session.query(Variable) + variables = session.scalars(select(Variable)).all() AirflowConsole().print_as(data=variables, output=args.output, mapper=lambda x: {"key": x.key}) @@ -107,7 +109,7 @@ def _variable_export_helper(filepath): """Helps export all the variables to the file.""" var_dict = {} with create_session() as session: - qry = session.query(Variable).all() + qry = session.scalars(select(Variable)) data = json.JSONDecoder() for var in qry: diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 958683447004..37a5f08b27e4 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -134,15 +134,13 @@ def get_templated_fields(cls, ti: TaskInstance, session: Session = NEW_SESSION) :param session: SqlAlchemy Session :return: Rendered Templated TI field """ - result = ( - session.query(cls.rendered_fields) - .filter( + result = session.scalar( + select(cls).where( cls.dag_id == ti.dag_id, cls.task_id == ti.task_id, cls.run_id == ti.run_id, cls.map_index == ti.map_index, ) - .one_or_none() ) if result: @@ -162,15 +160,13 @@ def get_k8s_pod_yaml(cls, ti: TaskInstance, session: Session = NEW_SESSION) -> d :param session: SqlAlchemy Session :return: Kubernetes Pod Yaml """ - result = ( - session.query(cls.k8s_pod_yaml) - .filter( + result = session.scalar( + select(cls).where( cls.dag_id == ti.dag_id, cls.task_id == ti.task_id, cls.run_id == ti.run_id, cls.map_index == ti.map_index, ) - .one_or_none() ) return result.k8s_pod_yaml if result else None @@ -243,7 +239,8 @@ def _do_delete_old_records( cls.task_id == task_id, tuple_not_in_condition( (cls.dag_id, cls.task_id, cls.run_id), - session.query(ti_clause.c.dag_id, ti_clause.c.task_id, ti_clause.c.run_id), + select(ti_clause.c.dag_id, ti_clause.c.task_id, ti_clause.c.run_id), + session=session, ), ) .execution_options(synchronize_session=False) diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 32a5a796d569..04af5bddc31c 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -22,14 +22,14 @@ import datetime import json import logging -from typing import TYPE_CHECKING, Any, Generator, Iterable +from typing import TYPE_CHECKING, Any, Generator, Iterable, overload import pendulum from dateutil import relativedelta from sqlalchemy import TIMESTAMP, PickleType, and_, event, false, nullsfirst, or_, true, tuple_ from sqlalchemy.dialects import mssql, mysql from sqlalchemy.exc import OperationalError -from sqlalchemy.sql import ColumnElement +from sqlalchemy.sql import ColumnElement, Select from sqlalchemy.sql.expression import ColumnOperators from sqlalchemy.types import JSON, Text, TypeDecorator, TypeEngine, UnicodeText @@ -515,11 +515,31 @@ def is_lock_not_available_error(error: OperationalError): return False +@overload def tuple_in_condition( columns: tuple[ColumnElement, ...], collection: Iterable[Any], ) -> ColumnOperators: - """Generates a tuple-in-collection operator to use in ``.filter()``. + ... + + +@overload +def tuple_in_condition( + columns: tuple[ColumnElement, ...], + collection: Select, + *, + session: Session, +) -> ColumnOperators: + ... + + +def tuple_in_condition( + columns: tuple[ColumnElement, ...], + collection: Iterable[Any] | Select, + *, + session: Session | None = None, +) -> ColumnOperators: + """Generates a tuple-in-collection operator to use in ``.where()``. For most SQL backends, this generates a simple ``([col, ...]) IN [condition]`` clause. This however does not work with MSSQL, where we need to expand to @@ -529,17 +549,43 @@ def tuple_in_condition( """ if settings.engine.dialect.name != "mssql": return tuple_(*columns).in_(collection) - clauses = [and_(*(c == v for c, v in zip(columns, values))) for values in collection] + if not isinstance(collection, Select): + rows = collection + elif session is None: + raise TypeError("session is required when passing in a subquery") + else: + rows = session.execute(collection) + clauses = [and_(*(c == v for c, v in zip(columns, values))) for values in rows] if not clauses: return false() return or_(*clauses) +@overload def tuple_not_in_condition( columns: tuple[ColumnElement, ...], collection: Iterable[Any], ) -> ColumnOperators: - """Generates a tuple-not-in-collection operator to use in ``.filter()``. + ... + + +@overload +def tuple_not_in_condition( + columns: tuple[ColumnElement, ...], + collection: Select, + *, + session: Session, +) -> ColumnOperators: + ... + + +def tuple_not_in_condition( + columns: tuple[ColumnElement, ...], + collection: Iterable[Any] | Select, + *, + session: Session | None = None, +) -> ColumnOperators: + """Generates a tuple-not-in-collection operator to use in ``.where()``. This is similar to ``tuple_in_condition`` except generating ``NOT IN``. @@ -547,7 +593,13 @@ def tuple_not_in_condition( """ if settings.engine.dialect.name != "mssql": return tuple_(*columns).not_in(collection) - clauses = [or_(*(c != v for c, v in zip(columns, values))) for values in collection] + if not isinstance(collection, Select): + rows = collection + elif session is None: + raise TypeError("session is required when passing in a subquery") + else: + rows = session.execute(collection) + clauses = [or_(*(c != v for c, v in zip(columns, values))) for values in rows] if not clauses: return true() return and_(*clauses) diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 7c2eb61d325e..74a7dcea0d0e 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -500,15 +500,15 @@ def test_mapped_task_render_with_template(self, dag_maker): assert 'echo "2022-01-01"' in output assert 'echo "2022-01-08"' in output - @mock.patch("sqlalchemy.orm.session.Session.query") + @mock.patch("airflow.cli.commands.task_command.select") + @mock.patch("airflow.cli.commands.task_command.Session.scalars") @mock.patch("airflow.cli.commands.task_command.DagRun") - def test_task_render_with_custom_timetable(self, mock_dagrun, mock_query): + def test_task_render_with_custom_timetable(self, mock_dagrun, mock_scalars, mock_select): """ when calling `tasks render` on dag with custom timetable, the DagRun object should be created with data_intervals. """ - mock_query.side_effect = sqlalchemy.exc.NoResultFound - + mock_scalars.side_effect = sqlalchemy.exc.NoResultFound task_command.task_render( self.parser.parse_args(["tasks", "render", "example_workday_timetable", "run_this", "2022-01-01"]) ) From fd2687d0d51fa444f63aac705d843f30e5922319 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 12 Jul 2023 14:50:20 +0800 Subject: [PATCH 164/533] Strong-type all single-state enum values (#32537) --- airflow/jobs/backfill_job_runner.py | 4 +-- airflow/jobs/scheduler_job_runner.py | 4 +-- airflow/models/dagrun.py | 2 +- airflow/models/pool.py | 8 +++--- airflow/models/taskinstance.py | 4 +-- airflow/operators/trigger_dagrun.py | 16 ++++++++---- airflow/sensors/external_task.py | 34 ++++++++++++++------------ airflow/ti_deps/dependencies_states.py | 30 +++++++++++------------ airflow/triggers/external_task.py | 3 ++- 9 files changed, 57 insertions(+), 48 deletions(-) diff --git a/airflow/jobs/backfill_job_runner.py b/airflow/jobs/backfill_job_runner.py index 35910b83ae99..162f10364287 100644 --- a/airflow/jobs/backfill_job_runner.py +++ b/airflow/jobs/backfill_job_runner.py @@ -696,7 +696,7 @@ def to_keep(key: TaskInstanceKey) -> bool: _dag_runs = ti_status.active_runs[:] for run in _dag_runs: run.update_state(session=session) - if run.state in State.finished: + if run.state in State.finished_dr_states: ti_status.finished_runs += 1 ti_status.active_runs.remove(run) executed_run_dates.append(run.execution_date) @@ -824,7 +824,7 @@ def _set_unfinished_dag_runs_to_failed( """ for dag_run in dag_runs: dag_run.update_state() - if dag_run.state not in State.finished: + if dag_run.state not in State.finished_dr_states: dag_run.set_state(DagRunState.FAILED) session.merge(dag_run) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 7b13ecc300dc..c0e687707617 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -625,7 +625,7 @@ def _enqueue_task_instances_with_queued_state(self, task_instances: list[TI], se """ # actually enqueue them for ti in task_instances: - if ti.dag_run.state in State.finished: + if ti.dag_run.state in State.finished_dr_states: ti.set_state(None, session=session) continue command = ti.command_as_list( @@ -1449,7 +1449,7 @@ def _schedule_dag_run( # TODO[HA]: Rename update_state -> schedule_dag_run, ?? something else? schedulable_tis, callback_to_run = dag_run.update_state(session=session, execute_callbacks=False) # Check if DAG not scheduled then skip interval calculation to same scheduler runtime - if dag_run.state in State.finished: + if dag_run.state in State.finished_dr_states: # Work out if we should allow creating a new DagRun now? if self._should_update_dag_next_dagruns(dag, dag_model, session=session): dag_model.calculate_dagrun_date_fields(dag, dag.get_run_data_interval(dag_run)) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index e3ed0bda00bb..2d3ff26d7f19 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -270,7 +270,7 @@ def set_state(self, state: DagRunState) -> None: raise ValueError(f"invalid DagRun state: {state}") if self._state != state: self._state = state - self.end_date = timezone.utcnow() if self._state in State.finished else None + self.end_date = timezone.utcnow() if self._state in State.finished_dr_states else None if state == DagRunState.QUEUED: self.queued_at = timezone.utcnow() diff --git a/airflow/models/pool.py b/airflow/models/pool.py index 83ec0368bd9a..0aab6fb634ca 100644 --- a/airflow/models/pool.py +++ b/airflow/models/pool.py @@ -175,12 +175,12 @@ def slots_stats( state_count_by_pool = session.execute( select(TaskInstance.pool, TaskInstance.state, func.sum(TaskInstance.pool_slots)) - .filter(TaskInstance.state.in_(list(EXECUTION_STATES))) + .filter(TaskInstance.state.in_(EXECUTION_STATES)) .group_by(TaskInstance.pool, TaskInstance.state) ) # calculate queued and running metrics - for (pool_name, state, count) in state_count_by_pool: + for pool_name, state, count in state_count_by_pool: # Some databases return decimal.Decimal here. count = int(count) @@ -188,9 +188,9 @@ def slots_stats( if not stats_dict: continue # TypedDict key must be a string literal, so we use if-statements to set value - if state == "running": + if state == TaskInstanceState.RUNNING: stats_dict["running"] = count - elif state == "queued": + elif state == TaskInstanceState.QUEUED: stats_dict["queued"] = count else: raise AirflowException(f"Unexpected state. Expected values: {EXECUTION_STATES}.") diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index ee1825c0633c..3ec15f449dbf 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1348,7 +1348,7 @@ def check_and_change_state_before_execution( self._try_number += 1 if not test_mode: - session.add(Log(State.RUNNING, self)) + session.add(Log(TaskInstanceState.RUNNING.value, self)) self.state = TaskInstanceState.RUNNING self.emit_state_change_metric(TaskInstanceState.RUNNING) @@ -1937,7 +1937,7 @@ def handle_failure( Stats.incr("ti_failures", tags=self.stats_tags) if not test_mode: - session.add(Log(State.FAILED, self)) + session.add(Log(TaskInstanceState.FAILED.value, self)) # Log failure duration session.add(TaskFail(ti=self)) diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py index 548ef9189418..5a7be1dcab7a 100644 --- a/airflow/operators/trigger_dagrun.py +++ b/airflow/operators/trigger_dagrun.py @@ -37,7 +37,7 @@ from airflow.utils.context import Context from airflow.utils.helpers import build_airflow_url_with_query from airflow.utils.session import provide_session -from airflow.utils.state import State +from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType XCOM_EXECUTION_DATE_ISO = "trigger_execution_date_iso" @@ -112,8 +112,8 @@ def __init__( reset_dag_run: bool = False, wait_for_completion: bool = False, poke_interval: int = 60, - allowed_states: list | None = None, - failed_states: list | None = None, + allowed_states: list[str] | None = None, + failed_states: list[str] | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: @@ -124,8 +124,14 @@ def __init__( self.reset_dag_run = reset_dag_run self.wait_for_completion = wait_for_completion self.poke_interval = poke_interval - self.allowed_states = allowed_states or [State.SUCCESS] - self.failed_states = failed_states or [State.FAILED] + if allowed_states: + self.allowed_states = [DagRunState(s) for s in allowed_states] + else: + self.allowed_states = [DagRunState.SUCCESS] + if failed_states: + self.failed_states = [DagRunState(s) for s in failed_states] + else: + self.failed_states = [DagRunState.FAILED] self._defer = deferrable if execution_date is not None and not isinstance(execution_date, (str, datetime.datetime)): diff --git a/airflow/sensors/external_task.py b/airflow/sensors/external_task.py index bacafc979c21..b5c1c1321f7e 100644 --- a/airflow/sensors/external_task.py +++ b/airflow/sensors/external_task.py @@ -37,7 +37,7 @@ from airflow.utils.helpers import build_airflow_url_with_query from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.sqlalchemy import tuple_in_condition -from airflow.utils.state import State +from airflow.utils.state import State, TaskInstanceState if TYPE_CHECKING: from sqlalchemy.orm import Query, Session @@ -76,26 +76,28 @@ class ExternalTaskSensor(BaseSensorOperator): without also having to clear the sensor). By default, the ExternalTaskSensor will not skip if the external task skips. - To change this, simply set ``skipped_states=[State.SKIPPED]``. Note that if - you are monitoring multiple tasks, and one enters error state and the other - enters a skipped state, then the external task will react to whichever one - it sees first. If both happen together, then the failed state takes priority. + To change this, simply set ``skipped_states=[TaskInstanceState.SKIPPED]``. + Note that if you are monitoring multiple tasks, and one enters error state + and the other enters a skipped state, then the external task will react to + whichever one it sees first. If both happen together, then the failed state + takes priority. It is possible to alter the default behavior by setting states which - cause the sensor to fail, e.g. by setting ``allowed_states=[State.FAILED]`` - and ``failed_states=[State.SUCCESS]`` you will flip the behaviour to get a - sensor which goes green when the external task *fails* and immediately goes - red if the external task *succeeds*! + cause the sensor to fail, e.g. by setting ``allowed_states=[DagRunState.FAILED]`` + and ``failed_states=[DagRunState.SUCCESS]`` you will flip the behaviour to + get a sensor which goes green when the external task *fails* and immediately + goes red if the external task *succeeds*! Note that ``soft_fail`` is respected when examining the failed_states. Thus if the external task enters a failed state and ``soft_fail == True`` the sensor will _skip_ rather than fail. As a result, setting ``soft_fail=True`` - and ``failed_states=[State.SKIPPED]`` will result in the sensor skipping if - the external task skips. However, this is a contrived example - consider - using ``skipped_states`` if you would like this behaviour. Using - ``skipped_states`` allows the sensor to skip if the target fails, but still - enter failed state on timeout. Using ``soft_fail == True`` as above will - cause the sensor to skip if the target fails, but also if it times out. + and ``failed_states=[DagRunState.SKIPPED]`` will result in the sensor + skipping if the external task skips. However, this is a contrived + example---consider using ``skipped_states`` if you would like this + behaviour. Using ``skipped_states`` allows the sensor to skip if the target + fails, but still enter failed state on timeout. Using ``soft_fail == True`` + as above will cause the sensor to skip if the target fails, but also if it + times out. :param external_dag_id: The dag_id that contains the task you want to wait for. (templated) @@ -146,7 +148,7 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - self.allowed_states = list(allowed_states) if allowed_states else [State.SUCCESS] + self.allowed_states = list(allowed_states) if allowed_states else [TaskInstanceState.SUCCESS.value] self.skipped_states = list(skipped_states) if skipped_states else [] self.failed_states = list(failed_states) if failed_states else [] diff --git a/airflow/ti_deps/dependencies_states.py b/airflow/ti_deps/dependencies_states.py index 543ce3528a1b..fd25d62f6d75 100644 --- a/airflow/ti_deps/dependencies_states.py +++ b/airflow/ti_deps/dependencies_states.py @@ -16,38 +16,38 @@ # under the License. from __future__ import annotations -from airflow.utils.state import State +from airflow.utils.state import TaskInstanceState EXECUTION_STATES = { - State.RUNNING, - State.QUEUED, + TaskInstanceState.RUNNING, + TaskInstanceState.QUEUED, } # In order to be able to get queued a task must have one of these states SCHEDULEABLE_STATES = { - State.NONE, - State.UP_FOR_RETRY, - State.UP_FOR_RESCHEDULE, + None, + TaskInstanceState.UP_FOR_RETRY, + TaskInstanceState.UP_FOR_RESCHEDULE, } RUNNABLE_STATES = { # For cases like unit tests and run manually - State.NONE, - State.UP_FOR_RETRY, - State.UP_FOR_RESCHEDULE, + None, + TaskInstanceState.UP_FOR_RETRY, + TaskInstanceState.UP_FOR_RESCHEDULE, # For normal scheduler/backfill cases - State.QUEUED, + TaskInstanceState.QUEUED, } QUEUEABLE_STATES = { - State.SCHEDULED, + TaskInstanceState.SCHEDULED, } BACKFILL_QUEUEABLE_STATES = { # For cases like unit tests and run manually - State.NONE, - State.UP_FOR_RESCHEDULE, - State.UP_FOR_RETRY, + None, + TaskInstanceState.UP_FOR_RESCHEDULE, + TaskInstanceState.UP_FOR_RETRY, # For normal backfill cases - State.SCHEDULED, + TaskInstanceState.SCHEDULED, } diff --git a/airflow/triggers/external_task.py b/airflow/triggers/external_task.py index 547556e31565..e739c7a7cb4e 100644 --- a/airflow/triggers/external_task.py +++ b/airflow/triggers/external_task.py @@ -27,6 +27,7 @@ from airflow.models import DagRun, TaskInstance from airflow.triggers.base import BaseTrigger, TriggerEvent from airflow.utils.session import NEW_SESSION, provide_session +from airflow.utils.state import DagRunState class TaskStateTrigger(BaseTrigger): @@ -110,7 +111,7 @@ class DagStateTrigger(BaseTrigger): def __init__( self, dag_id: str, - states: list[str], + states: list[DagRunState], execution_dates: list[datetime.datetime], poll_interval: float = 5.0, ): From a67427af3738218786f3745d9737f462c70d4233 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 12 Jul 2023 09:11:54 +0200 Subject: [PATCH 165/533] Fix get_blobs_list_async method to return BlobProperties (#32545) The get_blobs_list_async method advertised to return list of BlobProperties but returned list of blob names. This is a bug that has been detected by MyPy checks with the new Azure blob package. --- airflow/providers/microsoft/azure/CHANGELOG.rst | 10 ++++++++++ airflow/providers/microsoft/azure/hooks/wasb.py | 4 ++-- airflow/providers/microsoft/azure/provider.yaml | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index df78b4e7a88b..b05f1c931014 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -27,6 +27,16 @@ Changelog --------- +6.2.1 +..... + +.. note:: + Note: this version contains a fix to ``get_blobs_list_async`` method in ``WasbHook`` where it returned + a list of blob names, but advertised (via type hints) that it returns a list of ``BlobProperties`` objects. + This was a bug in the implementation and it was fixed in this release. However, if you were relying on the + previous behaviour, you might need to retrieve ``name`` property from the array elements returned by + this method. + 6.2.0 ..... diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 794da8dd8575..2281e184c22b 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -687,10 +687,10 @@ async def get_blobs_list_async( :param delimiter: filters objects based on the delimiter (for e.g '.csv') """ container = self._get_container_client(container_name) - blob_list = [] + blob_list: list[BlobProperties] = [] blobs = container.walk_blobs(name_starts_with=prefix, include=include, delimiter=delimiter, **kwargs) async for blob in blobs: - blob_list.append(blob.name) + blob_list.append(blob) return blob_list async def check_for_prefix_async(self, container_name: str, prefix: str, **kwargs: Any) -> bool: diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 441f51a5786b..a12088220485 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -22,6 +22,7 @@ description: | `Microsoft Azure `__ suspended: false versions: + - 6.2.1 - 6.2.0 - 6.1.2 - 6.1.1 From 40d54eac1a2f35167bdd179fda3fd018fe32d116 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 12 Jul 2023 09:22:09 +0200 Subject: [PATCH 166/533] Move CeleryExecutor to the celery provider (#32526) Originally Celery Executor and Celery Kubernetes executor have been part of the Airflow Core. However, with AIP-51 where some of the executor internals have been decoupled from the core, we can now move the Celery (and as next step Kubernetes) executors to the providers. This will allow to release fixes to the decoupled executors independently from releasing of the Airflow core version. Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Co-authored-by: Niko Oliveira --- .github/boring-cyborg.yml | 2 +- airflow/cli/cli_config.py | 6 +- airflow/cli/commands/celery_command.py | 2 +- airflow/config_templates/config.yml | 2 +- airflow/config_templates/default_airflow.cfg | 2 +- airflow/executors/__init__.py | 16 ++++ airflow/executors/executor_loader.py | 5 +- airflow/providers/celery/CHANGELOG.rst | 8 ++ .../providers/celery/executors/__init__.py | 36 ++++++++ .../celery}/executors/celery_executor.py | 12 +-- .../executors/celery_executor_utils.py | 15 +++- .../executors/celery_kubernetes_executor.py | 2 +- airflow/providers/celery/provider.yaml | 1 + .../templates/workers/worker-deployment.yaml | 2 +- .../logging-monitoring/check-health.rst | 4 +- .../executor/celery_kubernetes.rst | 2 +- docs/apache-airflow/extra-packages-ref.rst | 82 +++++++++---------- .../howto/docker-compose/docker-compose.yaml | 3 +- newsfragments/32526.significant.rst | 5 ++ ...e_commit_check_setup_extra_packages_ref.py | 2 +- scripts/in_container/_in_container_utils.sh | 9 ++ setup.py | 16 +++- tests/cli/conftest.py | 2 +- .../executors/test_celery_executor.py | 26 ++++-- tests/providers/celery/executors/__init__.py | 17 ++++ .../celery}/executors/test_celery_executor.py | 26 +++--- .../test_celery_kubernetes_executor.py | 4 +- tests/sensors/test_base.py | 4 +- tests/www/views/test_views_tasks.py | 2 +- 29 files changed, 224 insertions(+), 91 deletions(-) create mode 100644 airflow/providers/celery/executors/__init__.py rename airflow/{ => providers/celery}/executors/celery_executor.py (95%) rename airflow/{ => providers/celery}/executors/celery_executor_utils.py (94%) rename airflow/{ => providers/celery}/executors/celery_kubernetes_executor.py (99%) create mode 100644 newsfragments/32526.significant.rst create mode 100644 tests/providers/celery/executors/__init__.py rename tests/{ => providers/celery}/executors/test_celery_executor.py (91%) rename tests/{ => providers/celery}/executors/test_celery_kubernetes_executor.py (98%) diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 5aa495022b3f..989bb055df74 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -167,7 +167,7 @@ labelPRBasedOnFilePath: - airflow/kubernetes/**/* - airflow/kubernetes_executor_templates/**/* - airflow/executors/kubernetes_executor.py - - airflow/executors/celery_kubernetes_executor.py + - airflow/providers/celery/executors/celery_kubernetes_executor.py - docs/apache-airflow/core-concepts/executor/kubernetes.rst - docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst - docs/apache-airflow-providers-cncf-kubernetes/**/* diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 587934769ea5..046cbaad149a 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -67,7 +67,7 @@ def _check_value(self, action, value): executor_cls, _ = ExecutorLoader.import_executor_cls(executor) classes = () try: - from airflow.executors.celery_executor import CeleryExecutor + from airflow.providers.celery.executors.celery_executor import CeleryExecutor classes += (CeleryExecutor,) except ImportError: @@ -77,7 +77,9 @@ def _check_value(self, action, value): ) raise ArgumentError(action, message) try: - from airflow.executors.celery_kubernetes_executor import CeleryKubernetesExecutor + from airflow.providers.celery.executors.celery_kubernetes_executor import ( + CeleryKubernetesExecutor, + ) classes += (CeleryKubernetesExecutor,) except ImportError: diff --git a/airflow/cli/commands/celery_command.py b/airflow/cli/commands/celery_command.py index 2ac976d10c23..6d3dbe64bbba 100644 --- a/airflow/cli/commands/celery_command.py +++ b/airflow/cli/commands/celery_command.py @@ -34,7 +34,7 @@ from airflow import settings from airflow.configuration import conf -from airflow.executors.celery_executor import app as celery_app +from airflow.providers.celery.executors.celery_executor import app as celery_app from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging from airflow.utils.serve_logs import serve_logs diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 5336a88f4f1b..015da426c4f2 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2023,7 +2023,7 @@ celery: version_added: ~ type: string example: ~ - default: "airflow.executors.celery_executor" + default: "airflow.providers.celery.executors.celery_executor" worker_concurrency: description: | The concurrency that will be used when starting workers with the diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 7cae11065ba2..cff56508df4b 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -1037,7 +1037,7 @@ kubernetes_queue = kubernetes # This section only applies if you are using the CeleryExecutor in # ``[core]`` section above # The app name that will be used by celery -celery_app_name = airflow.executors.celery_executor +celery_app_name = airflow.providers.celery.executors.celery_executor # The concurrency that will be used when starting workers with the # ``airflow celery worker`` command. This defines the number of task instances that diff --git a/airflow/executors/__init__.py b/airflow/executors/__init__.py index 21ee94b03b9b..2c342d45f16d 100644 --- a/airflow/executors/__init__.py +++ b/airflow/executors/__init__.py @@ -15,3 +15,19 @@ # specific language governing permissions and limitations # under the License. """Executors.""" +from __future__ import annotations + +from airflow.utils.deprecation_tools import add_deprecated_classes + +__deprecated_classes = { + "celery_executor": { + "app": "airflow.providers.celery.executors.celery_executor_utils.app", + "CeleryExecutor": "airflow.providers.celery.executors.celery_executor.CeleryExecutor", + }, + "celery_kubernetes_executor": { + "CeleryKubernetesExecutor": "airflow.providers.celery.executors." + "celery_kubernetes_executor.CeleryKubernetesExecutor", + }, +} + +add_deprecated_classes(__deprecated_classes, __name__) diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 3ac49d46ca23..58cd9ae9168b 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -60,8 +60,9 @@ class ExecutorLoader: LOCAL_EXECUTOR: "airflow.executors.local_executor.LocalExecutor", LOCAL_KUBERNETES_EXECUTOR: "airflow.executors.local_kubernetes_executor.LocalKubernetesExecutor", SEQUENTIAL_EXECUTOR: "airflow.executors.sequential_executor.SequentialExecutor", - CELERY_EXECUTOR: "airflow.executors.celery_executor.CeleryExecutor", - CELERY_KUBERNETES_EXECUTOR: "airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor", + CELERY_EXECUTOR: "airflow.providers.celery.executors.celery_executor.CeleryExecutor", + CELERY_KUBERNETES_EXECUTOR: "airflow.providers.celery." + "executors.celery_kubernetes_executor.CeleryKubernetesExecutor", DASK_EXECUTOR: "airflow.executors.dask_executor.DaskExecutor", KUBERNETES_EXECUTOR: "airflow.executors.kubernetes_executor.KubernetesExecutor", DEBUG_EXECUTOR: "airflow.executors.debug_executor.DebugExecutor", diff --git a/airflow/providers/celery/CHANGELOG.rst b/airflow/providers/celery/CHANGELOG.rst index 85ccbc5fc4be..0d0038cc366a 100644 --- a/airflow/providers/celery/CHANGELOG.rst +++ b/airflow/providers/celery/CHANGELOG.rst @@ -27,6 +27,14 @@ Changelog --------- +3.3.0 +..... + +.. note:: + This provider release is the first release that has Celery Executor and + Celery Kubernetes Executor moved from the core ``apache-airflow`` package to a Celery + provider package. + 3.2.1 ..... diff --git a/airflow/providers/celery/executors/__init__.py b/airflow/providers/celery/executors/__init__.py new file mode 100644 index 000000000000..56c59e58e7b8 --- /dev/null +++ b/airflow/providers/celery/executors/__init__.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import packaging.version + +from airflow.exceptions import AirflowOptionalProviderFeatureException + +try: + from airflow import __version__ as airflow_version +except ImportError: + from airflow.version import version as airflow_version + +base_version = packaging.version.parse(airflow_version).base_version + +if packaging.version.parse(base_version) < packaging.version.parse("2.7.0"): + raise AirflowOptionalProviderFeatureException( + "Celery Executor from Celery Provider should only be used with Airflow 2.7.0+.\n" + f"This is Airflow {airflow_version} and Celery and CeleryKubernetesExecutor are " + f"available in the 'airflow.executors' package. You should not use " + f"the provider's executors in this version of Airflow." + ) diff --git a/airflow/executors/celery_executor.py b/airflow/providers/celery/executors/celery_executor.py similarity index 95% rename from airflow/executors/celery_executor.py rename to airflow/providers/celery/executors/celery_executor.py index b3a011c815a2..0786c9553853 100644 --- a/airflow/executors/celery_executor.py +++ b/airflow/providers/celery/executors/celery_executor.py @@ -64,7 +64,7 @@ def __getattr__(name): # celery_executor module without the time cost of its import and # construction if name == "app": - from airflow.executors.celery_executor_utils import app + from airflow.providers.celery.executors.celery_executor_utils import app return app raise AttributeError(f"module '{__name__}' has no attribute '{name}'") @@ -99,7 +99,7 @@ def __init__(self): self._sync_parallelism = conf.getint("celery", "SYNC_PARALLELISM") if self._sync_parallelism == 0: self._sync_parallelism = max(1, cpu_count() - 1) - from airflow.executors.celery_executor_utils import BulkStateFetcher + from airflow.providers.celery.executors.celery_executor_utils import BulkStateFetcher self.bulk_state_fetcher = BulkStateFetcher(self._sync_parallelism) self.tasks = {} @@ -118,7 +118,7 @@ def _num_tasks_per_send_process(self, to_send_count: int) -> int: return max(1, int(math.ceil(1.0 * to_send_count / self._sync_parallelism))) def _process_tasks(self, task_tuples: list[TaskTuple]) -> None: - from airflow.executors.celery_executor_utils import execute_command + from airflow.providers.celery.executors.celery_executor_utils import execute_command task_tuples_to_send = [task_tuple[:3] + (execute_command,) for task_tuple in task_tuples] first_task = next(t[3] for t in task_tuples_to_send) @@ -129,7 +129,7 @@ def _process_tasks(self, task_tuples: list[TaskTuple]) -> None: key_and_async_results = self._send_tasks_to_celery(task_tuples_to_send) self.log.debug("Sent all tasks.") - from airflow.executors.celery_executor_utils import ExceptionWithTraceback + from airflow.providers.celery.executors.celery_executor_utils import ExceptionWithTraceback for key, _, result in key_and_async_results: if isinstance(result, ExceptionWithTraceback) and isinstance( @@ -165,7 +165,7 @@ def _process_tasks(self, task_tuples: list[TaskTuple]) -> None: self.update_task_state(key, result.state, getattr(result, "info", None)) def _send_tasks_to_celery(self, task_tuples_to_send: list[TaskInstanceInCelery]): - from airflow.executors.celery_executor_utils import send_task_to_executor + from airflow.providers.celery.executors.celery_executor_utils import send_task_to_executor if len(task_tuples_to_send) == 1 or self._sync_parallelism == 1: # One tuple, or max one process -> send it in the main thread. @@ -304,7 +304,7 @@ def cleanup_stuck_queued_tasks(self, tis: list[TaskInstance]) -> list[str]: :return: List of readable task instances for a warning message """ readable_tis = [] - from airflow.executors.celery_executor_utils import app + from airflow.providers.celery.executors.celery_executor_utils import app for ti in tis: readable_tis.append(repr(ti)) diff --git a/airflow/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py similarity index 94% rename from airflow/executors/celery_executor_utils.py rename to airflow/providers/celery/executors/celery_executor_utils.py index 49112bba6ab4..c659d8c3770a 100644 --- a/airflow/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -27,6 +27,7 @@ import os import subprocess import traceback +import warnings from concurrent.futures import ProcessPoolExecutor from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional, Tuple @@ -40,7 +41,7 @@ import airflow.settings as settings from airflow.config_templates.default_celery import DEFAULT_CELERY_CONFIG from airflow.configuration import conf -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.executors.base_executor import BaseExecutor from airflow.models.taskinstance import TaskInstanceKey from airflow.stats import Stats @@ -66,7 +67,17 @@ else: celery_configuration = DEFAULT_CELERY_CONFIG -app = Celery(conf.get("celery", "CELERY_APP_NAME"), config_source=celery_configuration) +celery_app_name = conf.get("celery", "CELERY_APP_NAME") +if celery_app_name == "airflow.executors.celery_executor": + warnings.warn( + "The celery.CELERY_APP_NAME configuration uses deprecated package name: " + "'airflow.executors.celery_executor'. " + "Change it to `airflow.providers.celery.executors.celery_executor`, and " + "update the `-app` flag in your Celery Health Checks " + "to use `airflow.providers.celery.executors.celery_executor.app`.", + RemovedInAirflow3Warning, + ) +app = Celery(celery_app_name, config_source=celery_configuration) @celery_import_modules.connect diff --git a/airflow/executors/celery_kubernetes_executor.py b/airflow/providers/celery/executors/celery_kubernetes_executor.py similarity index 99% rename from airflow/executors/celery_kubernetes_executor.py rename to airflow/providers/celery/executors/celery_kubernetes_executor.py index d528c43fe53d..0522b12f41aa 100644 --- a/airflow/executors/celery_kubernetes_executor.py +++ b/airflow/providers/celery/executors/celery_kubernetes_executor.py @@ -22,8 +22,8 @@ from airflow.callbacks.base_callback_sink import BaseCallbackSink from airflow.callbacks.callback_requests import CallbackRequest from airflow.configuration import conf -from airflow.executors.celery_executor import CeleryExecutor from airflow.executors.kubernetes_executor import KubernetesExecutor +from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.utils.log.logging_mixin import LoggingMixin if TYPE_CHECKING: diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 83dc66ded0e8..b6b1e7e4e591 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 3.3.0 - 3.2.1 - 3.2.0 - 3.1.0 diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 75b2b1b60a4c..4e306745d7b7 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -205,7 +205,7 @@ spec: {{- else }} - sh - -c - - CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app airflow.executors.celery_executor.app inspect ping -d celery@$(hostname) + - CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d celery@$(hostname) {{- end }} {{- end }} ports: diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst index 58e883407866..78cf0f37254c 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst @@ -147,12 +147,12 @@ To check if the worker running on the local host is working correctly, run: .. code-block:: bash - celery --app airflow.executors.celery_executor.app inspect ping -d celery@${HOSTNAME} + celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d celery@${HOSTNAME} To check if the all workers in the cluster running is working correctly, run: .. code-block:: bash - celery --app airflow.executors.celery_executor.app inspect ping + celery --app airflow.providers.celery.executors.celery_executor.app inspect ping For more information, see: `Management Command-line Utilities (inspect/control) `__ and `Workers Guide `__ in the Celery documentation. diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst index 6e2471b14d2f..720fe7e743dd 100644 --- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst +++ b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst @@ -21,7 +21,7 @@ CeleryKubernetes Executor ========================= -The :class:`~airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor` allows users +The :class:`~airflow.providers.celery.executors.celery_kubernetes_executor.CeleryKubernetesExecutor` allows users to run simultaneously a ``CeleryExecutor`` and a ``KubernetesExecutor``. An executor is chosen to run a task based on the task's queue. diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index c8cfe872374b..027fbb657e41 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -37,47 +37,47 @@ These are core airflow extras that extend capabilities of core Airflow. They usu packages (with the exception of ``celery`` and ``cncf.kubernetes`` extras), they just install necessary python dependencies for the provided package. -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| extra | install command | enables | -+=====================+=====================================================+============================================================================+ -| aiobotocore | ``pip install 'apache-airflow[aiobotocore]'`` | Support for asynchronous (deferrable) operators for Amazon integration | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor (also installs the celery provider package!) | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| cgroups | ``pip install 'apache-airflow[cgroups]'`` | Needed To use CgroupTaskRunner | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| cncf.kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor (also installs the Kubernetes provider package) | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| dask | ``pip install 'apache-airflow[dask]'`` | DaskExecutor | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| deprecated_api | ``pip install 'apache-airflow[deprecated_api]'`` | Deprecated, experimental API that is replaced with the new REST API | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized services (Hadoop, Presto, Trino) | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| otel | ``pip install 'apache-airflow[otel]'`` | Required for OpenTelemetry metrics | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| pandas | ``pip install 'apache-airflow[pandas]'`` | Install Pandas library compatible with Airflow | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| statsd | ``pip install 'apache-airflow[statsd]'`` | Needed by StatsD metrics | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ -| virtualenv | ``pip install 'apache-airflow[virtualenv]'`` | Running python tasks in local virtualenv | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| extra | install command | enables | Preinstalled | ++=====================+=====================================================+============================================================================+==============+ +| aiobotocore | ``pip install 'apache-airflow[aiobotocore]'`` | Support for asynchronous (deferrable) operators for Amazon integration | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor (also installs the celery provider package!) | * | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| cgroups | ``pip install 'apache-airflow[cgroups]'`` | Needed To use CgroupTaskRunner | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| cncf.kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor (also installs the Kubernetes provider package) | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| dask | ``pip install 'apache-airflow[dask]'`` | DaskExecutor | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| deprecated_api | ``pip install 'apache-airflow[deprecated_api]'`` | Deprecated, experimental API that is replaced with the new REST API | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized services (Hadoop, Presto, Trino) | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| otel | ``pip install 'apache-airflow[otel]'`` | Required for OpenTelemetry metrics | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| pandas | ``pip install 'apache-airflow[pandas]'`` | Install Pandas library compatible with Airflow | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| statsd | ``pip install 'apache-airflow[statsd]'`` | Needed by StatsD metrics | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ +| virtualenv | ``pip install 'apache-airflow[virtualenv]'`` | Running python tasks in local virtualenv | | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ Providers extras diff --git a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml index 097b472f05b6..86b319b23383 100644 --- a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml +++ b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml @@ -149,9 +149,10 @@ services: <<: *airflow-common command: celery worker healthcheck: + # yamllint disable rule:line-length test: - "CMD-SHELL" - - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' + - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' interval: 30s timeout: 10s retries: 5 diff --git a/newsfragments/32526.significant.rst b/newsfragments/32526.significant.rst new file mode 100644 index 000000000000..8b3a71518ea8 --- /dev/null +++ b/newsfragments/32526.significant.rst @@ -0,0 +1,5 @@ +Default name of the Celery application changed from ``airflow.executors.celery_executor`` to ``airflow.providers.celery.executors.celery_executor`` + +You should change both your configuration and Health check command to use the new name: + * in configuration (``celery_app_name`` configuration in ``celery`` section) use ``airflow.providers.celery.executors.celery_executor`` + * in your Health check command use ``airflow.providers.celery.executors.celery_executor.app`` diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py index a523ac62c055..04b707f30ac8 100755 --- a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py +++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py @@ -209,7 +209,7 @@ def check_preinstalled_extras(console: Console) -> bool: :return: True if all ok, False otherwise """ preinstalled_providers_from_docs = get_preinstalled_providers_from_docs() - preinstalled_providers_from_setup = PREINSTALLED_PROVIDERS + preinstalled_providers_from_setup = [provider.split(">=")[0] for provider in PREINSTALLED_PROVIDERS] preinstalled_providers_table = Table() preinstalled_providers_table.add_column("PREINSTALLED_IN_SETUP", justify="right", style="cyan") diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index b368bb3aca2d..e01409bbe27a 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -163,7 +163,16 @@ function install_airflow_from_wheel() { exit 4 fi if [[ ${constraints_reference} == "none" ]]; then + set +e pip install "${airflow_package}${extras}" + res=$? + set -e + if [[ ${res} != "0" ]]; then + >&2 echo + >&2 echo "WARNING! Could not install airflow without constraints, trying to install it without dependencies in case some of the required providers are not yet released" + >&2 echo + pip install "${airflow_package}${extras}" --no-deps + fi else set +e pip install "${airflow_package}${extras}" --constraint \ diff --git a/setup.py b/setup.py index 21c0a7300c2d..756e3c8ac83d 100644 --- a/setup.py +++ b/setup.py @@ -729,6 +729,9 @@ def sort_extras_dependencies() -> dict[str, list[str]]: # Those providers do not have dependency on airflow2.0 because that would lead to circular dependencies. # This is not a problem for PIP but some tools (pipdeptree) show those as a warning. PREINSTALLED_PROVIDERS = [ + # TODO: When we release 3.3.0 version of celery provider we should change it to "celery>=3.3.0" here + # In order to make sure executors are available in the celery provider + "celery", "common.sql", "ftp", "http", @@ -744,8 +747,17 @@ def get_provider_package_name_from_package_id(package_id: str) -> str: :param package_id: id of the package (like amazon or microsoft.azure) :return: full name of package in PyPI """ - package_suffix = package_id.replace(".", "-") - return f"apache-airflow-providers-{package_suffix}" + version_spec = "" + if ">=" in package_id: + package, version = package_id.split(">=") + version_spec = f">={version}" + version_suffix = os.environ.get("VERSION_SUFFIX_FOR_PYPI") + if version_suffix: + version_spec += version_suffix + else: + package = package_id + package_suffix = package.replace(".", "-") + return f"apache-airflow-providers-{package_suffix}{version_spec}" def get_excluded_providers() -> list[str]: diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 0a0e2ec5cb99..3069726a059c 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -23,7 +23,7 @@ from airflow import models from airflow.cli import cli_parser -from airflow.executors import celery_executor, celery_kubernetes_executor +from airflow.providers.celery.executors import celery_executor, celery_kubernetes_executor from tests.test_utils.config import conf_vars # Create custom executors here because conftest is imported first diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index a89034d2af94..c4c3e5d6f04e 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -36,10 +36,10 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowTaskTimeout -from airflow.executors import celery_executor, celery_executor_utils from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.operators.bash import BashOperator +from airflow.providers.celery.executors import celery_executor, celery_executor_utils from airflow.utils.state import State from tests.test_utils import db @@ -68,8 +68,10 @@ def _prepare_app(broker_url=None, execute=None): test_config.update({"broker_url": broker_url}) test_app = Celery(broker_url, config_source=test_config) test_execute = test_app.task(execute) - patch_app = mock.patch("airflow.executors.celery_executor.app", test_app) - patch_execute = mock.patch("airflow.executors.celery_executor_utils.execute_command", test_execute) + patch_app = mock.patch("airflow.providers.celery.executors.celery_executor.app", test_app) + patch_execute = mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.execute_command", test_execute + ) backend = test_app.backend @@ -259,7 +261,7 @@ def __ne__(self, other): @pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestBulkStateFetcher: - bulk_state_fetcher_logger = "airflow.executors.celery_executor_utils.BulkStateFetcher" + bulk_state_fetcher_logger = "airflow.providers.celery.executors.celery_executor_utils.BulkStateFetcher" @mock.patch( "celery.backends.base.BaseKeyValueStoreBackend.mget", @@ -269,7 +271,9 @@ def test_should_support_kv_backend(self, mock_mget, caplog): caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) with _prepare_app(): mock_backend = BaseKeyValueStoreBackend(app=celery_executor.app) - with mock.patch("airflow.executors.celery_executor_utils.Celery.backend", mock_backend): + with mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.Celery.backend", mock_backend + ): caplog.clear() fetcher = celery_executor_utils.BulkStateFetcher() result = fetcher.get_many( @@ -292,7 +296,9 @@ def test_should_support_db_backend(self, mock_session, caplog): caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) with _prepare_app(): mock_backend = DatabaseBackend(app=celery_executor.app, url="sqlite3://") - with mock.patch("airflow.executors.celery_executor_utils.Celery.backend", mock_backend): + with mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.Celery.backend", mock_backend + ): caplog.clear() mock_session = mock_backend.ResultSession.return_value mock_session.query.return_value.filter.return_value.all.return_value = [ @@ -317,7 +323,9 @@ def test_should_retry_db_backend(self, mock_session, caplog): with _prepare_app(): mock_backend = DatabaseBackend(app=celery_executor.app, url="sqlite3://") - with mock.patch("airflow.executors.celery_executor_utils.Celery.backend", mock_backend): + with mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.Celery.backend", mock_backend + ): caplog.clear() mock_session = mock_backend.ResultSession.return_value mock_retry_db_result = mock_session.query.return_value.filter.return_value.all @@ -348,7 +356,9 @@ def test_should_support_base_backend(self, caplog): with _prepare_app(): mock_backend = mock.MagicMock(autospec=BaseBackend) - with mock.patch("airflow.executors.celery_executor_utils.Celery.backend", mock_backend): + with mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.Celery.backend", mock_backend + ): caplog.clear() fetcher = celery_executor_utils.BulkStateFetcher(1) result = fetcher.get_many( diff --git a/tests/providers/celery/executors/__init__.py b/tests/providers/celery/executors/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/tests/providers/celery/executors/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/executors/test_celery_executor.py b/tests/providers/celery/executors/test_celery_executor.py similarity index 91% rename from tests/executors/test_celery_executor.py rename to tests/providers/celery/executors/test_celery_executor.py index 11b461b76dca..91e3312421f9 100644 --- a/tests/executors/test_celery_executor.py +++ b/tests/providers/celery/executors/test_celery_executor.py @@ -34,11 +34,11 @@ from kombu.asynchronous import set_event_loop from airflow.configuration import conf -from airflow.executors import celery_executor, celery_executor_utils -from airflow.executors.celery_executor import CeleryExecutor from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.taskinstance import TaskInstance, TaskInstanceKey +from airflow.providers.celery.executors import celery_executor, celery_executor_utils +from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.utils import timezone from airflow.utils.state import State from tests.test_utils import db @@ -71,8 +71,10 @@ def _prepare_app(broker_url=None, execute=None): test_config.update({"broker_url": broker_url}) test_app = Celery(broker_url, config_source=test_config) test_execute = test_app.task(execute) - patch_app = mock.patch("airflow.executors.celery_executor.app", test_app) - patch_execute = mock.patch("airflow.executors.celery_executor_utils.execute_command", test_execute) + patch_app = mock.patch("airflow.providers.celery.executors.celery_executor.app", test_app) + patch_execute = mock.patch( + "airflow.providers.celery.executors.celery_executor_utils.execute_command", test_execute + ) backend = test_app.backend @@ -110,7 +112,9 @@ def test_supports_sentry(self): @pytest.mark.backend("mysql", "postgres") def test_exception_propagation(self, caplog): - caplog.set_level(logging.ERROR, logger="airflow.executors.celery_executor_utils.BulkStateFetcher") + caplog.set_level( + logging.ERROR, logger="airflow.providers.celery.executors.celery_executor_utils.BulkStateFetcher" + ) with _prepare_app(): executor = celery_executor.CeleryExecutor() executor.tasks = {"key": FakeCeleryResult()} @@ -118,8 +122,8 @@ def test_exception_propagation(self, caplog): assert celery_executor_utils.CELERY_FETCH_ERR_MSG_HEADER in caplog.text, caplog.record_tuples assert FAKE_EXCEPTION_MSG in caplog.text, caplog.record_tuples - @mock.patch("airflow.executors.celery_executor.CeleryExecutor.sync") - @mock.patch("airflow.executors.celery_executor.CeleryExecutor.trigger_tasks") + @mock.patch("airflow.providers.celery.executors.celery_executor.CeleryExecutor.sync") + @mock.patch("airflow.providers.celery.executors.celery_executor.CeleryExecutor.trigger_tasks") @mock.patch("airflow.executors.base_executor.Stats.gauge") def test_gauge_executor_metrics(self, mock_stats_gauge, mock_trigger_tasks, mock_sync): executor = celery_executor.CeleryExecutor() @@ -154,9 +158,9 @@ def test_command_validation(self, command, raise_exception): ) with mock.patch( - "airflow.executors.celery_executor_utils._execute_in_subprocess" + "airflow.providers.celery.executors.celery_executor_utils._execute_in_subprocess" ) as mock_subproc, mock.patch( - "airflow.executors.celery_executor_utils._execute_in_fork" + "airflow.providers.celery.executors.celery_executor_utils._execute_in_fork" ) as mock_fork, mock.patch( "celery.app.task.Task.request" ) as mock_task: @@ -224,7 +228,7 @@ def mock_celery_revoke(self): yield app.control.revoke @pytest.mark.backend("mysql", "postgres") - @mock.patch("airflow.executors.celery_executor.CeleryExecutor.fail") + @mock.patch("airflow.providers.celery.executors.celery_executor.CeleryExecutor.fail") def test_cleanup_stuck_queued_tasks(self, mock_fail): start_date = timezone.utcnow() - timedelta(days=2) @@ -255,7 +259,7 @@ def test_result_backend_sqlalchemy_engine_options(self, mock_celery): import importlib from airflow.config_templates import default_celery - from airflow.executors import celery_executor_utils + from airflow.providers.celery.executors import celery_executor_utils # reload celery conf to apply the new config importlib.reload(default_celery) diff --git a/tests/executors/test_celery_kubernetes_executor.py b/tests/providers/celery/executors/test_celery_kubernetes_executor.py similarity index 98% rename from tests/executors/test_celery_kubernetes_executor.py rename to tests/providers/celery/executors/test_celery_kubernetes_executor.py index ec222e71f70f..6302efea21da 100644 --- a/tests/executors/test_celery_kubernetes_executor.py +++ b/tests/providers/celery/executors/test_celery_kubernetes_executor.py @@ -23,9 +23,9 @@ from airflow.callbacks.callback_requests import CallbackRequest from airflow.configuration import conf -from airflow.executors.celery_executor import CeleryExecutor -from airflow.executors.celery_kubernetes_executor import CeleryKubernetesExecutor from airflow.executors.kubernetes_executor import KubernetesExecutor +from airflow.providers.celery.executors.celery_executor import CeleryExecutor +from airflow.providers.celery.executors.celery_kubernetes_executor import CeleryKubernetesExecutor KUBERNETES_QUEUE = CeleryKubernetesExecutor.KUBERNETES_QUEUE diff --git a/tests/sensors/test_base.py b/tests/sensors/test_base.py index 8761f1de4787..b4042945d326 100644 --- a/tests/sensors/test_base.py +++ b/tests/sensors/test_base.py @@ -24,8 +24,6 @@ import time_machine from airflow.exceptions import AirflowException, AirflowRescheduleException, AirflowSensorTimeout -from airflow.executors.celery_executor import CeleryExecutor -from airflow.executors.celery_kubernetes_executor import CeleryKubernetesExecutor from airflow.executors.debug_executor import DebugExecutor from airflow.executors.executor_constants import ( CELERY_EXECUTOR, @@ -44,6 +42,8 @@ from airflow.models import TaskReschedule from airflow.models.xcom import XCom from airflow.operators.empty import EmptyOperator +from airflow.providers.celery.executors.celery_executor import CeleryExecutor +from airflow.providers.celery.executors.celery_kubernetes_executor import CeleryKubernetesExecutor from airflow.sensors.base import BaseSensorOperator, PokeReturnValue, poke_mode_only from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep from airflow.utils import timezone diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 9b45fc6622be..43b229325051 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -29,10 +29,10 @@ from airflow import settings from airflow.exceptions import AirflowException -from airflow.executors.celery_executor import CeleryExecutor from airflow.models import DAG, DagBag, DagModel, TaskFail, TaskInstance, TaskReschedule from airflow.models.dagcode import DagCode from airflow.operators.bash import BashOperator +from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.security import permissions from airflow.utils import timezone from airflow.utils.log.logging_mixin import ExternalLoggingMixin From 04a6e850e7d4437fe93a60d713c1d82e8f23885f Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 12 Jul 2023 10:35:33 +0200 Subject: [PATCH 167/533] Fix data interval in cli.dags.trigger command output (#32548) --- airflow/api/client/local_client.py | 4 ++-- tests/api_experimental/client/test_local_client.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py index afdcd0abd668..b90039608f4c 100644 --- a/airflow/api/client/local_client.py +++ b/airflow/api/client/local_client.py @@ -43,8 +43,8 @@ def trigger_dag( "conf": dag_run.conf, "dag_id": dag_run.dag_id, "dag_run_id": dag_run.run_id, - "data_interval_end": dag_run.data_interval_start, - "data_interval_start": dag_run.data_interval_end, + "data_interval_start": dag_run.data_interval_start, + "data_interval_end": dag_run.data_interval_end, "end_date": dag_run.end_date, "external_trigger": dag_run.external_trigger, "last_scheduling_decision": dag_run.last_scheduling_decision, diff --git a/tests/api_experimental/client/test_local_client.py b/tests/api_experimental/client/test_local_client.py index 5527f00dea8e..575400a115de 100644 --- a/tests/api_experimental/client/test_local_client.py +++ b/tests/api_experimental/client/test_local_client.py @@ -147,8 +147,8 @@ def test_trigger_dag(self, mock): "conf": {}, "dag_id": test_dag_id, "dag_run_id": run_id, - "data_interval_end": EXECDATE, - "data_interval_start": EXECDATE + pendulum.duration(hours=1), + "data_interval_start": EXECDATE, + "data_interval_end": EXECDATE + pendulum.duration(hours=1), "end_date": None, "external_trigger": True, "last_scheduling_decision": None, From b3ce1161926efb880c3f525ac0a031ab4812fb95 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 12 Jul 2023 12:05:03 +0200 Subject: [PATCH 168/533] Fix async KPO by waiting pod termination in `execute_complete` before cleanup (#32467) * Fix async KPO by waiting pod termination in `execute_complete` before cleanup (#32467) --------- Signed-off-by: Hussein Awala --- .../cncf/kubernetes/operators/pod.py | 13 +-- .../providers/cncf/kubernetes/triggers/pod.py | 26 ++--- .../cncf/kubernetes/operators/test_pod.py | 99 +++++++++++++++++++ .../cncf/kubernetes/triggers/test_pod.py | 32 +----- .../cloud/triggers/test_kubernetes_engine.py | 12 +-- 5 files changed, 114 insertions(+), 68 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 124685e792e7..49940144b5dd 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -636,15 +636,11 @@ def invoke_defer_method(self): def execute_complete(self, context: Context, event: dict, **kwargs): pod = None - remote_pod = None try: pod = self.hook.get_pod( event["name"], event["namespace"], ) - # It is done to coincide with the current implementation of the general logic of the cleanup - # method. If it's going to be remade in future then it must be changed - remote_pod = pod if event["status"] in ("error", "failed", "timeout"): # fetch some logs when pod is failed if self.get_logs: @@ -661,16 +657,13 @@ def execute_complete(self, context: Context, event: dict, **kwargs): if self.do_xcom_push: xcom_sidecar_output = self.extract_xcom(pod=pod) - pod = self.pod_manager.await_pod_completion(pod) - # It is done to coincide with the current implementation of the general logic of - # the cleanup method. If it's going to be remade in future then it must be changed - remote_pod = pod return xcom_sidecar_output finally: - if pod is not None and remote_pod is not None: + pod = self.pod_manager.await_pod_completion(pod) + if pod is not None: self.post_complete_action( pod=pod, - remote_pod=remote_pod, + remote_pod=pod, ) def write_logs(self, pod: k8s.V1Pod): diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py b/airflow/providers/cncf/kubernetes/triggers/pod.py index 6fdf763eceda..6443dfb63f3c 100644 --- a/airflow/providers/cncf/kubernetes/triggers/pod.py +++ b/airflow/providers/cncf/kubernetes/triggers/pod.py @@ -154,23 +154,15 @@ async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] self.log.debug("Container %s status: %s", self.base_container_name, container_state) if container_state == ContainerState.TERMINATED: - if pod_status not in PodPhase.terminal_states: - self.log.info( - "Pod %s is still running. Sleeping for %s seconds.", - self.pod_name, - self.poll_interval, - ) - await asyncio.sleep(self.poll_interval) - else: - yield TriggerEvent( - { - "name": self.pod_name, - "namespace": self.pod_namespace, - "status": "success", - "message": "All containers inside pod have started successfully.", - } - ) - return + yield TriggerEvent( + { + "name": self.pod_name, + "namespace": self.pod_namespace, + "status": "success", + "message": "All containers inside pod have started successfully.", + } + ) + return elif self.should_wait(pod_phase=pod_status, container_state=container_state): self.log.info("Container is not completed and still working.") diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 1ce25190ba4c..15ca6553cd10 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -1389,10 +1389,12 @@ def test_async_create_pod_should_throw_exception(self, mocked_hook, mocked_clean ({"skip_on_exit_code": None}, 100, AirflowException, "Failed", "error"), ], ) + @patch(KUB_OP_PATH.format("pod_manager")) @patch(HOOK_CLASS) def test_async_create_pod_with_skip_on_exit_code_should_skip( self, mocked_hook, + mock_manager, extra_kwargs, actual_exit_code, expected_exc, @@ -1426,6 +1428,7 @@ def test_async_create_pod_with_skip_on_exit_code_should_skip( remote_pod.status.phase = pod_status remote_pod.status.container_statuses = [base_container, sidecar_container] mocked_hook.return_value.get_pod.return_value = remote_pod + mock_manager.await_pod_completion.return_value = remote_pod context = { "ti": MagicMock(), @@ -1608,3 +1611,99 @@ def test_cleanup_log_pod_spec_on_failure(self, log_pod_spec_on_failure, expect_m pod.status = V1PodStatus(phase=PodPhase.FAILED) with pytest.raises(AirflowException, match=expect_match): k.cleanup(pod, pod) + + +@pytest.mark.parametrize("do_xcom_push", [True, False]) +@patch(KUB_OP_PATH.format("extract_xcom")) +@patch(KUB_OP_PATH.format("post_complete_action")) +@patch(HOOK_CLASS) +def test_async_kpo_wait_termination_before_cleanup_on_success( + mocked_hook, post_complete_action, mock_extract_xcom, do_xcom_push +): + metadata = {"metadata.name": TEST_NAME, "metadata.namespace": TEST_NAMESPACE} + running_state = mock.MagicMock(**metadata, **{"status.phase": "Running"}) + succeeded_state = mock.MagicMock(**metadata, **{"status.phase": "Succeeded"}) + mocked_hook.return_value.get_pod.return_value = running_state + read_pod_mock = mocked_hook.return_value.core_v1_client.read_namespaced_pod + read_pod_mock.side_effect = [ + running_state, + running_state, + succeeded_state, + ] + + ti_mock = MagicMock() + + success_event = { + "status": "success", + "message": TEST_SUCCESS_MESSAGE, + "name": TEST_NAME, + "namespace": TEST_NAMESPACE, + } + + k = KubernetesPodOperator(task_id="task", deferrable=True, do_xcom_push=do_xcom_push) + k.execute_complete({"ti": ti_mock}, success_event) + + # check if it gets the pod + mocked_hook.return_value.get_pod.assert_called_once_with(TEST_NAME, TEST_NAMESPACE) + + # check if it pushes the xcom + assert ti_mock.xcom_push.call_count == 2 + ti_mock.xcom_push.assert_any_call(key="pod_name", value=TEST_NAME) + ti_mock.xcom_push.assert_any_call(key="pod_namespace", value=TEST_NAMESPACE) + + # assert that the xcom are extracted/not extracted + if do_xcom_push: + mock_extract_xcom.assert_called_once() + else: + mock_extract_xcom.assert_not_called() + + # check if it waits for the pod to complete + assert read_pod_mock.call_count == 3 + + # assert that the cleanup is called + post_complete_action.assert_called_once() + + +@pytest.mark.parametrize("do_xcom_push", [True, False]) +@patch(KUB_OP_PATH.format("extract_xcom")) +@patch(KUB_OP_PATH.format("post_complete_action")) +@patch(HOOK_CLASS) +def test_async_kpo_wait_termination_before_cleanup_on_failure( + mocked_hook, post_complete_action, mock_extract_xcom, do_xcom_push +): + metadata = {"metadata.name": TEST_NAME, "metadata.namespace": TEST_NAMESPACE} + running_state = mock.MagicMock(**metadata, **{"status.phase": "Running"}) + failed_state = mock.MagicMock(**metadata, **{"status.phase": "Failed"}) + mocked_hook.return_value.get_pod.return_value = running_state + read_pod_mock = mocked_hook.return_value.core_v1_client.read_namespaced_pod + read_pod_mock.side_effect = [ + running_state, + running_state, + failed_state, + ] + + ti_mock = MagicMock() + + success_event = {"status": "failed", "message": "error", "name": TEST_NAME, "namespace": TEST_NAMESPACE} + + post_complete_action.side_effect = AirflowException() + + k = KubernetesPodOperator(task_id="task", deferrable=True, do_xcom_push=do_xcom_push) + + with pytest.raises(AirflowException): + k.execute_complete({"ti": ti_mock}, success_event) + + # check if it gets the pod + mocked_hook.return_value.get_pod.assert_called_once_with(TEST_NAME, TEST_NAMESPACE) + + # assert that it does not push the xcom + ti_mock.xcom_push.assert_not_called() + + # assert that the xcom are not extracted + mock_extract_xcom.assert_not_called() + + # check if it waits for the pod to complete + assert read_pod_mock.call_count == 3 + + # assert that the cleanup is called + post_complete_action.assert_called_once() diff --git a/tests/providers/cncf/kubernetes/triggers/test_pod.py b/tests/providers/cncf/kubernetes/triggers/test_pod.py index 4ed731b42578..fbfff17278c7 100644 --- a/tests/providers/cncf/kubernetes/triggers/test_pod.py +++ b/tests/providers/cncf/kubernetes/triggers/test_pod.py @@ -96,8 +96,7 @@ def test_serialize(self, trigger): @mock.patch(f"{TRIGGER_PATH}.define_container_state") @mock.patch(f"{TRIGGER_PATH}._get_async_hook") async def test_run_loop_return_success_event(self, mock_hook, mock_method, trigger): - pod_mock = mock.MagicMock(**{"status.phase": "Succeeded"}) - mock_hook.return_value.get_pod.return_value = self._mock_pod_result(pod_mock) + mock_hook.return_value.get_pod.return_value = self._mock_pod_result(mock.MagicMock()) mock_method.return_value = ContainerState.TERMINATED expected_event = TriggerEvent( @@ -112,35 +111,6 @@ async def test_run_loop_return_success_event(self, mock_hook, mock_method, trigg assert actual_event == expected_event - @pytest.mark.asyncio - @mock.patch(f"{TRIGGER_PATH}.define_container_state") - @mock.patch(f"{TRIGGER_PATH}._get_async_hook") - async def test_run_loop_wait_pod_termination_before_returning_success_event( - self, mock_hook, mock_method, trigger - ): - running_state = mock.MagicMock(**{"status.phase": "Running"}) - succeeded_state = mock.MagicMock(**{"status.phase": "Succeeded"}) - mock_hook.return_value.get_pod.side_effect = [ - self._mock_pod_result(running_state), - self._mock_pod_result(running_state), - self._mock_pod_result(succeeded_state), - ] - mock_method.return_value = ContainerState.TERMINATED - - expected_event = TriggerEvent( - { - "name": POD_NAME, - "namespace": NAMESPACE, - "status": "success", - "message": "All containers inside pod have started successfully.", - } - ) - with mock.patch.object(asyncio, "sleep") as mock_sleep: - actual_event = await (trigger.run()).asend(None) - - assert actual_event == expected_event - assert mock_sleep.call_count == 2 - @pytest.mark.asyncio @mock.patch(f"{TRIGGER_PATH}.define_container_state") @mock.patch(f"{TRIGGER_PATH}._get_async_hook") diff --git a/tests/providers/google/cloud/triggers/test_kubernetes_engine.py b/tests/providers/google/cloud/triggers/test_kubernetes_engine.py index 154908a6c4b9..e695822d3863 100644 --- a/tests/providers/google/cloud/triggers/test_kubernetes_engine.py +++ b/tests/providers/google/cloud/triggers/test_kubernetes_engine.py @@ -110,13 +110,7 @@ def test_serialize_should_execute_successfully(self, trigger): async def test_run_loop_return_success_event_should_execute_successfully( self, mock_hook, mock_method, trigger ): - running_state = mock.MagicMock(**{"status.phase": "Running"}) - succeeded_state = mock.MagicMock(**{"status.phase": "Succeeded"}) - mock_hook.return_value.get_pod.side_effect = [ - self._mock_pod_result(running_state), - self._mock_pod_result(running_state), - self._mock_pod_result(succeeded_state), - ] + mock_hook.return_value.get_pod.return_value = self._mock_pod_result(mock.MagicMock()) mock_method.return_value = ContainerState.TERMINATED expected_event = TriggerEvent( @@ -127,11 +121,9 @@ async def test_run_loop_return_success_event_should_execute_successfully( "message": "All containers inside pod have started successfully.", } ) - with mock.patch.object(asyncio, "sleep") as mock_sleep: - actual_event = await (trigger.run()).asend(None) + actual_event = await (trigger.run()).asend(None) assert actual_event == expected_event - assert mock_sleep.call_count == 2 @pytest.mark.asyncio @mock.patch(f"{TRIGGER_KUB_PATH}.define_container_state") From a2ae2265ce960d65bc3c4bf805ee77954a1f895c Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 12 Jul 2023 19:03:02 +0800 Subject: [PATCH 169/533] Add JobState for job state constants (#32549) Also fixed a couple of cases where DagRunState and TaskInstanceState were incorrectly used for job state. --- airflow/cli/cli_config.py | 17 ++++++++++++----- airflow/cli/commands/jobs_command.py | 4 ++-- airflow/jobs/job.py | 16 ++++++++-------- airflow/jobs/scheduler_job_runner.py | 10 +++++----- airflow/models/taskinstance.py | 6 +++--- airflow/utils/state.py | 13 +++++++++++++ 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 046cbaad149a..92ba87c3de92 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -37,7 +37,7 @@ from airflow.settings import _ENABLE_AIP_44 from airflow.utils.cli import ColorMode from airflow.utils.module_loading import import_string -from airflow.utils.state import DagRunState +from airflow.utils.state import DagRunState, JobState from airflow.utils.timezone import parse as parsedate BUILD_DOCS = "BUILDING_AIRFLOW_DOCS" in os.environ @@ -281,9 +281,9 @@ def string_lower_type(val): ("--no-backfill",), help="filter all the backfill dagruns given the dag id", action="store_true" ) dagrun_states = tuple(state.value for state in DagRunState) -ARG_STATE = Arg( +ARG_DR_STATE = Arg( ("--state",), - help="Only list the dag runs corresponding to the state", + help="Only list the DAG runs corresponding to the state", metavar=", ".join(dagrun_states), choices=dagrun_states, ) @@ -291,6 +291,13 @@ def string_lower_type(val): # list_jobs ARG_DAG_ID_OPT = Arg(("-d", "--dag-id"), help="The id of the dag") ARG_LIMIT = Arg(("--limit",), help="Return a limited number of records") +job_states = tuple(state.value for state in JobState) +ARG_JOB_STATE = Arg( + ("--state",), + help="Only list the jobs corresponding to the state", + metavar=", ".join(job_states), + choices=job_states, +) # next_execution ARG_NUM_EXECUTIONS = Arg( @@ -1161,7 +1168,7 @@ class GroupCommand(NamedTuple): args=( ARG_DAG_ID_REQ_FLAG, ARG_NO_BACKFILL, - ARG_STATE, + ARG_DR_STATE, ARG_OUTPUT, ARG_VERBOSE, ARG_START_DATE, @@ -1172,7 +1179,7 @@ class GroupCommand(NamedTuple): name="list-jobs", help="List the jobs", func=lazy_load_command("airflow.cli.commands.dag_command.dag_list_jobs"), - args=(ARG_DAG_ID_OPT, ARG_STATE, ARG_LIMIT, ARG_OUTPUT, ARG_VERBOSE), + args=(ARG_DAG_ID_OPT, ARG_JOB_STATE, ARG_LIMIT, ARG_OUTPUT, ARG_VERBOSE), ), ActionCommand( name="state", diff --git a/airflow/cli/commands/jobs_command.py b/airflow/cli/commands/jobs_command.py index 339896d877f2..bcdd6df475d6 100644 --- a/airflow/cli/commands/jobs_command.py +++ b/airflow/cli/commands/jobs_command.py @@ -22,7 +22,7 @@ from airflow.jobs.job import Job from airflow.utils.net import get_hostname from airflow.utils.session import NEW_SESSION, provide_session -from airflow.utils.state import State +from airflow.utils.state import JobState @provide_session @@ -33,7 +33,7 @@ def check(args, session: Session = NEW_SESSION) -> None: if args.hostname and args.local: raise SystemExit("You can't use --hostname and --local at the same time") - query = select(Job).where(Job.state == State.RUNNING).order_by(Job.latest_heartbeat.desc()) + query = select(Job).where(Job.state == JobState.RUNNING).order_by(Job.latest_heartbeat.desc()) if args.job_type: query = query.where(Job.job_type == args.job_type) if args.hostname: diff --git a/airflow/jobs/job.py b/airflow/jobs/job.py index 99394cbc4553..add43e0aa5fb 100644 --- a/airflow/jobs/job.py +++ b/airflow/jobs/job.py @@ -40,7 +40,7 @@ from airflow.utils.platform import getuser from airflow.utils.session import NEW_SESSION, create_session, provide_session from airflow.utils.sqlalchemy import UtcDateTime -from airflow.utils.state import State +from airflow.utils.state import JobState def _resolve_dagrun_model(): @@ -132,7 +132,7 @@ def is_alive(self, grace_multiplier=2.1): else: health_check_threshold: int = self.heartrate * grace_multiplier return ( - self.state == State.RUNNING + self.state == JobState.RUNNING and (timezone.utcnow() - self.latest_heartbeat).total_seconds() < health_check_threshold ) @@ -181,7 +181,7 @@ def heartbeat( session.merge(self) previous_heartbeat = self.latest_heartbeat - if self.state in State.terminating_states: + if self.state in (JobState.SHUTDOWN, JobState.RESTARTING): # TODO: Make sure it is AIP-44 compliant self.kill() @@ -215,7 +215,7 @@ def heartbeat( def prepare_for_execution(self, session: Session = NEW_SESSION): """Prepares the job for execution.""" Stats.incr(self.__class__.__name__.lower() + "_start", 1, 1) - self.state = State.RUNNING + self.state = JobState.RUNNING self.start_date = timezone.utcnow() session.add(self) session.commit() @@ -251,7 +251,7 @@ def most_recent_job(job_type: str, session: Session = NEW_SESSION) -> Job | None .where(Job.job_type == job_type) .order_by( # Put "running" jobs at the front. - case({State.RUNNING: 0}, value=Job.state, else_=1), + case({JobState.RUNNING: 0}, value=Job.state, else_=1), Job.latest_heartbeat.desc(), ) .limit(1) @@ -308,12 +308,12 @@ def execute_job(job: Job | JobPydantic, execute_callable: Callable[[], int | Non try: ret = execute_callable() # In case of max runs or max duration - job.state = State.SUCCESS + job.state = JobState.SUCCESS except SystemExit: # In case of ^C or SIGTERM - job.state = State.SUCCESS + job.state = JobState.SUCCESS except Exception: - job.state = State.FAILED + job.state = JobState.FAILED raise return ret diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index c0e687707617..3b399d75b589 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -74,7 +74,7 @@ tuple_in_condition, with_row_locks, ) -from airflow.utils.state import DagRunState, State, TaskInstanceState +from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState from airflow.utils.types import DagRunType if TYPE_CHECKING: @@ -1586,10 +1586,10 @@ def adopt_or_reset_orphaned_tasks(self, session: Session = NEW_SESSION) -> int: update(Job) .where( Job.job_type == "SchedulerJob", - Job.state == State.RUNNING, + Job.state == JobState.RUNNING, Job.latest_heartbeat < (timezone.utcnow() - timedelta(seconds=timeout)), ) - .values(state=State.FAILED) + .values(state=JobState.FAILED) ).rowcount if num_failed: @@ -1605,7 +1605,7 @@ def adopt_or_reset_orphaned_tasks(self, session: Session = NEW_SESSION) -> int: # "or queued_by_job_id IS NONE") can go as soon as scheduler HA is # released. .outerjoin(TI.queued_by_job) - .where(or_(TI.queued_by_job_id.is_(None), Job.state != State.RUNNING)) + .where(or_(TI.queued_by_job_id.is_(None), Job.state != JobState.RUNNING)) .join(TI.dag_run) .where( DagRun.run_type != DagRunType.BACKFILL_JOB, @@ -1690,7 +1690,7 @@ def _find_zombies(self) -> None: .where(TI.state == TaskInstanceState.RUNNING) .where( or_( - Job.state != State.RUNNING, + Job.state != JobState.RUNNING, Job.latest_heartbeat < limit_dttm, ) ) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 3ec15f449dbf..3e328ec20858 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -57,6 +57,7 @@ inspect, or_, text, + update, ) from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.mutable import MutableDict @@ -123,7 +124,7 @@ tuple_in_condition, with_row_locks, ) -from airflow.utils.state import DagRunState, State, TaskInstanceState +from airflow.utils.state import DagRunState, JobState, State, TaskInstanceState from airflow.utils.task_group import MappedTaskGroup from airflow.utils.timeout import timeout from airflow.utils.xcom import XCOM_RETURN_KEY @@ -292,8 +293,7 @@ def clear_task_instances( if job_ids: from airflow.jobs.job import Job - for job in session.query(Job).filter(Job.id.in_(job_ids)).all(): - job.state = TaskInstanceState.RESTARTING + session.execute(update(Job).where(Job.id.in_(job_ids)).values(state=JobState.RESTARTING)) if activate_dag_runs is not None: warnings.warn( diff --git a/airflow/utils/state.py b/airflow/utils/state.py index fc74732accab..f18fd48c82fb 100644 --- a/airflow/utils/state.py +++ b/airflow/utils/state.py @@ -20,6 +20,19 @@ from enum import Enum +class JobState(str, Enum): + """All possible states that a Job can be in.""" + + RUNNING = "running" + SUCCESS = "success" + SHUTDOWN = "shutdown" + RESTARTING = "restarting" + FAILED = "failed" + + def __str__(self) -> str: + return self.value + + class TaskInstanceState(str, Enum): """All possible states that a Task Instance can be in. From 4d3c832c6afb0d5f85dbad85739bbd7476236f8d Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 12 Jul 2023 17:12:32 +0530 Subject: [PATCH 170/533] New breeze command to publish docs (#32495) --- .github/workflows/ci.yml | 14 +- BREEZE.rst | 45 +++ dev/README_RELEASE_AIRFLOW.md | 2 +- dev/README_RELEASE_HELM_CHART.md | 2 +- dev/README_RELEASE_PROVIDER_PACKAGES.md | 2 +- dev/breeze/README.md | 2 +- dev/breeze/setup.cfg | 1 + .../commands/release_management_commands.py | 59 +++- .../release_management_commands_config.py | 11 + .../src/airflow_breeze/utils/docs_errors.py | 117 +++++++ .../airflow_breeze/utils/helm_chart_utils.py | 34 ++ .../utils/publish_docs_builder.py | 296 ++++++++++++++++++ .../utils/publish_docs_helpers.py | 151 +++++++++ .../airflow_breeze/utils/spelling_checks.py | 188 +++++++++++ .../publish_provider_documentation.sh | 3 +- docs/publish_docs.py | 112 ------- images/breeze/output-commands-hash.txt | 9 +- images/breeze/output_release-management.svg | 48 +-- ...output_release-management_publish-docs.svg | 290 +++++++++++++++++ ...utput_setup_check-all-params-in-groups.svg | 14 +- ...output_setup_regenerate-command-images.svg | 8 +- 21 files changed, 1249 insertions(+), 159 deletions(-) create mode 100644 dev/breeze/src/airflow_breeze/utils/docs_errors.py create mode 100644 dev/breeze/src/airflow_breeze/utils/helm_chart_utils.py create mode 100644 dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py create mode 100644 dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py create mode 100644 dev/breeze/src/airflow_breeze/utils/spelling_checks.py delete mode 100755 docs/publish_docs.py create mode 100644 images/breeze/output_release-management_publish-docs.svg diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f83dfc000379..d62660e5362a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -630,7 +630,19 @@ jobs: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} docs-inventory- - name: "Build docs" - run: breeze build-docs ${{ needs.build-info.outputs.docs-filter-list-as-string }} + run: > + breeze build-docs + ${{ needs.build-info.outputs.docs-filter-list-as-string }} + --for-production + - name: "Clone airflow-site" + run: > + git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && + echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" + - name: "Publish docs" + run: > + breeze release-management publish-docs + --override-versioned + ${{ needs.build-info.outputs.docs-filter-list-as-string }} - name: Configure AWS credentials uses: ./.github/actions/configure-aws-credentials if: > diff --git a/BREEZE.rst b/BREEZE.rst index dd1993505f27..b31e7886de1d 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -462,6 +462,51 @@ Those are all available flags of ``build-docs`` command: :width: 100% :alt: Breeze build documentation +Publishing the documentation +-------------------------- + +To publish the documentation generated by ``build-docs`` in Breeze to ``airflow-site``, +use the ``release-management publish-docs`` command: + +.. code-block:: bash + + breeze release-management publish-docs + +The publishing documentation consists steps: + +* checking out the latest ``main`` of cloned ``airflow-site`` +* copying the documentation to ``airflow-site`` +* running post-docs scripts on the docs to generate back referencing HTML for new versions of docs + +.. code-block:: bash + + breeze release-management publish-docs --package-filter apache-airflow-providers-amazon + +The flag ``--package-filter`` can be used to selectively publish docs during a release. It can take +values such as apache-airflow, helm-chart, apache-airflow-providers, or any individual providers. +The documentation publication happens based on this flag. + +.. code-block:: bash + + breeze release-management publish-docs --override-versioned + +The flag ``--override-versioned`` is a boolean flag that is used to override the versioned directories +while publishing the documentation. + +.. code-block:: bash + + breeze release-management publish-docs --airflow-site-directory + +The flag ``--airflow-site-directory`` takes the path of the cloned ``airflow-site``. The command will +not proceed if this is an invalid path. + +Those are all available flags of ``release-management publish-docs`` command: + +.. image:: ./images/breeze/output_release-management_publish-docs.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/images/breeze/output_release-management_publish-docs.svg + :width: 100% + :alt: Breeze Publish documentation + Running static checks --------------------- diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 33f1fae0d339..ccce45a2da7a 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -804,7 +804,7 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo - Copy the documentation to the ``airflow-site`` repository, create commit, push changes, open a PR and merge it when the build is green. ```shell script - ./docs/publish_docs.py --package-filter apache-airflow --package-filter docker-stack + breeze release-management publish-docs --package-filter apache-airflow --package-filter docker-stack cd "${AIRFLOW_SITE_DIRECTORY}" cd post-docs python add-back-references.py airflow diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index e6d899432a67..727170ecb3e9 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -625,7 +625,7 @@ between the two repositories to be able to build the documentation. - Copy the documentation to the ``airflow-site`` repository. ```shell - ./docs/publish_docs.py --package-filter helm-chart + breeze release-management publish-docs --package-filter helm-chart ``` - Update `index.yaml` diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 9d89508a0882..b4e90105d5f6 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -408,7 +408,7 @@ All providers (including overriding documentation for doc-only changes): ```shell script cd "${AIRFLOW_REPO_ROOT}" -./docs/publish_docs.py \ +breeze release-management publish-docs \ --package-filter apache-airflow-providers \ --package-filter 'apache-airflow-providers-*' \ --override-versioned diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 69bdaf4cede6..a91598a0628b 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 5a58d062de9c220a74215d4beedb98228f93c5b9fc1ef590b3176605ab50197e446e6e38e5e889e1c4e46ba720314c54a0b1efb87964b55f3856bdceaa9852ce +Package config hash: c2bade421865f94fb91f652317d76f3686ad017067349265d8a1d465bbfc2f0796d570bf4777acfb31a7703f1ccb6b0a63c655b97cec9eb9fdeeb1676c804c02 --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index c1993ffe8ece..735962b80162 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -71,6 +71,7 @@ install_requires = twine wheel setuptools + jsonschema [options.packages.find] where=src diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 2f95e73c9e55..eec7e1d0eaf5 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -45,6 +45,7 @@ MOUNT_ALL, MOUNT_SELECTED, MULTI_PLATFORM, + get_available_documentation_packages, ) from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.ci_group import ci_group @@ -77,7 +78,7 @@ ) from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import Output, get_console -from airflow_breeze.utils.custom_param_types import BetterChoice +from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( check_remote_ghcr_io_commands, get_env_variables_for_docker_commands, @@ -103,6 +104,11 @@ generate_providers_metadata_for_package, get_related_providers, ) +from airflow_breeze.utils.publish_docs_builder import PublishDocsBuilder +from airflow_breeze.utils.publish_docs_helpers import ( + get_available_packages, + process_package_filters, +) from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.run_utils import ( RunCommandResult, @@ -769,6 +775,51 @@ def alias_image(image_from: str, image_to: str): ) +@release_management.command( + name="publish-docs", + help="Command to publish generated documentation to airflow-site", +) +@click.option("-s", "--override-versioned", help="Overrides versioned directories.", is_flag=True) +@click.option( + "-a", + "--airflow-site-directory", + envvar="AIRFLOW_SITE_DIRECTORY", + help="Local directory path of cloned airflow-site repo.", + required=True, +) +@click.option( + "--package-filter", + help="List of packages to consider.", + type=NotVerifiedBetterChoice(get_available_documentation_packages()), + multiple=True, +) +@option_verbose +@option_dry_run +def publish_docs( + override_versioned: bool, + airflow_site_directory: bool, + package_filter: tuple[str], +): + """Publishes documentation to airflow-site.""" + if not os.path.isdir(airflow_site_directory): + get_console().print( + "\n[error]location pointed by airflow_site_dir is not valid. " + "Provide the path of cloned airflow-site repo\n" + ) + + available_packages = get_available_packages() + package_filters = package_filter + + current_packages = process_package_filters(available_packages, package_filters) + print(f"Publishing docs for {len(current_packages)} package(s)") + for pkg in current_packages: + print(f" - {pkg}") + print() + for package_name in current_packages: + builder = PublishDocsBuilder(package_name=package_name, for_production=True) + builder.publish(override_versioned=override_versioned, airflow_site_dir=airflow_site_directory) + + @release_management.command( name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)." ) @@ -941,8 +992,8 @@ def release_prod_images( def is_package_in_dist(dist_files: list[str], package: str) -> bool: """Check if package has been prepared in dist folder.""" for file in dist_files: - if file.startswith(f'apache_airflow_providers_{package.replace(".","_")}') or file.startswith( - f'apache-airflow-providers-{package.replace(".","-")}' + if file.startswith(f'apache_airflow_providers_{package.replace(".", "_")}') or file.startswith( + f'apache-airflow-providers-{package.replace(".", "-")}' ): return True return False @@ -1110,7 +1161,7 @@ class ProviderPRInfo(NamedTuple): get_console().print() get_console().print( "Issue title: [yellow]Status of testing Providers that were " - f"prepared on { datetime.now().strftime('%B %d, %Y') }[/]" + f"prepared on {datetime.now().strftime('%B %d, %Y')}[/]" ) get_console().print() syntax = Syntax(issue_content, "markdown", theme="ansi_dark") diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index abbe671cb4a2..dcf7b35b43d9 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -25,6 +25,7 @@ "prepare-airflow-package", "release-prod-images", "generate-constraints", + "publish-docs", ], } @@ -140,6 +141,16 @@ ], } ], + "breeze release-management publish-docs": [ + { + "name": "Publish Docs", + "options": [ + "--override-versioned", + "--package-filter", + "--airflow-site-directory", + ], + }, + ], "breeze release-management generate-issue-content-providers": [ { "name": "Generate issue content flags", diff --git a/dev/breeze/src/airflow_breeze/utils/docs_errors.py b/dev/breeze/src/airflow_breeze/utils/docs_errors.py new file mode 100644 index 000000000000..441c4c13d12c --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/docs_errors.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from functools import total_ordering +from pathlib import Path +from typing import NamedTuple + +from rich.console import Console + +from airflow_breeze.utils.publish_docs_helpers import prepare_code_snippet + +CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) +ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() +DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") + +console = Console(force_terminal=True, color_system="standard", width=180) + + +@total_ordering +class DocBuildError(NamedTuple): + """Errors found in docs build.""" + + file_path: str | None + line_no: int | None + message: str + + def __eq__(self, other): + left = (self.file_path, self.line_no, self.message) + right = (other.file_path, other.line_no, other.message) + return left == right + + def __ne__(self, other): + return not self == other + + def __lt__(self, right): + file_path_a = self.file_path or "" + file_path_b = right.file_path or "" + line_no_a = self.line_no or 0 + line_no_b = right.line_no or 0 + left = (file_path_a, line_no_a, self.message) + right = (file_path_b, line_no_b, right.message) + return left < right + + +def display_errors_summary(build_errors: dict[str, list[DocBuildError]]) -> None: + """Displays summary of errors""" + console.print() + console.print("[red]" + "#" * 30 + " Start docs build errors summary " + "#" * 30 + "[/]") + console.print() + for package_name, errors in build_errors.items(): + if package_name: + console.print("=" * 30 + f" [info]{package_name}[/] " + "=" * 30) + else: + console.print("=" * 30, " [info]General[/] ", "=" * 30) + for warning_no, error in enumerate(sorted(errors), 1): + console.print("-" * 30, f"[red]Error {warning_no:3}[/]", "-" * 20) + console.print(error.message) + console.print() + if error.file_path and not error.file_path.endswith("") and error.line_no: + console.print( + f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)} ({error.line_no})" + ) + console.print() + console.print(prepare_code_snippet(error.file_path, error.line_no)) + elif error.file_path: + console.print(f"File path: {error.file_path}") + console.print() + console.print("[red]" + "#" * 30 + " End docs build errors summary " + "#" * 30 + "[/]") + console.print() + + +def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> list[DocBuildError]: + """ + Parses warnings from Sphinx. + + :param warning_text: warning to parse + :param docs_dir: documentation directory + :return: list of DocBuildErrors. + """ + sphinx_build_errors = [] + for sphinx_warning in warning_text.split("\n"): + if not sphinx_warning: + continue + warning_parts = sphinx_warning.split(":", 2) + if len(warning_parts) == 3: + try: + sphinx_build_errors.append( + DocBuildError( + file_path=os.path.join(docs_dir, warning_parts[0]), + line_no=int(warning_parts[1]), + message=warning_parts[2], + ) + ) + except Exception: + # If an exception occurred while parsing the warning message, display the raw warning message. + sphinx_build_errors.append( + DocBuildError(file_path=None, line_no=None, message=sphinx_warning) + ) + else: + sphinx_build_errors.append(DocBuildError(file_path=None, line_no=None, message=sphinx_warning)) + return sphinx_build_errors diff --git a/dev/breeze/src/airflow_breeze/utils/helm_chart_utils.py b/dev/breeze/src/airflow_breeze/utils/helm_chart_utils.py new file mode 100644 index 000000000000..ef9f1b8e2827 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/helm_chart_utils.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from pathlib import Path + +import yaml + +CHART_DIR = Path(__file__).resolve().parents[5] / "chart" +CHART_YAML_PATH = os.path.join(CHART_DIR, "Chart.yaml") + + +def chart_yaml() -> dict: + with open(CHART_YAML_PATH) as f: + return yaml.safe_load(f) + + +def chart_version() -> str: + return chart_yaml()["version"] diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py new file mode 100644 index 000000000000..24b5505f28c8 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import re +import shlex +import shutil +from glob import glob +from pathlib import Path +from subprocess import run + +from rich.console import Console + +from airflow_breeze.global_constants import get_airflow_version +from airflow_breeze.utils.docs_errors import DocBuildError, parse_sphinx_warnings +from airflow_breeze.utils.helm_chart_utils import chart_version +from airflow_breeze.utils.publish_docs_helpers import load_package_data, pretty_format_path +from airflow_breeze.utils.spelling_checks import SpellingError, parse_spelling_warnings + +PROCESS_TIMEOUT = 15 * 60 +CONSOLE_WIDTH = 180 + +ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() +DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") + +console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH) + + +class PublishDocsBuilder: + """Documentation builder for Airflow Docs Publishing.""" + + def __init__(self, package_name: str, for_production: bool): + self.package_name = package_name + self.for_production = for_production + + @property + def _doctree_dir(self) -> str: + return f"{DOCS_DIR}/_doctrees/docs/{self.package_name}" + + @property + def _inventory_cache_dir(self) -> str: + return f"{DOCS_DIR}/_inventory_cache" + + @property + def is_versioned(self): + """Is current documentation package versioned?""" + # Disable versioning. This documentation does not apply to any released product and we can update + # it as needed, i.e. with each new package of providers. + return self.package_name not in ("apache-airflow-providers", "docker-stack") + + @property + def _build_dir(self) -> str: + if self.is_versioned: + version = "stable" if self.for_production else "latest" + return f"{DOCS_DIR}/_build/docs/{self.package_name}/{version}" + else: + return f"{DOCS_DIR}/_build/docs/{self.package_name}" + + @property + def log_spelling_filename(self) -> str: + """Log from spelling job.""" + return os.path.join(self._build_dir, f"output-spelling-{self.package_name}.log") + + @property + def log_spelling_output_dir(self) -> str: + """Results from spelling job.""" + return os.path.join(self._build_dir, f"output-spelling-results-{self.package_name}") + + @property + def log_build_filename(self) -> str: + """Log from build job.""" + return os.path.join(self._build_dir, f"output-build-{self.package_name}.log") + + @property + def log_build_warning_filename(self) -> str: + """Warnings from build job.""" + return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log") + + @property + def _current_version(self): + if not self.is_versioned: + raise Exception("This documentation package is not versioned") + if self.package_name == "apache-airflow": + return get_airflow_version() + if self.package_name.startswith("apache-airflow-providers-"): + all_providers_yaml = load_package_data() + provider = next(p for p in all_providers_yaml if p["package-name"] == self.package_name) + return provider["versions"][0] + if self.package_name == "helm-chart": + return chart_version() + return Exception(f"Unsupported package: {self.package_name}") + + @property + def _publish_dir(self) -> str: + if self.is_versioned: + return f"docs-archive/{self.package_name}/{self._current_version}" + else: + return f"docs-archive/{self.package_name}" + + @property + def _src_dir(self) -> str: + return f"{DOCS_DIR}/{self.package_name}" + + def clean_files(self) -> None: + """Cleanup all artifacts generated by previous builds.""" + api_dir = os.path.join(self._src_dir, "_api") + + shutil.rmtree(api_dir, ignore_errors=True) + shutil.rmtree(self._build_dir, ignore_errors=True) + os.makedirs(api_dir, exist_ok=True) + os.makedirs(self._build_dir, exist_ok=True) + + def check_spelling(self, verbose: bool) -> list[SpellingError]: + """ + Checks spelling + + :param verbose: whether to show output while running + :return: list of errors + """ + spelling_errors = [] + os.makedirs(self._build_dir, exist_ok=True) + shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True) + os.makedirs(self.log_spelling_output_dir, exist_ok=True) + + build_cmd = [ + "sphinx-build", + "-W", # turn warnings into errors + "--color", # do emit colored output + "-T", # show full traceback on exception + "-b", # builder to use + "spelling", + "-c", + DOCS_DIR, + "-d", # path for the cached environment and doctree files + self._doctree_dir, + self._src_dir, # path to documentation source files + self.log_spelling_output_dir, + ] + + env = os.environ.copy() + env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.for_production: + env["AIRFLOW_FOR_PRODUCTION"] = "true" + if verbose: + console.print( + f"[info]{self.package_name:60}:[/] Executing cmd: ", + " ".join(shlex.quote(c) for c in build_cmd), + ) + console.print(f"[info]{self.package_name:60}:[/] The output is hidden until an error occurs.") + with open(self.log_spelling_filename, "w") as output: + completed_proc = run( + build_cmd, + cwd=self._src_dir, + env=env, + stdout=output if not verbose else None, + stderr=output if not verbose else None, + timeout=PROCESS_TIMEOUT, + ) + if completed_proc.returncode != 0: + spelling_errors.append( + SpellingError( + file_path=None, + line_no=None, + spelling=None, + suggestion=None, + context_line=None, + message=( + f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}." + ), + ) + ) + warning_text = "" + for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True): + with open(filepath) as spelling_file: + warning_text += spelling_file.read() + + spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir)) + console.print(f"[info]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]") + else: + if spelling_errors: + console.print( + f"[info]{self.package_name:60}:[/] [yellow]Finished spell-checking with warnings[/]" + ) + else: + console.print( + f"[info]{self.package_name:60}:[/] [green]Finished spell-checking successfully[/]" + ) + return spelling_errors + + def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: + """ + Build Sphinx documentation. + + :param verbose: whether to show output while running + :return: list of errors + """ + build_errors = [] + os.makedirs(self._build_dir, exist_ok=True) + + build_cmd = [ + "sphinx-build", + "-T", # show full traceback on exception + "--color", # do emit colored output + "-b", # builder to use + "html", + "-d", # path for the cached environment and doctree files + self._doctree_dir, + "-c", + DOCS_DIR, + "-w", # write warnings (and errors) to given file + self.log_build_warning_filename, + self._src_dir, + self._build_dir, # path to output directory + ] + env = os.environ.copy() + env["AIRFLOW_PACKAGE_NAME"] = self.package_name + if self.for_production: + env["AIRFLOW_FOR_PRODUCTION"] = "true" + if verbose: + console.print( + f"[info]{self.package_name:60}:[/] Executing cmd: ", + " ".join(shlex.quote(c) for c in build_cmd), + ) + else: + console.print( + f"[info]{self.package_name:60}:[/] Running sphinx. " + f"The output is hidden until an error occurs." + ) + with open(self.log_build_filename, "w") as output: + completed_proc = run( + build_cmd, + cwd=self._src_dir, + env=env, + stdout=output if not verbose else None, + stderr=output if not verbose else None, + timeout=PROCESS_TIMEOUT, + ) + if completed_proc.returncode != 0: + build_errors.append( + DocBuildError( + file_path=None, + line_no=None, + message=f"Sphinx returned non-zero exit status: {completed_proc.returncode}.", + ) + ) + if os.path.isfile(self.log_build_warning_filename): + with open(self.log_build_warning_filename) as warning_file: + warning_text = warning_file.read() + # Remove 7-bit C1 ANSI escape sequences + warning_text = re.sub(r"\x1B[@-_][0-?]*[ -/]*[@-~]", "", warning_text) + build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir)) + if build_errors: + console.print(f"[info]{self.package_name:60}:[/] [red]Finished docs building with errors[/]") + else: + console.print(f"[info]{self.package_name:60}:[/] [green]Finished docs building successfully[/]") + return build_errors + + def publish(self, override_versioned: bool, airflow_site_dir: str): + """Copy documentation packages files to airflow-site repository.""" + console.print(f"Publishing docs for {self.package_name}") + output_dir = os.path.join(airflow_site_dir, self._publish_dir) + pretty_source = pretty_format_path(self._build_dir, os.getcwd()) + pretty_target = pretty_format_path(output_dir, airflow_site_dir) + console.print(f"Copy directory: {pretty_source} => {pretty_target}") + if os.path.exists(output_dir): + if self.is_versioned: + if override_versioned: + console.print(f"Overriding previously existing {output_dir}! ") + else: + console.print( + f"Skipping previously existing {output_dir}! " + f"Delete it manually if you want to regenerate it!" + ) + console.print() + return + shutil.rmtree(output_dir) + shutil.copytree(self._build_dir, output_dir) + if self.is_versioned: + with open(os.path.join(output_dir, "..", "stable.txt"), "w") as stable_file: + stable_file.write(self._current_version) + console.print() diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py new file mode 100644 index 000000000000..d570b1992bf9 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import fnmatch +import json +import os +from glob import glob +from pathlib import Path +from typing import Any + +import yaml + +ROOT_DIR = Path(__file__).parents[5].resolve() +PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR / "airflow" / "provider.yaml.schema.json" + + +def _load_schema() -> dict[str, Any]: + with open(PROVIDER_DATA_SCHEMA_PATH) as schema_file: + content = json.load(schema_file) + return content + + +def _filepath_to_module(filepath: str): + return str(Path(filepath).relative_to(ROOT_DIR)).replace("/", ".") + + +def _filepath_to_system_tests(filepath: str): + return str( + ROOT_DIR + / "tests" + / "system" + / "providers" + / Path(filepath).relative_to(ROOT_DIR / "airflow" / "providers") + ) + + +def get_provider_yaml_paths(): + """Returns list of provider.yaml files""" + return sorted(glob(f"{ROOT_DIR}/airflow/providers/**/provider.yaml", recursive=True)) + + +def load_package_data() -> list[dict[str, Any]]: + """ + Load all data from providers files + + :return: A list containing the contents of all provider.yaml files. + """ + import jsonschema + + schema = _load_schema() + result = [] + for provider_yaml_path in get_provider_yaml_paths(): + with open(provider_yaml_path) as yaml_file: + provider = yaml.safe_load(yaml_file) + try: + jsonschema.validate(provider, schema=schema) + except jsonschema.ValidationError: + raise Exception(f"Unable to parse: {provider_yaml_path}.") + if provider["suspended"]: + continue + provider_yaml_dir = os.path.dirname(provider_yaml_path) + provider["python-module"] = _filepath_to_module(provider_yaml_dir) + provider["package-dir"] = provider_yaml_dir + provider["system-tests-dir"] = _filepath_to_system_tests(provider_yaml_dir) + result.append(provider) + return result + + +def get_available_packages(): + """Get list of all available packages to build.""" + all_providers_yaml = load_package_data() + provider_package_names = [ + provider["package-name"] for provider in all_providers_yaml if not provider.get("suspended") + ] + return [ + "apache-airflow", + "docker-stack", + *provider_package_names, + "apache-airflow-providers", + "helm-chart", + ] + + +def process_package_filters(available_packages: list[str], package_filters: list[str] | None): + """Filters the package list against a set of filters. + + A packet is returned if it matches at least one filter. The function keeps the order of the packages. + """ + if not package_filters: + return available_packages + + invalid_filters = [ + f for f in package_filters if not any(fnmatch.fnmatch(p, f) for p in available_packages) + ] + if invalid_filters: + raise SystemExit( + f"Some filters did not find any package: {invalid_filters}, Please check if they are correct." + ) + + return [p for p in available_packages if any(fnmatch.fnmatch(p, f) for f in package_filters)] + + +def pretty_format_path(path: str, start: str) -> str: + """Formats path nicely.""" + relpath = os.path.relpath(path, start) + if relpath == path: + return path + return f"{start}/{relpath}" + + +def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int = 5) -> str: + """ + Prepare code snippet with line numbers and a specific line marked. + + :param file_path: File name + :param line_no: Line number + :param context_lines_count: The number of lines that will be cut before and after. + :return: str + """ + with open(file_path) as text_file: + # Highlight code + code = text_file.read() + code_lines = code.split("\n") + # Prepend line number + code_lines = [ + f">{lno:3} | {line}" if line_no == lno else f"{lno:4} | {line}" + for lno, line in enumerate(code_lines, 1) + ] + # # Cut out the snippet + start_line_no = max(0, line_no - context_lines_count - 1) + end_line_no = line_no + context_lines_count + code_lines = code_lines[start_line_no:end_line_no] + # Join lines + code = "\n".join(code_lines) + return code diff --git a/dev/breeze/src/airflow_breeze/utils/spelling_checks.py b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py new file mode 100644 index 000000000000..42d9bf772375 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py @@ -0,0 +1,188 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import re +from functools import total_ordering +from pathlib import Path +from typing import NamedTuple + +from rich.console import Console + +from airflow_breeze.utils.publish_docs_helpers import prepare_code_snippet + +CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) +ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() +DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") + +console = Console(force_terminal=True, color_system="standard", width=180) + + +@total_ordering +class SpellingError(NamedTuple): + """Spelling errors found when building docs.""" + + file_path: str | None + line_no: int | None + spelling: str | None + suggestion: str | None + context_line: str | None + message: str + + def __eq__(self, other): + left = ( + self.file_path, + self.line_no, + self.spelling, + self.context_line, + self.message, + ) + right = ( + other.file_path, + other.line_no, + other.spelling, + other.context_line, + other.message, + ) + return left == right + + def __ne__(self, other): + return not self == other + + def __lt__(self, other): + file_path_a = self.file_path or "" + file_path_b = other.file_path or "" + line_no_a = self.line_no or 0 + line_no_b = other.line_no or 0 + context_line_a = self.context_line or "" + context_line_b = other.context_line or "" + left = (file_path_a, line_no_a, context_line_a, self.spelling, self.message) + right = ( + file_path_b, + line_no_b, + context_line_b, + other.spelling, + other.message, + ) + return left < right + + +def parse_spelling_warnings(warning_text: str, docs_dir: str) -> list[SpellingError]: + """ + Parses warnings from Sphinx. + + :param warning_text: warning to parse + :param docs_dir: documentation directory + :return: list of SpellingError. + """ + sphinx_spelling_errors = [] + for sphinx_warning in warning_text.split("\n"): + if not sphinx_warning: + continue + warning_parts = None + match = re.search(r"(.*):(\w*):\s\((\w*)\)\s?(\w*)\s?(.*)", sphinx_warning) + if match: + warning_parts = match.groups() + if warning_parts and len(warning_parts) == 5: + try: + sphinx_spelling_errors.append( + SpellingError( + file_path=os.path.join(docs_dir, warning_parts[0]), + line_no=int(warning_parts[1]) if warning_parts[1] not in ("None", "") else None, + spelling=warning_parts[2], + suggestion=warning_parts[3] if warning_parts[3] else None, + context_line=warning_parts[4], + message=sphinx_warning, + ) + ) + except Exception: + # If an exception occurred while parsing the warning message, display the raw warning message. + sphinx_spelling_errors.append( + SpellingError( + file_path=None, + line_no=None, + spelling=None, + suggestion=None, + context_line=None, + message=sphinx_warning, + ) + ) + else: + sphinx_spelling_errors.append( + SpellingError( + file_path=None, + line_no=None, + spelling=None, + suggestion=None, + context_line=None, + message=sphinx_warning, + ) + ) + return sphinx_spelling_errors + + +def display_spelling_error_summary(spelling_errors: dict[str, list[SpellingError]]) -> None: + """Displays summary of Spelling errors""" + console.print() + console.print("[red]" + "#" * 30 + " Start spelling errors summary " + "#" * 30 + "[/]") + console.print() + + for package_name, errors in sorted(spelling_errors.items()): + if package_name: + console.print("=" * 30, f" [info]{package_name}[/] ", "=" * 30) + else: + console.print("=" * 30, " [info]General[/] ", "=" * 30) + + for warning_no, error in enumerate(sorted(errors), 1): + console.print("-" * 30, f"Error {warning_no:3}", "-" * 30) + + _display_error(error) + + console.print("=" * 100) + console.print() + msg = """ +If there are spelling errors in the summary above, and the spelling is +correct, add the spelling to docs/spelling_wordlist.txt or use the +spelling directive. +Check https://sphinxcontrib-spelling.readthedocs.io/en/latest/customize.html#private-dictionaries +for more details. + +If there are no spelling errors in the summary above, there might be an +issue unrelated to spelling. Please review the traceback. + """ + console.print(msg) + console.print() + console.print + console.print("[red]" + "#" * 30 + " End docs build errors summary " + "#" * 30 + "[/]") + console.print + + +def _display_error(error: SpellingError): + console.print(error.message) + console.print() + if error.file_path: + console.print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)}") + if error.spelling: + console.print(f"Incorrect Spelling: '{error.spelling}'") + if error.suggestion: + console.print(f"Suggested Spelling: '{error.suggestion}'") + if error.context_line: + console.print(f"Line with Error: '{error.context_line}'") + if error.file_path and not error.file_path.endswith("") and error.line_no: + console.print(f"Line Number: {error.line_no}") + console.print(prepare_code_snippet(error.file_path, error.line_no)) diff --git a/dev/provider_packages/publish_provider_documentation.sh b/dev/provider_packages/publish_provider_documentation.sh index 3ffe3106c22e..cd7ef4433c96 100755 --- a/dev/provider_packages/publish_provider_documentation.sh +++ b/dev/provider_packages/publish_provider_documentation.sh @@ -28,6 +28,7 @@ do provider_filters+=("--package-filter" "apache-airflow-providers-${provider//./-}") done -./docs/publish_docs.py \ +breeze release-management build-docs \ "${provider_filters[@]}" + cd "${AIRFLOW_SITE_DIRECTORY}" diff --git a/docs/publish_docs.py b/docs/publish_docs.py deleted file mode 100755 index eeda3f80e45c..000000000000 --- a/docs/publish_docs.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import argparse -import os - -# isort:off (needed to workaround isort bug) -from exts.docs_build.docs_builder import AirflowDocsBuilder -from exts.docs_build.package_filter import process_package_filters -from exts.provider_yaml_utils import load_package_data - -# isort:on (needed to workaround isort bug) - -AIRFLOW_SITE_DIR = os.environ.get("AIRFLOW_SITE_DIRECTORY") - - -if __name__ != "__main__": - raise SystemExit( - "This file is intended to be executed as an executable program. You cannot use it as a module." - "To run this script, run the ./build_docs.py command" - ) - -if not ( - AIRFLOW_SITE_DIR - and os.path.isdir(AIRFLOW_SITE_DIR) - and os.path.isdir(os.path.join(AIRFLOW_SITE_DIR, "docs-archive")) -): - raise SystemExit( - "Before using this script, set the environment variable AIRFLOW_SITE_DIRECTORY. This variable " - "should contain the path to the airflow-site repository directory. " - "${AIRFLOW_SITE_DIRECTORY}/docs-archive must exists." - ) - -ALL_PROVIDER_YAMLS = load_package_data() - - -def get_available_packages(): - """Get list of all available packages to build.""" - provider_package_names = [ - provider["package-name"] for provider in ALL_PROVIDER_YAMLS if not provider.get("suspended") - ] - return [ - "apache-airflow", - "docker-stack", - *provider_package_names, - "apache-airflow-providers", - "helm-chart", - ] - - -def _get_parser(): - available_packages_list = " * " + "\n * ".join(get_available_packages()) - parser = argparse.ArgumentParser( - description="Copies the built documentation to airflow-site repository.", - epilog=f"List of supported documentation packages:\n{available_packages_list}", - ) - parser.formatter_class = argparse.RawTextHelpFormatter - parser.add_argument( - "--disable-checks", dest="disable_checks", action="store_true", help="Disables extra checks" - ) - parser.add_argument( - "--override-versioned", - dest="override_versioned", - action="store_true", - help="Overrides versioned directories", - ) - parser.add_argument( - "--package-filter", - action="append", - help=( - "Filter specifying for which packages the documentation is to be built. Wildcard are supported." - ), - ) - - return parser - - -def main(): - """Main code""" - args = _get_parser().parse_args() - available_packages = get_available_packages() - - package_filters = args.package_filter - - current_packages = process_package_filters(available_packages, package_filters) - print(f"Publishing docs for {len(current_packages)} package(s)") - for pkg in current_packages: - print(f" - {pkg}") - print() - for package_name in current_packages: - builder = AirflowDocsBuilder(package_name=package_name, for_production=True) - builder.publish(override_versioned=args.override_versioned) - - -main() diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 5320d90f547b..784550612ac8 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -43,21 +43,22 @@ release-management:install-provider-packages:a89493be1ae961c13469b5a25a605069 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5 release-management:prepare-provider-documentation:bc0b95c6a0f308273ab25f728afdb039 release-management:prepare-provider-packages:3e19a787c71d5515a2bc05539da0022f +release-management:publish-docs:4a7d78a846f7e58c6af4a55cbdf7104b release-management:release-prod-images:4d85a23a2175bf8894de5aedbdd85614 release-management:start-rc-process:108c9daabd5970fde86210b1669990cc release-management:start-release:ad13846c7573ddaaa59adc72f0f00284 release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:11bb87706beda348eec0182fb9138eb4 +release-management:473d554e19e108635deda89eb8e0bd05 sbom:generate-provider-requirements:1e7609a0ebafb839d30ec90e4254a628 sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64 sbom:d9c57d78e23a29ec9cd0139e230e1811 setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e -setup:check-all-params-in-groups:fa96f067af2bd80528ffd3f872ba6170 +setup:check-all-params-in-groups:c2ee7aa0ac0831c92167ed70c05f3bd3 setup:config:38ebaaf93ed42bc7b2a3000eeea2631d -setup:regenerate-command-images:a688d2d0ebb7e8cd88f94804257973f4 +setup:regenerate-command-images:76cc7b6568a1488a25e54fd5b4506b0a setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e -setup:201c30ea237ea013a9a209a77092a2e8 +setup:d5f6d65e2da4274ee76abf712e7f699e shell:48db0babd831d704b97118d053ee639a start-airflow:52ad9113f74b39c1ffbd37464e1415e6 static-checks:6ba0614c2c493b45bfc027c3570d737e diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index e44852bc2eaf..c02b1f93bf96 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -1,4 +1,4 @@ - + - + @@ -104,9 +104,12 @@ + + + - Command: release-management + Command: release-management @@ -117,26 +120,27 @@ -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare and manage Airflow releases -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│generate-issue-content-providers           Generates content for issue to test the release.                       │ -│generate-providers-metadata                Generates metadata for providers.                                      │ -│install-provider-packages                  Installs provider packages that can be found in dist.                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +│publish-docs                     Command to publish generated documentation to airflow-site                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-issue-content-providers           Generates content for issue to test the release.                       │ +│generate-providers-metadata                Generates metadata for providers.                                      │ +│install-provider-packages                  Installs provider packages that can be found in dist.                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_publish-docs.svg b/images/breeze/output_release-management_publish-docs.svg new file mode 100644 index 000000000000..f534f04e953c --- /dev/null +++ b/images/breeze/output_release-management_publish-docs.svg @@ -0,0 +1,290 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management publish-docs + + + + + + + + + + +Usage: breeze release-management publish-docs [OPTIONS] + +Command to publish generated documentation to airflow-site + +╭─ Publish Docs ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--override-versioned-sOverrides versioned directories.│ +│--package-filterList of packages to consider.                                                      │ +│(apache-airflow | apache-airflow-providers-airbyte |                               │ +│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |               │ +│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra | │ +│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |    │ +│apache-airflow-providers-apache-flink | apache-airflow-providers-apache-hdfs |     │ +│apache-airflow-providers-apache-hive | apache-airflow-providers-apache-impala |    │ +│apache-airflow-providers-apache-kafka | apache-airflow-providers-apache-kylin |    │ +│apache-airflow-providers-apache-livy | apache-airflow-providers-apache-pig |       │ +│apache-airflow-providers-apache-pinot | apache-airflow-providers-apache-spark |    │ +│apache-airflow-providers-apache-sqoop | apache-airflow-providers-apprise |         │ +│apache-airflow-providers-arangodb | apache-airflow-providers-asana |               │ +│apache-airflow-providers-atlassian-jira | apache-airflow-providers-celery |        │ +│apache-airflow-providers-cloudant | apache-airflow-providers-cncf-kubernetes |     │ +│apache-airflow-providers-common-sql | apache-airflow-providers-databricks |        │ +│apache-airflow-providers-datadog | apache-airflow-providers-dbt-cloud |            │ +│apache-airflow-providers-dingding | apache-airflow-providers-discord |             │ +│apache-airflow-providers-docker | apache-airflow-providers-elasticsearch |         │ +│apache-airflow-providers-exasol | apache-airflow-providers-facebook |              │ +│apache-airflow-providers-ftp | apache-airflow-providers-github |                   │ +│apache-airflow-providers-google | apache-airflow-providers-grpc |                  │ +│apache-airflow-providers-hashicorp | apache-airflow-providers-http |               │ +│apache-airflow-providers-imap | apache-airflow-providers-influxdb |                │ +│apache-airflow-providers-jdbc | apache-airflow-providers-jenkins |                 │ +│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql│ +│| apache-airflow-providers-microsoft-psrp |                                        │ +│apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |        │ +│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                  │ +│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                │ +│apache-airflow-providers-openlineage | apache-airflow-providers-opsgenie |         │ +│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |             │ +│apache-airflow-providers-papermill | apache-airflow-providers-plexus |             │ +│apache-airflow-providers-postgres | apache-airflow-providers-presto |              │ +│apache-airflow-providers-qubole | apache-airflow-providers-redis |                 │ +│apache-airflow-providers-salesforce | apache-airflow-providers-samba |             │ +│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |             │ +│apache-airflow-providers-sftp | apache-airflow-providers-singularity |             │ +│apache-airflow-providers-slack | apache-airflow-providers-smtp |                   │ +│apache-airflow-providers-snowflake | apache-airflow-providers-sqlite |             │ +│apache-airflow-providers-ssh | apache-airflow-providers-tableau |                  │ +│apache-airflow-providers-tabular | apache-airflow-providers-telegram |             │ +│apache-airflow-providers-trino | apache-airflow-providers-vertica |                │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                      │ +│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(TEXT)[required]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup_check-all-params-in-groups.svg b/images/breeze/output_setup_check-all-params-in-groups.svg index c684f857c9f9..780fd4d61c4e 100644 --- a/images/breeze/output_setup_check-all-params-in-groups.svg +++ b/images/breeze/output_setup_check-all-params-in-groups.svg @@ -157,13 +157,13 @@ │release-management:generate-constraints | release-management:generate-issue-content-providers |        │ │release-management:generate-providers-metadata | release-management:install-provider-packages |        │ │release-management:prepare-airflow-package | release-management:prepare-provider-documentation |       │ -│release-management:prepare-provider-packages | release-management:release-prod-images |                │ -│release-management:start-rc-process | release-management:start-release |                               │ -│release-management:verify-provider-packages | release-management | sbom:generate-provider-requirements │ -│| sbom:update-sbom-information | sbom | setup:autocomplete | setup:check-all-params-in-groups |        │ -│setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |  │ -│start-airflow | static-checks | testing:docker-compose-tests | testing:helm-tests |                    │ -│testing:integration-tests | testing:tests | testing)                                                   │ +│release-management:prepare-provider-packages | release-management:publish-docs |                       │ +│release-management:release-prod-images | release-management:start-rc-process |                         │ +│release-management:start-release | release-management:verify-provider-packages | release-management |  │ +│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |       │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade │ +│| setup:version | setup | shell | start-airflow | static-checks | testing:docker-compose-tests |       │ +│testing:helm-tests | testing:integration-tests | testing:tests | testing)                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/images/breeze/output_setup_regenerate-command-images.svg b/images/breeze/output_setup_regenerate-command-images.svg index dd7ee821952b..7ba3f4676871 100644 --- a/images/breeze/output_setup_regenerate-command-images.svg +++ b/images/breeze/output_setup_regenerate-command-images.svg @@ -170,10 +170,10 @@ │release-management:generate-constraints | release-management:generate-issue-content-providers |     │ │release-management:generate-providers-metadata | release-management:install-provider-packages |     │ │release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ -│release-management:prepare-provider-packages | release-management:release-prod-images |             │ -│release-management:start-rc-process | release-management:start-release |                            │ -│release-management:verify-provider-packages | release-management |                                  │ -│sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |    │ +│release-management:prepare-provider-packages | release-management:publish-docs |                    │ +│release-management:release-prod-images | release-management:start-rc-process |                      │ +│release-management:start-release | release-management:verify-provider-packages | release-management │ +│| sbom:generate-provider-requirements | sbom:update-sbom-information | sbom | setup:autocomplete |  │ │setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images |                 │ │setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks |                │ │testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ From b6888e275b1c32a0554b48515f68ba49ec7d9b17 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Wed, 12 Jul 2023 09:17:26 -0400 Subject: [PATCH 171/533] Add AuthManager.is_logged_in and use it to determine whether a user is logged in (#32428) --- airflow/api/auth/backend/session.py | 6 +++-- airflow/auth/managers/base_auth_manager.py | 5 ++++ airflow/auth/managers/fab/fab_auth_manager.py | 4 +++ airflow/www/auth.py | 4 +-- airflow/www/decorators.py | 3 ++- airflow/www/extensions/init_security.py | 4 +-- airflow/www/fab_security/manager.py | 6 ++--- airflow/www/security.py | 3 ++- airflow/www/templates/airflow/main.html | 2 +- .../templates/appbuilder/navbar_right.html | 2 +- airflow/www/views.py | 10 ++++---- .../auh/managers/fab/test_fab_auth_manager.py | 9 +++++++ tests/www/test_security.py | 25 +++++++++++++++---- 13 files changed, 60 insertions(+), 23 deletions(-) diff --git a/airflow/api/auth/backend/session.py b/airflow/api/auth/backend/session.py index 8cc75ccad024..c55f7484605c 100644 --- a/airflow/api/auth/backend/session.py +++ b/airflow/api/auth/backend/session.py @@ -20,7 +20,9 @@ from functools import wraps from typing import Any, Callable, TypeVar, cast -from flask import Response, g +from flask import Response + +from airflow.configuration import auth_manager CLIENT_AUTH: tuple[str, str] | Any | None = None @@ -37,7 +39,7 @@ def requires_authentication(function: T): @wraps(function) def decorated(*args, **kwargs): - if g.user.is_anonymous: + if not auth_manager.is_logged_in(): return Response("Unauthorized", 401, {}) return function(*args, **kwargs) diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index 6a4bb86f1666..462fe34d6304 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -33,3 +33,8 @@ class BaseAuthManager(LoggingMixin): def get_user_name(self) -> str: """Return the username associated to the user in session.""" ... + + @abstractmethod + def is_logged_in(self) -> bool: + """Return whether the user is logged in.""" + ... diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index 420385716822..b9f0c1e1df5a 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -39,3 +39,7 @@ def get_user_name(self) -> str: first_name = current_user.first_name or "" last_name = current_user.last_name or "" return f"{first_name} {last_name}".strip() + + def is_logged_in(self) -> bool: + """Return whether the user is logged in.""" + return current_user and not current_user.is_anonymous diff --git a/airflow/www/auth.py b/airflow/www/auth.py index 0056af55fd9b..82fb5d34c5c0 100644 --- a/airflow/www/auth.py +++ b/airflow/www/auth.py @@ -21,7 +21,7 @@ from flask import current_app, flash, g, redirect, render_template, request, url_for -from airflow.configuration import conf +from airflow.configuration import auth_manager, conf from airflow.utils.net import get_hostname T = TypeVar("T", bound=Callable) @@ -46,7 +46,7 @@ def decorated(*args, **kwargs): ) if appbuilder.sm.check_authorization(permissions, dag_id): return func(*args, **kwargs) - elif not g.user.is_anonymous and not g.user.perms: + elif auth_manager.is_logged_in() and not g.user.perms: return ( render_template( "airflow/no_roles_permissions.html", diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index c541ef32eba9..fc386d220f6b 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -29,6 +29,7 @@ from flask import after_this_request, g, request from pendulum.parsing.exceptions import ParserError +from airflow.configuration import auth_manager from airflow.models import Log from airflow.utils.log import secrets_masker from airflow.utils.session import create_session @@ -84,7 +85,7 @@ def wrapper(*args, **kwargs): __tracebackhide__ = True # Hide from pytest traceback. with create_session() as session: - if g.user.is_anonymous: + if not auth_manager.is_logged_in(): user = "anonymous" else: user = f"{g.user.username} ({g.user.get_full_name()})" diff --git a/airflow/www/extensions/init_security.py b/airflow/www/extensions/init_security.py index ab558bf4f447..ba57f99b1448 100644 --- a/airflow/www/extensions/init_security.py +++ b/airflow/www/extensions/init_security.py @@ -22,7 +22,7 @@ from flask import g, redirect, url_for from flask_login import logout_user -from airflow.configuration import conf +from airflow.configuration import auth_manager, conf from airflow.exceptions import AirflowConfigException, AirflowException log = logging.getLogger(__name__) @@ -68,6 +68,6 @@ def init_api_experimental_auth(app): def init_check_user_active(app): @app.before_request def check_user_active(): - if g.user is not None and not g.user.is_anonymous and not g.user.is_active: + if auth_manager.is_logged_in() and not g.user.is_active: logout_user() return redirect(url_for(app.appbuilder.sm.auth_view.endpoint + ".login")) diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index be0c4c63e39f..00d38064363e 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -73,7 +73,7 @@ from flask_login import AnonymousUserMixin, LoginManager, current_user from werkzeug.security import check_password_hash, generate_password_hash -from airflow.configuration import conf +from airflow.configuration import auth_manager, conf from airflow.www.fab_security.sqla.models import Action, Permission, RegisterUser, Resource, Role, User # This product contains a modified portion of 'Flask App Builder' developed by Daniel Vaz Gaspar. @@ -540,7 +540,7 @@ def auth_rate_limit(self) -> str: @property def current_user(self): """Current user object.""" - if current_user.is_authenticated: + if auth_manager.is_logged_in(): return g.user elif current_user_jwt: return current_user_jwt @@ -1415,7 +1415,7 @@ def _get_user_permission_resources( return result def get_user_menu_access(self, menu_names: list[str] | None = None) -> set[str]: - if current_user.is_authenticated: + if auth_manager.is_logged_in(): return self._get_user_permission_resources(g.user, "menu_access", resource_names=menu_names) elif current_user_jwt: return self._get_user_permission_resources( diff --git a/airflow/www/security.py b/airflow/www/security.py index 890648009c83..33188328fc09 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -24,6 +24,7 @@ from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload +from airflow.configuration import auth_manager from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.models import DagBag, DagModel from airflow.security import permissions @@ -338,7 +339,7 @@ def get_accessible_dag_ids( if not user_actions: user_actions = [permissions.ACTION_CAN_EDIT, permissions.ACTION_CAN_READ] - if user.is_anonymous: + if not auth_manager.is_logged_in(): roles = user.roles else: if (permissions.ACTION_CAN_EDIT in user_actions and self.can_edit_all_dags(user)) or ( diff --git a/airflow/www/templates/airflow/main.html b/airflow/www/templates/airflow/main.html index ffebe05c00b1..c977276940ad 100644 --- a/airflow/www/templates/airflow/main.html +++ b/airflow/www/templates/airflow/main.html @@ -91,7 +91,7 @@ {% endblock %} {% block footer %} - {% if not current_user.is_anonymous %} + {% if auth_manager.is_logged_in() %} {% set version_label = 'Version' %}
    diff --git a/airflow/www/templates/appbuilder/navbar_right.html b/airflow/www/templates/appbuilder/navbar_right.html index 1ade0847a65e..3bbf73eee36e 100644 --- a/airflow/www/templates/appbuilder/navbar_right.html +++ b/airflow/www/templates/appbuilder/navbar_right.html @@ -63,7 +63,7 @@ -{% if not current_user.is_anonymous %} +{% if auth_manager.is_logged_in() %}
  • diff --git a/airflow/www/views.py b/airflow/www/views.py index e198e390ace7..a18429a1a93e 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -83,7 +83,7 @@ set_dag_run_state_to_success, set_state, ) -from airflow.configuration import AIRFLOW_CONFIG, conf +from airflow.configuration import AIRFLOW_CONFIG, auth_manager, conf from airflow.datasets import Dataset from airflow.exceptions import ( AirflowConfigException, @@ -617,13 +617,13 @@ def show_traceback(error): return ( render_template( "airflow/traceback.html", - python_version=sys.version.split(" ")[0] if g.user.is_authenticated else "redact", - airflow_version=version if g.user.is_authenticated else "redact", + python_version=sys.version.split(" ")[0] if auth_manager.is_logged_in() else "redact", + airflow_version=version if auth_manager.is_logged_in() else "redact", hostname=get_hostname() - if conf.getboolean("webserver", "EXPOSE_HOSTNAME") and g.user.is_authenticated + if conf.getboolean("webserver", "EXPOSE_HOSTNAME") and auth_manager.is_logged_in() else "redact", info=traceback.format_exc() - if conf.getboolean("webserver", "EXPOSE_STACKTRACE") and g.user.is_authenticated + if conf.getboolean("webserver", "EXPOSE_STACKTRACE") and auth_manager.is_logged_in() else "Error! Please contact server admin.", ), 500, diff --git a/tests/auh/managers/fab/test_fab_auth_manager.py b/tests/auh/managers/fab/test_fab_auth_manager.py index 16e5151495c9..4f24b1297ef7 100644 --- a/tests/auh/managers/fab/test_fab_auth_manager.py +++ b/tests/auh/managers/fab/test_fab_auth_manager.py @@ -17,6 +17,7 @@ from __future__ import annotations from unittest import mock +from unittest.mock import Mock import pytest @@ -46,3 +47,11 @@ def test_get_user_name(self, mock_current_user, first_name, last_name, expected, mock_current_user.return_value = user assert auth_manager.get_user_name() == expected + + @mock.patch("flask_login.utils._get_user") + def test_is_logged_in(self, mock_current_user, auth_manager): + user = Mock() + user.is_anonymous.return_value = True + mock_current_user.return_value = user + + assert auth_manager.is_logged_in() is False diff --git a/tests/www/test_security.py b/tests/www/test_security.py index 48e02a3cbb10..97583418ee19 100644 --- a/tests/www/test_security.py +++ b/tests/www/test_security.py @@ -21,6 +21,7 @@ import datetime import logging from unittest import mock +from unittest.mock import patch import pytest import time_machine @@ -28,6 +29,7 @@ from flask_appbuilder.views import BaseView, ModelView from sqlalchemy import Column, Date, Float, Integer, String +from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.base import Base @@ -303,8 +305,12 @@ def test_verify_public_role_has_no_permissions(security_manager): assert public.permissions == [] -def test_verify_default_anon_user_has_no_accessible_dag_ids(app, session, security_manager): +@patch.object(FabAuthManager, "is_logged_in") +def test_verify_default_anon_user_has_no_accessible_dag_ids( + mock_is_logged_in, app, session, security_manager +): with app.app_context(): + mock_is_logged_in.return_value = False user = AnonymousUser() app.config["AUTH_ROLE_PUBLIC"] = "Public" assert security_manager.get_user_roles(user) == {security_manager.get_public_role()} @@ -331,15 +337,19 @@ def test_verify_default_anon_user_has_no_access_to_specific_dag(app, session, se assert has_dag_perm(permissions.ACTION_CAN_EDIT, dag_id, user) is False +@patch.object(FabAuthManager, "is_logged_in") @pytest.mark.parametrize( "mock_dag_models", [["test_dag_id_1", "test_dag_id_2", "test_dag_id_3"]], indirect=True, ) -def test_verify_anon_user_with_admin_role_has_all_dag_access(app, security_manager, mock_dag_models): +def test_verify_anon_user_with_admin_role_has_all_dag_access( + mock_is_logged_in, app, security_manager, mock_dag_models +): test_dag_ids = mock_dag_models with app.app_context(): app.config["AUTH_ROLE_PUBLIC"] = "Admin" + mock_is_logged_in.return_value = False user = AnonymousUser() assert security_manager.get_user_roles(user) == {security_manager.get_public_role()} @@ -374,7 +384,6 @@ def test_verify_anon_user_with_admin_role_has_access_to_each_dag( def test_get_user_roles(app_builder, security_manager): user = mock.MagicMock() - user.is_anonymous = False roles = app_builder.sm.find_role("Admin") user.roles = roles assert security_manager.get_user_roles(user) == roles @@ -449,7 +458,8 @@ def test_get_current_user_permissions(app): assert len(user.perms) == 0 -def test_get_accessible_dag_ids(app, security_manager, session): +@patch.object(FabAuthManager, "is_logged_in") +def test_get_accessible_dag_ids(mock_is_logged_in, app, security_manager, session): role_name = "MyRole1" permission_action = [permissions.ACTION_CAN_READ] dag_id = "dag_id" @@ -465,6 +475,7 @@ def test_get_accessible_dag_ids(app, security_manager, session): (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), ], ) as user: + mock_is_logged_in.return_value = True dag_model = DagModel(dag_id=dag_id, fileloc="/tmp/dag_.py", schedule_interval="2 2 * * *") session.add(dag_model) session.commit() @@ -476,7 +487,10 @@ def test_get_accessible_dag_ids(app, security_manager, session): assert security_manager.get_accessible_dag_ids(user) == {"dag_id"} -def test_dont_get_inaccessible_dag_ids_for_dag_resource_permission(app, security_manager, session): +@patch.object(FabAuthManager, "is_logged_in") +def test_dont_get_inaccessible_dag_ids_for_dag_resource_permission( + mock_is_logged_in, app, security_manager, session +): # In this test case, # get_readable_dag_ids() don't return DAGs to which the user has CAN_EDIT action username = "Monsieur User" @@ -492,6 +506,7 @@ def test_dont_get_inaccessible_dag_ids_for_dag_resource_permission(app, security (permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG), ], ) as user: + mock_is_logged_in.return_value = True dag_model = DagModel(dag_id=dag_id, fileloc="/tmp/dag_.py", schedule_interval="2 2 * * *") session.add(dag_model) session.commit() From f675c4d34c64235358605bc92d61eaf0204c4fee Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 12 Jul 2023 18:28:48 +0200 Subject: [PATCH 172/533] Fix fetching inventory from s3 for airflow packages (#32562) The #32495 changed the way to run docs - with using --for-production flag, which means that the docs are generated under "stable" link rather than "latest" - in order to test document publishing. We could change it back to latest, but the problem revealed that we do not have to actually make a distinction there. It's ok to keep it all as "stable" - no problem with that - there is no distinction between latest and stable other than different link and we never mix the two (our production docs have only stable), so it's ok to switch everything to stable. As a follow-up `--for-production` flag will be removed and we will switch everything to stable for all documentation building. For now fetching docs from stable link should fix the problem. --- docs/exts/docs_build/fetch_inventories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py index c2f902564691..6db368f0d0b5 100644 --- a/docs/exts/docs_build/fetch_inventories.py +++ b/docs/exts/docs_build/fetch_inventories.py @@ -43,7 +43,7 @@ EXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, "_inventory_cache", "expiration-date") S3_DOC_URL = "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com" -S3_DOC_URL_VERSIONED = S3_DOC_URL + "/docs/{package_name}/latest/objects.inv" +S3_DOC_URL_VERSIONED = S3_DOC_URL + "/docs/{package_name}/stable/objects.inv" S3_DOC_URL_NON_VERSIONED = S3_DOC_URL + "/docs/{package_name}/objects.inv" From dd43b3916dca9f5ed056d4dc3bfc027faba627ca Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 12 Jul 2023 22:01:20 +0530 Subject: [PATCH 173/533] Removing outdated documentations (#32559) --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index b4e90105d5f6..615722130712 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -387,22 +387,6 @@ You will need to change it manually to see the docs - Copy the documentation to the ``airflow-site`` repository -**NOTE** In order to run the publish documentation you need to activate virtualenv where you installed -apache-airflow with doc extra: - -* `pip install '.[doc_gen]'` - -If you don't have virtual env set you can do: - -```shell script -cd -virtualenv providers - -source venv/providers/bin/activate - -pip install 'apache-airflow[doc_gen]' -``` - All providers (including overriding documentation for doc-only changes): ```shell script From fcbf15947f2f271d502bc6f4b9e3d0bada072d84 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Wed, 12 Jul 2023 17:49:43 +0100 Subject: [PATCH 174/533] Fix breaking change when Active Directory ID is used as host in WASB (#32560) * Fix breaking change when Active Directory ID is used as host in WASB A recent change: https://github.com/apache/airflow/commit/46ee1c2c8d3d0e5793f42fd10bcd80150caa538b, to build account url if not provided broke backward compatibility in Wasb hook by assuming that host is always provided as a fully qualified url. Previously, we generate the account url as the last resort to getting the connection. This change restores the previous behaviour of generating a URL if the host was provided with an ID instead of a URL * Update tests/providers/microsoft/azure/hooks/test_wasb.py * rstrip trailing slash in url --- .../providers/microsoft/azure/hooks/wasb.py | 8 ++++- .../microsoft/azure/hooks/test_wasb.py | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 2281e184c22b..6c4b76cdc47b 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -210,13 +210,19 @@ def get_conn(self) -> BlobServiceClient: if sas_token.startswith("https"): return BlobServiceClient(account_url=sas_token, **extra) else: - return BlobServiceClient(account_url=f"{account_url}/{sas_token}", **extra) + if not account_url.startswith("https://"): + # TODO: require url in the host field in the next major version? + account_url = f"https://{conn.login}.blob.core.windows.net" + return BlobServiceClient(account_url=f"{account_url.rstrip('/')}/{sas_token}", **extra) # Fall back to old auth (password) or use managed identity if not provided. credential = conn.password if not credential: credential = DefaultAzureCredential() self.log.info("Using DefaultAzureCredential as credential") + if not account_url.startswith("https://"): + # TODO: require url in the host field in the next major version? + account_url = f"https://{conn.login}.blob.core.windows.net/" return BlobServiceClient( account_url=account_url, credential=credential, diff --git a/tests/providers/microsoft/azure/hooks/test_wasb.py b/tests/providers/microsoft/azure/hooks/test_wasb.py index 672d20d3a65e..9e9ce9a0d06e 100644 --- a/tests/providers/microsoft/azure/hooks/test_wasb.py +++ b/tests/providers/microsoft/azure/hooks/test_wasb.py @@ -200,6 +200,42 @@ def test_azure_directory_connection(self): assert isinstance(hook.get_conn(), BlobServiceClient) assert isinstance(hook.get_conn().credential, ClientSecretCredential) + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.BlobServiceClient") + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.DefaultAzureCredential") + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook.get_connection") + def test_active_directory_ID_used_as_host(self, mock_get_conn, mock_credential, mock_blob_service_client): + hook = WasbHook(wasb_conn_id="testconn") + mock_get_conn.return_value = Connection( + conn_id="testconn", + conn_type=self.connection_type, + login="testaccountname", + host="testaccountID", + ) + hook.get_conn() + assert mock_blob_service_client.call_args == mock.call( + account_url="https://testaccountname.blob.core.windows.net/", + credential=mock_credential.return_value, + ) + + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.BlobServiceClient") + @mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook.get_connection") + def test_sas_token_provided_and_active_directory_ID_used_as_host( + self, mock_get_conn, mock_blob_service_client + ): + hook = WasbHook(wasb_conn_id="testconn") + mock_get_conn.return_value = Connection( + conn_id="testconn", + conn_type=self.connection_type, + login="testaccountname", + host="testaccountID", + extra=json.dumps({"sas_token": "SAStoken"}), + ) + hook.get_conn() + assert mock_blob_service_client.call_args == mock.call( + account_url="https://testaccountname.blob.core.windows.net/SAStoken", + sas_token="SAStoken", + ) + @pytest.mark.parametrize( argnames="conn_id_str", argvalues=[ From e7f59a913e1fcf9052e69f62af9fe23901f1a358 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Wed, 12 Jul 2023 22:08:02 +0300 Subject: [PATCH 175/533] Prepare docs for July 2023 2nd wave of Providers (#32566) * Prepare docs for July 2023 2nd wave of Providers * clean sftp CHANGELOG.rst --- airflow/providers/amazon/CHANGELOG.rst | 8 +++++ airflow/providers/amazon/__init__.py | 2 +- airflow/providers/amazon/provider.yaml | 1 + .../providers/cncf/kubernetes/CHANGELOG.rst | 17 ++++++++++ airflow/providers/cncf/kubernetes/__init__.py | 2 +- .../providers/cncf/kubernetes/provider.yaml | 1 + airflow/providers/google/CHANGELOG.rst | 33 +++++++++++++++++++ airflow/providers/google/__init__.py | 2 +- airflow/providers/google/provider.yaml | 1 + airflow/providers/http/CHANGELOG.rst | 21 ++++++++++++ airflow/providers/http/__init__.py | 2 +- airflow/providers/http/provider.yaml | 1 + .../providers/microsoft/azure/CHANGELOG.rst | 14 ++++++++ airflow/providers/microsoft/azure/__init__.py | 2 +- airflow/providers/sftp/CHANGELOG.rst | 17 ++++++++++ airflow/providers/sftp/__init__.py | 2 +- airflow/providers/sftp/provider.yaml | 1 + airflow/providers/snowflake/CHANGELOG.rst | 13 ++++++++ airflow/providers/snowflake/__init__.py | 2 +- airflow/providers/snowflake/provider.yaml | 1 + .../commits.rst | 14 +++++++- .../apache-airflow-providers-amazon/index.rst | 6 ++-- .../commits.rst | 16 ++++++++- .../index.rst | 2 +- .../commits.rst | 26 ++++++++++++++- .../apache-airflow-providers-google/index.rst | 7 ++-- .../changelog.rst | 2 -- .../apache-airflow-providers-http/commits.rst | 18 ++++++++++ docs/apache-airflow-providers-http/index.rst | 2 +- .../commits.rst | 17 +++++++++- .../index.rst | 6 ++-- .../changelog.rst | 2 -- .../apache-airflow-providers-sftp/commits.rst | 18 ++++++++++ docs/apache-airflow-providers-sftp/index.rst | 6 ++-- .../commits.rst | 16 ++++++++- .../index.rst | 6 ++-- 36 files changed, 274 insertions(+), 33 deletions(-) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index f0a03a85c001..00867788794a 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -26,6 +26,14 @@ Changelog --------- +8.3.1 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Append region info to S3ToRedshitOperator if present (#32328)`` + 8.3.0 ..... diff --git a/airflow/providers/amazon/__init__.py b/airflow/providers/amazon/__init__.py index 204607dbae69..1aff2c342595 100644 --- a/airflow/providers/amazon/__init__.py +++ b/airflow/providers/amazon/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "8.3.0" +__version__ = "8.3.1" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 3622896504da..e8ba23afc4d9 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 8.3.1 - 8.3.0 - 8.2.0 - 8.1.0 diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index ace9bacf2006..a356a6c6dcfe 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -27,6 +27,23 @@ Changelog --------- +7.3.0 +..... + +Features +~~~~~~~~ + +* ``Logging from all containers in KubernetesOperatorPod (#31663)`` + +Bug Fixes +~~~~~~~~~ + +* ``Fix async KPO by waiting pod termination in 'execute_complete' before cleanup (#32467)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``D205 Support - Providers: Stragglers and new additions (#32447)`` + 7.2.0 ..... diff --git a/airflow/providers/cncf/kubernetes/__init__.py b/airflow/providers/cncf/kubernetes/__init__.py index 3235c02fd586..ce49dc19b7a4 100644 --- a/airflow/providers/cncf/kubernetes/__init__.py +++ b/airflow/providers/cncf/kubernetes/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "7.2.0" +__version__ = "7.3.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index f65527ded795..86ab5f70160b 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 7.3.0 - 7.2.0 - 7.1.0 - 7.0.0 diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 6bf553b3ebd1..2a588dd5be7c 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -27,6 +27,39 @@ Changelog --------- +10.4.0 +...... + +Features +~~~~~~~~ + +Implement deferrable mode for S3ToGCSOperator + +Bug Fixes +~~~~~~~~~ + +* ``Bugfix GCSToGCSOperator when copy files to folder without wildcard (#32486)`` +* ``Fix 'cache_control' parameter of upload function in 'GCSHook' (#32440)`` +* ``Fix BigQuery transfer operators to respect project_id arguments (#32232)`` +* ``Fix the gcp_gcs_delete_objects on empty list (#32383)`` +* ``Fix endless loop of defer in cloud_build (#32387)`` +* ``Fix GCSToGCSOperator copy without wildcard and exact_match=True (#32376)`` + +Misc +~~~~ + +* ``Allow a destination folder to be provided (#31885)`` +* ``Moves 'AzureBlobStorageToGCSOperator' from Azure to Google provider (#32306)`` +* ``Give better link to job configuration docs in BigQueryInsertJobOperator (#31736)`` + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Combine 8 into 1 (#29462)`` + * ``D205 Support - Provider: Google (#32356)`` + * ``Migrating Google AutoML example_dags to sys tests (#32368)`` + * ``build(pre-commit): check deferrable default value (#32370)`` + 10.3.0 ...... diff --git a/airflow/providers/google/__init__.py b/airflow/providers/google/__init__.py index 0bf8186b4a75..a400c70d9c10 100644 --- a/airflow/providers/google/__init__.py +++ b/airflow/providers/google/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "10.3.0" +__version__ = "10.4.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index ea1d7573757c..5a7b272f420e 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -30,6 +30,7 @@ description: | suspended: false versions: + - 10.4.0 - 10.3.0 - 10.2.0 - 10.1.1 diff --git a/airflow/providers/http/CHANGELOG.rst b/airflow/providers/http/CHANGELOG.rst index d8d88f5f8088..0a8282b7d1fa 100644 --- a/airflow/providers/http/CHANGELOG.rst +++ b/airflow/providers/http/CHANGELOG.rst @@ -27,6 +27,27 @@ Changelog --------- +4.5.0 +..... + +Features +~~~~~~~~ + +* ``Add deferrable mode to SimpleHttpOperator (#32448)`` + +Bug Fixes +~~~~~~~~~ + +* ``Fix headers passed into HttpAsyncHook (#32409)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` + * ``Improve provider documentation and README structure (#32125)`` + 4.4.2 ..... diff --git a/airflow/providers/http/__init__.py b/airflow/providers/http/__init__.py index 9f58801b5ac6..0945a15cf76e 100644 --- a/airflow/providers/http/__init__.py +++ b/airflow/providers/http/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.4.2" +__version__ = "4.5.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 84772a7ba389..fc2e752f5b0b 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.5.0 - 4.4.2 - 4.4.1 - 4.4.0 diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index b05f1c931014..f79ee08f7e84 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -37,6 +37,20 @@ Changelog previous behaviour, you might need to retrieve ``name`` property from the array elements returned by this method. +Bug Fixes +~~~~~~~~~ + +* ``Fix breaking change when Active Directory ID is used as host in WASB (#32560)`` +* ``Fix get_blobs_list_async method to return BlobProperties (#32545)`` + +Misc +~~~~ + +* ``Moves 'AzureBlobStorageToGCSOperator' from Azure to Google provider (#32306)`` + +.. Review and move the new changes to one of the sections above: + * ``D205 Support - Providers: Stragglers and new additions (#32447)`` + 6.2.0 ..... diff --git a/airflow/providers/microsoft/azure/__init__.py b/airflow/providers/microsoft/azure/__init__.py index 40f553f79fb7..9efb9241a1da 100644 --- a/airflow/providers/microsoft/azure/__init__.py +++ b/airflow/providers/microsoft/azure/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "6.2.0" +__version__ = "6.2.1" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/sftp/CHANGELOG.rst b/airflow/providers/sftp/CHANGELOG.rst index 951358a53b76..eb6dada8be8f 100644 --- a/airflow/providers/sftp/CHANGELOG.rst +++ b/airflow/providers/sftp/CHANGELOG.rst @@ -27,6 +27,23 @@ Changelog --------- +4.4.0 +..... + +Features +~~~~~~~~ + +* ``Adds sftp_sensor decorator (#32457)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` + * ``D205 Support - Providers: Pagerduty to SMTP (inclusive) (#32358)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``Improve provider documentation and README structure (#32125)`` + * ``Remove redundant Operator suffix from sensor name (#32475)`` + 4.3.1 ..... diff --git a/airflow/providers/sftp/__init__.py b/airflow/providers/sftp/__init__.py index 667c882cb7de..b79b8d17a338 100644 --- a/airflow/providers/sftp/__init__.py +++ b/airflow/providers/sftp/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.3.1" +__version__ = "4.4.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index 84b8dd738fad..db83671cc107 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.4.0 - 4.3.1 - 4.3.0 - 4.2.4 diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/airflow/providers/snowflake/CHANGELOG.rst index 94e11b6bc269..e23ae628be74 100644 --- a/airflow/providers/snowflake/CHANGELOG.rst +++ b/airflow/providers/snowflake/CHANGELOG.rst @@ -27,6 +27,19 @@ Changelog --------- +4.3.1 +..... + +Bug Fixes +~~~~~~~~~ + +* ``Fix an issue that crashes Airflow Webserver when passed invalid private key path to Snowflake (#32016)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``build(pre-commit): check deferrable default value (#32370)`` + * ``D205 Support - Providers: Snowflake to Zendesk (inclusive) (#32359)`` + 4.3.0 ..... diff --git a/airflow/providers/snowflake/__init__.py b/airflow/providers/snowflake/__init__.py index 53291daf0a5a..226ebb68bd22 100644 --- a/airflow/providers/snowflake/__init__.py +++ b/airflow/providers/snowflake/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.3.0" +__version__ = "4.3.1" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index cad5d16d6f4c..d33e5532167d 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.3.1 - 4.3.0 - 4.2.0 - 4.1.0 diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index 538c13b75604..256accb171b4 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -28,14 +28,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-12 ``Append region info to S3ToRedshitOperator if present (#32328)`` +================================================================================================= =========== ================================================================= + 8.3.0 ..... -Latest change: 2023-07-08 +Latest change: 2023-07-09 ================================================================================================= =========== ============================================================================================================ Commit Committed Subject ================================================================================================= =========== ============================================================================================================ +`cd1ac927a7 `_ 2023-07-09 ``Prepare docs for July 2023 RC3 amazon provider (#32455)`` `c99c03135c `_ 2023-07-08 ``D205 Support - Providers: Stragglers and new additions (#32447)`` `8c6751ff0f `_ 2023-07-07 ``quick fix on RDS operator to prevent parameter collision (#32436)`` `869f3a93a8 `_ 2023-07-07 ``Remove ability to specify arbitrary hook params in AWS RDS trigger (#32386)`` diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index b018d27cb0a7..2e8cd21503ac 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -82,7 +82,7 @@ Package apache-airflow-providers-amazon Amazon integration (including `Amazon Web Services (AWS) `__). -Release: 8.3.0 +Release: 8.3.1 Provider package ---------------- @@ -157,5 +157,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-amazon 8.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-amazon 8.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 8.3.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 8.3.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst index 7dbbb8dfbef7..55d5a3e3adc4 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst @@ -28,14 +28,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-12 ``Fix async KPO by waiting pod termination in 'execute_complete' before cleanup (#32467)`` +`c99c03135c `_ 2023-07-08 ``D205 Support - Providers: Stragglers and new additions (#32447)`` +`9a0f41ba53 `_ 2023-07-06 ``Logging from all containers in KubernetesOperatorPod (#31663)`` +================================================================================================= =========== ========================================================================================== + 7.2.0 ..... -Latest change: 2023-07-05 +Latest change: 2023-07-06 ================================================================================================= =========== ========================================================================= Commit Committed Subject ================================================================================================= =========== ========================================================================= +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index d087d1d75759..95ce846ab13e 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-cncf-kubernetes `Kubernetes `__ -Release: 7.2.0 +Release: 7.3.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-google/commits.rst b/docs/apache-airflow-providers-google/commits.rst index ae000fb1f329..3536fa3ca8b4 100644 --- a/docs/apache-airflow-providers-google/commits.rst +++ b/docs/apache-airflow-providers-google/commits.rst @@ -35,14 +35,38 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-11 ``Bugfix GCSToGCSOperator when copy files to folder without wildcard (#32486)`` +`86c6cc9a9a `_ 2023-07-11 ``Combine 8 into 1 (#29462)`` +`3a8da4b55b `_ 2023-07-11 ``Allow a destination folder to be provided (#31885)`` +`0f73647bda `_ 2023-07-08 ``D205 Support - Provider: Google (#32356)`` +`e70bee00cd `_ 2023-07-08 ``Fix 'cache_control' parameter of upload function in 'GCSHook' (#32440)`` +`257136786c `_ 2023-07-08 ``Moves 'AzureBlobStorageToGCSOperator' from Azure to Google provider (#32306)`` +`723eb7d453 `_ 2023-07-07 ``Give better link to job configuration docs in BigQueryInsertJobOperator (#31736)`` +`6c854dcb16 `_ 2023-07-07 ``Migrating Google AutoML example_dags to sys tests (#32368)`` +`53d9421905 `_ 2023-07-07 ``build(pre-commit): check deferrable default value (#32370)`` +`2d690de110 `_ 2023-07-07 ``Fix BigQuery transfer operators to respect project_id arguments (#32232)`` +`e7587b3369 `_ 2023-07-06 ``Fix the gcp_gcs_delete_objects on empty list (#32383)`` +`becfb3c64f `_ 2023-07-06 ``Fix endless loop of defer in cloud_build (#32387)`` +`e4757d6dfa `_ 2023-07-06 ``Fix GCSToGCSOperator copy without wildcard and exact_match=True (#32376)`` +================================================================================================= =========== ==================================================================================== + 10.3.0 ...... -Latest change: 2023-07-05 +Latest change: 2023-07-06 ================================================================================================= =========== ============================================================================================================= Commit Committed Subject ================================================================================================= =========== ============================================================================================================= +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `f8593503cb `_ 2023-07-05 ``Add default_deferrable config (#31712)`` diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 6721dfc0399e..0c2fca3e2cdc 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -88,7 +88,7 @@ Google services including: - `Google Workspace `__ (formerly Google Suite) -Release: 10.3.0 +Release: 10.4.0 Provider package ---------------- @@ -149,6 +149,7 @@ PIP package Version required ``google-cloud-spanner`` ``>=3.11.1`` ``google-cloud-speech`` ``>=2.18.0`` ``google-cloud-storage`` ``>=2.7.0`` +``google-cloud-storage-transfer`` ``>=1.4.1`` ``google-cloud-tasks`` ``>=2.13.0`` ``google-cloud-texttospeech`` ``>=2.14.1`` ``google-cloud-translate`` ``>=3.11.0`` @@ -206,5 +207,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-google 10.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-google 10.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.4.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 10.4.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-http/changelog.rst b/docs/apache-airflow-providers-http/changelog.rst index ae63287e513b..b7e2b8d1c4d1 100644 --- a/docs/apache-airflow-providers-http/changelog.rst +++ b/docs/apache-airflow-providers-http/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/http/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-http/commits.rst b/docs/apache-airflow-providers-http/commits.rst index 4618cdfa3fb5..d7e7e5757a83 100644 --- a/docs/apache-airflow-providers-http/commits.rst +++ b/docs/apache-airflow-providers-http/commits.rst @@ -28,6 +28,23 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-09 ``Add deferrable mode to SimpleHttpOperator (#32448)`` +`358e6e8fa1 `_ 2023-07-06 ``Fix headers passed into HttpAsyncHook (#32409)`` +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` +`3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`1240dcc167 `_ 2023-07-05 ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ================================================================= + 4.4.2 ..... @@ -36,6 +53,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` diff --git a/docs/apache-airflow-providers-http/index.rst b/docs/apache-airflow-providers-http/index.rst index 70d28ec4d685..09004b389641 100644 --- a/docs/apache-airflow-providers-http/index.rst +++ b/docs/apache-airflow-providers-http/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-http `Hypertext Transfer Protocol (HTTP) `__ -Release: 4.4.2 +Release: 4.5.0 Provider package ---------------- diff --git a/docs/apache-airflow-providers-microsoft-azure/commits.rst b/docs/apache-airflow-providers-microsoft-azure/commits.rst index 10822915c2a8..591c6975adc2 100644 --- a/docs/apache-airflow-providers-microsoft-azure/commits.rst +++ b/docs/apache-airflow-providers-microsoft-azure/commits.rst @@ -28,14 +28,29 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-12 ``Fix breaking change when Active Directory ID is used as host in WASB (#32560)`` +`a67427af37 `_ 2023-07-12 ``Fix get_blobs_list_async method to return BlobProperties (#32545)`` +`c99c03135c `_ 2023-07-08 ``D205 Support - Providers: Stragglers and new additions (#32447)`` +`257136786c `_ 2023-07-08 ``Moves 'AzureBlobStorageToGCSOperator' from Azure to Google provider (#32306)`` +================================================================================================= =========== ================================================================================= + 6.2.0 ..... -Latest change: 2023-07-05 +Latest change: 2023-07-06 ================================================================================================= =========== ====================================================================================== Commit Committed Subject ================================================================================================= =========== ====================================================================================== +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `1240dcc167 `_ 2023-07-05 ``D205 Support - Providers: GRPC to Oracle (inclusive) (#32357)`` diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst index c2ee6d69df49..003c0bc41bb2 100644 --- a/docs/apache-airflow-providers-microsoft-azure/index.rst +++ b/docs/apache-airflow-providers-microsoft-azure/index.rst @@ -81,7 +81,7 @@ Package apache-airflow-providers-microsoft-azure `Microsoft Azure `__ -Release: 6.2.0 +Release: 6.2.1 Provider package ---------------- @@ -151,5 +151,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-microsoft-azure 6.2.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-microsoft-azure 6.2.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 6.2.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-microsoft-azure 6.2.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-sftp/changelog.rst b/docs/apache-airflow-providers-sftp/changelog.rst index 3f0ce545ca8c..96edf683bca7 100644 --- a/docs/apache-airflow-providers-sftp/changelog.rst +++ b/docs/apache-airflow-providers-sftp/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/sftp/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-sftp/commits.rst b/docs/apache-airflow-providers-sftp/commits.rst index af7828a52bf2..9a6acdf88823 100644 --- a/docs/apache-airflow-providers-sftp/commits.rst +++ b/docs/apache-airflow-providers-sftp/commits.rst @@ -28,6 +28,23 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-10 ``Remove redundant Operator suffix from sensor name (#32475)`` +`a3fa12ce3c `_ 2023-07-10 ``Adds sftp_sensor decorator (#32457)`` +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` +`e45bee8840 `_ 2023-07-05 ``D205 Support - Providers: Pagerduty to SMTP (inclusive) (#32358)`` +`3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== ==================================================================== + 4.3.1 ..... @@ -36,6 +53,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `9276310a43 `_ 2023-06-05 ``Improve docstrings in providers (#31681)`` `a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` diff --git a/docs/apache-airflow-providers-sftp/index.rst b/docs/apache-airflow-providers-sftp/index.rst index 72fbf2546b7e..ef1c42226872 100644 --- a/docs/apache-airflow-providers-sftp/index.rst +++ b/docs/apache-airflow-providers-sftp/index.rst @@ -69,7 +69,7 @@ Package apache-airflow-providers-sftp `SSH File Transfer Protocol (SFTP) `__ -Release: 4.3.1 +Release: 4.4.0 Provider package ---------------- @@ -121,5 +121,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-sftp 4.3.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-sftp 4.3.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-sftp 4.4.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-sftp 4.4.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-snowflake/commits.rst b/docs/apache-airflow-providers-snowflake/commits.rst index f9b7e0cad3ef..fcaf1247f51f 100644 --- a/docs/apache-airflow-providers-snowflake/commits.rst +++ b/docs/apache-airflow-providers-snowflake/commits.rst @@ -28,14 +28,28 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-09 ``Fix an issue that crashes Airflow Webserver when passed invalid private key path to Snowflake (#32016)`` +`53d9421905 `_ 2023-07-07 ``build(pre-commit): check deferrable default value (#32370)`` +`21e8f878a3 `_ 2023-07-06 ``D205 Support - Providers: Snowflake to Zendesk (inclusive) (#32359)`` +================================================================================================= =========== ========================================================================================================== + 4.3.0 ..... -Latest change: 2023-07-05 +Latest change: 2023-07-06 ================================================================================================= =========== ================================================================ Commit Committed Subject ================================================================================================= =========== ================================================================ +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `891c2e4019 `_ 2023-07-04 ``Add Deferrable switch to SnowflakeSqlApiOperator (#31596)`` diff --git a/docs/apache-airflow-providers-snowflake/index.rst b/docs/apache-airflow-providers-snowflake/index.rst index fa2eddc05fe8..7f5b1f705c5c 100644 --- a/docs/apache-airflow-providers-snowflake/index.rst +++ b/docs/apache-airflow-providers-snowflake/index.rst @@ -77,7 +77,7 @@ Package apache-airflow-providers-snowflake `Snowflake `__ -Release: 4.3.0 +Release: 4.3.1 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-snowflake 4.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-snowflake 4.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 4.3.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-snowflake 4.3.1 wheel package `_ (`asc `__, `sha512 `__) From 5ef886e97608ab8c97fa70c3fcd067b2748be4f6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 12 Jul 2023 21:41:19 +0200 Subject: [PATCH 176/533] Remove "--for-production" flag from docs building (#32564) We had two types of documentation building: * latest * for production But in fact they never overlapped and we were never mixing the two We used the latest for all development work and for production for releasing to "airflow.apache.org". However as we saw in #32562 (triggered by #32495 changing the build in main to run with `--for-production` flag - we actually do not need the "latest" builds at all. Everything can be build with "for production" by default. This change removes the `--for-production` flag entirely, leaving the "for production" build mode as the only one available. --- .github/workflows/ci.yml | 1 - dev/README_RELEASE_AIRFLOW.md | 2 +- dev/README_RELEASE_HELM_CHART.md | 2 +- dev/README_RELEASE_PROVIDER_PACKAGES.md | 9 +- .../commands/developer_commands.py | 13 +- .../commands/developer_commands_config.py | 1 - .../commands/release_management_commands.py | 2 +- .../airflow_breeze/params/doc_build_params.py | 3 - .../utils/publish_docs_builder.py | 9 +- .../build_provider_documentation.sh | 1 - docs/build_docs.py | 40 +----- docs/conf.py | 26 ++-- docs/exts/airflow_intersphinx.py | 5 +- .../docs_build/dev_index_template.html.jinja2 | 8 +- docs/exts/docs_build/docs_builder.py | 9 +- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-commands.svg | 2 +- images/breeze/output_build-docs.svg | 134 ++++++++---------- 18 files changed, 91 insertions(+), 178 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d62660e5362a..bc0c1838252b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -633,7 +633,6 @@ jobs: run: > breeze build-docs ${{ needs.build-info.outputs.docs-filter-list-as-string }} - --for-production - name: "Clone airflow-site" run: > git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index ccce45a2da7a..b24c2feff1af 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -792,7 +792,7 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo ```shell script cd "${AIRFLOW_REPO_ROOT}" - breeze build-docs --package-filter apache-airflow --package-filter docker-stack --clean-build --for-production + breeze build-docs --package-filter apache-airflow --package-filter docker-stack --clean-build ``` - Now you can preview the documentation. diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index 727170ecb3e9..cf664c3d84e2 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -613,7 +613,7 @@ between the two repositories to be able to build the documentation. ```shell cd "${AIRFLOW_REPO_ROOT}" git checkout helm-chart/${VERSION} - breeze build-docs --package-filter helm-chart --clean-build --for-production + breeze build-docs --package-filter helm-chart --clean-build ``` - Now you can preview the documentation. diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 615722130712..65bf93558617 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -347,7 +347,7 @@ git pull --rebase ```shell script cd "${AIRFLOW_REPO_ROOT}" -breeze build-docs --clean-build --for-production --package-filter apache-airflow-providers \ +breeze build-docs --clean-build --package-filter apache-airflow-providers \ --package-filter 'apache-airflow-providers-*' ``` @@ -359,7 +359,7 @@ If we want to just release some providers you can release them in this way: ```shell script cd "${AIRFLOW_REPO_ROOT}" -breeze build-docs --clean-build --for-production \ +breeze build-docs --clean-build \ --package-filter apache-airflow-providers \ --package-filter 'apache-airflow-providers-PACKAGE1' \ --package-filter 'apache-airflow-providers-PACKAGE2' \ @@ -380,11 +380,6 @@ If you have providers as list of provider ids because you just released them, yo ./docs/start_doc_server.sh ``` -You should navigate the providers and make sure the docs render properly. -Note: if you used ``--for-production`` then default of url paths goes to ``latest`` -thus viewing the pages will result in 404 file not found error. -You will need to change it manually to see the docs - - Copy the documentation to the ``airflow-site`` repository All providers (including overriding documentation for doc-only changes): diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index cc81cd5b4881..59d95e5f88f8 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -340,12 +340,6 @@ def start_airflow( "before the build - useful for a clean build.", is_flag=True, ) -@click.option( - "--for-production", - help="Builds documentation for official release i.e. all links point to stable version. " - "Implies --clean-build", - is_flag=True, -) @click.option( "--one-pass-only", help="Builds documentation in one pass only. This is useful for debugging sphinx errors.", @@ -357,17 +351,12 @@ def start_airflow( def build_docs( docs_only: bool, spellcheck_only: bool, - for_production: bool, builder: str, clean_build: bool, one_pass_only: bool, package_filter: tuple[str], github_repository: str, ): - """Build documentation in the container.""" - if for_production and not clean_build: - get_console().print("\n[warning]When building docs for production, clean-build is forced\n") - clean_build = True perform_environment_checks() cleanup_python_generated_files() params = BuildCiParams( @@ -385,7 +374,7 @@ def build_docs( package_filter=package_filter, docs_only=docs_only, spellcheck_only=spellcheck_only, - for_production=for_production, + one_pass_only=one_pass_only, skip_environment_initialization=True, ) extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index cbc9e4348b6b..81c99e793f95 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -185,7 +185,6 @@ "--docs-only", "--spellcheck-only", "--clean-build", - "--for-production", "--one-pass-only", "--package-filter", "--github-repository", diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index eec7e1d0eaf5..565e323c4d40 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -816,7 +816,7 @@ def publish_docs( print(f" - {pkg}") print() for package_name in current_packages: - builder = PublishDocsBuilder(package_name=package_name, for_production=True) + builder = PublishDocsBuilder(package_name=package_name) builder.publish(override_versioned=override_versioned, airflow_site_dir=airflow_site_directory) diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py b/dev/breeze/src/airflow_breeze/params/doc_build_params.py index 4f6998813d0b..158e93964534 100644 --- a/dev/breeze/src/airflow_breeze/params/doc_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py @@ -27,7 +27,6 @@ class DocBuildParams: package_filter: tuple[str] docs_only: bool spellcheck_only: bool - for_production: bool skip_environment_initialization: bool = False one_pass_only: bool = False github_actions = os.environ.get("GITHUB_ACTIONS", "false") @@ -39,8 +38,6 @@ def args_doc_builder(self) -> list[str]: doc_args.append("--docs-only") if self.spellcheck_only: doc_args.append("--spellcheck-only") - if self.for_production: - doc_args.append("--for-production") if self.one_pass_only: doc_args.append("--one-pass-only") if AIRFLOW_BRANCH != "main": diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py index 24b5505f28c8..5a64cbd9976b 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py @@ -44,9 +44,8 @@ class PublishDocsBuilder: """Documentation builder for Airflow Docs Publishing.""" - def __init__(self, package_name: str, for_production: bool): + def __init__(self, package_name: str): self.package_name = package_name - self.for_production = for_production @property def _doctree_dir(self) -> str: @@ -66,7 +65,7 @@ def is_versioned(self): @property def _build_dir(self) -> str: if self.is_versioned: - version = "stable" if self.for_production else "latest" + version = "stable" return f"{DOCS_DIR}/_build/docs/{self.package_name}/{version}" else: return f"{DOCS_DIR}/_build/docs/{self.package_name}" @@ -154,8 +153,6 @@ def check_spelling(self, verbose: bool) -> list[SpellingError]: env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name - if self.for_production: - env["AIRFLOW_FOR_PRODUCTION"] = "true" if verbose: console.print( f"[info]{self.package_name:60}:[/] Executing cmd: ", @@ -229,8 +226,6 @@ def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: ] env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name - if self.for_production: - env["AIRFLOW_FOR_PRODUCTION"] = "true" if verbose: console.print( f"[info]{self.package_name:60}:[/] Executing cmd: ", diff --git a/dev/provider_packages/build_provider_documentation.sh b/dev/provider_packages/build_provider_documentation.sh index e59cf87e694a..7d8a55b8a5b2 100755 --- a/dev/provider_packages/build_provider_documentation.sh +++ b/dev/provider_packages/build_provider_documentation.sh @@ -29,7 +29,6 @@ do done .breeze build-docs \ - --for-production \ --clean-build \ "${provider_filters[@]}" cd "${AIRFLOW_SITE_DIRECTORY}" diff --git a/docs/build_docs.py b/docs/build_docs.py index bc66fc793f4d..84ddb4d22cac 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -135,12 +135,6 @@ def _get_parser(): parser.add_argument( "--spellcheck-only", dest="spellcheck_only", action="store_true", help="Only perform spellchecking" ) - parser.add_argument( - "--for-production", - dest="for_production", - action="store_true", - help="Builds documentation for official release i.e. all links point to stable version", - ) parser.add_argument( "-j", "--jobs", @@ -173,7 +167,6 @@ class BuildSpecification(NamedTuple): """Specification of single build.""" package_name: str - for_production: bool verbose: bool @@ -195,9 +188,7 @@ class SpellCheckResult(NamedTuple): def perform_docs_build_for_single_package(build_specification: BuildSpecification) -> BuildDocsResult: """Performs single package docs build.""" - builder = AirflowDocsBuilder( - package_name=build_specification.package_name, for_production=build_specification.for_production - ) + builder = AirflowDocsBuilder(package_name=build_specification.package_name) console.print(f"[info]{build_specification.package_name:60}:[/] Building documentation") result = BuildDocsResult( package_name=build_specification.package_name, @@ -211,9 +202,7 @@ def perform_docs_build_for_single_package(build_specification: BuildSpecificatio def perform_spell_check_for_single_package(build_specification: BuildSpecification) -> SpellCheckResult: """Performs single package spell check.""" - builder = AirflowDocsBuilder( - package_name=build_specification.package_name, for_production=build_specification.for_production - ) + builder = AirflowDocsBuilder(package_name=build_specification.package_name) console.print(f"[info]{build_specification.package_name:60}:[/] Checking spelling started") result = SpellCheckResult( package_name=build_specification.package_name, @@ -230,7 +219,6 @@ def build_docs_for_packages( current_packages: list[str], docs_only: bool, spellcheck_only: bool, - for_production: bool, jobs: int, verbose: bool, ) -> tuple[dict[str, list[DocBuildError]], dict[str, list[SpellingError]]]: @@ -240,7 +228,7 @@ def build_docs_for_packages( with with_group("Cleaning documentation files"): for package_name in current_packages: console.print(f"[info]{package_name:60}:[/] Cleaning files") - builder = AirflowDocsBuilder(package_name=package_name, for_production=for_production) + builder = AirflowDocsBuilder(package_name=package_name) builder.clean_files() if jobs > 1: run_in_parallel( @@ -248,7 +236,6 @@ def build_docs_for_packages( all_spelling_errors, current_packages, docs_only, - for_production, jobs, spellcheck_only, verbose, @@ -259,7 +246,6 @@ def build_docs_for_packages( all_spelling_errors, current_packages, docs_only, - for_production, spellcheck_only, verbose, ) @@ -271,7 +257,6 @@ def run_sequentially( all_spelling_errors, current_packages, docs_only, - for_production, spellcheck_only, verbose, ): @@ -281,7 +266,6 @@ def run_sequentially( build_result = perform_docs_build_for_single_package( build_specification=BuildSpecification( package_name=package_name, - for_production=for_production, verbose=verbose, ) ) @@ -293,7 +277,6 @@ def run_sequentially( spellcheck_result = perform_spell_check_for_single_package( build_specification=BuildSpecification( package_name=package_name, - for_production=for_production, verbose=verbose, ) ) @@ -307,7 +290,6 @@ def run_in_parallel( all_spelling_errors, current_packages, docs_only, - for_production, jobs, spellcheck_only, verbose, @@ -317,7 +299,6 @@ def run_in_parallel( if not spellcheck_only: run_docs_build_in_parallel( all_build_errors=all_build_errors, - for_production=for_production, current_packages=current_packages, verbose=verbose, pool=pool, @@ -325,7 +306,6 @@ def run_in_parallel( if not docs_only: run_spell_check_in_parallel( all_spelling_errors=all_spelling_errors, - for_production=for_production, current_packages=current_packages, verbose=verbose, pool=pool, @@ -345,7 +325,6 @@ def print_build_output(result: BuildDocsResult): def run_docs_build_in_parallel( all_build_errors: dict[str, list[DocBuildError]], - for_production: bool, current_packages: list[str], verbose: bool, pool, @@ -358,7 +337,6 @@ def run_docs_build_in_parallel( doc_build_specifications.append( BuildSpecification( package_name=package_name, - for_production=for_production, verbose=verbose, ) ) @@ -385,7 +363,6 @@ def print_spelling_output(result: SpellCheckResult): def run_spell_check_in_parallel( all_spelling_errors: dict[str, list[SpellingError]], - for_production: bool, current_packages: list[str], verbose: bool, pool, @@ -395,9 +372,7 @@ def run_spell_check_in_parallel( with with_group("Scheduling spell checking of documentation"): for package_name in current_packages: console.print(f"[info]{package_name:60}:[/] Scheduling spellchecking") - spell_check_specifications.append( - BuildSpecification(package_name=package_name, for_production=for_production, verbose=verbose) - ) + spell_check_specifications.append(BuildSpecification(package_name=package_name, verbose=verbose)) with with_group("Running spell checking of documentation"): console.print() result_list = pool.map(perform_spell_check_for_single_package, spell_check_specifications) @@ -455,7 +430,6 @@ def main(): disable_provider_checks = args.disable_provider_checks disable_checks = args.disable_checks package_filters = args.package_filter - for_production = args.for_production with with_group("Available packages"): for pkg in sorted(available_packages): @@ -489,7 +463,6 @@ def main(): current_packages=priority_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, - for_production=for_production, jobs=jobs, verbose=args.verbose, ) @@ -506,7 +479,6 @@ def main(): current_packages=current_packages if len(priority_packages) > 1 else normal_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, - for_production=for_production, jobs=jobs, verbose=args.verbose, ) @@ -522,7 +494,6 @@ def main(): all_spelling_errors, args, docs_only, - for_production, jobs, package_build_errors, package_spelling_errors, @@ -535,7 +506,6 @@ def main(): all_spelling_errors, args, docs_only, - for_production, jobs, package_build_errors, package_spelling_errors, @@ -566,7 +536,6 @@ def retry_building_docs_if_needed( all_spelling_errors, args, docs_only, - for_production, jobs, package_build_errors, package_spelling_errors, @@ -588,7 +557,6 @@ def retry_building_docs_if_needed( current_packages=to_retry_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, - for_production=for_production, jobs=jobs, verbose=args.verbose, ) diff --git a/docs/conf.py b/docs/conf.py index 5029f4c0dcc3..9f1645e42599 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,6 @@ CONF_DIR = pathlib.Path(__file__).parent.absolute() INVENTORY_CACHE_DIR = CONF_DIR / "_inventory_cache" ROOT_DIR = CONF_DIR.parent -FOR_PRODUCTION = os.environ.get("AIRFLOW_FOR_PRODUCTION", "false") == "true" # By default (e.g. on RTD), build docs for `airflow` package PACKAGE_NAME = os.environ.get("AIRFLOW_PACKAGE_NAME", "apache-airflow") @@ -343,7 +342,7 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): "searchbox.html", "globaltoc.html", ] - if FOR_PRODUCTION and PACKAGE_VERSION != "devel" + if PACKAGE_VERSION != "devel" else [ "searchbox.html", "globaltoc.html", @@ -358,16 +357,15 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): html_theme_options: dict[str, Any] = {"hide_website_buttons": True, "sidebar_includehidden": True} -if FOR_PRODUCTION: - html_theme_options["navbar_links"] = [ - {"href": "/community/", "text": "Community"}, - {"href": "/meetups/", "text": "Meetups"}, - {"href": "/docs/", "text": "Documentation"}, - {"href": "/use-cases/", "text": "Use-cases"}, - {"href": "/announcements/", "text": "Announcements"}, - {"href": "/blog/", "text": "Blog"}, - {"href": "/ecosystem/", "text": "Ecosystem"}, - ] +html_theme_options["navbar_links"] = [ + {"href": "/community/", "text": "Community"}, + {"href": "/meetups/", "text": "Meetups"}, + {"href": "/docs/", "text": "Documentation"}, + {"href": "/use-cases/", "text": "Use-cases"}, + {"href": "/announcements/", "text": "Announcements"}, + {"href": "/blog/", "text": "Blog"}, + {"href": "/ecosystem/", "text": "Ecosystem"}, +] # A dictionary of values to pass into the template engine's context for all pages. html_context = { @@ -447,10 +445,6 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): "config_ctx": {"configs": configs, "deprecated_options": deprecated_options}, "quick_start_ctx": { "doc_root_url": f"https://airflow.apache.org/docs/apache-airflow/{PACKAGE_VERSION}/" - if FOR_PRODUCTION - else ( - "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com/docs/apache-airflow/latest/" - ) }, "official_download_page": { "base_url": f"https://downloads.apache.org/airflow/{PACKAGE_VERSION}", diff --git a/docs/exts/airflow_intersphinx.py b/docs/exts/airflow_intersphinx.py index b9255d2e9552..129d037e7a0c 100644 --- a/docs/exts/airflow_intersphinx.py +++ b/docs/exts/airflow_intersphinx.py @@ -42,8 +42,7 @@ def _create_init_py(app, config): def _generate_provider_intersphinx_mapping(): airflow_mapping = {} - for_production = os.environ.get("AIRFLOW_FOR_PRODUCTION", "false") == "true" - current_version = "stable" if for_production else "latest" + current_version = "stable" for provider in load_package_data(): package_name = provider["package-name"] @@ -71,7 +70,7 @@ def _generate_provider_intersphinx_mapping(): airflow_mapping[pkg_name] = ( # base URI - f'/docs/{pkg_name}/{"stable" if for_production else "latest"}/', + f"/docs/{pkg_name}/stable/", (doc_inventory if os.path.exists(doc_inventory) else cache_inventory,), ) for pkg_name in ["apache-airflow-providers", "docker-stack"]: diff --git a/docs/exts/docs_build/dev_index_template.html.jinja2 b/docs/exts/docs_build/dev_index_template.html.jinja2 index 6a52742951e5..63f1bb9130eb 100644 --- a/docs/exts/docs_build/dev_index_template.html.jinja2 +++ b/docs/exts/docs_build/dev_index_template.html.jinja2 @@ -49,7 +49,7 @@
    • -

    apache-airflow

    +

    apache-airflow

    Apache Airflow Core, which includes webserver, scheduler, CLI and other components that are needed for minimal Airflow installation.

    @@ -63,7 +63,7 @@

    @@ -81,10 +81,10 @@
    - Helm Chart - logo + Helm Chart - logo
    -

    Helm Chart

    +

    Helm Chart

    It will help you set up your own Airflow on a cloud/on-prem k8s environment and leverage its scalable nature to support a large group of users. Thanks to Kubernetes, we are not tied to a specific cloud provider.

    diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index f03f432a01c6..375e9db8c528 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -43,9 +43,8 @@ class AirflowDocsBuilder: """Documentation builder for Airflow.""" - def __init__(self, package_name: str, for_production: bool): + def __init__(self, package_name: str): self.package_name = package_name - self.for_production = for_production @property def _doctree_dir(self) -> str: @@ -65,7 +64,7 @@ def is_versioned(self): @property def _build_dir(self) -> str: if self.is_versioned: - version = "stable" if self.for_production else "latest" + version = "stable" return f"{DOCS_DIR}/_build/docs/{self.package_name}/{version}" else: return f"{DOCS_DIR}/_build/docs/{self.package_name}" @@ -154,8 +153,6 @@ def check_spelling(self, verbose: bool) -> list[SpellingError]: env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name - if self.for_production: - env["AIRFLOW_FOR_PRODUCTION"] = "true" if verbose: console.print( f"[info]{self.package_name:60}:[/] Executing cmd: ", @@ -229,8 +226,6 @@ def build_sphinx_docs(self, verbose: bool) -> list[DocBuildError]: ] env = os.environ.copy() env["AIRFLOW_PACKAGE_NAME"] = self.package_name - if self.for_production: - env["AIRFLOW_FOR_PRODUCTION"] = "true" if verbose: console.print( f"[info]{self.package_name:60}:[/] Executing cmd: ", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 784550612ac8..a1cd835a9aec 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,7 +2,7 @@ # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:344261ca3aa7ff31e098b1d88280566a -build-docs:891c3f3a4e5558e96083204ed3512c44 +build-docs:b45888f44f40dfd47bd918ea4ffedbe1 ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96 ci:free-space:49af17b032039c05c41a7a8283f365cc ci:get-workflow-info:8246038093359b9c3c110043419473e2 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 708e4d6758c5..ea72db06740e 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -247,7 +247,7 @@ │start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ │if contents of www directory changed.                                                           │ │static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ +│build-docs        │ │down              Stop running breeze environment.                                                                │ │shell             Enter breeze environment. this is the default command use when no other is selected.            │ │exec              Joins the interactive shell of running airflow container.                                       │ diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index cc5f824d72f3..f0475ed9c40a 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -1,4 +1,4 @@ - + ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -│publish-docs                     Command to publish generated documentation to airflow-site                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│generate-issue-content-providers           Generates content for issue to test the release.                       │ -│generate-providers-metadata                Generates metadata for providers.                                      │ -│install-provider-packages                  Installs provider packages that can be found in dist.                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Airflow release commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-airflow-package      Prepare sdist/whl package of Airflow.                                                │ +│create-minor-branch          Create a new version branch and update the default branches in main                  │ +│start-rc-process             Start RC process                                                                     │ +│start-release                Start Airflow release process                                                        │ +│release-prod-images          Release production images to DockerHub (needs DockerHub permissions).                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Providers release commands ─────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-provider-documentation      Prepare CHANGELOG, README and COMMITS information for providers.              │ +│prepare-provider-packages           Prepare sdist/whl packages of Airflow Providers.                              │ +│install-provider-packages           Installs provider packages that can be found in dist.                         │ +│verify-provider-packages            Verifies if all provider code is following expectations for providers.        │ +│generate-providers-metadata         Generates metadata for providers.                                             │ +│generate-issue-content-providers    Generates content for issue to test the release.                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-constraints    Generates pinned constraint files with all extras from setup.py in parallel.              │ +│publish-docs            Command to publish generated documentation to airflow-site                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_create-minor-branch.svg b/images/breeze/output_release-management_create-minor-branch.svg new file mode 100644 index 000000000000..8980e6598a44 --- /dev/null +++ b/images/breeze/output_release-management_create-minor-branch.svg @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management create-minor-branch + + + + + + + + + + +Usage: breeze release-management create-minor-branch [OPTIONS] + +Create a new version branch and update the default branches in main + +╭─ Create minor branch flags ──────────────────────────────────────────────────────────────────────────────────────────╮ +│*--version-branchThe version branch you want to create e.g 2-4(TEXT)[required]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_start-rc-process.svg b/images/breeze/output_release-management_start-rc-process.svg new file mode 100644 index 000000000000..0d70e8a16f92 --- /dev/null +++ b/images/breeze/output_release-management_start-rc-process.svg @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management start-rc-process + + + + + + + + + + +Usage: breeze release-management start-rc-process [OPTIONS] + +Start the process for releasing a new RC. + +╭─ Start RC process flags ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│*--versionThe release candidate version e.g. 2.4.3rc1(TEXT)[required]│ +│*--previous-versionPrevious version released e.g. 2.4.2(TEXT)[required]│ +│--github-tokenGitHub token to use in generating issue for testing of release candidate(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_start-release.svg b/images/breeze/output_release-management_start-release.svg new file mode 100644 index 000000000000..da50f43b4cec --- /dev/null +++ b/images/breeze/output_release-management_start-release.svg @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management start-release + + + + + + + + + + +Usage: breeze release-management start-release [OPTIONS] + +Start the process of releasing an Airflow version. This command will guide you through the release process. + +╭─ Start release flags ────────────────────────────────────────────────────────────────────────────────────────────────╮ +│*--release-candidate(TEXT)[required]│ +│*--previous-release(TEXT)[required]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + From ae29225ff54e8f3bc4fa6934c59b84852c93c713 Mon Sep 17 00:00:00 2001 From: Daniel Hoherd Date: Wed, 12 Jul 2023 18:53:36 -0400 Subject: [PATCH 179/533] Allow templates in cleanup.schedule (#32570) --- chart/templates/cleanup/cleanup-cronjob.yaml | 2 +- chart/values.schema.json | 2 +- chart/values.yaml | 7 ++- helm_tests/airflow_aux/test_cleanup_pods.py | 48 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/chart/templates/cleanup/cleanup-cronjob.yaml b/chart/templates/cleanup/cleanup-cronjob.yaml index fb81dc875478..57cfdf7be480 100644 --- a/chart/templates/cleanup/cleanup-cronjob.yaml +++ b/chart/templates/cleanup/cleanup-cronjob.yaml @@ -43,7 +43,7 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: - schedule: "{{ .Values.cleanup.schedule }}" + schedule: "{{ tpl .Values.cleanup.schedule . }}" # The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run concurrencyPolicy: Forbid {{- if .Values.cleanup.failedJobsHistoryLimit }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 06b6971fb322..cb33bb7b4dbd 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -5822,7 +5822,7 @@ "default": false }, "schedule": { - "description": "Cleanup schedule.", + "description": "Cleanup schedule (templated).", "type": "string", "default": "*/15 * * * *" }, diff --git a/chart/values.yaml b/chart/values.yaml index 2f79afb14074..d0cfdc3a171d 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1969,8 +1969,13 @@ limits: [] # This runs as a CronJob to cleanup old pods. cleanup: enabled: false - # Run every 15 minutes + # Run every 15 minutes (templated). schedule: "*/15 * * * *" + # To select a random-ish, deterministic starting minute between 3 and 12 inclusive for each release: + # '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 * * * *' + # To select the last digit of unix epoch time as the starting minute on each deploy: + # '{{- now | unixEpoch | trunc -1 -}}-59/* * * * *' + # Command to use when running the cleanup cronjob (templated). command: ~ # Args to use when running the cleanup cronjob (templated). diff --git a/helm_tests/airflow_aux/test_cleanup_pods.py b/helm_tests/airflow_aux/test_cleanup_pods.py index 60859ac3d432..013662e5f9ed 100644 --- a/helm_tests/airflow_aux/test_cleanup_pods.py +++ b/helm_tests/airflow_aux/test_cleanup_pods.py @@ -22,6 +22,54 @@ from tests.charts.helm_template_generator import render_chart +class TestCleanupDeployment: + """Tests cleanup pods deployments.""" + + def test_should_have_a_schedule_with_defaults(self): + doc = render_chart( + values={ + "cleanup": {"enabled": True}, + }, + show_only=["templates/cleanup/cleanup-cronjob.yaml"], + )[0] + + assert doc["spec"]["schedule"] == "*/15 * * * *" + + cron_tests = [ + ("release-name", "*/5 * * * *", "*/5 * * * *"), + ("something-else", "@hourly", "@hourly"), + ( + "custom-name", + '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 * * * *', + "7-59/15 * * * *", + ), + ( + "airflow-rules", + '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 * * * *', + "10-59/15 * * * *", + ), + ] + + @pytest.mark.parametrize( + "release_name,schedule_value,schedule_result", + cron_tests, + ids=[x[0] for x in cron_tests], + ) + def test_should_work_with_custom_schedule_string(self, release_name, schedule_value, schedule_result): + doc = render_chart( + name=release_name, + values={ + "cleanup": { + "enabled": True, + "schedule": schedule_value, + }, + }, + show_only=["templates/cleanup/cleanup-cronjob.yaml"], + )[0] + + assert doc["spec"]["schedule"] == schedule_result + + class TestCleanupPods: """Tests cleanup of pods.""" From 4a12bdb2451aa53626da05d197d85c2a34d459fc Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jens-scheffler-bosch@users.noreply.github.com> Date: Thu, 13 Jul 2023 05:55:57 +0200 Subject: [PATCH 180/533] Bugfix, prevent force of unpause on trigger DAG (#32456) --- airflow/www/templates/airflow/dag.html | 4 ++-- airflow/www/templates/airflow/dags.html | 2 +- airflow/www/views.py | 15 +++++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 66531c607e58..754e17262a0f 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -221,9 +221,9 @@

    {% if 'dag_id' in request.args %} - {{ page_title }}

    {# Use dag_id instead of dag.dag_id, because the DAG might not exist in the webserver's DagBag #} {% if dag %} - diff --git a/airflow/www/views.py b/airflow/www/views.py index a18429a1a93e..a62a966eebb1 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2159,10 +2159,17 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): recent_confs=recent_confs, ) - if unpause and dag.get_is_paused(): - dag_model = models.DagModel.get_dagmodel(dag_id) - if dag_model is not None: - dag_model.set_is_paused(is_paused=False) + if dag.get_is_paused(): + if unpause or not ui_fields_defined: + flash(f"Unpaused DAG {dag_id}.") + dag_model = models.DagModel.get_dagmodel(dag_id) + if dag_model is not None: + dag_model.set_is_paused(is_paused=False) + else: + flash( + f"DAG {dag_id} is paused, unpause if you want to have the triggered run being executed.", + "warning", + ) try: dag.create_dagrun( From bf68e1060b0214ee195c61f9d7be992161e25589 Mon Sep 17 00:00:00 2001 From: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:49:52 -0600 Subject: [PATCH 181/533] Fix permissions on `/blocked` endpoint (#32571) This endpoint is used to update the UI with how many dagruns are running out of the max active dagruns per dag. It requires no elevated perms. This also fixes the test coverage on this endpoint. --- airflow/www/views.py | 2 +- tests/www/views/test_views_acl.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/airflow/www/views.py b/airflow/www/views.py index a62a966eebb1..db618989ff72 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2391,7 +2391,7 @@ def dagrun_clear(self, *, session: Session = NEW_SESSION): @expose("/blocked", methods=["POST"]) @auth.has_access( [ - (permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_RUN), ] ) diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index f445d0fe8340..30e2a975c4cf 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -98,6 +98,7 @@ def acl_app(app): "all_dag_role": [ (permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG), (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG), + (permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_RUN), (permissions.ACTION_CAN_READ, permissions.RESOURCE_TASK_INSTANCE), (permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE), ], @@ -641,16 +642,21 @@ def test_failure(dag_faker_client, url, unexpected_content): def test_blocked_success(client_all_dags_dagruns): - resp = client_all_dags_dagruns.post("blocked", follow_redirects=True) + resp = client_all_dags_dagruns.post("blocked") check_content_in_response("example_bash_operator", resp) def test_blocked_success_for_all_dag_user(all_dag_user_client): - resp = all_dag_user_client.post("blocked", follow_redirects=True) + resp = all_dag_user_client.post("blocked") check_content_in_response("example_bash_operator", resp) check_content_in_response("example_subdag_operator", resp) +def test_blocked_viewer(viewer_client): + resp = viewer_client.post("blocked") + check_content_in_response("example_bash_operator", resp) + + @pytest.mark.parametrize( "dags_to_block, unexpected_dag_ids", [ @@ -670,11 +676,7 @@ def test_blocked_success_when_selecting_dags( dags_to_block, unexpected_dag_ids, ): - resp = admin_client.post( - "blocked", - data={"dag_ids": dags_to_block}, - follow_redirects=True, - ) + resp = admin_client.post("blocked", data={"dag_ids": dags_to_block}) assert resp.status_code == 200 for dag_id in unexpected_dag_ids: check_content_not_in_response(dag_id, resp) From 145b16caaa43f0c42bffd97344df916c602cddde Mon Sep 17 00:00:00 2001 From: David Date: Thu, 13 Jul 2023 02:29:07 -0400 Subject: [PATCH 182/533] Fix S3ToRedshiftOperator does not support default values on UPSERT (#32558) Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- airflow/providers/amazon/aws/transfers/s3_to_redshift.py | 2 +- tests/providers/amazon/aws/transfers/test_s3_to_redshift.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py index b42b2d8cbb7f..b80b5bed3276 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py @@ -178,7 +178,7 @@ def execute(self, context: Context) -> None: where_statement = " AND ".join([f"{self.table}.{k} = {copy_destination}.{k}" for k in keys]) sql = [ - f"CREATE TABLE {copy_destination} (LIKE {destination});", + f"CREATE TABLE {copy_destination} (LIKE {destination} INCLUDING DEFAULTS);", copy_statement, "BEGIN;", f"DELETE FROM {destination} USING {copy_destination} WHERE {where_statement};", diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py b/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py index f73acf661e69..dff441f0a6e1 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_redshift.py @@ -222,7 +222,7 @@ def test_upsert(self, mock_run, mock_session, mock_connection, mock_hook): ; """ transaction = f""" - CREATE TABLE #{table} (LIKE {schema}.{table}); + CREATE TABLE #{table} (LIKE {schema}.{table} INCLUDING DEFAULTS); {copy_statement} BEGIN; DELETE FROM {schema}.{table} USING #{table} WHERE {table}.id = #{table}.id; From e01323635a88ecf313a415ea41d32d6d28fa0794 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 13 Jul 2023 16:30:09 +0200 Subject: [PATCH 183/533] Add deprecation info to the providers modules and classes docstring (#32536) * Add deprecation info to providers modules and classes docstring Signed-off-by: Hussein Awala --------- Signed-off-by: Hussein Awala --- airflow/providers/apache/drill/operators/drill.py | 4 ++++ .../providers/apache/druid/operators/druid_check.py | 2 +- .../cncf/kubernetes/operators/kubernetes_pod.py | 1 + .../cncf/kubernetes/triggers/kubernetes_pod.py | 1 + .../providers/elasticsearch/hooks/elasticsearch.py | 2 +- airflow/providers/google/cloud/operators/dataflow.py | 2 ++ airflow/providers/google/cloud/sensors/bigquery.py | 12 +++++++++++- airflow/providers/google/cloud/sensors/gcs.py | 6 ++++-- airflow/providers/jdbc/operators/jdbc.py | 4 ++++ .../microsoft/azure/sensors/data_factory.py | 6 ++++++ airflow/providers/microsoft/azure/sensors/wasb.py | 5 +++++ .../microsoft/azure/transfers/local_to_adls.py | 3 ++- airflow/providers/microsoft/mssql/operators/mssql.py | 4 ++++ airflow/providers/mysql/operators/mysql.py | 4 ++++ airflow/providers/oracle/operators/oracle.py | 4 ++++ airflow/providers/postgres/operators/postgres.py | 4 ++++ airflow/providers/snowflake/operators/snowflake.py | 4 ++++ airflow/providers/sqlite/operators/sqlite.py | 4 ++++ airflow/providers/trino/operators/trino.py | 4 ++++ airflow/providers/vertica/operators/vertica.py | 4 ++++ .../providers/google/cloud/sensors/test_bigquery.py | 2 +- 21 files changed, 75 insertions(+), 7 deletions(-) diff --git a/airflow/providers/apache/drill/operators/drill.py b/airflow/providers/apache/drill/operators/drill.py index 3ab760b6db8f..e654d6b6a51f 100644 --- a/airflow/providers/apache/drill/operators/drill.py +++ b/airflow/providers/apache/drill/operators/drill.py @@ -28,6 +28,10 @@ class DrillOperator(SQLExecuteQueryOperator): """ Executes the provided SQL in the identified Drill environment. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:DrillOperator` diff --git a/airflow/providers/apache/druid/operators/druid_check.py b/airflow/providers/apache/druid/operators/druid_check.py index 25813384c222..c6bf855cf652 100644 --- a/airflow/providers/apache/druid/operators/druid_check.py +++ b/airflow/providers/apache/druid/operators/druid_check.py @@ -27,7 +27,7 @@ class DruidCheckOperator(SQLCheckOperator): """ This class is deprecated. - Please use `airflow.providers.common.sql.operators.sql.SQLCheckOperator`. + Please use :class:`airflow.providers.common.sql.operators.sql.SQLCheckOperator`. """ def __init__(self, druid_broker_conn_id: str = "druid_broker_default", **kwargs): diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py index 0c593a17bf32..36993895bc31 100644 --- a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.operators.pod` instead.""" from __future__ import annotations import warnings diff --git a/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py index 83ece6c0be3a..3cb60f7bd12a 100644 --- a/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.triggers.pod` instead.""" from __future__ import annotations import warnings diff --git a/airflow/providers/elasticsearch/hooks/elasticsearch.py b/airflow/providers/elasticsearch/hooks/elasticsearch.py index 978eac8f5ca8..c3a792eb697e 100644 --- a/airflow/providers/elasticsearch/hooks/elasticsearch.py +++ b/airflow/providers/elasticsearch/hooks/elasticsearch.py @@ -142,7 +142,7 @@ class ElasticsearchHook(ElasticsearchSQLHook): """ This class is deprecated and was renamed to ElasticsearchSQLHook. - Please use `airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook`. + Please use :class:`airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook`. """ def __init__(self, *args, **kwargs): diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py index 0a225a47b623..e6710c9f9a00 100644 --- a/airflow/providers/google/cloud/operators/dataflow.py +++ b/airflow/providers/google/cloud/operators/dataflow.py @@ -173,6 +173,7 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator): Start a Java Cloud Dataflow batch job; the parameters of the operation will be passed to the job. This class is deprecated. + Please use :class:`providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator`. Example usage: @@ -1023,6 +1024,7 @@ class DataflowCreatePythonJobOperator(GoogleCloudBaseOperator): for instances, project and zone information, which apply to all dataflow operators in the DAG. This class is deprecated. + Please use :class:`providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator`. .. seealso:: diff --git a/airflow/providers/google/cloud/sensors/bigquery.py b/airflow/providers/google/cloud/sensors/bigquery.py index d13748da1807..d4f15fac1bf9 100644 --- a/airflow/providers/google/cloud/sensors/bigquery.py +++ b/airflow/providers/google/cloud/sensors/bigquery.py @@ -256,6 +256,11 @@ class BigQueryTableExistenceAsyncSensor(BigQueryTableExistenceSensor): """ Checks for the existence of a table in Google Big Query. + This class is deprecated and will be removed in a future release. + + Please use :class:`airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor` + and set *deferrable* attribute to *True* instead. + :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook must provide access to the specified project. @@ -291,6 +296,11 @@ class BigQueryTableExistencePartitionAsyncSensor(BigQueryTablePartitionExistence """ Checks for the existence of a partition within a table in Google BigQuery. + This class is deprecated and will be removed in a future release. + + Please use :class:`airflow.providers.google.cloud.sensors.bigquery.BigQueryTablePartitionExistenceSensor` + and set *deferrable* attribute to *True* instead. + :param project_id: The Google cloud project in which to look for the table. The connection supplied to the hook must provide access to the specified project. @@ -316,7 +326,7 @@ def __init__(self, **kwargs): warnings.warn( "Class `BigQueryTableExistencePartitionAsyncSensor` is deprecated and " "will be removed in a future release. " - "Please use `BigQueryTableExistencePartitionSensor` and " + "Please use `BigQueryTablePartitionExistenceSensor` and " "set `deferrable` attribute to `True` instead", AirflowProviderDeprecationWarning, ) diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/airflow/providers/google/cloud/sensors/gcs.py index 34df5f52570f..e427d1c6bef0 100644 --- a/airflow/providers/google/cloud/sensors/gcs.py +++ b/airflow/providers/google/cloud/sensors/gcs.py @@ -133,8 +133,10 @@ class GCSObjectExistenceAsyncSensor(GCSObjectExistenceSensor): """ Checks for the existence of a file in Google Cloud Storage. - Class `GCSObjectExistenceAsyncSensor` is deprecated and will be removed in a future release. - Please use `GCSObjectExistenceSensor` and set `deferrable` attribute to `True` instead. + This class is deprecated and will be removed in a future release. + + Please use :class:`airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor` + and set *deferrable* attribute to *True* instead. :param bucket: The Google Cloud Storage bucket where the object is. :param object: The name of the object to check in the Google cloud storage bucket. diff --git a/airflow/providers/jdbc/operators/jdbc.py b/airflow/providers/jdbc/operators/jdbc.py index 4e94d69ba77b..fae69b2af437 100644 --- a/airflow/providers/jdbc/operators/jdbc.py +++ b/airflow/providers/jdbc/operators/jdbc.py @@ -30,6 +30,10 @@ class JdbcOperator(SQLExecuteQueryOperator): Requires jaydebeapi. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator` instead. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:JdbcOperator` diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/airflow/providers/microsoft/azure/sensors/data_factory.py index f0651494bfac..4caa26a99d84 100644 --- a/airflow/providers/microsoft/azure/sensors/data_factory.py +++ b/airflow/providers/microsoft/azure/sensors/data_factory.py @@ -127,6 +127,12 @@ class AzureDataFactoryPipelineRunStatusAsyncSensor(AzureDataFactoryPipelineRunSt """ Checks the status of a pipeline run asynchronously. + This class is deprecated and will be removed in a future release. + + Please use + :class:`airflow.providers.microsoft.azure.sensors.data_factory.AzureDataFactoryPipelineRunStatusSensor` + and set *deferrable* attribute to *True* instead. + :param azure_data_factory_conn_id: The connection identifier for connecting to Azure Data Factory. :param run_id: The pipeline run identifier. :param resource_group_name: The resource group name. diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py index 18ac2331e018..5abd88a129ac 100644 --- a/airflow/providers/microsoft/azure/sensors/wasb.py +++ b/airflow/providers/microsoft/azure/sensors/wasb.py @@ -112,6 +112,11 @@ class WasbBlobAsyncSensor(WasbBlobSensor): """ Polls asynchronously for the existence of a blob in a WASB container. + This class is deprecated and will be removed in a future release. + + Please use :class:`airflow.providers.microsoft.azure.sensors.wasb.WasbBlobSensor` + and set *deferrable* attribute to *True* instead. + :param container_name: name of the container in which the blob should be searched for :param blob_name: name of the blob to check existence for :param wasb_conn_id: the connection identifier for connecting to Azure WASB diff --git a/airflow/providers/microsoft/azure/transfers/local_to_adls.py b/airflow/providers/microsoft/azure/transfers/local_to_adls.py index 7eee5009afcc..dc65857c7ca0 100644 --- a/airflow/providers/microsoft/azure/transfers/local_to_adls.py +++ b/airflow/providers/microsoft/azure/transfers/local_to_adls.py @@ -103,7 +103,8 @@ class LocalToAzureDataLakeStorageOperator(LocalFilesystemToADLSOperator): """ This class is deprecated. - Please use `airflow.providers.microsoft.azure.transfers.local_to_adls.LocalFilesystemToADLSOperator`. + Please use + :class:`airflow.providers.microsoft.azure.transfers.local_to_adls.LocalFilesystemToADLSOperator`. """ def __init__(self, *args, **kwargs): diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/airflow/providers/microsoft/mssql/operators/mssql.py index 043162f3ebc3..dc06b36c9a56 100644 --- a/airflow/providers/microsoft/mssql/operators/mssql.py +++ b/airflow/providers/microsoft/mssql/operators/mssql.py @@ -37,6 +37,10 @@ class MsSqlOperator(SQLExecuteQueryOperator): If conn_type is ``'odbc'``, then :py:class:`~airflow.providers.odbc.hooks.odbc.OdbcHook` is used. Otherwise, :py:class:`~airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook` is used. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + :param sql: the sql code to be executed (templated) :param mssql_conn_id: reference to a specific mssql database :param parameters: (optional) the parameters to render the SQL query with. diff --git a/airflow/providers/mysql/operators/mysql.py b/airflow/providers/mysql/operators/mysql.py index b21c9cdaf420..7c8e1a09a457 100644 --- a/airflow/providers/mysql/operators/mysql.py +++ b/airflow/providers/mysql/operators/mysql.py @@ -28,6 +28,10 @@ class MySqlOperator(SQLExecuteQueryOperator): """ Executes sql code in a specific MySQL database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:MySqlOperator` diff --git a/airflow/providers/oracle/operators/oracle.py b/airflow/providers/oracle/operators/oracle.py index 53396db986e3..87bd7494b9cb 100644 --- a/airflow/providers/oracle/operators/oracle.py +++ b/airflow/providers/oracle/operators/oracle.py @@ -36,6 +36,10 @@ class OracleOperator(SQLExecuteQueryOperator): """ Executes sql code in a specific Oracle database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + :param sql: the sql code to be executed. Can receive a str representing a sql statement, a list of str (sql statements), or reference to a template file. Template reference are recognized by str ending in '.sql' diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py index ef92b51dda75..71f49ef7f8bc 100644 --- a/airflow/providers/postgres/operators/postgres.py +++ b/airflow/providers/postgres/operators/postgres.py @@ -28,6 +28,10 @@ class PostgresOperator(SQLExecuteQueryOperator): """ Executes sql code in a specific Postgres database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + :param sql: the SQL code to be executed as a single string, or a list of str (sql statements), or a reference to a template file. Template references are recognized by str ending in '.sql' diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 61e60b5781fb..29b57e54eb97 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -42,6 +42,10 @@ class SnowflakeOperator(SQLExecuteQueryOperator): """ Executes SQL code in a Snowflake database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:SnowflakeOperator` diff --git a/airflow/providers/sqlite/operators/sqlite.py b/airflow/providers/sqlite/operators/sqlite.py index 58642246d8c0..f764cc01ebb9 100644 --- a/airflow/providers/sqlite/operators/sqlite.py +++ b/airflow/providers/sqlite/operators/sqlite.py @@ -28,6 +28,10 @@ class SqliteOperator(SQLExecuteQueryOperator): """ Executes sql code in a specific Sqlite database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:SqliteOperator` diff --git a/airflow/providers/trino/operators/trino.py b/airflow/providers/trino/operators/trino.py index 7affe22dc163..7f90bf9947b4 100644 --- a/airflow/providers/trino/operators/trino.py +++ b/airflow/providers/trino/operators/trino.py @@ -32,6 +32,10 @@ class TrinoOperator(SQLExecuteQueryOperator): """ Executes sql code using a specific Trino query Engine. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:TrinoOperator` diff --git a/airflow/providers/vertica/operators/vertica.py b/airflow/providers/vertica/operators/vertica.py index 3ef8fea1fdea..2316f1b074ef 100644 --- a/airflow/providers/vertica/operators/vertica.py +++ b/airflow/providers/vertica/operators/vertica.py @@ -28,6 +28,10 @@ class VerticaOperator(SQLExecuteQueryOperator): """ Executes sql code in a specific Vertica database. + This class is deprecated. + + Please use :class:`airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator`. + :param vertica_conn_id: reference to a specific Vertica database :param sql: the SQL code to be executed as a single string, or a list of str (sql statements), or a reference to a template file. diff --git a/tests/providers/google/cloud/sensors/test_bigquery.py b/tests/providers/google/cloud/sensors/test_bigquery.py index cd6f9b49422f..5fe40227c53a 100644 --- a/tests/providers/google/cloud/sensors/test_bigquery.py +++ b/tests/providers/google/cloud/sensors/test_bigquery.py @@ -330,7 +330,7 @@ class TestBigQueryTableExistencePartitionAsyncSensor: depcrecation_message = ( "Class `BigQueryTableExistencePartitionAsyncSensor` is deprecated and " "will be removed in a future release. " - "Please use `BigQueryTableExistencePartitionSensor` and " + "Please use `BigQueryTablePartitionExistenceSensor` and " "set `deferrable` attribute to `True` instead" ) From 44b4a3755deea23f8ff0eb1316db880b1d64812c Mon Sep 17 00:00:00 2001 From: VladaZakharova <80038284+VladaZakharova@users.noreply.github.com> Date: Thu, 13 Jul 2023 18:37:11 +0200 Subject: [PATCH 184/533] Add deferrable mode to `BeamRunPythonPipelineOperator` (#31471) * Add deferrable mode to BeamRunPythonPipelineOperator * Fix docs * Fix deferrable parameter in init * fix system test --- airflow/providers/apache/beam/hooks/beam.py | 190 +++++++++++++++++- .../providers/apache/beam/operators/beam.py | 113 +++++++++-- airflow/providers/apache/beam/provider.yaml | 5 + .../apache/beam/triggers/__init__.py | 16 ++ .../providers/apache/beam/triggers/beam.py | 116 +++++++++++ .../operators.rst | 29 +++ .../providers/apache/beam/hooks/test_beam.py | 177 +++++++++++++++- .../apache/beam/operators/test_beam.py | 155 ++++++++++++-- .../apache/beam/triggers/__init__.py | 16 ++ .../apache/beam/triggers/test_beam.py | 107 ++++++++++ .../providers/apache/beam/example_python.py | 11 +- .../apache/beam/example_python_async.py | 134 ++++++++++++ 12 files changed, 1023 insertions(+), 46 deletions(-) create mode 100644 airflow/providers/apache/beam/triggers/__init__.py create mode 100644 airflow/providers/apache/beam/triggers/beam.py create mode 100644 tests/providers/apache/beam/triggers/__init__.py create mode 100644 tests/providers/apache/beam/triggers/test_beam.py create mode 100644 tests/system/providers/apache/beam/example_python_async.py diff --git a/airflow/providers/apache/beam/hooks/beam.py b/airflow/providers/apache/beam/hooks/beam.py index 264d9076e0ea..d92ade2050c9 100644 --- a/airflow/providers/apache/beam/hooks/beam.py +++ b/airflow/providers/apache/beam/hooks/beam.py @@ -18,8 +18,10 @@ """This module contains a Apache Beam Hook.""" from __future__ import annotations +import asyncio import contextlib import copy +import functools import json import logging import os @@ -27,8 +29,8 @@ import shlex import shutil import subprocess +import tempfile import textwrap -from tempfile import TemporaryDirectory from typing import Callable from packaging.version import Version @@ -222,7 +224,7 @@ def start_python_pipeline( If a value is passed to this parameter, a new virtual environment has been created with additional packages installed. - You could also install the apache-beam package if it is not installed on your system or you want + You could also install the apache-beam package if it is not installed on your system, or you want to use a different version. :param py_system_site_packages: Whether to include system_site_packages in your virtualenv. See virtualenv documentation for more information. @@ -251,7 +253,7 @@ def start_python_pipeline( """ ) raise AirflowException(warning_invalid_environment) - tmp_dir = exit_stack.enter_context(TemporaryDirectory(prefix="apache-beam-venv")) + tmp_dir = exit_stack.enter_context(tempfile.TemporaryDirectory(prefix="apache-beam-venv")) py_interpreter = prepare_virtualenv( venv_directory=tmp_dir, python_bin=py_interpreter, @@ -381,3 +383,185 @@ def start_go_pipeline_with_binary( command_prefix=command_prefix, process_line_callback=process_line_callback, ) + + +class BeamAsyncHook(BeamHook): + """ + Asynchronous hook for Apache Beam. + :param runner: Runner type. + """ + + def __init__( + self, + runner: str, + ) -> None: + self.runner = runner + super().__init__(runner=self.runner) + + @staticmethod + async def _create_tmp_dir(prefix: str) -> str: + """Helper method to create temporary directory.""" + # Creating separate thread to create temporary directory + loop = asyncio.get_running_loop() + partial_func = functools.partial(tempfile.mkdtemp, prefix=prefix) + tmp_dir = await loop.run_in_executor(None, partial_func) + return tmp_dir + + @staticmethod + async def _cleanup_tmp_dir(tmp_dir: str) -> None: + """ + Helper method to delete temporary directory after finishing work with it. + Is uses `rmtree` method to recursively remove the temporary directory. + """ + shutil.rmtree(tmp_dir) + + async def start_python_pipeline_async( + self, + variables: dict, + py_file: str, + py_options: list[str] | None = None, + py_interpreter: str = "python3", + py_requirements: list[str] | None = None, + py_system_site_packages: bool = False, + ): + """ + Starts Apache Beam python pipeline. + + :param variables: Variables passed to the pipeline. + :param py_file: Path to the python file to execute. + :param py_options: Additional options. + :param py_interpreter: Python version of the Apache Beam pipeline. + If None, this defaults to the python3. + To track python versions supported by beam and related + issues check: https://issues.apache.org/jira/browse/BEAM-1251 + :param py_requirements: Additional python package(s) to install. + If a value is passed to this parameter, a new virtual environment has been created with + additional packages installed. + You could also install the apache-beam package if it is not installed on your system, or you want + to use a different version. + :param py_system_site_packages: Whether to include system_site_packages in your virtualenv. + See virtualenv documentation for more information. + This option is only relevant if the ``py_requirements`` parameter is not None. + """ + if "labels" in variables: + variables["labels"] = [f"{key}={value}" for key, value in variables["labels"].items()] + + # Creating temporary directory + tmp_dir = await self._create_tmp_dir(prefix="apache-beam-venv") + + async with contextlib.AsyncExitStack() as exit_stack: + if py_requirements is not None: + if not py_requirements and not py_system_site_packages: + warning_invalid_environment = textwrap.dedent( + """\ + Invalid method invocation. You have disabled inclusion of system packages and empty + list required for installation, so it is not possible to create a valid virtual + environment. In the virtual environment, apache-beam package must be installed for + your job to be executed. + + To fix this problem: + * install apache-beam on the system, then set parameter py_system_site_packages + to True, + * add apache-beam to the list of required packages in parameter py_requirements. + """ + ) + raise AirflowException(warning_invalid_environment) + + # Pushing asynchronous callback to ensure the cleanup of the temporary + # directory when the asynchronous context is exited + exit_stack.push_async_callback(self._cleanup_tmp_dir, tmp_dir) + + py_interpreter = prepare_virtualenv( + venv_directory=tmp_dir, + python_bin=py_interpreter, + system_site_packages=py_system_site_packages, + requirements=py_requirements, + ) + command_prefix: list[str] = [py_interpreter] + (py_options or []) + [py_file] + beam_version = ( + subprocess.check_output( + [py_interpreter, "-c", "import apache_beam; print(apache_beam.__version__)"] + ) + .decode() + .strip() + ) + self.log.info("Beam version: %s", beam_version) + impersonate_service_account = variables.get("impersonate_service_account") + if impersonate_service_account: + if Version(beam_version) < Version("2.39.0") or True: + raise AirflowException( + "The impersonateServiceAccount option requires Apache Beam 2.39.0 or newer." + ) + return_code = await self.start_pipeline_async( + variables=variables, + command_prefix=command_prefix, + ) + return return_code + + async def start_pipeline_async( + self, + variables: dict, + command_prefix: list[str], + working_directory: str | None = None, + ) -> int: + cmd = command_prefix + [ + f"--runner={self.runner}", + ] + if variables: + cmd.extend(beam_options_to_args(variables)) + return await self.run_beam_command_async( + cmd=cmd, + working_directory=working_directory, + log=self.log, + ) + + async def run_beam_command_async( + self, + cmd: list[str], + log: logging.Logger, + working_directory: str | None = None, + ) -> int: + """ + Function responsible for running pipeline command in subprocess. + + :param cmd: Parts of the command to be run in subprocess + :param working_directory: Working directory + :param log: logger. + """ + cmd_str_representation = " ".join(shlex.quote(c) for c in cmd) + log.info("Running command: %s", cmd_str_representation) + + # Creating a separate asynchronous process + process = await asyncio.create_subprocess_shell( + cmd_str_representation, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + cwd=working_directory, + ) + # Waits for Apache Beam pipeline to complete. + log.info("Start waiting for Apache Beam process to complete.") + + # Creating separate threads for stdout and stderr + stdout_task = asyncio.create_task(self.read_logs(process.stdout)) + stderr_task = asyncio.create_task(self.read_logs(process.stderr)) + + # Waiting for the both tasks to complete + await asyncio.gather(stdout_task, stderr_task) + + # Wait for the process to complete and return return_code + return_code = await process.wait() + log.info("Process exited with return code: %s", return_code) + + if return_code != 0: + raise AirflowException(f"Apache Beam process failed with return code {return_code}") + return return_code + + async def read_logs(self, stream_reader): + while True: + line = await stream_reader.readline() + if not line: + break + decoded_line = line.decode().strip() + self.log.info(decoded_line) diff --git a/airflow/providers/apache/beam/operators/beam.py b/airflow/providers/apache/beam/operators/beam.py index 68e697254779..cc7d217abbea 100644 --- a/airflow/providers/apache/beam/operators/beam.py +++ b/airflow/providers/apache/beam/operators/beam.py @@ -18,6 +18,8 @@ """This module contains Apache Beam operators.""" from __future__ import annotations +import asyncio +import contextlib import copy import os import stat @@ -26,11 +28,13 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import ExitStack from functools import partial -from typing import TYPE_CHECKING, Callable, Sequence +from typing import IO, TYPE_CHECKING, Any, Callable, Sequence from airflow import AirflowException +from airflow.configuration import conf from airflow.models import BaseOperator from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType +from airflow.providers.apache.beam.triggers.beam import BeamPipelineTrigger from airflow.providers.google.cloud.hooks.dataflow import ( DataflowHook, process_line_and_extract_dataflow_job_id_callback, @@ -144,7 +148,7 @@ class BeamBasePipelineOperator(BaseOperator, BeamDataflowMixin, ABC): When defining labels (labels option), you can also provide a dictionary. :param gcp_conn_id: Optional. The connection ID to use connecting to Google Cloud Storage if python file is on GCS. - :param dataflow_config: Dataflow configuration, used when runner type is set to DataflowRunner, + :param dataflow_config: Dataflow's configuration, used when runner type is set to DataflowRunner, (optional) defaults to None. """ @@ -167,7 +171,7 @@ def __init__( self.dataflow_config = DataflowConfiguration(**dataflow_config) else: self.dataflow_config = dataflow_config or DataflowConfiguration() - self.beam_hook: BeamHook | None = None + self.beam_hook: BeamHook self.dataflow_hook: DataflowHook | None = None self.dataflow_job_id: str | None = None @@ -237,8 +241,8 @@ class BeamRunPythonPipelineOperator(BeamBasePipelineOperator): to use a different version. :param py_system_site_packages: Whether to include system_site_packages in your virtualenv. See virtualenv documentation for more information. - This option is only relevant if the ``py_requirements`` parameter is not None. + :param deferrable: Run operator in the deferrable mode: checks for the state using asynchronous calls. """ template_fields: Sequence[str] = ( @@ -264,6 +268,7 @@ def __init__( py_system_site_packages: bool = False, gcp_conn_id: str = "google_cloud_default", dataflow_config: DataflowConfiguration | dict | None = None, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__( @@ -283,35 +288,42 @@ def __init__( self.pipeline_options.setdefault("labels", {}).update( {"airflow-version": "v" + version.replace(".", "-").replace("+", "-")} ) + self.deferrable = deferrable def execute(self, context: Context): """Execute the Apache Beam Pipeline.""" ( - is_dataflow, - dataflow_job_name, - snake_case_pipeline_options, - process_line_callback, + self.is_dataflow, + self.dataflow_job_name, + self.snake_case_pipeline_options, + self.process_line_callback, ) = self._init_pipeline_options(format_pipeline_options=True, job_name_variable_key="job_name") - if not self.beam_hook: raise AirflowException("Beam hook is not defined.") + # Check deferrable parameter passed to the operator + # to determine type of run - asynchronous or synchronous + if self.deferrable: + asyncio.run(self.execute_async(context)) + else: + return self.execute_sync(context) + def execute_sync(self, context: Context): with ExitStack() as exit_stack: if self.py_file.lower().startswith("gs://"): gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id) tmp_gcs_file = exit_stack.enter_context(gcs_hook.provide_file(object_url=self.py_file)) self.py_file = tmp_gcs_file.name - if is_dataflow and self.dataflow_hook: + if self.is_dataflow and self.dataflow_hook: with self.dataflow_hook.provide_authorized_gcloud(): self.beam_hook.start_python_pipeline( - variables=snake_case_pipeline_options, + variables=self.snake_case_pipeline_options, py_file=self.py_file, py_options=self.py_options, py_interpreter=self.py_interpreter, py_requirements=self.py_requirements, py_system_site_packages=self.py_system_site_packages, - process_line_callback=process_line_callback, + process_line_callback=self.process_line_callback, ) DataflowJobLink.persist( self, @@ -320,25 +332,82 @@ def execute(self, context: Context): self.dataflow_config.location, self.dataflow_job_id, ) - if dataflow_job_name and self.dataflow_config.location: - self.dataflow_hook.wait_for_done( - job_name=dataflow_job_name, - location=self.dataflow_config.location, - job_id=self.dataflow_job_id, - multiple_jobs=False, - project_id=self.dataflow_config.project_id, - ) return {"dataflow_job_id": self.dataflow_job_id} else: self.beam_hook.start_python_pipeline( - variables=snake_case_pipeline_options, + variables=self.snake_case_pipeline_options, py_file=self.py_file, py_options=self.py_options, py_interpreter=self.py_interpreter, py_requirements=self.py_requirements, py_system_site_packages=self.py_system_site_packages, - process_line_callback=process_line_callback, + process_line_callback=self.process_line_callback, + ) + + async def execute_async(self, context: Context): + # Creating a new event loop to manage I/O operations asynchronously + loop = asyncio.get_event_loop() + if self.py_file.lower().startswith("gs://"): + gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id) + # Running synchronous `enter_context()` method in a separate + # thread using the default executor `None`. The `run_in_executor()` function returns the + # file object, which is created using gcs function `provide_file()`, asynchronously. + # This means we can perform asynchronous operations with this file. + create_tmp_file_call = gcs_hook.provide_file(object_url=self.py_file) + tmp_gcs_file: IO[str] = await loop.run_in_executor( + None, contextlib.ExitStack().enter_context, create_tmp_file_call + ) + self.py_file = tmp_gcs_file.name + + if self.is_dataflow and self.dataflow_hook: + DataflowJobLink.persist( + self, + context, + self.dataflow_config.project_id, + self.dataflow_config.location, + self.dataflow_job_id, + ) + with self.dataflow_hook.provide_authorized_gcloud(): + self.defer( + trigger=BeamPipelineTrigger( + variables=self.snake_case_pipeline_options, + py_file=self.py_file, + py_options=self.py_options, + py_interpreter=self.py_interpreter, + py_requirements=self.py_requirements, + py_system_site_packages=self.py_system_site_packages, + runner=self.runner, + ), + method_name="execute_complete", ) + else: + self.defer( + trigger=BeamPipelineTrigger( + variables=self.snake_case_pipeline_options, + py_file=self.py_file, + py_options=self.py_options, + py_interpreter=self.py_interpreter, + py_requirements=self.py_requirements, + py_system_site_packages=self.py_system_site_packages, + runner=self.runner, + ), + method_name="execute_complete", + ) + + def execute_complete(self, context: Context, event: dict[str, Any]): + """ + Callback for when the trigger fires - returns immediately. + Relies on trigger to throw an exception, otherwise it assumes execution was + successful. + """ + if event["status"] == "error": + raise AirflowException(event["message"]) + self.log.info( + "%s completed with response %s ", + self.task_id, + event["message"], + ) + return {"dataflow_job_id": self.dataflow_job_id} def on_kill(self) -> None: if self.dataflow_hook and self.dataflow_job_id: diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index d177e251d7c6..dbc59a95af0b 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -63,6 +63,11 @@ hooks: python-modules: - airflow.providers.apache.beam.hooks.beam +triggers: + - integration-name: Apache Beam + python-modules: + - airflow.providers.apache.beam.triggers.beam + additional-extras: - name: google dependencies: diff --git a/airflow/providers/apache/beam/triggers/__init__.py b/airflow/providers/apache/beam/triggers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/airflow/providers/apache/beam/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/apache/beam/triggers/beam.py b/airflow/providers/apache/beam/triggers/beam.py new file mode 100644 index 000000000000..2926a17b4588 --- /dev/null +++ b/airflow/providers/apache/beam/triggers/beam.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Any, AsyncIterator + +from airflow.providers.apache.beam.hooks.beam import BeamAsyncHook +from airflow.triggers.base import BaseTrigger, TriggerEvent + + +class BeamPipelineTrigger(BaseTrigger): + """ + Trigger to perform checking the pipeline status until it reaches terminate state. + + :param variables: Variables passed to the pipeline. + :param py_file: Path to the python file to execute. + :param py_options: Additional options. + :param py_interpreter: Python version of the Apache Beam pipeline. If `None`, this defaults to the + python3. To track python versions supported by beam and related issues + check: https://issues.apache.org/jira/browse/BEAM-1251 + :param py_requirements: Additional python package(s) to install. + If a value is passed to this parameter, a new virtual environment has been created with + additional packages installed. + + You could also install the apache-beam package if it is not installed on your system, or you want + to use a different version. + :param py_system_site_packages: Whether to include system_site_packages in your virtualenv. + See virtualenv documentation for more information. + + This option is only relevant if the ``py_requirements`` parameter is not None. + :param runner: Runner on which pipeline will be run. By default, "DirectRunner" is being used. + Other possible options: DataflowRunner, SparkRunner, FlinkRunner, PortableRunner. + See: :class:`~providers.apache.beam.hooks.beam.BeamRunnerType` + See: https://beam.apache.org/documentation/runners/capability-matrix/ + """ + + def __init__( + self, + variables: dict, + py_file: str, + py_options: list[str] | None = None, + py_interpreter: str = "python3", + py_requirements: list[str] | None = None, + py_system_site_packages: bool = False, + runner: str = "DirectRunner", + ): + super().__init__() + self.variables = variables + self.py_file = py_file + self.py_options = py_options + self.py_interpreter = py_interpreter + self.py_requirements = py_requirements + self.py_system_site_packages = py_system_site_packages + self.runner = runner + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serializes BeamPipelineTrigger arguments and classpath.""" + return ( + "airflow.providers.apache.beam.triggers.beam.BeamPipelineTrigger", + { + "variables": self.variables, + "py_file": self.py_file, + "py_options": self.py_options, + "py_interpreter": self.py_interpreter, + "py_requirements": self.py_requirements, + "py_system_site_packages": self.py_system_site_packages, + "runner": self.runner, + }, + ) + + async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] + """Gets current pipeline status and yields a TriggerEvent.""" + hook = self._get_async_hook() + while True: + try: + return_code = await hook.start_python_pipeline_async( + variables=self.variables, + py_file=self.py_file, + py_options=self.py_options, + py_interpreter=self.py_interpreter, + py_requirements=self.py_requirements, + py_system_site_packages=self.py_system_site_packages, + ) + if return_code == 0: + yield TriggerEvent( + { + "status": "success", + "message": "Pipeline has finished SUCCESSFULLY", + } + ) + return + else: + yield TriggerEvent({"status": "error", "message": "Operation failed"}) + return + + except Exception as e: + self.log.exception("Exception occurred while checking for pipeline state") + yield TriggerEvent({"status": "error", "message": str(e)}) + return + + def _get_async_hook(self) -> BeamAsyncHook: + return BeamAsyncHook(runner=self.runner) diff --git a/docs/apache-airflow-providers-apache-beam/operators.rst b/docs/apache-airflow-providers-apache-beam/operators.rst index fedf6b02655e..f7487fb26460 100644 --- a/docs/apache-airflow-providers-apache-beam/operators.rst +++ b/docs/apache-airflow-providers-apache-beam/operators.rst @@ -61,6 +61,23 @@ Python Pipelines with DirectRunner :start-after: [START howto_operator_start_python_direct_runner_pipeline_gcs_file] :end-before: [END howto_operator_start_python_direct_runner_pipeline_gcs_file] +You can use deferrable mode for this action in order to run the operator asynchronously. It will give you a +possibility to free up the worker when it knows it has to wait, and hand off the job of resuming Operator to a Trigger. +As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a +lot less resources wasted on idle Operators or Sensors: + +.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_start_python_direct_runner_pipeline_local_file_async] + :end-before: [END howto_operator_start_python_direct_runner_pipeline_local_file_async] + +.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_start_python_direct_runner_pipeline_gcs_file_async] + :end-before: [END howto_operator_start_python_direct_runner_pipeline_gcs_file_async] + Python Pipelines with DataflowRunner ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -76,6 +93,18 @@ Python Pipelines with DataflowRunner :start-after: [START howto_operator_start_python_dataflow_runner_pipeline_async_gcs_file] :end-before: [END howto_operator_start_python_dataflow_runner_pipeline_async_gcs_file] + +You can use deferrable mode for this action in order to run the operator asynchronously. It will give you a +possibility to free up the worker when it knows it has to wait, and hand off the job of resuming Operator to a Trigger. +As a result, while it is suspended (deferred), it is not taking up a worker slot and your cluster will have a +lot less resources wasted on idle Operators or Sensors: + +.. exampleinclude:: /../../tests/system/providers/apache/beam/example_python_async.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_start_python_dataflow_runner_pipeline_gcs_file_async] + :end-before: [END howto_operator_start_python_dataflow_runner_pipeline_gcs_file_async] + | | diff --git a/tests/providers/apache/beam/hooks/test_beam.py b/tests/providers/apache/beam/hooks/test_beam.py index 3e3e895cf900..1cfe71121ca5 100644 --- a/tests/providers/apache/beam/hooks/test_beam.py +++ b/tests/providers/apache/beam/hooks/test_beam.py @@ -21,12 +21,17 @@ import re import subprocess from unittest import mock -from unittest.mock import ANY, MagicMock +from unittest.mock import ANY, AsyncMock, MagicMock import pytest from airflow.exceptions import AirflowException -from airflow.providers.apache.beam.hooks.beam import BeamHook, beam_options_to_args, run_beam_command +from airflow.providers.apache.beam.hooks.beam import ( + BeamAsyncHook, + BeamHook, + beam_options_to_args, + run_beam_command, +) PY_FILE = "apache_beam.examples.wordcount" JAR_FILE = "unitest.jar" @@ -391,3 +396,171 @@ class TestBeamOptionsToArgs: def test_beam_options_to_args(self, options, expected_args): args = beam_options_to_args(options) assert args == expected_args + + +class TestBeamAsyncHook: + @pytest.mark.asyncio + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook.run_beam_command_async") + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook._create_tmp_dir") + async def test_start_python_pipeline(self, mock_create_dir, mock_runner): + hook = BeamAsyncHook(runner=DEFAULT_RUNNER) + mock_create_dir.return_value = AsyncMock() + mock_runner.return_value = 0 + + await hook.start_python_pipeline_async( + variables=copy.deepcopy(BEAM_VARIABLES_PY), + py_file=PY_FILE, + py_options=PY_OPTIONS, + ) + + expected_cmd = [ + "python3", + "-m", + PY_FILE, + f"--runner={DEFAULT_RUNNER}", + "--output=gs://test/output", + "--labels=foo=bar", + ] + mock_create_dir.assert_called_once() + mock_runner.assert_called_once_with( + cmd=expected_cmd, + working_directory=None, + log=ANY, + ) + + @pytest.mark.asyncio + @mock.patch("airflow.providers.apache.beam.hooks.beam.subprocess.check_output", return_value=b"2.35.0") + async def test_start_python_pipeline_unsupported_option(self, mock_check_output): + hook = BeamAsyncHook(runner=DEFAULT_RUNNER) + + with pytest.raises( + AirflowException, + match=re.escape("The impersonateServiceAccount option requires Apache Beam 2.39.0 or newer."), + ): + await hook.start_python_pipeline_async( + variables={ + "impersonate_service_account": "test@impersonation.com", + }, + py_file="/tmp/file.py", + py_options=["-m"], + py_interpreter="python3", + py_requirements=None, + py_system_site_packages=False, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "py_interpreter", + [ + pytest.param("python", id="default python"), + pytest.param("python2", id="major python version 2.x"), + pytest.param("python3", id="major python version 3.x"), + pytest.param("python3.6", id="major.minor python version"), + ], + ) + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook.run_beam_command_async") + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook._create_tmp_dir") + @mock.patch("airflow.providers.apache.beam.hooks.beam.subprocess.check_output", return_value=b"2.39.0") + async def test_start_python_pipeline_with_custom_interpreter( + self, mock_check_output, mock_create_dir, mock_runner, py_interpreter + ): + hook = BeamAsyncHook(runner=DEFAULT_RUNNER) + mock_create_dir.return_value = AsyncMock() + mock_runner.return_value = 0 + + await hook.start_python_pipeline_async( + variables=copy.deepcopy(BEAM_VARIABLES_PY), + py_file=PY_FILE, + py_options=PY_OPTIONS, + py_interpreter=py_interpreter, + ) + + expected_cmd = [ + py_interpreter, + "-m", + PY_FILE, + f"--runner={DEFAULT_RUNNER}", + "--output=gs://test/output", + "--labels=foo=bar", + ] + mock_runner.assert_called_once_with( + cmd=expected_cmd, + working_directory=None, + log=ANY, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "current_py_requirements, current_py_system_site_packages", + [ + pytest.param("foo-bar", False, id="requirements without system site-packages"), + pytest.param("foo-bar", True, id="requirements with system site-packages"), + pytest.param([], True, id="only system site-packages"), + ], + ) + @mock.patch(BEAM_STRING.format("prepare_virtualenv")) + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook.run_beam_command_async") + @mock.patch("airflow.providers.apache.beam.hooks.beam.subprocess.check_output", return_value=b"2.39.0") + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook._create_tmp_dir") + @mock.patch("airflow.providers.apache.beam.hooks.beam.BeamAsyncHook._cleanup_tmp_dir") + async def test_start_python_pipeline_with_non_empty_py_requirements_and_without_system_packages( + self, + mock_cleanup_dir, + mock_create_dir, + mock_check_output, + mock_runner, + mock_virtualenv, + current_py_requirements, + current_py_system_site_packages, + ): + hook = BeamAsyncHook(runner=DEFAULT_RUNNER) + mock_create_dir.return_value = AsyncMock() + mock_virtualenv.return_value = "/dummy_dir/bin/python" + mock_cleanup_dir.return_value = AsyncMock() + + await hook.start_python_pipeline_async( + variables=copy.deepcopy(BEAM_VARIABLES_PY), + py_file=PY_FILE, + py_options=PY_OPTIONS, + py_requirements=current_py_requirements, + py_system_site_packages=current_py_system_site_packages, + ) + + expected_cmd = [ + "/dummy_dir/bin/python", + "-m", + PY_FILE, + f"--runner={DEFAULT_RUNNER}", + "--output=gs://test/output", + "--labels=foo=bar", + ] + mock_runner.assert_called_once_with( + cmd=expected_cmd, + working_directory=None, + log=ANY, + ) + mock_virtualenv.assert_called_once_with( + venv_directory=mock.ANY, + python_bin="python3", + system_site_packages=current_py_system_site_packages, + requirements=current_py_requirements, + ) + mock_create_dir.assert_called_once() + + @pytest.mark.asyncio + @mock.patch(BEAM_STRING.format("run_beam_command")) + @mock.patch("airflow.providers.apache.beam.hooks.beam.subprocess.check_output", return_value=b"2.39.0") + async def test_start_python_pipeline_with_empty_py_requirements_and_without_system_packages( + self, mock_check_output, mock_runner + ): + hook = BeamAsyncHook(runner=DEFAULT_RUNNER) + + with pytest.raises(AirflowException, match=r"Invalid method invocation\."): + await hook.start_python_pipeline_async( + variables=copy.deepcopy(BEAM_VARIABLES_PY), + py_file=PY_FILE, + py_options=PY_OPTIONS, + py_requirements=[], + ) + + mock_runner.assert_not_called() diff --git a/tests/providers/apache/beam/operators/test_beam.py b/tests/providers/apache/beam/operators/test_beam.py index 9bdf016f48ab..c5949dd8686e 100644 --- a/tests/providers/apache/beam/operators/test_beam.py +++ b/tests/providers/apache/beam/operators/test_beam.py @@ -22,11 +22,13 @@ import pytest +from airflow.exceptions import AirflowException, TaskDeferred from airflow.providers.apache.beam.operators.beam import ( BeamRunGoPipelineOperator, BeamRunJavaPipelineOperator, BeamRunPythonPipelineOperator, ) +from airflow.providers.apache.beam.triggers.beam import BeamPipelineTrigger from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration from airflow.version import version @@ -150,13 +152,6 @@ def test_exec_dataflow_runner(self, gcs_hook, dataflow_hook_mock, beam_hook_mock py_system_site_packages=False, process_line_callback=mock.ANY, ) - dataflow_hook_mock.return_value.wait_for_done.assert_called_once_with( - job_id=self.operator.dataflow_job_id, - job_name=job_name, - location="us-central1", - multiple_jobs=False, - project_id=dataflow_config.project_id, - ) dataflow_hook_mock.return_value.provide_authorized_gcloud.assert_called_once_with() @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowJobLink.persist") @@ -268,13 +263,6 @@ def test_exec_dataflow_runner(self, gcs_hook, dataflow_hook_mock, beam_hook_mock job_class=JOB_CLASS, process_line_callback=mock.ANY, ) - dataflow_hook_mock.return_value.wait_for_done.assert_called_once_with( - job_id=self.operator.dataflow_job_id, - job_name=job_name, - location="us-central1", - multiple_jobs=False, - project_id=dataflow_hook_mock.return_value.project_id, - ) @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowJobLink.persist") @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") @@ -696,3 +684,142 @@ def test_on_kill_direct_runner(self, _, dataflow_mock, __): self.operator.execute(None) self.operator.on_kill() dataflow_cancel_job.assert_not_called() + + +class TestBeamRunPythonPipelineOperatorAsync: + def setup_method(self): + self.operator = BeamRunPythonPipelineOperator( + task_id=TASK_ID, + py_file=PY_FILE, + py_options=PY_OPTIONS, + default_pipeline_options=DEFAULT_OPTIONS, + pipeline_options=ADDITIONAL_OPTIONS, + deferrable=True, + ) + + def test_init(self): + """Test BeamRunPythonPipelineOperator instance is properly initialized.""" + assert self.operator.task_id == TASK_ID + assert self.operator.py_file == PY_FILE + assert self.operator.runner == DEFAULT_RUNNER + assert self.operator.py_options == PY_OPTIONS + assert self.operator.py_interpreter == PY_INTERPRETER + assert self.operator.default_pipeline_options == DEFAULT_OPTIONS + assert self.operator.pipeline_options == EXPECTED_ADDITIONAL_OPTIONS + + @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.GCSHook") + def test_async_execute_should_execute_successfully(self, gcs_hook, beam_hook_mock): + """ + Asserts that a task is deferred and the BeamPipelineTrigger will be fired + when the BeamRunPythonPipelineOperator is executed in deferrable mode when deferrable=True. + """ + with pytest.raises(TaskDeferred) as exc: + self.operator.execute(context=mock.MagicMock()) + + assert isinstance(exc.value.trigger, BeamPipelineTrigger), "Trigger is not a BeamPipelineTrigger" + + def test_async_execute_should_throw_exception(self): + """Tests that an AirflowException is raised in case of error event""" + + with pytest.raises(AirflowException): + self.operator.execute_complete( + context=mock.MagicMock(), event={"status": "error", "message": "test failure message"} + ) + + def test_async_execute_logging_should_execute_successfully(self): + """Asserts that logging occurs as expected""" + + with mock.patch.object(self.operator.log, "info") as mock_log_info: + self.operator.execute_complete( + context=mock.MagicMock(), + event={"status": "success", "message": "Pipeline has finished SUCCESSFULLY"}, + ) + mock_log_info.assert_called_with( + "%s completed with response %s ", TASK_ID, "Pipeline has finished SUCCESSFULLY" + ) + + @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.GCSHook") + def test_async_execute_direct_runner(self, gcs_hook, beam_hook_mock): + """ + Test BeamHook is created and the right args are passed to + start_python_workflow when executing direct runner. + """ + gcs_provide_file = gcs_hook.return_value.provide_file + with pytest.raises(TaskDeferred): + self.operator.execute(context=mock.MagicMock()) + beam_hook_mock.assert_called_once_with(runner=DEFAULT_RUNNER) + gcs_provide_file.assert_called_once_with(object_url=PY_FILE) + + @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowJobLink.persist") + @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.GCSHook") + def test_exec_dataflow_runner(self, gcs_hook, dataflow_hook_mock, beam_hook_mock, persist_link_mock): + """ + Test DataflowHook is created and the right args are passed to + start_python_dataflow when executing Dataflow runner. + """ + + dataflow_config = DataflowConfiguration(impersonation_chain=TEST_IMPERSONATION_ACCOUNT) + self.operator.runner = "DataflowRunner" + self.operator.dataflow_config = dataflow_config + gcs_provide_file = gcs_hook.return_value.provide_file + magic_mock = mock.MagicMock() + with pytest.raises(TaskDeferred): + self.operator.execute(context=magic_mock) + + job_name = dataflow_hook_mock.build_dataflow_job_name.return_value + dataflow_hook_mock.assert_called_once_with( + gcp_conn_id=dataflow_config.gcp_conn_id, + poll_sleep=dataflow_config.poll_sleep, + impersonation_chain=dataflow_config.impersonation_chain, + drain_pipeline=dataflow_config.drain_pipeline, + cancel_timeout=dataflow_config.cancel_timeout, + wait_until_finished=dataflow_config.wait_until_finished, + ) + expected_options = { + "project": dataflow_hook_mock.return_value.project_id, + "job_name": job_name, + "staging_location": "gs://test/staging", + "output": "gs://test/output", + "labels": {"foo": "bar", "airflow-version": TEST_VERSION}, + "region": "us-central1", + "impersonate_service_account": TEST_IMPERSONATION_ACCOUNT, + } + gcs_provide_file.assert_called_once_with(object_url=PY_FILE) + persist_link_mock.assert_called_once_with( + self.operator, + magic_mock, + expected_options["project"], + expected_options["region"], + self.operator.dataflow_job_id, + ) + beam_hook_mock.return_value.start_python_pipeline.assert_not_called() + dataflow_hook_mock.return_value.provide_authorized_gcloud.assert_called_once_with() + + @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowJobLink.persist") + @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.GCSHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowHook") + def test_on_kill_dataflow_runner(self, dataflow_hook_mock, _, __, ___): + self.operator.runner = "DataflowRunner" + dataflow_cancel_job = dataflow_hook_mock.return_value.cancel_job + with pytest.raises(TaskDeferred): + self.operator.execute(context=mock.MagicMock()) + self.operator.dataflow_job_id = JOB_ID + self.operator.on_kill() + dataflow_cancel_job.assert_called_once_with( + job_id=JOB_ID, project_id=self.operator.dataflow_config.project_id + ) + + @mock.patch("airflow.providers.apache.beam.operators.beam.BeamHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.DataflowHook") + @mock.patch("airflow.providers.apache.beam.operators.beam.GCSHook") + def test_on_kill_direct_runner(self, _, dataflow_mock, __): + dataflow_cancel_job = dataflow_mock.return_value.cancel_job + with pytest.raises(TaskDeferred): + self.operator.execute(mock.MagicMock()) + self.operator.on_kill() + dataflow_cancel_job.assert_not_called() diff --git a/tests/providers/apache/beam/triggers/__init__.py b/tests/providers/apache/beam/triggers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/providers/apache/beam/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/apache/beam/triggers/test_beam.py b/tests/providers/apache/beam/triggers/test_beam.py new file mode 100644 index 000000000000..ce22dda2152c --- /dev/null +++ b/tests/providers/apache/beam/triggers/test_beam.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.apache.beam.triggers.beam import BeamPipelineTrigger +from airflow.triggers.base import TriggerEvent + +HOOK_STATUS_STR = "airflow.providers.apache.beam.hooks.beam.BeamAsyncHook.start_python_pipeline_async" +CLASSPATH = "airflow.providers.apache.beam.triggers.beam.BeamPipelineTrigger" + +TASK_ID = "test_task" +LOCATION = "test-location" +INSTANCE_NAME = "airflow-test-instance" +INSTANCE = {"type": "BASIC", "displayName": INSTANCE_NAME} +PROJECT_ID = "test_project_id" +TEST_VARIABLES = {"output": "gs://bucket_test/output", "labels": {"airflow-version": "v2-7-0-dev0"}} +TEST_PY_FILE = "apache_beam.examples.wordcount" +TEST_PY_OPTIONS = [] +TEST_PY_INTERPRETER = "python3" +TEST_PY_REQUIREMENTS = ["apache-beam[gcp]==2.46.0"] +TEST_PY_PACKAGES = False +TEST_RUNNER = "DirectRunner" + + +@pytest.fixture +def trigger(): + return BeamPipelineTrigger( + variables=TEST_VARIABLES, + py_file=TEST_PY_FILE, + py_options=TEST_PY_OPTIONS, + py_interpreter=TEST_PY_INTERPRETER, + py_requirements=TEST_PY_REQUIREMENTS, + py_system_site_packages=TEST_PY_PACKAGES, + runner=TEST_RUNNER, + ) + + +class TestBeamPipelineTrigger: + def test_beam_trigger_serialization_should_execute_successfully(self, trigger): + """ + Asserts that the BeamPipelineTrigger correctly serializes its arguments + and classpath. + """ + classpath, kwargs = trigger.serialize() + assert classpath == CLASSPATH + assert kwargs == { + "variables": TEST_VARIABLES, + "py_file": TEST_PY_FILE, + "py_options": TEST_PY_OPTIONS, + "py_interpreter": TEST_PY_INTERPRETER, + "py_requirements": TEST_PY_REQUIREMENTS, + "py_system_site_packages": TEST_PY_PACKAGES, + "runner": TEST_RUNNER, + } + + @pytest.mark.asyncio + @mock.patch(HOOK_STATUS_STR) + async def test_beam_trigger_on_success_should_execute_successfully(self, mock_pipeline_status, trigger): + """ + Tests the BeamPipelineTrigger only fires once the job execution reaches a successful state. + """ + mock_pipeline_status.return_value = 0 + generator = trigger.run() + actual = await generator.asend(None) + assert TriggerEvent({"status": "success", "message": "Pipeline has finished SUCCESSFULLY"}) == actual + + @pytest.mark.asyncio + @mock.patch(HOOK_STATUS_STR) + async def test_beam_trigger_error_should_execute_successfully(self, mock_pipeline_status, trigger): + """ + Test that BeamPipelineTrigger fires the correct event in case of an error. + """ + mock_pipeline_status.return_value = 1 + + generator = trigger.run() + actual = await generator.asend(None) + assert TriggerEvent({"status": "error", "message": "Operation failed"}) == actual + + @pytest.mark.asyncio + @mock.patch(HOOK_STATUS_STR) + async def test_beam_trigger_exception_should_execute_successfully(self, mock_pipeline_status, trigger): + """ + Test that BeamPipelineTrigger fires the correct event in case of an error. + """ + mock_pipeline_status.side_effect = Exception("Test exception") + + generator = trigger.run() + actual = await generator.asend(None) + assert TriggerEvent({"status": "error", "message": "Test exception"}) == actual diff --git a/tests/system/providers/apache/beam/example_python.py b/tests/system/providers/apache/beam/example_python.py index 50c469c73df6..929844ec5464 100644 --- a/tests/system/providers/apache/beam/example_python.py +++ b/tests/system/providers/apache/beam/example_python.py @@ -47,7 +47,7 @@ task_id="start_python_pipeline_local_direct_runner", py_file="apache_beam.examples.wordcount", py_options=["-m"], - py_requirements=["apache-beam[gcp]==2.26.0"], + py_requirements=["apache-beam[gcp]==2.46.0"], py_interpreter="python3", py_system_site_packages=False, ) @@ -59,7 +59,7 @@ py_file=GCS_PYTHON, py_options=[], pipeline_options={"output": GCS_OUTPUT}, - py_requirements=["apache-beam[gcp]==2.26.0"], + py_requirements=["apache-beam[gcp]==2.46.0"], py_interpreter="python3", py_system_site_packages=False, ) @@ -76,7 +76,7 @@ "output": GCS_OUTPUT, }, py_options=[], - py_requirements=["apache-beam[gcp]==2.26.0"], + py_requirements=["apache-beam[gcp]==2.46.0"], py_interpreter="python3", py_system_site_packages=False, dataflow_config=DataflowConfiguration( @@ -90,7 +90,7 @@ py_file="apache_beam.examples.wordcount", runner="SparkRunner", py_options=["-m"], - py_requirements=["apache-beam[gcp]==2.26.0"], + py_requirements=["apache-beam[gcp]==2.46.0"], py_interpreter="python3", py_system_site_packages=False, ) @@ -103,7 +103,7 @@ pipeline_options={ "output": "/tmp/start_python_pipeline_local_flink_runner", }, - py_requirements=["apache-beam[gcp]==2.26.0"], + py_requirements=["apache-beam[gcp]==2.46.0"], py_interpreter="python3", py_system_site_packages=False, ) @@ -113,6 +113,7 @@ start_python_pipeline_local_direct_runner, start_python_pipeline_direct_runner, ] + >> start_python_pipeline_dataflow_runner >> start_python_pipeline_local_flink_runner >> start_python_pipeline_local_spark_runner ) diff --git a/tests/system/providers/apache/beam/example_python_async.py b/tests/system/providers/apache/beam/example_python_async.py new file mode 100644 index 000000000000..520e8adc4cdf --- /dev/null +++ b/tests/system/providers/apache/beam/example_python_async.py @@ -0,0 +1,134 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG for Apache Beam operators +""" +from __future__ import annotations + +from airflow import models +from airflow.providers.apache.beam.operators.beam import BeamRunPythonPipelineOperator +from airflow.providers.google.cloud.operators.dataflow import DataflowConfiguration +from tests.system.providers.apache.beam.utils import ( + DEFAULT_ARGS, + GCP_PROJECT_ID, + GCS_OUTPUT, + GCS_PYTHON, + GCS_STAGING, + GCS_TMP, + START_DATE, +) + +with models.DAG( + dag_id="dataflow_native_python_async", + start_date=START_DATE, + schedule=None, # Override to match your needs + catchup=False, + default_args=DEFAULT_ARGS, + tags=["example"], +) as dag: + + # [START howto_operator_start_python_direct_runner_pipeline_local_file_async] + start_python_pipeline_local_direct_runner = BeamRunPythonPipelineOperator( + task_id="start_python_pipeline_local_direct_runner", + py_file="apache_beam.examples.wordcount", + py_options=["-m"], + py_requirements=["apache-beam[gcp]==2.46.0"], + py_interpreter="python3", + py_system_site_packages=False, + deferrable=True, + ) + # [END howto_operator_start_python_direct_runner_pipeline_local_file_async] + + # [START howto_operator_start_python_direct_runner_pipeline_gcs_file_async] + start_python_pipeline_direct_runner = BeamRunPythonPipelineOperator( + task_id="start_python_pipeline_direct_runner", + py_file=GCS_PYTHON, + py_options=[], + pipeline_options={"output": GCS_OUTPUT}, + py_requirements=["apache-beam[gcp]==2.46.0"], + py_interpreter="python3", + py_system_site_packages=False, + deferrable=True, + ) + # [END howto_operator_start_python_direct_runner_pipeline_gcs_file_async] + + # [START howto_operator_start_python_dataflow_runner_pipeline_gcs_file_async] + start_python_pipeline_dataflow_runner = BeamRunPythonPipelineOperator( + task_id="start_python_pipeline_dataflow_runner", + runner="DataflowRunner", + py_file=GCS_PYTHON, + pipeline_options={ + "tempLocation": GCS_TMP, + "stagingLocation": GCS_STAGING, + "output": GCS_OUTPUT, + }, + py_options=[], + py_requirements=["apache-beam[gcp]==2.46.0"], + py_interpreter="python3", + py_system_site_packages=False, + dataflow_config=DataflowConfiguration( + job_name="{{task.task_id}}", project_id=GCP_PROJECT_ID, location="us-central1" + ), + deferrable=True, + ) + # [END howto_operator_start_python_dataflow_runner_pipeline_gcs_file_async] + + # [START howto_operator_start_python_pipeline_local_runner_spark_runner_async] + start_python_pipeline_local_spark_runner = BeamRunPythonPipelineOperator( + task_id="start_python_pipeline_local_spark_runner", + py_file="apache_beam.examples.wordcount", + runner="SparkRunner", + py_options=["-m"], + py_requirements=["apache-beam[gcp]==2.46.0"], + py_interpreter="python3", + py_system_site_packages=False, + deferrable=True, + ) + # [END howto_operator_start_python_pipeline_local_runner_spark_runner_async] + + # [START howto_operator_start_python_pipeline_local_runner_flink_runner_async] + start_python_pipeline_local_flink_runner = BeamRunPythonPipelineOperator( + task_id="start_python_pipeline_local_flink_runner", + py_file="apache_beam.examples.wordcount", + runner="FlinkRunner", + py_options=["-m"], + pipeline_options={ + "output": "/tmp/start_python_pipeline_local_flink_runner", + }, + py_requirements=["apache-beam[gcp]==2.46.0"], + py_interpreter="python3", + py_system_site_packages=False, + deferrable=True, + ) + # [END howto_operator_start_python_pipeline_local_runner_flink_runner_async] + + ( + [ + start_python_pipeline_local_direct_runner, + start_python_pipeline_direct_runner, + ] + >> start_python_pipeline_dataflow_runner + >> start_python_pipeline_local_flink_runner + >> start_python_pipeline_local_spark_runner + ) + + +from tests.system.utils import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From a58dcbb41d0c54d70f1d1a18e08935d61fa9c25f Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Fri, 14 Jul 2023 05:26:51 +0800 Subject: [PATCH 185/533] Add setup/teardown info to grid/graph view (#32578) * Add setup/teardown info to grid/graph view * Spellcheck * Refactor setup/teardown edge marking --------- Co-authored-by: Jed Cunningham --- airflow/utils/dag_edges.py | 5 ++++ .../www/static/js/components/Graph/Edge.tsx | 2 ++ airflow/www/static/js/dag/InstanceTooltip.tsx | 3 ++ .../www/static/js/dag/details/graph/Node.tsx | 29 +++++++++++++++---- .../js/dag/details/taskInstance/Details.tsx | 10 +++++++ airflow/www/static/js/types/index.ts | 2 ++ airflow/www/static/js/utils/graph.ts | 1 + airflow/www/views.py | 7 +++++ tests/utils/test_task_group.py | 28 ++++++++++++++++++ 9 files changed, 82 insertions(+), 5 deletions(-) diff --git a/airflow/utils/dag_edges.py b/airflow/utils/dag_edges.py index 470b4cdaab49..7e31762ad91b 100644 --- a/airflow/utils/dag_edges.py +++ b/airflow/utils/dag_edges.py @@ -101,6 +101,7 @@ def collect_edges(task_group): # Collect all the edges between individual tasks edges = set() + setup_teardown_edges = set() tasks_to_trace: list[Operator] = dag.roots while tasks_to_trace: @@ -108,6 +109,8 @@ def collect_edges(task_group): for task in tasks_to_trace: for child in task.downstream_list: edge = (task.task_id, child.task_id) + if task.is_setup and child.is_teardown: + setup_teardown_edges.add(edge) if edge in edges: continue edges.add(edge) @@ -120,6 +123,8 @@ def collect_edges(task_group): for source_id, target_id in sorted(edges.union(edges_to_add) - edges_to_skip): record = {"source_id": source_id, "target_id": target_id} label = dag.get_edge_info(source_id, target_id).get("label") + if (source_id, target_id) in setup_teardown_edges: + record["is_setup_teardown"] = True if label: record["label"] = label result.append(record) diff --git a/airflow/www/static/js/components/Graph/Edge.tsx b/airflow/www/static/js/components/Graph/Edge.tsx index 0205bf49cd3f..910d2d3e436a 100644 --- a/airflow/www/static/js/components/Graph/Edge.tsx +++ b/airflow/www/static/js/components/Graph/Edge.tsx @@ -35,6 +35,7 @@ interface EdgeProps { id: string; labels?: ElkLabel[]; layoutOptions?: LayoutOptions; + isSetupTeardown?: boolean; }; }; } @@ -63,6 +64,7 @@ const CustomEdge = ({ data }: EdgeProps) => { x={(d) => d.x || 0} y={(d) => d.y || 0} data={[s.startPoint, ...(s.bendPoints || []), s.endPoint]} + strokeDasharray={rest.isSetupTeardown ? "10,5" : undefined} /> ))} diff --git a/airflow/www/static/js/dag/InstanceTooltip.tsx b/airflow/www/static/js/dag/InstanceTooltip.tsx index 4765927abb79..0503cb95085d 100644 --- a/airflow/www/static/js/dag/InstanceTooltip.tsx +++ b/airflow/www/static/js/dag/InstanceTooltip.tsx @@ -62,6 +62,9 @@ const InstanceTooltip = ({ return ( Task Id: {taskId} + {!!group.setupTeardownType && ( + Type: {group.setupTeardownType} + )} {group.tooltip && {group.tooltip}} {isMapped && totalTasks > 0 && ( diff --git a/airflow/www/static/js/dag/details/graph/Node.tsx b/airflow/www/static/js/dag/details/graph/Node.tsx index 8d160b041739..3655dd27c7e9 100644 --- a/airflow/www/static/js/dag/details/graph/Node.tsx +++ b/airflow/www/static/js/dag/details/graph/Node.tsx @@ -18,7 +18,7 @@ */ import React from "react"; -import { Box, Text, Flex } from "@chakra-ui/react"; +import { Box, Text, Flex, useTheme } from "@chakra-ui/react"; import { Handle, NodeProps, Position } from "reactflow"; import { SimpleStatus } from "src/dag/StatusBox"; @@ -28,6 +28,10 @@ import { getGroupAndMapSummary, hoverDelay } from "src/utils"; import Tooltip from "src/components/Tooltip"; import InstanceTooltip from "src/dag/InstanceTooltip"; import { useContainerRef } from "src/context/containerRef"; +import { + MdOutlineArrowCircleUp, + MdOutlineArrowCircleDown, +} from "react-icons/md"; export interface CustomNodeProps { label: string; @@ -42,6 +46,7 @@ export interface CustomNodeProps { onToggleCollapse: () => void; isOpen?: boolean; isActive?: boolean; + setupTeardownType?: "setup" | "teardown"; } export const BaseNode = ({ @@ -58,8 +63,10 @@ export const BaseNode = ({ onToggleCollapse, isOpen, isActive, + setupTeardownType, }, }: NodeProps) => { + const { colors } = useTheme(); const { onSelect } = useSelection(); const containerRef = useContainerRef(); @@ -114,10 +121,22 @@ export const BaseNode = ({ p={2} flexWrap="wrap" > - - - {taskName} - + + + + {taskName} + + {setupTeardownType === "setup" && ( + + )} + {setupTeardownType === "teardown" && ( + + )} + {!!instance && instance.state && ( diff --git a/airflow/www/static/js/dag/details/taskInstance/Details.tsx b/airflow/www/static/js/dag/details/taskInstance/Details.tsx index 58964d3ec121..5cf1c5cfb99b 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Details.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/Details.tsx @@ -133,6 +133,16 @@ const Details = ({ instance, group, dagId }: Props) => { + {!!group.setupTeardownType && ( +
    + + + + )} {mappedStates && totalTasks > 0 && ( + + + +
    Type + + {group.setupTeardownType} + +
    diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts index 08929d33c400..09a7ca4f9bf7 100644 --- a/airflow/www/static/js/types/index.ts +++ b/airflow/www/static/js/types/index.ts @@ -99,6 +99,7 @@ interface Task { operator?: string; hasOutletDatasets?: boolean; triggerRule?: API.TriggerRule; + setupTeardownType?: "setup" | "teardown"; } type RunOrdering = ( @@ -137,6 +138,7 @@ export interface WebserverEdge { label?: string; sourceId: string; targetId: string; + isSetupTeardown?: boolean; } interface DatasetListItem extends API.Dataset { diff --git a/airflow/www/static/js/utils/graph.ts b/airflow/www/static/js/utils/graph.ts index 2dd970bf8c10..abcf0c548257 100644 --- a/airflow/www/static/js/utils/graph.ts +++ b/airflow/www/static/js/utils/graph.ts @@ -169,6 +169,7 @@ const generateGraph = ({ id: `${e.sourceId}-${e.targetId}`, sources: [e.sourceId], targets: [e.targetId], + isSetupTeardown: e.isSetupTeardown, labels: e.label ? [ { diff --git a/airflow/www/views.py b/airflow/www/views.py index db618989ff72..762db4759a02 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -383,6 +383,12 @@ def set_overall_state(record): else: instances = list(map(_get_summary, grouped_tis.get(item.task_id, []))) + setup_teardown_type = {} + if item.is_setup is True: + setup_teardown_type["setupTeardownType"] = "setup" + elif item.is_teardown is True: + setup_teardown_type["setupTeardownType"] = "teardown" + return { "id": item.task_id, "instances": instances, @@ -392,6 +398,7 @@ def set_overall_state(record): "has_outlet_datasets": any(isinstance(i, Dataset) for i in (item.outlets or [])), "operator": item.operator_name, "trigger_rule": item.trigger_rule, + **setup_teardown_type, } # Task Group diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index eb1e77e93f29..c2baec25d3e3 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -536,6 +536,34 @@ def test_dag_edges(): ] +def test_dag_edges_setup_teardown(): + execution_date = pendulum.parse("20200101") + with DAG("test_dag_edges", start_date=execution_date) as dag: + setup1 = EmptyOperator(task_id="setup1").as_setup() + teardown1 = EmptyOperator(task_id="teardown1").as_teardown() + + with setup1 >> teardown1: + EmptyOperator(task_id="task1") + + with TaskGroup("group_a"): + setup2 = EmptyOperator(task_id="setup2").as_setup() + teardown2 = EmptyOperator(task_id="teardown2").as_teardown() + + with setup2 >> teardown2: + EmptyOperator(task_id="task2") + + edges = dag_edges(dag) + + assert sorted((e["source_id"], e["target_id"], e.get("is_setup_teardown")) for e in edges) == [ + ("group_a.setup2", "group_a.task2", None), + ("group_a.setup2", "group_a.teardown2", True), + ("group_a.task2", "group_a.teardown2", None), + ("setup1", "task1", None), + ("setup1", "teardown1", True), + ("task1", "teardown1", None), + ] + + def test_duplicate_group_id(): from airflow.exceptions import DuplicateTaskIdFound From 41164dd663c003c6be80abdf3b2180ec930a82e4 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Fri, 14 Jul 2023 06:25:02 +0800 Subject: [PATCH 186/533] Fix dags list counts (#32579) --- airflow/www/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/www/views.py b/airflow/www/views.py index 762db4759a02..85375a518b3f 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -804,9 +804,7 @@ def index(self): is_paused_count = dict( session.execute( - select(DagModel.is_paused, func.count(DagModel.dag_id)) - .group_by(DagModel.is_paused) - .select_from(all_dags) + select(DagModel.is_paused, func.count(DagModel.dag_id)).group_by(DagModel.is_paused) ).all() ) From f1fc6dc4b9bd496ddd25898eea63d83f12cb6ad0 Mon Sep 17 00:00:00 2001 From: Sanjay M Date: Fri, 14 Jul 2023 00:50:36 +0100 Subject: [PATCH 187/533] Fix rendering of dagRunTimeout (#32565) * #32442 fixed rendering of dagruntimeout * #32442 renamed ViewScheduleInterval to ViewTimeDelta --- ...wScheduleInterval.tsx => ViewTimeDelta.tsx} | 4 ++-- airflow/www/static/js/dag/details/Dag.tsx | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) rename airflow/www/static/js/components/{ViewScheduleInterval.tsx => ViewTimeDelta.tsx} (94%) diff --git a/airflow/www/static/js/components/ViewScheduleInterval.tsx b/airflow/www/static/js/components/ViewTimeDelta.tsx similarity index 94% rename from airflow/www/static/js/components/ViewScheduleInterval.tsx rename to airflow/www/static/js/components/ViewTimeDelta.tsx index 68bc5cf1f5b4..2a0bb868c580 100644 --- a/airflow/www/static/js/components/ViewScheduleInterval.tsx +++ b/airflow/www/static/js/components/ViewTimeDelta.tsx @@ -24,7 +24,7 @@ interface Props { data: Record; } -const ViewScheduleInterval = ({ data }: Props) => { +const ViewTimeDelta = ({ data }: Props) => { const genericTimeDeltaUnits = [ "days", "day", @@ -58,4 +58,4 @@ const ViewScheduleInterval = ({ data }: Props) => { ); }; -export default ViewScheduleInterval; +export default ViewTimeDelta; diff --git a/airflow/www/static/js/dag/details/Dag.tsx b/airflow/www/static/js/dag/details/Dag.tsx index 836c8960b389..1dc6326f59fe 100644 --- a/airflow/www/static/js/dag/details/Dag.tsx +++ b/airflow/www/static/js/dag/details/Dag.tsx @@ -45,7 +45,7 @@ import { } from "src/utils"; import { useGridData, useDagDetails } from "src/api"; import Time from "src/components/Time"; -import ViewScheduleInterval from "src/components/ViewScheduleInterval"; +import ViewTimeDelta from "src/components/ViewTimeDelta"; import type { TaskState } from "src/types"; import type { DAG, DAGDetail } from "src/types/api-generated"; @@ -74,6 +74,7 @@ const Dag = () => { "tags", "owners", "params", + "dagRunTimeout", ]; const listParams = new URLSearchParamsWrapper({ @@ -282,7 +283,7 @@ const Dag = () => { {dagDetailsData.scheduleInterval?.value} ) : ( // for TimeDelta and RelativeDelta - { )}
    Dag run timeout + {dagDetailsData.dagRunTimeout?.type === undefined ? ( + null + ) : ( + // for TimeDelta and RelativeDelta + + )} +
    Params From bb5307850d93df16f442bfce2f388c90d63c6f9a Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 13 Jul 2023 17:52:27 -0700 Subject: [PATCH 188/533] Fix short circuit operator re teardowns (#32538) Short circuit operator should not skip teardown tasks. Note that if there's a setup downstream that is skipped from this, sometimes the teardown will end up skipped by the scheduler in accordance with trigger rules, but that's handled by the scheduler. Also, did a little optimization so that most of the time we don't need to create an intermediate list of to-be-skipped task in this operator's execute (only when debugging do we need to "materialize" the list first). Also consolidated logic so that we only have one call to `skip` instead of two different ones, one in each side of an if / else. --- airflow/models/baseoperator.py | 5 +- airflow/operators/python.py | 46 +++++++---- tests/operators/test_python.py | 142 ++++++++++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 20 deletions(-) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index ccbcb12efb37..40fe3b8f0bd6 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -75,7 +75,7 @@ from airflow.models.param import ParamsDict from airflow.models.pool import Pool from airflow.models.taskinstance import TaskInstance, clear_task_instances -from airflow.models.taskmixin import DAGNode, DependencyMixin +from airflow.models.taskmixin import DependencyMixin from airflow.serialization.enums import DagAttributeTypes from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep @@ -100,6 +100,7 @@ import jinja2 # Slow import. from airflow.models.dag import DAG + from airflow.models.operator import Operator from airflow.models.taskinstancekey import TaskInstanceKey from airflow.models.xcom_arg import XComArg from airflow.utils.task_group import TaskGroup @@ -1373,7 +1374,7 @@ def dry_run(self) -> None: self.log.info("Rendering template for %s", field) self.log.info(content) - def get_direct_relatives(self, upstream: bool = False) -> Iterable[DAGNode]: + def get_direct_relatives(self, upstream: bool = False) -> Iterable[Operator]: """ Get list of the direct relatives to the current task, upstream or downstream. diff --git a/airflow/operators/python.py b/airflow/operators/python.py index a0ea5be80ccb..2db5ca69c326 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -18,6 +18,7 @@ from __future__ import annotations import inspect +import logging import os import pickle import shutil @@ -30,7 +31,7 @@ from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent -from typing import Any, Callable, Collection, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Callable, Collection, Iterable, Mapping, Sequence, cast import dill @@ -49,6 +50,9 @@ from airflow.utils.process_utils import execute_in_subprocess from airflow.utils.python_virtualenv import prepare_virtualenv, write_python_script +if TYPE_CHECKING: + from pendulum.datetime import DateTime + def task(python_callable: Callable | None = None, multiple_outputs: bool | None = None, **kwargs): """Deprecated. Use :func:`airflow.decorators.task` instead. @@ -251,27 +255,35 @@ def execute(self, context: Context) -> Any: self.log.info("Proceeding with downstream tasks...") return condition - downstream_tasks = context["task"].get_flat_relatives(upstream=False) - self.log.debug("Downstream task IDs %s", downstream_tasks) + if not self.downstream_task_ids: + self.log.info("No downstream tasks; nothing to do.") + return - if downstream_tasks: - dag_run = context["dag_run"] - execution_date = dag_run.execution_date + dag_run = context["dag_run"] + def get_tasks_to_skip(): if self.ignore_downstream_trigger_rules is True: - self.log.info("Skipping all downstream tasks...") - self.skip(dag_run, execution_date, downstream_tasks, map_index=context["ti"].map_index) + tasks = context["task"].get_flat_relatives(upstream=False) else: - self.log.info("Skipping downstream tasks while respecting trigger rules...") - # Explicitly setting the state of the direct, downstream task(s) to "skipped" and letting the - # Scheduler handle the remaining downstream task(s) appropriately. - self.skip( - dag_run, - execution_date, - context["task"].get_direct_relatives(upstream=False), - map_index=context["ti"].map_index, - ) + tasks = context["task"].get_direct_relatives(upstream=False) + for t in tasks: + if not t.is_teardown: + yield t + + to_skip = get_tasks_to_skip() + # this let's us avoid an intermediate list unless debug logging + if self.log.getEffectiveLevel() <= logging.DEBUG: + self.log.debug("Downstream task IDs %s", to_skip := list(get_tasks_to_skip())) + + self.log.info("Skipping downstream tasks") + + self.skip( + dag_run=dag_run, + execution_date=cast("DateTime", dag_run.execution_date), + tasks=to_skip, + map_index=context["ti"].map_index, + ) self.log.info("Done.") diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index 498be2eb526d..1df74fef4fc5 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -26,7 +26,9 @@ from collections import namedtuple from datetime import date, datetime, timedelta from subprocess import CalledProcessError +from typing import Generator from unittest import mock +from unittest.mock import MagicMock import pytest from slugify import slugify @@ -35,7 +37,7 @@ from airflow.exceptions import AirflowException, DeserializingResultError, RemovedInAirflow3Warning from airflow.models import DAG, DagRun, TaskInstance as TI from airflow.models.baseoperator import BaseOperator -from airflow.models.taskinstance import clear_task_instances, set_current_context +from airflow.models.taskinstance import TaskInstance, clear_task_instances, set_current_context from airflow.operators.empty import EmptyOperator from airflow.operators.python import ( BranchPythonOperator, @@ -1145,3 +1147,141 @@ def test_get_context_in_old_style_context_task(self): with DAG(dag_id="edge_case_context_dag", default_args=DEFAULT_ARGS, schedule="@once"): op = PythonOperator(python_callable=get_all_the_context, task_id="get_all_the_context") op.run(ignore_first_depends_on_past=True, ignore_ti_state=True) + + +class TestShortCircuitWithTeardown: + @pytest.mark.parametrize( + "ignore_downstream_trigger_rules, with_teardown, should_skip, expected", + [ + (False, True, True, ["op2"]), + (False, True, False, []), + (False, False, True, ["op2"]), + (False, False, False, []), + (True, True, True, ["op2", "op3"]), + (True, True, False, []), + (True, False, True, ["op2", "op3", "op4"]), + (True, False, False, []), + ], + ) + def test_short_circuit_with_teardowns( + self, dag_maker, ignore_downstream_trigger_rules, should_skip, with_teardown, expected + ): + with dag_maker() as dag: + op1 = ShortCircuitOperator( + task_id="op1", + python_callable=lambda: not should_skip, + ignore_downstream_trigger_rules=ignore_downstream_trigger_rules, + ) + op2 = PythonOperator(task_id="op2", python_callable=print) + op3 = PythonOperator(task_id="op3", python_callable=print) + op4 = PythonOperator(task_id="op4", python_callable=print) + if with_teardown: + op4.as_teardown() + op1 >> op2 >> op3 >> op4 + op1.skip = MagicMock() + dagrun = dag_maker.create_dagrun() + tis = dagrun.get_task_instances() + ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0] + ti._run_raw_task() + expected_tasks = {dag.task_dict[x] for x in expected} + if should_skip: + # we can't use assert_called_with because it's a set and therefore not ordered + actual_skipped = set(op1.skip.call_args.kwargs["tasks"]) + assert actual_skipped == expected_tasks + else: + op1.skip.assert_not_called() + + @pytest.mark.parametrize("config", ["sequence", "parallel"]) + def test_short_circuit_with_teardowns_complicated(self, dag_maker, config): + with dag_maker(): + s1 = PythonOperator(task_id="s1", python_callable=print).as_setup() + s2 = PythonOperator(task_id="s2", python_callable=print).as_setup() + op1 = ShortCircuitOperator( + task_id="op1", + python_callable=lambda: False, + ) + op2 = PythonOperator(task_id="op2", python_callable=print) + t1 = PythonOperator(task_id="t1", python_callable=print).as_teardown(setups=s1) + t2 = PythonOperator(task_id="t2", python_callable=print).as_teardown(setups=s2) + if config == "sequence": + s1 >> op1 >> s2 >> op2 >> [t1, t2] + elif config == "parallel": + s1 >> op1 >> s2 >> op2 >> t2 >> t1 + else: + raise ValueError("unexpected") + op1.skip = MagicMock() + dagrun = dag_maker.create_dagrun() + tis = dagrun.get_task_instances() + ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0] + ti._run_raw_task() + # we can't use assert_called_with because it's a set and therefore not ordered + actual_skipped = set(op1.skip.call_args.kwargs["tasks"]) + assert actual_skipped == {s2, op2} + + def test_short_circuit_with_teardowns_complicated_2(self, dag_maker): + with dag_maker(): + s1 = PythonOperator(task_id="s1", python_callable=print).as_setup() + s2 = PythonOperator(task_id="s2", python_callable=print).as_setup() + op1 = ShortCircuitOperator( + task_id="op1", + python_callable=lambda: False, + ) + op2 = PythonOperator(task_id="op2", python_callable=print) + op3 = PythonOperator(task_id="op3", python_callable=print) + t1 = PythonOperator(task_id="t1", python_callable=print).as_teardown(setups=s1) + t2 = PythonOperator(task_id="t2", python_callable=print).as_teardown(setups=s2) + s1 >> op1 >> op3 >> t1 + s2 >> op2 >> t2 + + # this is the weird, maybe nonsensical part + # in this case we don't want to skip t2 since it should run + op1 >> t2 + op1.skip = MagicMock() + dagrun = dag_maker.create_dagrun() + tis = dagrun.get_task_instances() + ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0] + ti._run_raw_task() + # we can't use assert_called_with because it's a set and therefore not ordered + actual_kwargs = op1.skip.call_args.kwargs + actual_skipped = set(actual_kwargs["tasks"]) + assert actual_kwargs["execution_date"] == dagrun.logical_date + assert actual_skipped == {op3} + + @pytest.mark.parametrize("level", [logging.DEBUG, logging.INFO]) + def test_short_circuit_with_teardowns_debug_level(self, dag_maker, level, clear_db): + """ + When logging is debug we convert to a list to log the tasks skipped + before passing them to the skip method. + """ + with dag_maker(): + s1 = PythonOperator(task_id="s1", python_callable=print).as_setup() + s2 = PythonOperator(task_id="s2", python_callable=print).as_setup() + op1 = ShortCircuitOperator( + task_id="op1", + python_callable=lambda: False, + ) + op1.log.setLevel(level) + op2 = PythonOperator(task_id="op2", python_callable=print) + op3 = PythonOperator(task_id="op3", python_callable=print) + t1 = PythonOperator(task_id="t1", python_callable=print).as_teardown(setups=s1) + t2 = PythonOperator(task_id="t2", python_callable=print).as_teardown(setups=s2) + s1 >> op1 >> op3 >> t1 + s2 >> op2 >> t2 + + # this is the weird, maybe nonsensical part + # in this case we don't want to skip t2 since it should run + op1 >> t2 + op1.skip = MagicMock() + dagrun = dag_maker.create_dagrun() + tis = dagrun.get_task_instances() + ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0] + ti._run_raw_task() + # we can't use assert_called_with because it's a set and therefore not ordered + actual_kwargs = op1.skip.call_args.kwargs + actual_skipped = actual_kwargs["tasks"] + if level <= logging.DEBUG: + assert isinstance(actual_skipped, list) + else: + assert isinstance(actual_skipped, Generator) + assert set(actual_skipped) == {op3} + assert actual_kwargs["execution_date"] == dagrun.logical_date From 8ed8b0a7ee768d6429a34cb92e8118566012811f Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 14 Jul 2023 20:23:25 +0530 Subject: [PATCH 189/533] Fixing airflow_site_dir type (#32593) --- .../src/airflow_breeze/commands/release_management_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index a52eb9e595dd..7c089eb3ad5e 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -785,7 +785,7 @@ def alias_image(image_from: str, image_to: str): @option_dry_run def publish_docs( override_versioned: bool, - airflow_site_directory: bool, + airflow_site_directory: str, package_filter: tuple[str], ): """Publishes documentation to airflow-site.""" From 0584d1342a9b74484b4e43eade850ad67caa7a23 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 14 Jul 2023 23:36:03 +0800 Subject: [PATCH 190/533] Convert pull request template to YAML (#32597) --- .github/PULL_REQUEST_TEMPLATE.md | 42 ------------------------------ .github/PULL_REQUEST_TEMPLATE.yml | 43 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 42 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 011b4caa5e71..000000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,42 +0,0 @@ - - - - ---- -**^ Add meaningful description above** - -Read the **[Pull Request Guidelines](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#pull-request-guidelines)** for more information. -In case of fundamental code changes, an Airflow Improvement Proposal ([AIP](https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Improvement+Proposals)) is needed. -In case of a new dependency, check compliance with the [ASF 3rd Party License Policy](https://www.apache.org/legal/resolved.html#category-x). -In case of backwards incompatible changes please leave a note in a newsfragment file, named `{pr_number}.significant.rst` or `{issue_number}.significant.rst`, in [newsfragments](https://github.com/apache/airflow/tree/main/newsfragments). diff --git a/.github/PULL_REQUEST_TEMPLATE.yml b/.github/PULL_REQUEST_TEMPLATE.yml new file mode 100644 index 000000000000..978e5b601a69 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.yml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +name: Pull request +description: File a pull request +body: + - type: markdown + # yamllint disable rule:line-length + value: > + Thank you for contributing! Please make sure that your code changes are covered with tests. In case of new features or big changes, remember to adjust the documentation. + + In case of an existing issue, reference it using one of the following: + closes: #ISSUE + related: #ISSUE + + Feel free to ping committers for the review! + + How to write a good git commit message: http://chris.beams.io/posts/git-commit/ + # yamllint enable rule:line-length + - type: textarea + placeholder: Add meaningful description here + - type: markdown + # yamllint disable rule:line-length + value: > + Read the **[Pull Request Guidelines](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#pull-request-guidelines)** for more information. + In case of fundamental code changes, an Airflow Improvement Proposal ([AIP](https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Improvement+Proposals)) is needed. + In case of a new dependency, check compliance with the [ASF 3rd Party License Policy](https://www.apache.org/legal/resolved.html#category-x). + In case of backwards incompatible changes please leave a note in a newsfragment file, named `{pr_number}.significant.rst` or `{issue_number}.significant.rst`, in [newsfragments](https://github.com/apache/airflow/tree/main/newsfragments). + # yamllint enable rule:line-length From 1a1753c7246a2b35b993aad659f5551afd7e0215 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 14 Jul 2023 16:44:33 +0100 Subject: [PATCH 191/533] Support setting dependencies for tasks called outside TaskGroup context manager (#32351) * Support setting dependencies for tasks called outside TaskGroup context manager Currently, you must instantiate a classic operator or call a decorated operator inside the context manager before it will link up with the context manager. For example, tasks 1 and 2 below will be outside the group1 context: ``` task1 = BashOperator(task_id="task1", bash_command="echo task1") task2 = BashOperator(task_id="task2", bash_command="echo task2") with TaskGroup('group1'): task1 >> task2 ``` This PR addresses the above such that when you do that, the tasks will be inside the group1 context. For a single task, you can do: ``` task1 = BashOperator(task_id="task1", bash_command="echo task1") with TaskGroup('group1') as scope: scope.add(task1) ``` * make a refactor * fixup! make a refactor * apply suggestions from code review * Relax reference resolution type check * fixup! Relax reference resolution type check --------- Co-authored-by: Tzu-ping Chung --- airflow/models/abstractoperator.py | 9 ++++ airflow/models/taskmixin.py | 45 +++++++++++----- airflow/models/xcom_arg.py | 10 ++++ airflow/utils/edgemodifier.py | 9 ++++ airflow/utils/task_group.py | 30 ++++++++++- tests/utils/test_task_group.py | 87 +++++++++++++++++++++++++++++- 6 files changed, 173 insertions(+), 17 deletions(-) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 73d9204aab30..9a8f88ce7df2 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -51,6 +51,7 @@ from airflow.models.mappedoperator import MappedOperator from airflow.models.operator import Operator from airflow.models.taskinstance import TaskInstance + from airflow.utils.task_group import TaskGroup DEFAULT_OWNER: str = conf.get_mandatory_value("operators", "default_owner") DEFAULT_POOL_SLOTS: int = 1 @@ -382,6 +383,14 @@ def iter_mapped_task_groups(self) -> Iterator[MappedTaskGroup]: yield parent parent = parent.task_group + def add_to_taskgroup(self, task_group: TaskGroup) -> None: + """Add the task to the given task group. + + :meta private: + """ + if self.node_id not in task_group.children: + task_group.add(self) + def get_closest_mapped_task_group(self) -> MappedTaskGroup | None: """Get the mapped task group "closest" to this task in the DAG. diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py index 658759a9b763..3a4f17974b91 100644 --- a/airflow/models/taskmixin.py +++ b/airflow/models/taskmixin.py @@ -97,12 +97,14 @@ def __lshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): """Implements Task << Task.""" self.set_upstream(other) self.set_setup_teardown_ctx_dependencies(other) + self.set_taskgroup_ctx_dependencies(other) return other def __rshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): """Implements Task >> Task.""" self.set_downstream(other) self.set_setup_teardown_ctx_dependencies(other) + self.set_taskgroup_ctx_dependencies(other) return other def __rrshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): @@ -115,24 +117,39 @@ def __rlshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): self.__rshift__(other) return self + @abstractmethod + def add_to_taskgroup(self, task_group: TaskGroup) -> None: + """Add the task to the given task group.""" + raise NotImplementedError() + + @classmethod + def _iter_references(cls, obj: Any) -> Iterable[tuple[DependencyMixin, str]]: + from airflow.models.baseoperator import AbstractOperator + from airflow.utils.mixins import ResolveMixin + + if isinstance(obj, AbstractOperator): + yield obj, "operator" + elif isinstance(obj, ResolveMixin): + yield from obj.iter_references() + elif isinstance(obj, Sequence): + for o in obj: + yield from cls._iter_references(o) + def set_setup_teardown_ctx_dependencies(self, other: DependencyMixin | Sequence[DependencyMixin]): if not SetupTeardownContext.active: return - from airflow.models.xcom_arg import PlainXComArg - - op1 = self - if isinstance(self, PlainXComArg): - op1 = self.operator - SetupTeardownContext.update_context_map(op1) - if isinstance(other, Sequence): - for op in other: - if isinstance(op, PlainXComArg): - op = op.operator - SetupTeardownContext.update_context_map(op) + for op, _ in self._iter_references([self, other]): + SetupTeardownContext.update_context_map(op) + + def set_taskgroup_ctx_dependencies(self, other: DependencyMixin | Sequence[DependencyMixin]): + from airflow.utils.task_group import TaskGroupContext + + if not TaskGroupContext.active: return - if isinstance(other, PlainXComArg): - other = other.operator - SetupTeardownContext.update_context_map(other) + task_group = TaskGroupContext.get_current_task_group(None) + for op, _ in self._iter_references([self, other]): + if task_group: + op.add_to_taskgroup(task_group) class TaskMixin(DependencyMixin): diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py index c7778ac6760a..154878a4dbe6 100644 --- a/airflow/models/xcom_arg.py +++ b/airflow/models/xcom_arg.py @@ -42,6 +42,7 @@ if TYPE_CHECKING: from airflow.models.dag import DAG from airflow.models.operator import Operator + from airflow.utils.task_group import TaskGroup # Callable objects contained by MapXComArg. We only accept callables from # the user, but deserialize them into strings in a serialized XComArg for @@ -207,6 +208,15 @@ def resolve(self, context: Context, session: Session = NEW_SESSION) -> Any: """ raise NotImplementedError() + def add_to_taskgroup(self, task_group: TaskGroup) -> None: + """Add the task to the given task group. + + :meta private: + """ + for op, _ in self.iter_references(): + if op.node_id not in task_group.children: + task_group.add(op) + def __enter__(self): if not self.operator.is_setup and not self.operator.is_teardown: raise AirflowException("Only setup/teardown tasks can be used as context managers.") diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py index b693e1a1bee2..c6d8065c455d 100644 --- a/airflow/utils/edgemodifier.py +++ b/airflow/utils/edgemodifier.py @@ -172,6 +172,15 @@ def add_edge_info(self, dag, upstream_id: str, downstream_id: str): """ dag.set_edge_info(upstream_id, downstream_id, {"label": self.label}) + def add_to_taskgroup(self, task_group: TaskGroup) -> None: + """No-op, since we're not a task. + + We only add tasks to TaskGroups and not EdgeModifiers, but we need + this to satisfy the interface. + + :meta private: + """ + # Factory functions def Label(label: str): diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index d59ecaf40b4d..e55a4abbe1c0 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -207,13 +207,17 @@ def __iter__(self): else: yield child - def add(self, task: DAGNode) -> None: + def add(self, task: DAGNode) -> DAGNode: """Add a task to this TaskGroup. :meta private: """ from airflow.models.abstractoperator import AbstractOperator + if TaskGroupContext.active: + if task.task_group and task.task_group != self: + task.task_group.children.pop(task.node_id, None) + task.task_group = self existing_tg = task.task_group if isinstance(task, AbstractOperator) and existing_tg is not None and existing_tg != self: raise TaskAlreadyInTaskGroup(task.node_id, existing_tg.node_id, self.node_id) @@ -237,6 +241,7 @@ def add(self, task: DAGNode) -> None: raise AirflowException("Cannot add a non-empty TaskGroup") self.children[key] = task + return task def _remove(self, task: DAGNode) -> None: key = task.node_id @@ -543,6 +548,26 @@ def iter_tasks(self) -> Iterator[AbstractOperator]: f"Encountered a DAGNode that is not a TaskGroup or an AbstractOperator: {type(child)}" ) + def add_task(self, task: AbstractOperator) -> None: + """Add a task to the task group. + + :param task: the task to add + """ + if not TaskGroupContext.active: + raise AirflowException( + "Using this method on a task group that's not a context manager is not supported." + ) + task.add_to_taskgroup(self) + + def add_to_taskgroup(self, task_group: TaskGroup) -> None: + """No-op, since we're not a task. + + We only add tasks to TaskGroups and not TaskGroup, but we need + this to satisfy the interface. + + :meta private: + """ + class MappedTaskGroup(TaskGroup): """A mapped task group. @@ -613,6 +638,7 @@ def get_mapped_ti_count(self, run_id: str, *, session: Session) -> int: class TaskGroupContext: """TaskGroup context is used to keep the current TaskGroup when TaskGroup is used as ContextManager.""" + active: bool = False _context_managed_task_group: TaskGroup | None = None _previous_context_managed_task_groups: list[TaskGroup] = [] @@ -622,6 +648,7 @@ def push_context_managed_task_group(cls, task_group: TaskGroup): if cls._context_managed_task_group: cls._previous_context_managed_task_groups.append(cls._context_managed_task_group) cls._context_managed_task_group = task_group + cls.active = True @classmethod def pop_context_managed_task_group(cls) -> TaskGroup | None: @@ -631,6 +658,7 @@ def pop_context_managed_task_group(cls) -> TaskGroup | None: cls._context_managed_task_group = cls._previous_context_managed_task_groups.pop() else: cls._context_managed_task_group = None + cls.active = False return old_task_group @classmethod diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index c2baec25d3e3..4475e7141a47 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -22,8 +22,8 @@ import pendulum import pytest -from airflow.decorators import dag, task_group as task_group_decorator -from airflow.exceptions import TaskAlreadyInTaskGroup +from airflow.decorators import dag, task as task_decorator, task_group as task_group_decorator +from airflow.exceptions import AirflowException, TaskAlreadyInTaskGroup from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg @@ -34,6 +34,20 @@ from airflow.utils.task_group import TaskGroup, task_group_to_dict from tests.models import DEFAULT_DATE + +def make_task(name, type_="classic"): + if type_ == "classic": + return BashOperator(task_id=name, bash_command="echo 1") + + else: + + @task_decorator + def my_task(): + pass + + return my_task.override(task_id=name)() + + EXPECTED_JSON = { "id": None, "value": { @@ -1465,3 +1479,72 @@ def test_task_group_arrow_with_setups_teardowns(): tg1 >> w2 assert t1.downstream_task_ids == set() assert w1.downstream_task_ids == {"tg1.t1", "w2"} + + +def test_tasks_defined_outside_taskgrooup(dag_maker): + # Test that classic tasks defined outside a task group are added to the root task group + # when the relationships are defined inside the task group + with dag_maker() as dag: + t1 = make_task("t1") + t2 = make_task("t2") + t3 = make_task("t3") + with TaskGroup(group_id="tg1"): + t1 >> t2 >> t3 + dag.validate() + assert dag.task_group.children.keys() == {"tg1"} + assert dag.task_group.children["tg1"].children.keys() == {"t1", "t2", "t3"} + assert dag.task_group.children["tg1"].children["t1"].upstream_task_ids == set() + assert dag.task_group.children["tg1"].children["t1"].downstream_task_ids == {"t2"} + assert dag.task_group.children["tg1"].children["t2"].upstream_task_ids == {"t1"} + assert dag.task_group.children["tg1"].children["t2"].downstream_task_ids == {"t3"} + assert dag.task_group.children["tg1"].children["t3"].upstream_task_ids == {"t2"} + assert dag.task_group.children["tg1"].children["t3"].downstream_task_ids == set() + + # Test that decorated tasks defined outside a task group are added to the root task group + # when relationships are defined inside the task group + with dag_maker() as dag: + t1 = make_task("t1", type_="decorated") + t2 = make_task("t2", type_="decorated") + t3 = make_task("t3", type_="decorated") + with TaskGroup(group_id="tg1"): + t1 >> t2 >> t3 + dag.validate() + assert dag.task_group.children.keys() == {"tg1"} + assert dag.task_group.children["tg1"].children.keys() == {"t1", "t2", "t3"} + assert dag.task_group.children["tg1"].children["t1"].upstream_task_ids == set() + assert dag.task_group.children["tg1"].children["t1"].downstream_task_ids == {"t2"} + assert dag.task_group.children["tg1"].children["t2"].upstream_task_ids == {"t1"} + assert dag.task_group.children["tg1"].children["t2"].downstream_task_ids == {"t3"} + assert dag.task_group.children["tg1"].children["t3"].upstream_task_ids == {"t2"} + assert dag.task_group.children["tg1"].children["t3"].downstream_task_ids == set() + + # Test adding single decorated task defined outside a task group to a task group + with dag_maker() as dag: + t1 = make_task("t1", type_="decorated") + with TaskGroup(group_id="tg1") as tg1: + tg1.add_task(t1) + dag.validate() + assert dag.task_group.children.keys() == {"tg1"} + assert dag.task_group.children["tg1"].children.keys() == {"t1"} + assert dag.task_group.children["tg1"].children["t1"].upstream_task_ids == set() + assert dag.task_group.children["tg1"].children["t1"].downstream_task_ids == set() + + # Test adding single classic task defined outside a task group to a task group + with dag_maker() as dag: + t1 = make_task("t1") + with TaskGroup(group_id="tg1") as tg1: + tg1.add_task(t1) + dag.validate() + assert dag.task_group.children.keys() == {"tg1"} + assert dag.task_group.children["tg1"].children.keys() == {"t1"} + assert dag.task_group.children["tg1"].children["t1"].upstream_task_ids == set() + assert dag.task_group.children["tg1"].children["t1"].downstream_task_ids == set() + + with pytest.raises( + AirflowException, + match="Using this method on a task group that's not a context manager is not supported.", + ): + with dag_maker(): + t1 = make_task("t1") + tg1 = TaskGroup(group_id="tg1") + tg1.add_task(t1) From 945f48a1fdace8825f3949e5227bed0af2fd38ff Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 14 Jul 2023 22:06:14 +0530 Subject: [PATCH 192/533] Implement the breeze add-back-references command (#32594) Co-authored-by: Jarek Potiuk --- BREEZE.rst | 32 ++++ .../commands/release_management_commands.py | 46 +++++ .../release_management_commands_config.py | 6 + .../utils/add_back_references.py | 162 ++++++++++++++++++ images/breeze/output-commands-hash.txt | 9 +- images/breeze/output_release-management.svg | 22 ++- ...release-management_add-back-references.svg | 122 +++++++++++++ ...utput_setup_check-all-params-in-groups.svg | 44 ++--- ...output_setup_regenerate-command-images.svg | 50 +++--- 9 files changed, 441 insertions(+), 52 deletions(-) create mode 100644 dev/breeze/src/airflow_breeze/utils/add_back_references.py create mode 100644 images/breeze/output_release-management_add-back-references.svg diff --git a/BREEZE.rst b/BREEZE.rst index 78051287765b..367146259f93 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2020,6 +2020,38 @@ Those are all available flags of ``release-management publish-docs`` command: :width: 100% :alt: Breeze Publish documentation +Adding back referencing HTML for the documentation +"""""""""""""""""""""""""""""""""""""""""""""""""" + +To add back references to the documentation generated by ``build-docs`` in Breeze to ``airflow-site``, +use the ``release-management add-back-references`` command. This is important to support backward compatibility +the airflow documentation. + +.. code-block:: bash + + breeze release-management add-back-references + +.. code-block:: bash + + release-management add-back-references --airflow-site-directory DIRECTORY + +The flag ``--airflow-site-directory`` takes the path of the cloned ``airflow-site``. The command will +not proceed if this is an invalid path. + +.. code-block:: bash + + breeze release-management publish-docs --gen-type [airflow | providers | helm] + +The flag ``--gen-type`` is a string of which type of back references to generate. It can take the values of +airflow, providers, or helm. The command will fail if this is incorrect. + +Those are all available flags of ``release-management add-back-references`` command: + +.. image:: .images/breeze/output_release-management_add-back-references.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/images/breeze/output_release-management_add-back-references.svg + :width: 100% + :alt: Breeze Add Back References + Generating constraints """""""""""""""""""""" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 7c089eb3ad5e..84fe2517bd8f 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -46,6 +46,10 @@ get_available_documentation_packages, ) from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.add_back_references import ( + GenerationType, + start_generating_back_references, +) from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.common_options import ( argument_packages, @@ -808,6 +812,48 @@ def publish_docs( builder.publish(override_versioned=override_versioned, airflow_site_dir=airflow_site_directory) +@release_management.command( + name="add-back-references", + help="Command to add back references for documentation to make it backward compatible", +) +@click.option( + "-a", + "--airflow-site-directory", + envvar="AIRFLOW_SITE_DIRECTORY", + help="Local directory path of cloned airflow-site repo.", + required=True, +) +@click.option( + "-g", + "--gen-type", + help="Type of back references to generate, supports: [airflow | providers | helm]", + type=str, + required=True, +) +@option_verbose +@option_dry_run +def add_back_references( + airflow_site_directory: bool, + gen_type: str, +): + """Adds back references for documentation generated by build-docs and publish-docs""" + if not os.path.isdir(airflow_site_directory): + get_console().print( + "\n[error]location pointed by airflow_site_dir is not valid. " + "Provide the path of cloned airflow-site repo\n" + ) + sys.exit(1) + + gen = GenerationType[gen_type] + if gen not in GenerationType: + get_console().print( + "\n[error]invalid type of doc generation required. Pass one of [airflow | providers | helm]\n" + ) + sys.exit(1) + + start_generating_back_references(gen, airflow_site_directory) + + @release_management.command( name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)." ) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 0bd911e745c9..38196ea48a44 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -169,6 +169,12 @@ ], }, ], + "breeze release-management add-back-references": [ + { + "name": "Add Back References to Docs", + "options": ["--airflow-site-directory", "--gen-type"], + }, + ], "breeze release-management generate-issue-content-providers": [ { "name": "Generate issue content flags", diff --git a/dev/breeze/src/airflow_breeze/utils/add_back_references.py b/dev/breeze/src/airflow_breeze/utils/add_back_references.py new file mode 100644 index 000000000000..11396488711f --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/add_back_references.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import enum +import os +import tempfile +from pathlib import Path +from urllib.error import URLError +from urllib.request import urlopen + +from rich import print + +airflow_redirects_link = ( + "https://raw.githubusercontent.com/apache/airflow/main/docs/apache-airflow/redirects.txt" +) +helm_redirects_link = "https://raw.githubusercontent.com/apache/airflow/main/docs/helm-chart/redirects.txt" + + +# types of generations supported +class GenerationType(enum.Enum): + airflow = 1 + helm = 2 + providers = 3 + + +def download_file(url): + try: + temp_dir = Path(tempfile.mkdtemp(prefix="temp_dir", suffix="")) + file_name = temp_dir / "redirects.txt" + filedata = urlopen(url) + data = filedata.read() + with open(file_name, "wb") as f: + f.write(data) + return True, file_name + except URLError as e: + if e.reason == "Not Found": + print(f"[blue]The {url} does not exist. Skipping.") + else: + print(f"[yellow]Could not download file {url}: {e}") + return False, "no-file" + + +def construct_old_to_new_tuple_mapping(file_name: Path) -> list[tuple[str, str]]: + old_to_new_tuples: list[tuple[str, str]] = [] + with open(file_name) as f: + file_content = [] + lines = f.readlines() + # Skip empty line + + for line in lines: + if not line.strip(): + continue + + # Skip comments + if line.startswith("#"): + continue + + line = line.rstrip() + file_content.append(line) + + old_path, new_path = line.split(" ") + old_path = old_path.replace(".rst", ".html") + new_path = new_path.replace(".rst", ".html") + + old_to_new_tuples.append((old_path, new_path)) + return old_to_new_tuples + + +def get_redirect_content(url: str): + return f'' + + +def get_github_redirects_url(provider_name: str): + return f"https://raw.githubusercontent.com/apache/airflow/main/docs/{provider_name}/redirects.txt" + + +def get_provider_docs_path(docs_archive_path, provider_name: str): + return docs_archive_path + "/" + provider_name + + +def create_back_reference_html(back_ref_url, path): + content = get_redirect_content(back_ref_url) + + if Path(path).exists(): + print(f"Skipping file:{path}, redirects already exist") + return + + # creating a back reference html file + with open(path, "w") as f: + f.write(content) + print(f"[green]Created back reference redirect: {path}") + + +def generate_back_references(link: str, base_path: str): + is_downloaded, file_name = download_file(link) + if not is_downloaded: + old_to_new: list[tuple[str, str]] = [] + else: + print(f"Constructs old to new mapping from redirects.txt for {base_path}") + old_to_new = construct_old_to_new_tuple_mapping(file_name) + old_to_new.append(("index.html", "changelog.html")) + old_to_new.append(("index.html", "security.html")) + + versions = [f.path.split("/")[-1] for f in os.scandir(base_path) if f.is_dir()] + + for version in versions: + print(f"Processing {base_path}, version: {version}") + versioned_provider_path = base_path + "/" + version + + for old, new in old_to_new: + # only if old file exists, add the back reference + if os.path.exists(versioned_provider_path + "/" + old): + split_new_path = new.split("/") + file_name = new.split("/")[-1] + dest_dir = versioned_provider_path + "/" + "/".join(split_new_path[: len(split_new_path) - 1]) + + # finds relative path of old file with respect to new and handles case of different file + # names also + relative_path = os.path.relpath(old, new) + # remove one directory level because file path was used above + relative_path = relative_path.replace("../", "", 1) + + os.makedirs(dest_dir, exist_ok=True) + dest_file_path = dest_dir + "/" + file_name + create_back_reference_html(relative_path, dest_file_path) + + +def start_generating_back_references(gen_type, airflow_site_directory): + docs_archive_path = airflow_site_directory + "/docs-archive" + airflow_docs_path = docs_archive_path + "/apache-airflow" + helm_docs_path = docs_archive_path + "/helm-chart" + + if gen_type == GenerationType.airflow: + generate_back_references(airflow_redirects_link, airflow_docs_path) + elif gen_type == GenerationType.helm: + generate_back_references(helm_redirects_link, helm_docs_path) + elif gen_type == GenerationType.providers: + all_providers = [ + f.path.split("/")[-1] + for f in os.scandir(docs_archive_path) + if f.is_dir() and "providers" in f.name + ] + for p in all_providers: + print(f"Processing airflow provider: {p}") + generate_back_references( + get_github_redirects_url(p), get_provider_docs_path(docs_archive_path, p) + ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 1890af05405e..17bca23d512e 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -35,6 +35,7 @@ prod-image:build:58ab53c2f5e799bbe05d6fac26d001bf prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 prod-image:2da480c21d7eb120075489dc1da0f55b +release-management:add-back-references:8dab6a30076a55f2d31c6d22a94e0ccb release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485 release-management:generate-issue-content-providers:9cea234261547208ac812464d3e4a598 @@ -48,17 +49,17 @@ release-management:release-prod-images:4d85a23a2175bf8894de5aedbdd85614 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:b53a6b829715075b5570de4760887a2a +release-management:077bbb9b6bc15e9700735a3e25bfee1a sbom:generate-provider-requirements:1e7609a0ebafb839d30ec90e4254a628 sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64 sbom:d9c57d78e23a29ec9cd0139e230e1811 setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e -setup:check-all-params-in-groups:c2ee7aa0ac0831c92167ed70c05f3bd3 +setup:check-all-params-in-groups:76b3b1d3726ca2a446bab0668b1c50f5 setup:config:38ebaaf93ed42bc7b2a3000eeea2631d -setup:regenerate-command-images:76cc7b6568a1488a25e54fd5b4506b0a +setup:regenerate-command-images:9217287065e8d7b64d36bc6a80da1e7a setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e -setup:d5f6d65e2da4274ee76abf712e7f699e +setup:3bf520bc5f968107d5022679d7be7c7f shell:48db0babd831d704b97118d053ee639a start-airflow:52ad9113f74b39c1ffbd37464e1415e6 static-checks:6ba0614c2c493b45bfc027c3570d737e diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index e751de1f99de..54715bce8428 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -1,4 +1,4 @@ - + SQLite as metadata DB in production – it should only be used for dev/testing. We recommend using Postgres or MySQL. - Click here for more information. + Click here for more information. {% endcall %} {% endif %} {% if production_executor_warning | default(false) %} {% call show_message(category='warning', dismissible=false) %} Do not use the {{ production_executor_warning }} in production. - Click here for more information. + Click here for more information. {% endcall %} {% endif %} {% if otel_on | default(false) %} {% call show_message(category='warning', dismissible=false) %} - OpenTelemetry support is experimental. + OpenTelemetry support is experimental. Click here for more information. {% endcall %} {% endif %} @@ -273,7 +273,7 @@

    {{ page_title }}

    {% for owner in dag.owners.split(",") %}
    - {{ version_label }}: {% if airflow_version %}v{{ airflow_version }}{% else %} N/A{% endif %} + {{ version_label }}: {% if airflow_version %}v{{ airflow_version }}{% else %} N/A{% endif %} {% if git_version %}
    Git Version: {{ git_version }}{% endif %}
    diff --git a/airflow/www/templates/airflow/plugin.html b/airflow/www/templates/airflow/plugin.html index 0e91a8e63bb1..5bf0d02f35b5 100644 --- a/airflow/www/templates/airflow/plugin.html +++ b/airflow/www/templates/airflow/plugin.html @@ -27,7 +27,7 @@

    {{ title }}

    {% if plugins|length == 0 %}

    No plugins loaded. Learn more in the - plugins documentation. + plugins documentation.

    {% endif %} {% for plugin in plugins %} diff --git a/airflow/www/templates/airflow/providers.html b/airflow/www/templates/airflow/providers.html index ec86ae536cff..a1fcb30ba7e1 100644 --- a/airflow/www/templates/airflow/providers.html +++ b/airflow/www/templates/airflow/providers.html @@ -27,7 +27,7 @@

    {{ title }}

    {% if providers|length == 0 %}

    No providers loaded. Learn more in the - provider packages documentation. + provider packages documentation.

    {% endif %} diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index f65a34ff0ba2..601cd22d344f 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -4427,9 +4427,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001349: - version "1.0.30001439" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001439.tgz" - integrity sha512-1MgUzEkoMO6gKfXflStpYgZDlFM7M/ck/bgfVCACO5vnAf0fXoNVHdWtqGU+MYca+4bL9Z5bpOVmR33cWW9G2A== + version "1.0.30001516" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001516.tgz#621b1be7d85a8843ee7d210fd9d87b52e3daab3a" + integrity sha512-Wmec9pCBY8CWbmI4HsjBeQLqDTqV91nFVR83DnZpYyRnPI1wePDsTg0bGLPC5VU/3OIZV1fmxEea1b+tFKe86g== ccount@^2.0.0: version "2.0.1" From f1e1cdcc3b2826e68ba133f350300b5065bbca33 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 16 Jul 2023 17:09:09 +0200 Subject: [PATCH 209/533] Add option for storing configuration description in providers (#32629) This is mostly for the capacity of adding configuration to provider.yaml files. This change adds capavilty of adding "config" option to provider yaml and provider_info. This is not yet used, but it has been extracted as separate PR from #32604 in an attempt to make it more reviewable and readable. --- airflow/provider.yaml.schema.json | 710 +++++++++++++++++------------- airflow/provider_info.schema.json | 285 ++++++++---- airflow/providers_manager.py | 45 +- 3 files changed, 630 insertions(+), 410 deletions(-) diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index a1d0822af63a..509e5dfce62c 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -1,345 +1,437 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "package-name": { - "description": "Package name available under which the package is available in the PyPI repository.", - "type": "string" - }, - "name": { - "description": "Provider name", - "type": "string" - }, - "description": { - "description": "Information about the package in RST format", - "type": "string" - }, - "versions": { - "description": "List of available versions in PyPI. Sorted descending according to release date.", - "type": "array", - "items": { - "type": "string" - } - }, - "suspended": { - "description": "If set to true, the provider is suspended and it's not a candidate for release nor contributes dependencies to constraint calculations/CI image. Tests are excluded.", - "type:": "boolean" - }, - "dependencies": { - "description": "Dependencies that should be added to the provider", - "type": "array", - "items": { - "type": "string" - } - }, - "excluded-python-versions": { - "description": "List of python versions excluded for that provider", - "type": "array", - "items": { - "type": "string" - } - }, - "integrations": { - "description": "List of integrations supported by the provider.", - "type": "array", - "items": { - "type": "object", - "properties": { - "integration-name": { - "type": "string", - "description": "Name of the integration." - }, - "external-doc-url": { - "type": "string", - "description": "URL to external documentation for the integration." - }, - "how-to-guide": { - "description": "List of paths to how-to-guide for the integration. The path must start with '/docs/'", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "package-name": { + "description": "Package name available under which the package is available in the PyPI repository.", + "type": "string" + }, + "name": { + "description": "Provider name", + "type": "string" + }, + "description": { + "description": "Information about the package in RST format", + "type": "string" + }, + "versions": { + "description": "List of available versions in PyPI. Sorted descending according to release date.", "type": "array", "items": { - "type": "string" + "type": "string" } - }, - "logo": { - "description": "Path to the logo for the integration. The path must start with '/integration-logos/'", - "type": "string" - }, - "tags": { - "description": "List of tags describing the integration. While we're using RST, only one tag is supported per integration.", + }, + "suspended": { + "description": "If set to true, the provider is suspended and it's not a candidate for release nor contributes dependencies to constraint calculations/CI image. Tests are excluded.", + "type:": "boolean" + }, + "dependencies": { + "description": "Dependencies that should be added to the provider", "type": "array", "items": { - "type": "string", - "enum": [ - "alibaba", - "apache", - "aws", - "azure", - "dbt", - "gcp", - "gmp", - "google", - "kafka", - "protocol", - "service", - "software", - "yandex" - ] - }, - "minItems": 1, - "maxItems": 1 - } + "type": "string" + } }, - "additionalProperties": false, - "required": [ - "integration-name", - "external-doc-url", - "tags" - ] - } - }, - "operators": { - "type": "array", - "items": { - "type": "object", - "properties": { - "integration-name": { - "type": "string", - "description": "Integration name. It must have a matching item in the 'integration' section of any provider." - }, - "python-modules": { - "description": "List of python modules containing the operators.", + "excluded-python-versions": { + "description": "List of python versions excluded for that provider", + "type": "array", + "items": { + "type": "string" + } + }, + "integrations": { + "description": "List of integrations supported by the provider.", "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Name of the integration." + }, + "external-doc-url": { + "type": "string", + "description": "URL to external documentation for the integration." + }, + "how-to-guide": { + "description": "List of paths to how-to-guide for the integration. The path must start with '/docs/'", + "type": "array", + "items": { + "type": "string" + } + }, + "logo": { + "description": "Path to the logo for the integration. The path must start with '/integration-logos/'", + "type": "string" + }, + "tags": { + "description": "List of tags describing the integration. While we're using RST, only one tag is supported per integration.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "alibaba", + "apache", + "aws", + "azure", + "dbt", + "gcp", + "gmp", + "google", + "kafka", + "protocol", + "service", + "software", + "yandex" + ] + }, + "minItems": 1, + "maxItems": 1 + } + }, + "additionalProperties": false, + "required": [ + "integration-name", + "external-doc-url", + "tags" + ] } - } }, - "additionalProperties": false, - "required": [ - "integration-name", - "python-modules" - ] - } - }, - "sensors": { - "type": "array", - "items": { - "type": "object", - "properties": { - "integration-name": { - "type": "string", - "description": "Integration name. It must have a matching item in the 'integration' section of any provider." - }, - "python-modules": { - "description": "List of python modules containing the sensors.", + "operators": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-modules": { + "description": "List of python modules containing the operators.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "integration-name", + "python-modules" + ] } - } }, - "required": [ - "integration-name", - "python-modules" - ], - "additionalProperties": true - } - }, - "hooks": { - "type": "array", - "items": { - "type": "object", - "properties": { - "integration-name": { - "type": "string", - "description": "Integration name. It must have a matching item in the 'integration' section of any provider." - }, - "python-modules": { - "description": "List of python modules containing the hooks.", + "sensors": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-modules": { + "description": "List of python modules containing the sensors.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "integration-name", + "python-modules" + ], + "additionalProperties": true } - } }, - "additionalProperties": false, - "required": [ - "integration-name", - "python-modules" - ] - } - }, - "transfers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "how-to-guide": { - "description": "Path to how-to-guide for the transfer. The path must start with '/docs/'", - "type": "string" - }, - "source-integration-name": { - "type": "string", - "description": "Integration name. It must have a matching item in the 'integration' section of any provider." - }, - "target-integration-name": { - "type": "string", - "description": "Target integration name. It must have a matching item in the 'integration' section of any provider." - }, - "python-module": { - "type": "string", - "description": "List of python modules containing the transfers." - } + "hooks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-modules": { + "description": "List of python modules containing the hooks.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "integration-name", + "python-modules" + ] + } }, - "additionalProperties": false, - "required": [ - "source-integration-name", - "target-integration-name", - "python-module" - ] - } - }, - "triggers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "integration-name": { - "type": "string", - "description": "Integration name. It must have a matching item in the 'integration' section of any provider." - }, - "python-modules": { - "description": "List of Python modules containing the triggers.", + "transfers": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "how-to-guide": { + "description": "Path to how-to-guide for the transfer. The path must start with '/docs/'", + "type": "string" + }, + "source-integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "target-integration-name": { + "type": "string", + "description": "Target integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-module": { + "type": "string", + "description": "List of python modules containing the transfers." + } + }, + "additionalProperties": false, + "required": [ + "source-integration-name", + "target-integration-name", + "python-module" + ] } - } }, - "additionalProperties": false, - "required": [ - "integration-name", - "python-modules" - ] - } - }, - "connection-types": { - "type": "array", - "description": "Array of connection types mapped to hook class names", - "items": { - "type": "object", - "properties": { - "connection-type": { - "description": "Type of connection defined by the provider", - "type": "string" - }, - "hook-class-name": { - "description": "Hook class name that implements the connection type", - "type": "string" - } - }, - "required": [ - "connection-type", - "hook-class-name" - ] - } - }, - "extra-links": { - "type": "array", - "description": "Operator class names that provide extra link functionality", - "items": { - "type": "string" - } - }, - "additional-extras": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "description": "Name of the extra", - "type": "string" - }, - "dependencies": { - "description": "Dependencies that should be added for the extra", + "triggers": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-modules": { + "description": "List of Python modules containing the triggers.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "integration-name", + "python-modules" + ] } - } }, - "required": [ "name", "dependencies"] - }, - - "description": "Additional extras that the provider should have. Replaces auto-generated cross-provider extras, if matching the same prefix, so that you can specify boundaries for existing dependencies." - }, - "task-decorators": { - "type": "array", - "description": "Decorators to use with the TaskFlow API. Can be accessed by users via '@task.'", - "items": { - "name": { + "connection-types": { + "type": "array", + "description": "Array of connection types mapped to hook class names", + "items": { + "type": "object", + "properties": { + "connection-type": { + "description": "Type of connection defined by the provider", + "type": "string" + }, + "hook-class-name": { + "description": "Hook class name that implements the connection type", + "type": "string" + } + }, + "required": [ + "connection-type", + "hook-class-name" + ] + } + }, + "extra-links": { + "type": "array", + "description": "Operator class names that provide extra link functionality", + "items": { "type": "string" + } + }, + "additional-extras": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "description": "Name of the extra", + "type": "string" + }, + "dependencies": { + "description": "Dependencies that should be added for the extra", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "dependencies" + ] }, - "path": { + "description": "Additional extras that the provider should have. Replaces auto-generated cross-provider extras, if matching the same prefix, so that you can specify boundaries for existing dependencies." + }, + "task-decorators": { + "type": "array", + "description": "Decorators to use with the TaskFlow API. Can be accessed by users via '@task.'", + "items": { + "name": { + "type": "string" + }, + "path": { + "type": "string" + } + } + }, + "secrets-backends": { + "type": "array", + "description": "Secrets Backend class names", + "items": { "type": "string" } + }, + "logging": { + "type": "array", + "description": "Logging Task Handlers class names", + "items": { + "type": "string" + } + }, + "auth-backends": { + "type": "array", + "description": "API Auth Backend module names", + "items": { + "type": "string" + } + }, + "notifications": { + "type": "array", + "description": "Notification class names", + "items": { + "type": "string" + } + }, + "executors": { + "type": "array", + "description": "Executor class names", + "items": { + "type": "string" + } + }, + "config": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": { + "type": [ + "string", + "null" + ] + }, + "options": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/option" + } + }, + "renamed": { + "type": "object", + "properties": { + "previous_name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + }, + "required": [ + "description", + "options" + ], + "additionalProperties": false + } + }, + "plugins": { + "type": "array", + "description": "Plugins exposed by the provider", + "items": { + "name": { + "type": "string" + }, + "plugin-class": { + "type": "string" + } + } } }, - "secrets-backends": { - "type": "array", - "description": "Secrets Backend class names", - "items": { - "type": "string" - } - }, - "logging": { - "type": "array", - "description": "Logging Task Handlers class names", - "items": { - "type": "string" - } - }, - "auth-backends": { - "type": "array", - "description": "API Auth Backend module names", - "items": { - "type": "string" - } - }, - "notifications": { - "type": "array", - "description": "Notification class names", - "items": { - "type": "string" - } - }, - "executors": { - "type": "array", - "description": "Executor class names", - "items": { - "type": "string" - } - }, - "plugins": { - "type": "array", - "description": "Plugins exposed by the provider", - "items": { - "name": { - "type": "string" - }, - "plugin-class": { - "type": "string" + "additionalProperties": false, + "definitions": { + "option": { + "type": "object", + "properties": { + "description": { + "type": [ + "string", + "null" + ] + }, + "version_added": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": "string", + "enum": [ + "string", + "boolean", + "integer", + "float" + ] + }, + "example": { + "type": [ + "string", + "null", + "number" + ] + }, + "default": { + "type": [ + "string", + "null", + "number" + ] + }, + "sensitive": { + "type": "boolean", + "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values" + } + }, + "required": [ + "description", + "version_added", + "type", + "example", + "default" + ], + "additional_properties": false } - } - } - }, - "additionalProperties": false, - "required": [ - "name", - "package-name", - "description", - "suspended", - "dependencies", - "versions" - ] + }, + "required": [ + "name", + "package-name", + "description", + "suspended", + "dependencies", + "versions" + ] } diff --git a/airflow/provider_info.schema.json b/airflow/provider_info.schema.json index 86b0430cc36f..9a5dcaefbe20 100644 --- a/airflow/provider_info.schema.json +++ b/airflow/provider_info.schema.json @@ -1,105 +1,198 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "package-name": { - "description": "Package name available under which the package is available in the PyPI repository.", - "type": "string" - }, - "name": { - "description": "Provider name", - "type": "string" - }, - "description": { - "description": "Information about the package in RST format", - "type": "string" - }, - "hook-class-names": { - "type": "array", - "description": "Hook class names that provide connection types to core (deprecated by connection-types)", - "items": { - "type": "string" - }, - "deprecated": { - "description": "The hook-class-names property has been deprecated in favour of connection-types which is more performant version allowing to only import individual Hooks rather than all hooks at once", - "deprecatedVersion": "2.2.0" - } - }, - "connection-types": { - "type": "array", - "description": "Map of connection types mapped to hook class names.", - "items": { - "type": "object", - "properties": { - "connection-type": { - "description": "Type of connection defined by the provider", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "package-name": { + "description": "Package name available under which the package is available in the PyPI repository.", "type": "string" - }, - "hook-class-name": { - "description": "Hook class name that implements the connection type", + }, + "name": { + "description": "Provider name", "type": "string" - } - } - }, - "required": ["connection-type", "hook-class-name"] - }, - "extra-links": { - "type": "array", - "description": "Operator class names that provide extra link functionality", - "items": { - "type": "string" - } - }, - "secrets-backends": { - "type": "array", - "description": "Secrets Backend class names", - "items": { - "type": "string" - } - }, - "logging": { - "type": "array", - "description": "Logging Task Handlers class names", - "items": { - "type": "string" - } - }, - "auth-backends": { - "type": "array", - "description": "API Auth Backend module names", - "items": { - "type": "string" - } - }, - "notifications": { - "type": "array", - "description": "Notification class names", - "items": { - "type": "string" - } - }, - "executors": { - "type": "array", - "description": "Executor class names", - "items": { - "type": "string" - } - }, - "task-decorators": { - "type": "array", - "description": "Apply custom decorators to the TaskFlow API. Can be accessed by users via '@task.'", - "items": { - "name": { + }, + "description": { + "description": "Information about the package in RST format", + "type": "string" + }, + "hook-class-names": { + "type": "array", + "description": "Hook class names that provide connection types to core (deprecated by connection-types)", + "items": { "type": "string" }, - "path": { + "deprecated": { + "description": "The hook-class-names property has been deprecated in favour of connection-types which is more performant version allowing to only import individual Hooks rather than all hooks at once", + "deprecatedVersion": "2.2.0" + } + }, + "connection-types": { + "type": "array", + "description": "Map of connection types mapped to hook class names.", + "items": { + "type": "object", + "properties": { + "connection-type": { + "description": "Type of connection defined by the provider", + "type": "string" + }, + "hook-class-name": { + "description": "Hook class name that implements the connection type", + "type": "string" + } + } + }, + "required": [ + "connection-type", + "hook-class-name" + ] + }, + "extra-links": { + "type": "array", + "description": "Operator class names that provide extra link functionality", + "items": { "type": "string" } + }, + "secrets-backends": { + "type": "array", + "description": "Secrets Backend class names", + "items": { + "type": "string" + } + }, + "logging": { + "type": "array", + "description": "Logging Task Handlers class names", + "items": { + "type": "string" + } + }, + "auth-backends": { + "type": "array", + "description": "API Auth Backend module names", + "items": { + "type": "string" + } + }, + "notifications": { + "type": "array", + "description": "Notification class names", + "items": { + "type": "string" + } + }, + "executors": { + "type": "array", + "description": "Executor class names", + "items": { + "type": "string" + } + }, + "config": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": { + "type": [ + "string", + "null" + ] + }, + "options": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/option" + } + }, + "renamed": { + "type": "object", + "properties": { + "previous_name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + }, + "required": [ + "description", + "options" + ], + "additionalProperties": false + } + }, + "task-decorators": { + "type": "array", + "description": "Apply custom decorators to the TaskFlow API. Can be accessed by users via '@task.'", + "items": { + "name": { + "type": "string" + }, + "path": { + "type": "string" + } + } } - } - }, - "required": [ - "name", - "description" - ] + }, + "definitions": { + "option": { + "type": "object", + "properties": { + "description": { + "type": [ + "string", + "null" + ] + }, + "version_added": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": "string", + "enum": [ + "string", + "boolean", + "integer", + "float" + ] + }, + "example": { + "type": [ + "string", + "null", + "number" + ] + }, + "default": { + "type": [ + "string", + "null", + "number" + ] + }, + "sensitive": { + "type": "boolean", + "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values" + } + }, + "required": [ + "description", + "version_added", + "type", + "example", + "default" + ], + "additional_properties": false + } + }, + "required": [ + "name", + "description" + ] } diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index abd14d27b58c..4f2f063599f9 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -130,12 +130,23 @@ def __contains__(self, key): return key in self._raw_dict +def _read_schema_from_resources_or_local_file(filename: str) -> dict: + try: + with resource_files("airflow").joinpath(filename).open("rb") as f: + schema = json.load(f) + except FileNotFoundError: + import pathlib + + with (pathlib.Path(__file__).parent / filename).open("rb") as f: + schema = json.load(f) + return schema + + def _create_provider_info_schema_validator(): """Creates JSON schema validator from the provider_info.schema.json.""" import jsonschema - with resource_files("airflow").joinpath("provider_info.schema.json").open("rb") as f: - schema = json.load(f) + schema = _read_schema_from_resources_or_local_file("provider_info.schema.json") cls = jsonschema.validators.validator_for(schema) validator = cls(schema) return validator @@ -145,8 +156,7 @@ def _create_customized_form_field_behaviours_schema_validator(): """Creates JSON schema validator from the customized_form_field_behaviours.schema.json.""" import jsonschema - with resource_files("airflow").joinpath("customized_form_field_behaviours.schema.json").open("rb") as f: - schema = json.load(f) + schema = _read_schema_from_resources_or_local_file("customized_form_field_behaviours.schema.json") cls = jsonschema.validators.validator_for(schema) validator = cls(schema) return validator @@ -390,6 +400,7 @@ def __init__(self): self._logging_class_name_set: set[str] = set() self._secrets_backend_class_name_set: set[str] = set() self._executor_class_name_set: set[str] = set() + self._provider_configs: dict[str, dict[str, Any]] = {} self._api_auth_backend_module_names: set[str] = set() self._trigger_info_set: set[TriggerInfo] = set() self._provider_schema_validator = _create_provider_info_schema_validator() @@ -462,6 +473,16 @@ def initialize_providers_executors(self): self.initialize_providers_list() self._discover_executors() + @provider_info_cache("config") + def initialize_providers_configuration(self): + """Lazy initialization of providers configuration information.""" + self.initialize_providers_list() + self._discover_config() + # Now update conf with the new provider configuration from providers + from airflow.configuration import conf + + conf.load_provider_configuration() + @provider_info_cache("auth_backends") def initialize_providers_auth_backends(self): """Lazy initialization of providers API auth_backends information.""" @@ -554,7 +575,6 @@ def _add_provider_info_from_local_source_file(self, path, package_name) -> None: with open(path) as provider_yaml_file: provider_info = yaml.safe_load(provider_yaml_file) self._provider_schema_validator.validate(provider_info) - version = provider_info["versions"][0] if package_name not in self._provider_dict: self._provider_dict[package_name] = ProviderInfo(version, provider_info, "source") @@ -965,6 +985,12 @@ def _discover_executors(self) -> None: if _correctness_check(provider_package, executors_class_name, provider): self._executor_class_name_set.add(executors_class_name) + def _discover_config(self) -> None: + """Retrieve all configs defined in the providers.""" + for provider_package, provider in self._provider_dict.items(): + if provider.data.get("config"): + self._provider_configs[provider_package] = provider.data.get("config") + @provider_info_cache("triggers") def initialize_providers_triggers(self): """Initialization of providers triggers.""" @@ -1053,3 +1079,12 @@ def auth_backend_module_names(self) -> list[str]: def executor_class_names(self) -> list[str]: self.initialize_providers_executors() return sorted(self._executor_class_name_set) + + @property + def provider_configs(self) -> list[tuple[str, dict[str, Any]]]: + self.initialize_providers_configuration() + return sorted(self._provider_configs.items(), key=lambda x: x[0]) + + @property + def already_initialized_provider_configs(self) -> list[tuple[str, dict[str, Any]]]: + return sorted(self._provider_configs.items(), key=lambda x: x[0]) From 09c5114f67f152c44b555ee408d76b26cfe0691f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 16 Jul 2023 22:21:35 +0200 Subject: [PATCH 210/533] Refactor preparation of extension documentation (#32632) The code to prepare extensions have been largely repeated. This small refactor will make it easier to maintain it and make it easier to add a new extension. --- docs/exts/auth_backend.rst.jinja2 | 2 +- docs/exts/connections.rst.jinja2 | 2 +- docs/exts/extra_links.rst.jinja2 | 2 +- docs/exts/logging.rst.jinja2 | 2 +- docs/exts/operators_and_hooks_ref.py | 249 ++++++++++----------------- docs/exts/secret_backend.rst.jinja2 | 2 +- 6 files changed, 93 insertions(+), 166 deletions(-) diff --git a/docs/exts/auth_backend.rst.jinja2 b/docs/exts/auth_backend.rst.jinja2 index d14b115d0ed0..781e4b1a936e 100644 --- a/docs/exts/auth_backend.rst.jinja2 +++ b/docs/exts/auth_backend.rst.jinja2 @@ -20,7 +20,7 @@ {{ provider_dict['name'] }} {{ header_separator * (provider_dict['name']|length) }} -{% for backend in provider_dict['auth_backends'] -%} +{% for backend in provider_dict['auth-backends'] -%} - :class:`~{{ backend }}` {% endfor -%} diff --git a/docs/exts/connections.rst.jinja2 b/docs/exts/connections.rst.jinja2 index 1b4f63d0d20f..0e9ba6adaf47 100644 --- a/docs/exts/connections.rst.jinja2 +++ b/docs/exts/connections.rst.jinja2 @@ -20,7 +20,7 @@ {{ provider_dict['name'] }} {{ header_separator * (provider_dict['name']|length) }} -{% for backend in provider_dict['connection_types'] -%} +{% for backend in provider_dict['connection-types'] -%} - `{{ backend['connection-type'] }}`: :class:`~{{ backend['hook-class-name'] }}` {% endfor -%} diff --git a/docs/exts/extra_links.rst.jinja2 b/docs/exts/extra_links.rst.jinja2 index be085fd1a39a..31e3221ac569 100644 --- a/docs/exts/extra_links.rst.jinja2 +++ b/docs/exts/extra_links.rst.jinja2 @@ -20,7 +20,7 @@ {{ provider_dict['name'] }} {{ header_separator * (provider_dict['name']|length) }} -{% for extra_link in provider_dict['extra_links'] -%} +{% for extra_link in provider_dict['extra-links'] -%} - :class:`~{{ extra_link }}` {% endfor -%} diff --git a/docs/exts/logging.rst.jinja2 b/docs/exts/logging.rst.jinja2 index 0ed076cf57ba..aa7ece198f80 100644 --- a/docs/exts/logging.rst.jinja2 +++ b/docs/exts/logging.rst.jinja2 @@ -22,7 +22,7 @@ :doc:`{{ provider }}:logging/index` -{% for handler in provider_dict['handlers'] -%} +{% for handler in provider_dict['logging'] -%} - :class:`~{{ handler }}` {% endfor -%} diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index 9b78674b9d02..cc47cb46fe69 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -127,7 +127,6 @@ def _prepare_operators_data(tags: set[str] | None): def _render_operator_content(*, tags: set[str] | None, header_separator: str): tabular_data = _prepare_operators_data(tags) - return _render_template( "operators_and_hooks_ref.rst.jinja2", items=tabular_data, header_separator=header_separator ) @@ -168,147 +167,11 @@ def _prepare_transfer_data(tags: set[str] | None): def _render_transfer_content(*, tags: set[str] | None, header_separator: str): tabular_data = _prepare_transfer_data(tags) - return _render_template( "operators_and_hooks_ref-transfers.rst.jinja2", items=tabular_data, header_separator=header_separator ) -def _prepare_logging_data(): - package_data = load_package_data() - all_logging = {} - for provider in package_data: - logging_handlers = provider.get("logging") - if logging_handlers: - package_name = provider["package-name"] - all_logging[package_name] = {"name": provider["name"], "handlers": logging_handlers} - return all_logging - - -def _render_logging_content(*, header_separator: str): - tabular_data = _prepare_logging_data() - - return _render_template("logging.rst.jinja2", items=tabular_data, header_separator=header_separator) - - -def _prepare_auth_backend_data(): - package_data = load_package_data() - all_auth_backends = {} - for provider in package_data: - auth_backends_list = provider.get("auth-backends") - if auth_backends_list: - package_name = provider["package-name"] - all_auth_backends[package_name] = {"name": provider["name"], "auth_backends": auth_backends_list} - return all_auth_backends - - -def _render_auth_backend_content(*, header_separator: str): - tabular_data = _prepare_auth_backend_data() - - return _render_template("auth_backend.rst.jinja2", items=tabular_data, header_separator=header_separator) - - -def _prepare_secrets_backend_data(): - package_data = load_package_data() - all_secret_backends = {} - for provider in package_data: - secret_backends_list = provider.get("secrets-backends") - if secret_backends_list: - package_name = provider["package-name"] - all_secret_backends[package_name] = { - "name": provider["name"], - "secrets_backends": secret_backends_list, - } - return all_secret_backends - - -def _render_secrets_backend_content(*, header_separator: str): - tabular_data = _prepare_secrets_backend_data() - - return _render_template( - "secret_backend.rst.jinja2", items=tabular_data, header_separator=header_separator - ) - - -def _prepare_connections_data(): - package_data = load_package_data() - all_connections = {} - for provider in package_data: - connections_list = provider.get("connection-types") - if connections_list: - package_name = provider["package-name"] - all_connections[package_name] = { - "name": provider["name"], - "connection_types": connections_list, - } - return all_connections - - -def _render_connections_content(*, header_separator: str): - tabular_data = _prepare_connections_data() - - return _render_template("connections.rst.jinja2", items=tabular_data, header_separator=header_separator) - - -def _prepare_extra_links_data(): - package_data = load_package_data() - all_extra_links = {} - for provider in package_data: - extra_link_list = provider.get("extra-links") - if extra_link_list: - package_name = provider["package-name"] - all_extra_links[package_name] = { - "name": provider["name"], - "extra_links": extra_link_list, - } - return all_extra_links - - -def _render_extra_links_content(*, header_separator: str): - tabular_data = _prepare_extra_links_data() - - return _render_template("extra_links.rst.jinja2", items=tabular_data, header_separator=header_separator) - - -def _prepare_notifications_data(): - package_data = load_package_data() - all_notifiers = {} - for provider in package_data: - notifications = provider.get("notifications") - if notifications: - package_name = provider["package-name"] - all_notifiers[package_name] = { - "name": provider["name"], - "notifications": notifications, - } - return all_notifiers - - -def _prepare_executors_data(): - package_data = load_package_data() - all_executors = {} - for provider in package_data: - if executors := provider.get("executors"): - package_name = provider["package-name"] - all_executors[package_name] = { - "name": provider["name"], - "executors": executors, - } - return all_executors - - -def _render_notification_content(*, header_separator: str): - tabular_data = _prepare_notifications_data() - - return _render_template("notifications.rst.jinja2", items=tabular_data, header_separator=header_separator) - - -def _render_executors_content(*, header_separator: str): - tabular_data = _prepare_executors_data() - - return _render_template("executors.rst.jinja2", items=tabular_data, header_separator=header_separator) - - class BaseJinjaReferenceDirective(Directive): """The base directive for OperatorsHooksReferenceDirective and TransfersReferenceDirective""" @@ -341,10 +204,24 @@ def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAU raise NotImplementedError("Tou need to override render_content method.") +def _common_render_list_content(*, header_separator: str, resource_type: str, template: str): + tabular_data = { + provider["package-name"]: { + "name": provider["name"], + resource_type: provider.get(resource_type, []), + } + for provider in load_package_data() + if provider.get(resource_type) is not None + } + return _render_template(template, items=tabular_data, header_separator=header_separator) + + class OperatorsHooksReferenceDirective(BaseJinjaReferenceDirective): """Generates a list of operators, sensors, hooks""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: return _render_operator_content( tags=tags, header_separator=header_separator, @@ -354,7 +231,9 @@ def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAU class TransfersReferenceDirective(BaseJinjaReferenceDirective): """Generate a list of transfer operators""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: return _render_transfer_content( tags=tags, header_separator=header_separator, @@ -364,63 +243,85 @@ def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAU class LoggingDirective(BaseJinjaReferenceDirective): """Generate list of logging handlers""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_logging_content( - header_separator=header_separator, + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( + header_separator=header_separator, resource_type="logging", template="logging.rst.jinja2" ) class AuthBackendDirective(BaseJinjaReferenceDirective): """Generate list of auth backend handlers""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_auth_backend_content( + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( header_separator=header_separator, + resource_type="auth-backends", + template="auth_backend.rst.jinja2", ) class SecretsBackendDirective(BaseJinjaReferenceDirective): """Generate list of secret backend handlers""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_secrets_backend_content( + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( header_separator=header_separator, + resource_type="secrets-backends", + template="secret_backend.rst.jinja2", ) class ConnectionsDirective(BaseJinjaReferenceDirective): """Generate list of connections""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_connections_content( + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( header_separator=header_separator, + resource_type="connection-types", + template="connections.rst.jinja2", ) class ExtraLinksDirective(BaseJinjaReferenceDirective): """Generate list of extra links""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_extra_links_content( - header_separator=header_separator, + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( + header_separator=header_separator, resource_type="extra-links", template="extra_links.rst.jinja2" ) class NotificationsDirective(BaseJinjaReferenceDirective): """Generate list of notifiers""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_notification_content( + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( header_separator=header_separator, + resource_type="notifications", + template="notifications.rst.jinja2", ) class ExecutorsDirective(BaseJinjaReferenceDirective): """Generate list of executors""" - def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): - return _render_executors_content( - header_separator=header_separator, + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + return _common_render_list_content( + header_separator=header_separator, resource_type="executors", template="executors.rst.jinja2" ) @@ -475,35 +376,61 @@ def transfers(tag: Iterable[str], header_separator: str): @option_header_separator def logging(header_separator: str): """Renders Logger content""" - print(_render_logging_content(header_separator=header_separator)) + print( + _common_render_list_content( + header_separator=header_separator, resource_type="logging", template="logging.rst.jinja2" + ) + ) @cli.command() @option_header_separator def auth_backends(header_separator: str): """Renders Logger content""" - print(_render_auth_backend_content(header_separator=header_separator)) + print( + _common_render_list_content( + header_separator=header_separator, + resource_type="auth-backends", + template="auth_backend.rst.jinja2", + ) + ) @cli.command() @option_header_separator def secret_backends(header_separator: str): """Renders Secret Backends content""" - print(_render_secrets_backend_content(header_separator=header_separator)) + print( + _common_render_list_content( + header_separator=header_separator, + resource_type="secrets-backends", + template="secret_backend.rst.jinja2", + ) + ) @cli.command() @option_header_separator def connections(header_separator: str): """Renders Connections content""" - print(_render_connections_content(header_separator=header_separator)) + print( + _common_render_list_content( + header_separator=header_separator, + resource_type="connection-types", + template="connections.rst.jinja2", + ) + ) @cli.command() @option_header_separator def extra_links(header_separator: str): """Renders Extra links content""" - print(_render_extra_links_content(header_separator=header_separator)) + print( + _common_render_list_content( + header_separator=header_separator, resource_type="extra-links", template="extra_links.rst.jinja2" + ) + ) if __name__ == "__main__": diff --git a/docs/exts/secret_backend.rst.jinja2 b/docs/exts/secret_backend.rst.jinja2 index 52646c22f970..77bd6cf3d9f4 100644 --- a/docs/exts/secret_backend.rst.jinja2 +++ b/docs/exts/secret_backend.rst.jinja2 @@ -20,7 +20,7 @@ {{ provider_dict['name'] }} {{ header_separator * (provider_dict['name']|length) }} -{% for backend in provider_dict['secrets_backends'] -%} +{% for backend in provider_dict['secrets-backends'] -%} - :class:`~{{ backend }}` {% endfor -%} From 4a44343731144a7a7dc7fff7e3ed01663d4dd2e1 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Mon, 17 Jul 2023 08:23:25 +0300 Subject: [PATCH 211/533] Prepare RC3 for elasticsearch provider release (#32636) --- airflow/providers/elasticsearch/CHANGELOG.rst | 12 ++++++++++++ .../commits.rst | 12 ++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/airflow/providers/elasticsearch/CHANGELOG.rst b/airflow/providers/elasticsearch/CHANGELOG.rst index 33e5bbcd2fa6..a7daf1475314 100644 --- a/airflow/providers/elasticsearch/CHANGELOG.rst +++ b/airflow/providers/elasticsearch/CHANGELOG.rst @@ -38,11 +38,23 @@ Breaking changes * ``Deprecate the 2 non-official elasticsearch libraries (#31920)`` +Bug Fixes +~~~~~~~~~ + +* ``Fix unsound type hint in ElasticsearchTaskHandler.es_read (#32509)`` + +Misc +~~~~ + +* ``Fix Failing ES Remote Logging (#32438)`` + .. Review and move the new changes to one of the sections above: * ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` * ``Improve provider documentation and README structure (#32125)`` * ``Remove spurious headers for provider changelogs (#32373)`` * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``Add deprecation info to the providers modules and classes docstring (#32536)`` + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` 4.5.1 ..... diff --git a/docs/apache-airflow-providers-elasticsearch/commits.rst b/docs/apache-airflow-providers-elasticsearch/commits.rst index 5a44d297a58c..60a78a115b69 100644 --- a/docs/apache-airflow-providers-elasticsearch/commits.rst +++ b/docs/apache-airflow-providers-elasticsearch/commits.rst @@ -31,17 +31,21 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-15 ``Fix unsound type hint in ElasticsearchTaskHandler.es_read (#32509)`` +`41e611929e `_ 2023-07-15 ``Fix Failing ES Remote Logging (#32438)`` +`e01323635a `_ 2023-07-13 ``Add deprecation info to the providers modules and classes docstring (#32536)`` +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` `3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` `cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` `d1aa509bbd `_ 2023-06-28 ``D205 Support - Providers: Databricks to Github (inclusive) (#32243)`` `09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` `b28c90354f `_ 2023-06-24 ``Deprecate the 2 non-official elasticsearch libraries (#31920)`` -================================================================================================= =========== ======================================================================= +================================================================================================= =========== ================================================================================ 4.5.1 ..... From 6064a6e3e78605d1fe79e2acfa473e860312d685 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:55:23 -0400 Subject: [PATCH 212/533] Fix refuse_to_run_test_from_wrongly_named_files to handle system tests (#32655) --- tests/conftest.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 98dcb3be960b..9fdda1467cf7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -909,8 +909,16 @@ def clear_lru_cache(): @pytest.fixture(autouse=True) def refuse_to_run_test_from_wrongly_named_files(request): + dirname: str = request.node.fspath.dirname filename: str = request.node.fspath.basename - if not request.node.fspath.basename.startswith("test_"): + is_system_test: bool = "tests/system/" in dirname + if is_system_test and not request.node.fspath.basename.startswith("example_"): + raise Exception( + f"All test method files in tests/system must start with 'example_'. Seems that {filename} " + f"contains {request.function} that looks like a test case. Please rename the file to " + f"follow the example_* pattern if you want to run the tests in it." + ) + if not is_system_test and not request.node.fspath.basename.startswith("test_"): raise Exception( f"All test method files in tests/ must start with 'test_'. Seems that {filename} " f"contains {request.function} that looks like a test case. Please rename the file to " From 113018ec99e5d414ac0abf5bc29431bfa8f070bb Mon Sep 17 00:00:00 2001 From: Syed Hussain <103602455+syedahsn@users.noreply.github.com> Date: Mon, 17 Jul 2023 14:57:25 -0700 Subject: [PATCH 213/533] Deferrable mode for EKS Create/Delete Operator (#32355) --- airflow/providers/amazon/aws/operators/eks.py | 147 ++++++++++++++- airflow/providers/amazon/aws/triggers/eks.py | 173 +++++++++++++++++- .../operators/eks.rst | 2 + .../amazon/aws/operators/test_eks.py | 33 ++++ .../providers/amazon/aws/triggers/test_eks.py | 17 ++ .../example_eks_with_fargate_in_one_step.py | 3 + 6 files changed, 367 insertions(+), 8 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/eks.py b/airflow/providers/amazon/aws/operators/eks.py index 6858f801121b..e7931cc07a07 100644 --- a/airflow/providers/amazon/aws/operators/eks.py +++ b/airflow/providers/amazon/aws/operators/eks.py @@ -21,7 +21,7 @@ import warnings from ast import literal_eval from datetime import timedelta -from typing import TYPE_CHECKING, List, Sequence, cast +from typing import TYPE_CHECKING, Any, List, Sequence, cast from botocore.exceptions import ClientError, WaiterError @@ -30,8 +30,10 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.eks import EksHook from airflow.providers.amazon.aws.triggers.eks import ( + EksCreateClusterTrigger, EksCreateFargateProfileTrigger, EksCreateNodegroupTrigger, + EksDeleteClusterTrigger, EksDeleteFargateProfileTrigger, EksDeleteNodegroupTrigger, ) @@ -187,6 +189,9 @@ class EksCreateClusterOperator(BaseOperator): (templated) :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check cluster state :param waiter_max_attempts: The maximum number of attempts to check cluster state + :param deferrable: If True, the operator will wait asynchronously for the job to complete. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False) """ @@ -225,6 +230,7 @@ def __init__( wait_for_completion: bool = False, aws_conn_id: str = DEFAULT_CONN_ID, region: str | None = None, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), waiter_delay: int = 30, waiter_max_attempts: int = 40, **kwargs, @@ -237,7 +243,7 @@ def __init__( self.nodegroup_role_arn = nodegroup_role_arn self.fargate_pod_execution_role_arn = fargate_pod_execution_role_arn self.create_fargate_profile_kwargs = create_fargate_profile_kwargs or {} - self.wait_for_completion = wait_for_completion + self.wait_for_completion = False if deferrable else wait_for_completion self.waiter_delay = waiter_delay self.waiter_max_attempts = waiter_max_attempts self.aws_conn_id = aws_conn_id @@ -246,6 +252,7 @@ def __init__( self.create_nodegroup_kwargs = create_nodegroup_kwargs or {} self.fargate_selectors = fargate_selectors or [{"namespace": DEFAULT_NAMESPACE_NAME}] self.fargate_profile_name = fargate_profile_name + self.deferrable = deferrable super().__init__( **kwargs, ) @@ -274,12 +281,25 @@ def execute(self, context: Context): # Short circuit early if we don't need to wait to attach compute # and the caller hasn't requested to wait for the cluster either. - if not self.compute and not self.wait_for_completion: + if not any([self.compute, self.wait_for_completion, self.deferrable]): return None - self.log.info("Waiting for EKS Cluster to provision. This will take some time.") + self.log.info("Waiting for EKS Cluster to provision. This will take some time.") client = self.eks_hook.conn + if self.deferrable: + self.defer( + trigger=EksCreateClusterTrigger( + cluster_name=self.cluster_name, + aws_conn_id=self.aws_conn_id, + region_name=self.region, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + method_name="deferrable_create_cluster_next", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) + try: client.get_waiter("cluster_active").wait( name=self.cluster_name, @@ -311,6 +331,89 @@ def execute(self, context: Context): subnets=cast(List[str], self.resources_vpc_config.get("subnetIds")), ) + def deferrable_create_cluster_next(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "failed": + self.log.error("Cluster failed to start and will be torn down.") + self.eks_hook.delete_cluster(name=self.cluster_name) + self.defer( + trigger=EksDeleteClusterTrigger( + cluster_name=self.cluster_name, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region_name=self.region, + force_delete_compute=False, + ), + method_name="execute_failed", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) + elif event["status"] == "success": + self.log.info("Cluster is ready to provision compute.") + _create_compute( + compute=self.compute, + cluster_name=self.cluster_name, + aws_conn_id=self.aws_conn_id, + region=self.region, + wait_for_completion=self.wait_for_completion, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + nodegroup_name=self.nodegroup_name, + nodegroup_role_arn=self.nodegroup_role_arn, + create_nodegroup_kwargs=self.create_nodegroup_kwargs, + fargate_profile_name=self.fargate_profile_name, + fargate_pod_execution_role_arn=self.fargate_pod_execution_role_arn, + fargate_selectors=self.fargate_selectors, + create_fargate_profile_kwargs=self.create_fargate_profile_kwargs, + subnets=cast(List[str], self.resources_vpc_config.get("subnetIds")), + ) + if self.compute == "fargate": + self.defer( + trigger=EksCreateFargateProfileTrigger( + cluster_name=self.cluster_name, + fargate_profile_name=self.fargate_profile_name, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region=self.region, + ), + method_name="execute_complete", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) + else: + self.defer( + trigger=EksCreateNodegroupTrigger( + nodegroup_name=self.nodegroup_name, + cluster_name=self.cluster_name, + aws_conn_id=self.aws_conn_id, + region_name=self.region, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + method_name="execute_complete", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) + + def execute_failed(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.info("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "delteted": + self.log.info("Cluster deleted") + raise event["exception"] + + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + resource = "fargate profile" if self.compute == "fargate" else self.compute + if event is None: + self.log.info("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] != "success": + raise AirflowException(f"Error creating {resource}: {event}") + + self.log.info("%s created successfully", resource) + class EksCreateNodegroupOperator(BaseOperator): """ @@ -564,6 +667,11 @@ class EksDeleteClusterOperator(BaseOperator): maintained on each worker node). :param region: Which AWS region the connection should use. (templated) If this is None or empty then the default boto3 behaviour is used. + :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check cluster state + :param waiter_max_attempts: The maximum number of attempts to check cluster state + :param deferrable: If True, the operator will wait asynchronously for the cluster to be deleted. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False) """ @@ -582,13 +690,19 @@ def __init__( wait_for_completion: bool = False, aws_conn_id: str = DEFAULT_CONN_ID, region: str | None = None, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + waiter_delay: int = 30, + waiter_max_attempts: int = 40, **kwargs, ) -> None: self.cluster_name = cluster_name self.force_delete_compute = force_delete_compute - self.wait_for_completion = wait_for_completion + self.wait_for_completion = False if deferrable else wait_for_completion self.aws_conn_id = aws_conn_id self.region = region + self.deferrable = deferrable + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts super().__init__(**kwargs) def execute(self, context: Context): @@ -596,8 +710,20 @@ def execute(self, context: Context): aws_conn_id=self.aws_conn_id, region_name=self.region, ) - - if self.force_delete_compute: + if self.deferrable: + self.defer( + trigger=EksDeleteClusterTrigger( + cluster_name=self.cluster_name, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region_name=self.region, + force_delete_compute=self.force_delete_compute, + ), + method_name="execute_complete", + timeout=timedelta(seconds=self.waiter_delay * self.waiter_max_attempts), + ) + elif self.force_delete_compute: self.delete_any_nodegroups(eks_hook) self.delete_any_fargate_profiles(eks_hook) @@ -645,6 +771,13 @@ def delete_any_fargate_profiles(self, eks_hook) -> None: ) self.log.info(SUCCESS_MSG.format(compute=FARGATE_FULL_NAME)) + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error. Event is None") + raise AirflowException("Trigger error. Event is None") + elif event["status"] == "success": + self.log.info("Cluster deleted successfully.") + class EksDeleteNodegroupOperator(BaseOperator): """ diff --git a/airflow/providers/amazon/aws/triggers/eks.py b/airflow/providers/amazon/aws/triggers/eks.py index a6fb75eb80fa..ff99b512001e 100644 --- a/airflow/providers/amazon/aws/triggers/eks.py +++ b/airflow/providers/amazon/aws/triggers/eks.py @@ -17,11 +17,178 @@ from __future__ import annotations import warnings +from typing import Any from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.eks import EksHook from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger +from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait +from airflow.triggers.base import TriggerEvent + + +class EksCreateClusterTrigger(AwsBaseWaiterTrigger): + """ + Trigger for EksCreateClusterOperator. + + The trigger will asynchronously wait for the cluster to be created. + + :param cluster_name: The name of the EKS cluster + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: Which AWS region the connection should use. + If this is None or empty then the default boto3 behaviour is used. + """ + + def __init__( + self, + cluster_name: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + ): + super().__init__( + serialized_fields={"cluster_name": cluster_name, "region_name": region_name}, + waiter_name="cluster_active", + waiter_args={"name": cluster_name}, + failure_message="Error checking Eks cluster", + status_message="Eks cluster status is", + status_queries=["cluster.status"], + return_value=None, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) + + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class EksDeleteClusterTrigger(AwsBaseWaiterTrigger): + """ + Trigger for EksDeleteClusterOperator. + + The trigger will asynchronously wait for the cluster to be deleted. If there are + any nodegroups or fargate profiles associated with the cluster, they will be deleted + before the cluster is deleted. + + :param cluster_name: The name of the EKS cluster + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: Which AWS region the connection should use. + If this is None or empty then the default boto3 behaviour is used. + :param force_delete_compute: If True, any nodegroups or fargate profiles associated + with the cluster will be deleted before the cluster is deleted. + """ + + def __init__( + self, + cluster_name, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + force_delete_compute: bool, + ): + self.cluster_name = cluster_name + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts + self.aws_conn_id = aws_conn_id + self.region_name = region_name + self.force_delete_compute = force_delete_compute + + def serialize(self) -> tuple[str, dict[str, Any]]: + return ( + self.__class__.__module__ + "." + self.__class__.__qualname__, + { + "cluster_name": self.cluster_name, + "waiter_delay": str(self.waiter_delay), + "waiter_max_attempts": str(self.waiter_max_attempts), + "aws_conn_id": self.aws_conn_id, + "region_name": self.region_name, + "force_delete_compute": self.force_delete_compute, + }, + ) + + def hook(self) -> AwsGenericHook: + return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + async def run(self): + async with self.hook.async_conn as client: + waiter = client.get_waiter("cluster_deleted") + if self.force_delete_compute: + await self.delete_any_nodegroups(client=client) + await self.delete_any_fargate_profiles(client=client) + await client.delete_cluster(name=self.cluster_name) + await async_wait( + waiter=waiter, + waiter_delay=int(self.waiter_delay), + waiter_max_attempts=int(self.waiter_max_attempts), + args={"name": self.cluster_name}, + failure_message="Error deleting cluster", + status_message="Status of cluster is", + status_args=["cluster.status"], + ) + + yield TriggerEvent({"status": "deleted"}) + + async def delete_any_nodegroups(self, client) -> None: + """ + Deletes all EKS Nodegroups for a provided Amazon EKS Cluster. + + All the EKS Nodegroups are deleted simultaneously. We wait for + all Nodegroups to be deleted before returning. + """ + nodegroups = await client.list_nodegroups(clusterName=self.cluster_name) + if nodegroups.get("nodegroups", None): + self.log.info("Deleting nodegroups") + # ignoring attr-defined here because aws_base hook defines get_waiter for all hooks + waiter = self.hook.get_waiter( # type: ignore[attr-defined] + "all_nodegroups_deleted", deferrable=True, client=client + ) + for group in nodegroups["nodegroups"]: + await client.delete_nodegroup(clusterName=self.cluster_name, nodegroupName=group) + await async_wait( + waiter=waiter, + waiter_delay=int(self.waiter_delay), + waiter_max_attempts=int(self.waiter_max_attempts), + args={"clusterName": self.cluster_name}, + failure_message=f"Error deleting nodegroup for cluster {self.cluster_name}", + status_message="Deleting nodegroups associated with the cluster", + status_args=["nodegroups"], + ) + self.log.info("All nodegroups deleted") + else: + self.log.info("No nodegroups associated with cluster %s", self.cluster_name) + + async def delete_any_fargate_profiles(self, client) -> None: + """ + Deletes all EKS Fargate profiles for a provided Amazon EKS Cluster. + + EKS Fargate profiles must be deleted one at a time, so we must wait + for one to be deleted before sending the next delete command. + """ + fargate_profiles = await client.list_fargate_profiles(clusterName=self.cluster_name) + if fargate_profiles.get("fargateProfileNames"): + self.log.info("Waiting for Fargate profiles to delete. This will take some time.") + for profile in fargate_profiles["fargateProfileNames"]: + await client.delete_fargate_profile(clusterName=self.cluster_name, fargateProfileName=profile) + await async_wait( + waiter=client.get_waiter("fargate_profile_deleted"), + waiter_delay=int(self.waiter_delay), + waiter_max_attempts=int(self.waiter_max_attempts), + args={"clusterName": self.cluster_name, "fargateProfileName": profile}, + failure_message=f"Error deleting fargate profile for cluster {self.cluster_name}", + status_message="Status of fargate profile is", + status_args=["fargateProfile.status"], + ) + self.log.info("All Fargate profiles deleted") + else: + self.log.info(f"No Fargate profiles associated with cluster {self.cluster_name}") class EksCreateFargateProfileTrigger(AwsBaseWaiterTrigger): @@ -145,7 +312,11 @@ def __init__( region_name: str | None, ): super().__init__( - serialized_fields={"cluster_name": cluster_name, "nodegroup_name": nodegroup_name}, + serialized_fields={ + "cluster_name": cluster_name, + "nodegroup_name": nodegroup_name, + "region_name": region_name, + }, waiter_name="nodegroup_active", waiter_args={"clusterName": cluster_name, "nodegroupName": nodegroup_name}, failure_message="Error creating nodegroup", diff --git a/docs/apache-airflow-providers-amazon/operators/eks.rst b/docs/apache-airflow-providers-amazon/operators/eks.rst index cff682db2aa8..9f1cc9df61ee 100644 --- a/docs/apache-airflow-providers-amazon/operators/eks.rst +++ b/docs/apache-airflow-providers-amazon/operators/eks.rst @@ -76,6 +76,7 @@ Create an Amazon EKS cluster and AWS Fargate profile in one step To create an Amazon EKS cluster and an AWS Fargate profile in one command, you can use :class:`~airflow.providers.amazon.aws.operators.eks.EksCreateClusterOperator`. +You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. Note: An AWS IAM role with the following permissions is required: ``ec2.amazon.aws.com`` must be in the Trusted Relationships @@ -97,6 +98,7 @@ Delete an Amazon EKS Cluster To delete an existing Amazon EKS Cluster you can use :class:`~airflow.providers.amazon.aws.operators.eks.EksDeleteClusterOperator`. +You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eks_with_nodegroups.py :language: python diff --git a/tests/providers/amazon/aws/operators/test_eks.py b/tests/providers/amazon/aws/operators/test_eks.py index 8381a26ef151..e537ab20a333 100644 --- a/tests/providers/amazon/aws/operators/test_eks.py +++ b/tests/providers/amazon/aws/operators/test_eks.py @@ -337,6 +337,34 @@ def test_fargate_compute_missing_fargate_pod_execution_role_arn(self): ): missing_fargate_pod_execution_role_arn.execute({}) + @mock.patch.object(EksHook, "create_cluster") + def test_eks_create_cluster_short_circuit_early(self, mock_create_cluster, caplog): + mock_create_cluster.return_value = None + eks_create_cluster_operator = EksCreateClusterOperator( + task_id=TASK_ID, + **self.create_cluster_params, + compute=None, + wait_for_completion=False, + deferrable=False, + ) + eks_create_cluster_operator.execute({}) + assert len(caplog.records) == 0 + + @mock.patch.object(EksHook, "create_cluster") + def test_eks_create_cluster_with_deferrable(self, mock_create_cluster, caplog): + mock_create_cluster.return_value = None + + eks_create_cluster_operator = EksCreateClusterOperator( + task_id=TASK_ID, + **self.create_cluster_params, + compute=None, + wait_for_completion=False, + deferrable=True, + ) + with pytest.raises(TaskDeferred): + eks_create_cluster_operator.execute({}) + assert "Waiting for EKS Cluster to provision. This will take some time." in caplog.messages + class TestEksCreateFargateProfileOperator: def setup_method(self) -> None: @@ -542,6 +570,11 @@ def test_existing_cluster_not_in_use_with_wait( mock_waiter.assert_called_with(mock.ANY, name=CLUSTER_NAME) assert_expected_waiter_type(mock_waiter, "ClusterDeleted") + def test_eks_delete_cluster_operator_with_deferrable(self): + self.delete_cluster_operator.deferrable = True + with pytest.raises(TaskDeferred): + self.delete_cluster_operator.execute({}) + class TestEksDeleteNodegroupOperator: def setup_method(self) -> None: diff --git a/tests/providers/amazon/aws/triggers/test_eks.py b/tests/providers/amazon/aws/triggers/test_eks.py index 045519aea57e..023f8d2a97d4 100644 --- a/tests/providers/amazon/aws/triggers/test_eks.py +++ b/tests/providers/amazon/aws/triggers/test_eks.py @@ -19,8 +19,10 @@ import pytest from airflow.providers.amazon.aws.triggers.eks import ( + EksCreateClusterTrigger, EksCreateFargateProfileTrigger, EksCreateNodegroupTrigger, + EksDeleteClusterTrigger, EksDeleteFargateProfileTrigger, EksDeleteNodegroupTrigger, ) @@ -68,6 +70,21 @@ class TestEksTriggers: waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, region_name=TEST_REGION, ), + EksCreateClusterTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_DELAY, + aws_conn_id=TEST_AWS_CONN_ID, + region_name=TEST_REGION, + ), + EksDeleteClusterTrigger( + cluster_name=TEST_CLUSTER_IDENTIFIER, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_DELAY, + aws_conn_id=TEST_AWS_CONN_ID, + region_name=TEST_REGION, + force_delete_compute=True, + ), ], ) def test_serialize_recreate(self, trigger): diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py index ae67a26588bd..6b9907d83610 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -81,6 +81,9 @@ # Opting to use the same ARN for the cluster and the pod here, # but a different ARN could be configured and passed if desired. fargate_pod_execution_role_arn=fargate_pod_role_arn, + deferrable=True, + waiter_delay=30, + wait_for_completion=399, ) # [END howto_operator_eks_create_cluster_with_fargate_profile] From 9d0006fb9ee9c1b4742613fa3c1e82064eda6ba8 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Mon, 17 Jul 2023 16:06:56 -0700 Subject: [PATCH 214/533] D205 Support - Utils (#32591) Co-author: shubham22 Co-authored-by: Niko Oliveira --- airflow/utils/cli.py | 18 ++++---- airflow/utils/cli_action_loggers.py | 6 ++- airflow/utils/configuration.py | 3 +- airflow/utils/dag_cycle_tester.py | 5 ++- airflow/utils/db.py | 24 +++++------ airflow/utils/db_cleanup.py | 5 ++- airflow/utils/dot_renderer.py | 1 + airflow/utils/edgemodifier.py | 21 ++++------ airflow/utils/email.py | 7 ++-- airflow/utils/file.py | 22 +++++----- airflow/utils/hashlib_wrapper.py | 6 +-- airflow/utils/helpers.py | 16 ++----- airflow/utils/json.py | 6 +-- airflow/utils/log/colored_log.py | 7 ++-- airflow/utils/log/file_processor_handler.py | 12 +++--- airflow/utils/log/file_task_handler.py | 16 +++---- airflow/utils/log/logging_mixin.py | 25 +++++------ airflow/utils/log/non_caching_file_handler.py | 26 ++++++------ airflow/utils/log/trigger_handler.py | 3 +- airflow/utils/mixins.py | 5 ++- airflow/utils/module_loading.py | 5 ++- airflow/utils/net.py | 9 ++-- airflow/utils/operator_helpers.py | 10 +++-- airflow/utils/operator_resources.py | 10 ++--- airflow/utils/platform.py | 8 +--- airflow/utils/process_utils.py | 1 + airflow/utils/session.py | 1 + airflow/utils/sqlalchemy.py | 38 +++++++++-------- airflow/utils/state.py | 5 +-- airflow/utils/task_group.py | 42 ++++++++----------- airflow/utils/timezone.py | 1 + airflow/utils/types.py | 5 +-- 32 files changed, 176 insertions(+), 193 deletions(-) diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index c800ed71fcd8..354c2b236c07 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -61,9 +61,10 @@ def _check_cli_args(args): def action_cli(func=None, check_db=True): def action_logging(f: T) -> T: """ - Decorates function to execute function at the same time submitting action_logging - but in CLI context. It will call action logger callbacks twice, - one for pre-execution and the other one for post-execution. + Decorates function to execute function at the same time submitting action_logging but in CLI context. + + It will call action logger callbacks twice, one for + pre-execution and the other one for post-execution. Action logger will be called with below keyword parameters: sub_command : name of sub-command @@ -84,8 +85,7 @@ def action_logging(f: T) -> T: @functools.wraps(f) def wrapper(*args, **kwargs): """ - An wrapper for cli functions. It assumes to have Namespace instance - at 1st positional argument. + A wrapper for cli functions; assumes Namespace instance as first positional argument. :param args: Positional argument. It assumes to have Namespace instance at 1st positional argument @@ -126,7 +126,8 @@ def wrapper(*args, **kwargs): def _build_metrics(func_name, namespace): """ - Builds metrics dict from function args + Builds metrics dict from function args. + It assumes that function arguments is from airflow.bin.cli module's function and has Namespace instance where it optionally contains "dag_id", "task_id", and "execution_date". @@ -359,10 +360,7 @@ def should_ignore_depends_on_past(args) -> bool: def suppress_logs_and_warning(f: T) -> T: - """ - Decorator to suppress logging and warning messages - in cli functions. - """ + """Decorator to suppress logging and warning messages in cli functions.""" @functools.wraps(f) def _wrapper(*args, **kwargs): diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index 02c311d40ab3..575f4bfb08ca 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -16,8 +16,10 @@ # specific language governing permissions and limitations # under the License. """ -An Action Logger module. Singleton pattern has been applied into this module -so that registered callbacks can be used all through the same python process. +An Action Logger module. + +Singleton pattern has been applied into this module so that registered +callbacks can be used all through the same python process. """ from __future__ import annotations diff --git a/airflow/utils/configuration.py b/airflow/utils/configuration.py index 98fb6388e64b..84c5e946ee1a 100644 --- a/airflow/utils/configuration.py +++ b/airflow/utils/configuration.py @@ -27,8 +27,7 @@ def tmp_configuration_copy(chmod=0o600, include_env=True, include_cmds=True): """ - Returns a path for a temporary file including a full copy of the configuration - settings. + Returns a path for a temporary file including a full copy of the configuration settings. :param include_env: Should the value of configuration from ``AIRFLOW__`` environment variables be included or not diff --git a/airflow/utils/dag_cycle_tester.py b/airflow/utils/dag_cycle_tester.py index 8f150dc0a349..4cea52200da1 100644 --- a/airflow/utils/dag_cycle_tester.py +++ b/airflow/utils/dag_cycle_tester.py @@ -33,8 +33,9 @@ def test_cycle(dag: DAG) -> None: """ A wrapper function of `check_cycle` for backward compatibility purpose. - New code should use `check_cycle` instead since this function name `test_cycle` starts with 'test_' and - will be considered as a unit test by pytest, resulting in failure. + + New code should use `check_cycle` instead since this function name `test_cycle` starts + with 'test_' and will be considered as a unit test by pytest, resulting in failure. """ from warnings import warn diff --git a/airflow/utils/db.py b/airflow/utils/db.py index eb4c863dc3e3..f4c2ae8b1bd0 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -1006,10 +1006,10 @@ def check_username_duplicates(session: Session) -> Iterable[str]: def reflect_tables(tables: list[Base | str] | None, session): """ - When running checks prior to upgrades, we use reflection to determine current state of the - database. - This function gets the current state of each table in the set of models provided and returns - a SqlAlchemy metadata object containing them. + When running checks prior to upgrades, we use reflection to determine current state of the database. + + This function gets the current state of each table in the set of models + provided and returns a SqlAlchemy metadata object containing them. """ import sqlalchemy.schema @@ -1180,6 +1180,7 @@ def _create_table_as( ): """ Create a new table with rows from query. + We have to handle CTAS differently for different dialects. """ from sqlalchemy import column, select, table @@ -1256,10 +1257,7 @@ def _move_dangling_data_to_new_table( def _dangling_against_dag_run(session, source_table, dag_run): - """ - Given a source table, we generate a subquery that will return 1 for every row that - has a dagrun. - """ + """Given a source table, we generate a subquery that will return 1 for every row that has a dagrun.""" source_to_dag_run_join_cond = and_( source_table.c.dag_id == dag_run.c.dag_id, source_table.c.execution_date == dag_run.c.execution_date, @@ -1274,8 +1272,7 @@ def _dangling_against_dag_run(session, source_table, dag_run): def _dangling_against_task_instance(session, source_table, dag_run, task_instance): """ - Given a source table, we generate a subquery that will return 1 for every row that - has a valid task instance (and associated dagrun). + Given a source table, generate a subquery that will return 1 for every row that has a valid task instance. This is used to identify rows that need to be removed from tables prior to adding a TI fk. @@ -1367,8 +1364,8 @@ def _move_duplicate_data_to_new_table( def check_bad_references(session: Session) -> Iterable[str]: """ - Starting in Airflow 2.2, we began a process of replacing `execution_date` with `run_id` - in many tables. + Starting in Airflow 2.2, we began a process of replacing `execution_date` with `run_id` in many tables. + Here we go through each table and look for records that can't be mapped to a dag run. When we find such "dangling" rows we back them up in a special table and delete them from the main table. @@ -1383,6 +1380,8 @@ def check_bad_references(session: Session) -> Iterable[str]: @dataclass class BadReferenceConfig: """ + Bad reference config class. + :param bad_rows_func: function that returns subquery which determines whether bad rows exist :param join_tables: table objects referenced in subquery :param ref_table: information-only identifier for categorizing the missing ref @@ -1552,6 +1551,7 @@ def upgradedb( session: Session = NEW_SESSION, ): """ + Upgrades the DB. :param to_revision: Optional Alembic revision ID to upgrade *to*. If omitted, upgrades to latest revision. diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 22fb0020e72a..36bced947348 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. """ -This module took inspiration from the community maintenance dag +This module took inspiration from the community maintenance dag. + +See: (https://github.com/teamclairvoyant/airflow-maintenance-dags/blob/4e5c7682a808082561d60cbc9cafaa477b0d8c65/db-cleanup/airflow-db-cleanup.py). """ from __future__ import annotations @@ -341,6 +343,7 @@ def _print_config(*, configs: dict[str, _TableConfig]): def _suppress_with_logging(table, session): """ Suppresses errors but logs them. + Also stores the exception instance so it can be referred to after exiting context. """ try: diff --git a/airflow/utils/dot_renderer.py b/airflow/utils/dot_renderer.py index d3b329cb5453..f168566f7e33 100644 --- a/airflow/utils/dot_renderer.py +++ b/airflow/utils/dot_renderer.py @@ -38,6 +38,7 @@ def _refine_color(color: str): """ Converts color in #RGB (12 bits) format to #RRGGBB (32 bits), if it possible. + Otherwise, it returns the original value. Graphviz does not support colors in #RGB format. :param color: Text representation of color diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py index c6d8065c455d..14aa3ef25e45 100644 --- a/airflow/utils/edgemodifier.py +++ b/airflow/utils/edgemodifier.py @@ -24,8 +24,9 @@ class EdgeModifier(DependencyMixin): """ - Class that represents edge information to be added between two - tasks/operators. Has shorthand factory functions, like Label("hooray"). + Class that represents edge information to be added between two tasks/operators. + + Has shorthand factory functions, like Label("hooray"). Current implementation supports t1 >> Label("Success route") >> t2 @@ -77,8 +78,9 @@ def _save_nodes( def _convert_streams_to_task_groups(self): """ - Both self._upstream and self._downstream are required to determine if - we should convert a node to a TaskGroup or leave it as a DAGNode. + Convert a node to a TaskGroup or leave it as a DAGNode. + + Requires both self._upstream and self._downstream. To do this, we keep a set of group_ids seen among the streams. If we find that the nodes are from the same TaskGroup, we will leave them as DAGNodes and not @@ -121,8 +123,7 @@ def set_upstream( edge_modifier: EdgeModifier | None = None, ): """ - Sets the given task/list onto the upstream attribute, and then checks if - we have both sides so we can resolve the relationship. + Set the given task/list onto the upstream attribute, then attempt to resolve the relationship. Providing this also provides << via DependencyMixin. """ @@ -139,8 +140,7 @@ def set_downstream( edge_modifier: EdgeModifier | None = None, ): """ - Sets the given task/list onto the downstream attribute, and then checks if - we have both sides so we can resolve the relationship. + Set the given task/list onto the downstream attribute, then attempt to resolve the relationship. Providing this also provides >> via DependencyMixin. """ @@ -154,10 +154,7 @@ def set_downstream( def update_relative( self, other: DependencyMixin, upstream: bool = True, edge_modifier: EdgeModifier | None = None ) -> None: - """ - Called if we're not the "main" side of a relationship; we still run the - same logic, though. - """ + """Called if we're not the "main" side of a relationship; we still run the same logic, though.""" if upstream: self.set_upstream(other) else: diff --git a/airflow/utils/email.py b/airflow/utils/email.py index e807b8f75d9f..37e028fbd99e 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -321,9 +321,10 @@ def _get_smtp_connection(host: str, port: int, timeout: int, with_ssl: bool) -> def _get_email_list_from_str(addresses: str) -> list[str]: """ - Extract a list of email addresses from a string. The string - can contain multiple email addresses separated by - any of the following delimiters: ',' or ';'. + Extract a list of email addresses from a string. + + The string can contain multiple email addresses separated + by any of the following delimiters: ',' or ';'. :param addresses: A string containing one or more email addresses. :return: A list of email addresses. diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 08b4048ad58a..fb17ee6a8d57 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -41,8 +41,9 @@ class _IgnoreRule(Protocol): @staticmethod def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule | None: """ - Build an ignore rule from the supplied pattern where base_dir - and definition_file should be absolute paths. + Build an ignore rule from the supplied pattern. + + ``base_dir`` and ``definition_file`` should be absolute paths. """ @staticmethod @@ -134,8 +135,9 @@ def TemporaryDirectory(*args, **kwargs): def mkdirs(path, mode): """ - Creates the directory specified by path, creating intermediate directories - as necessary. If directory already exists, this is a no-op. + Creates the directory specified by path, creating intermediate directories as necessary. + + If directory already exists, this is a no-op. :param path: The directory to create :param mode: The mode to give to the directory e.g. 0o755, ignores umask @@ -164,10 +166,7 @@ def correct_maybe_zipped(fileloc: str | Path) -> str | Path: def correct_maybe_zipped(fileloc: None | str | Path) -> None | str | Path: - """ - If the path contains a folder with a .zip suffix, then - the folder is treated as a zip archive and path to zip is returned. - """ + """If the path contains a folder with a .zip suffix, treat it as a zip archive and return path.""" if not fileloc: return fileloc search_ = ZIP_REGEX.search(str(fileloc)) @@ -182,8 +181,10 @@ def correct_maybe_zipped(fileloc: None | str | Path) -> None | str | Path: def open_maybe_zipped(fileloc, mode="r"): """ - Opens the given file. If the path contains a folder with a .zip suffix, then - the folder is treated as a zip archive, opening the file inside the archive. + Opens the given file. + + If the path contains a folder with a .zip suffix, then the folder + is treated as a zip archive, opening the file inside the archive. :return: a file object, as in `open`, or as in `ZipFile.open`. """ @@ -332,6 +333,7 @@ def find_dag_file_paths(directory: str | os.PathLike[str], safe_mode: bool) -> l def might_contain_dag(file_path: str, safe_mode: bool, zip_file: zipfile.ZipFile | None = None) -> bool: """ Check whether a Python file contains Airflow DAGs. + When safe_mode is off (with False value), this function always returns True. If might_contain_dag_callable isn't specified, it uses airflow default heuristic diff --git a/airflow/utils/hashlib_wrapper.py b/airflow/utils/hashlib_wrapper.py index 25f02e5c3ad7..09850c565cb0 100644 --- a/airflow/utils/hashlib_wrapper.py +++ b/airflow/utils/hashlib_wrapper.py @@ -28,11 +28,9 @@ def md5(__string: ReadableBuffer = b"") -> hashlib._Hash: """ - Safely allows calling the hashlib.md5 function with the "usedforsecurity" disabled - when specified in the configuration. + Safely allows calling the hashlib.md5 function when "usedforsecurity" is disabled in the configuration. - :param string: The data to hash. - Default to empty str byte. + :param string: The data to hash. Default to empty str byte. :return: The hashed value. """ if PY39: diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index a29cf07a5919..b900ca1033c7 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -120,10 +120,7 @@ def is_container(obj: Any) -> bool: def as_tuple(obj: Any) -> tuple: - """ - If obj is a container, returns obj as a tuple. - Otherwise, returns a tuple containing obj. - """ + """Return obj as a tuple if obj is a container, otherwise return a tuple containing obj.""" if is_container(obj): return tuple(obj) else: @@ -139,10 +136,7 @@ def chunks(items: list[T], chunk_size: int) -> Generator[list[T], None, None]: def reduce_in_chunks(fn: Callable[[S, list[T]], S], iterable: list[T], initializer: S, chunk_size: int = 0): - """ - Reduce the given list of items by splitting it into chunks - of the given size and passing each chunk through the reducer. - """ + """Split the list of items into chunks of a given size and pass each chunk through the reducer.""" if len(iterable) == 0: return initializer if chunk_size == 0: @@ -172,8 +166,7 @@ def parse_template_string(template_string: str) -> tuple[str | None, jinja2.Temp def render_log_filename(ti: TaskInstance, try_number, filename_template) -> str: """ - Given task instance, try_number, filename_template, return the rendered log - filename. + Given task instance, try_number, filename_template, return the rendered log filename. :param ti: task instance :param try_number: try_number of the task @@ -327,8 +320,7 @@ def is_set(val): def prune_dict(val: Any, mode="strict"): """ - Given dict ``val``, returns new dict based on ``val`` with all - empty elements removed. + Given dict ``val``, returns new dict based on ``val`` with all empty elements removed. What constitutes "empty" is controlled by the ``mode`` parameter. If mode is 'strict' then only ``None`` elements will be removed. If mode is ``truthy``, then element ``x`` diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 3a0a70fb7585..7f05c8778d27 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -105,11 +105,7 @@ def encode(self, o: Any) -> str: class XComDecoder(json.JSONDecoder): - """ - This decoder deserializes dicts to objects if they contain - the `__classname__` key otherwise it will return the dict - as is. - """ + """Deserialize dicts to objects if they contain the `__classname__` key, otherwise return the dict.""" def __init__(self, *args, **kwargs) -> None: if not kwargs.get("object_hook"): diff --git a/airflow/utils/log/colored_log.py b/airflow/utils/log/colored_log.py index aea7c255e618..3afa39410d04 100644 --- a/airflow/utils/log/colored_log.py +++ b/airflow/utils/log/colored_log.py @@ -42,9 +42,10 @@ class CustomTTYColoredFormatter(TTYColoredFormatter, TimezoneAware): """ - Custom log formatter which extends `colored.TTYColoredFormatter` - by adding attributes to message arguments and coloring error - traceback. + Custom log formatter. + + Extends `colored.TTYColoredFormatter` by adding attributes + to message arguments and coloring error traceback. """ def __init__(self, *args, **kwargs): diff --git a/airflow/utils/log/file_processor_handler.py b/airflow/utils/log/file_processor_handler.py index 79efb387a304..cda24cdd06be 100644 --- a/airflow/utils/log/file_processor_handler.py +++ b/airflow/utils/log/file_processor_handler.py @@ -30,9 +30,10 @@ class FileProcessorHandler(logging.Handler): """ - FileProcessorHandler is a python log handler that handles - dag processor logs. It creates and delegates log handling - to `logging.FileHandler` after receiving dag processor context. + FileProcessorHandler is a python log handler that handles dag processor logs. + + It creates and delegates log handling to `logging.FileHandler` + after receiving dag processor context. :param base_log_folder: Base log folder to place logs. :param filename_template: template filename string @@ -108,8 +109,9 @@ def _get_log_directory(self): def _symlink_latest_log_directory(self): """ - Create symbolic link to the current day's log directory to - allow easy access to the latest scheduler log files. + Create symbolic link to the current day's log directory. + + Allows easy access to the latest scheduler log files. :return: None """ diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 9184a2042091..5d791aaa0cca 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -134,10 +134,10 @@ def _interleave_logs(*logs): class FileTaskHandler(logging.Handler): """ - FileTaskHandler is a python log handler that handles and reads - task instance logs. It creates and delegates log handling - to `logging.FileHandler` after receiving task instance context. - It reads logs from task instance's host machine. + FileTaskHandler is a python log handler that handles and reads task instance logs. + + It creates and delegates log handling to `logging.FileHandler` after receiving task + instance context. It reads logs from task instance's host machine. :param base_log_folder: Base log folder to place logs. :param filename_template: template filename string @@ -278,8 +278,7 @@ def _read( metadata: dict[str, Any] | None = None, ): """ - Template method that contains custom logic of reading - logs given the try_number. + Template method that contains custom logic of reading logs given the try_number. :param ti: task instance record :param try_number: current try_number to read log from @@ -462,8 +461,9 @@ def _prepare_log_folder(self, directory: Path): def _init_file(self, ti): """ - Create log directory and give it permissions that are configured. See above _prepare_log_folder - method for more detailed explanation. + Create log directory and give it permissions that are configured. + + See above _prepare_log_folder method for more detailed explanation. :param ti: task instance object :return: relative log path of the given task instance diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 646c73f1b4bc..97e1a29ff6e2 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -54,10 +54,7 @@ def __getattr__(name): def remove_escape_codes(text: str) -> str: - """ - Remove ANSI escapes codes from string. It's used to remove - "colors" from log messages. - """ + """Remove ANSI escapes codes from string; used to remove "colors" from log messages.""" return ANSI_ESCAPE.sub("", text) @@ -118,15 +115,15 @@ def supports_external_link(self) -> bool: # IO generics (and apparently it has not even been intended) # See more: https://giters.com/python/typeshed/issues/6077 class StreamLogWriter(IOBase, IO[str]): # type: ignore[misc] - """Allows to redirect stdout and stderr to logger.""" + """ + Allows to redirect stdout and stderr to logger. + + :param log: The log level method to write to, ie. log.debug, log.warning + """ encoding: None = None def __init__(self, logger, level): - """ - :param log: The log level method to write to, ie. log.debug, log.warning - :return: - """ self.logger = logger self.level = level self._buffer = "" @@ -141,8 +138,9 @@ def close(self): @property def closed(self): """ - Returns False to indicate that the stream is not closed, as it will be - open for the duration of Airflow's lifecycle. + Return False to indicate that the stream is not closed. + + Streams will be open for the duration of Airflow's lifecycle. For compatibility with the io.IOBase interface. """ @@ -174,6 +172,7 @@ def flush(self): def isatty(self): """ Returns False to indicate the fd is not connected to a tty(-like) device. + For compatibility reasons. """ return False @@ -181,8 +180,10 @@ def isatty(self): class RedirectStdHandler(StreamHandler): """ + Custom StreamHandler that uses current sys.stderr/stdout as the stream for logging. + This class is like a StreamHandler using sys.stderr/stdout, but uses - whatever sys.stderr/stderr is currently set to rather than the value of + whatever sys.stderr/stdout is currently set to rather than the value of sys.stderr/stdout at handler construction time, except when running a task in a kubernetes executor pod. """ diff --git a/airflow/utils/log/non_caching_file_handler.py b/airflow/utils/log/non_caching_file_handler.py index 46fe69a99bc7..aa0ca9864e2e 100644 --- a/airflow/utils/log/non_caching_file_handler.py +++ b/airflow/utils/log/non_caching_file_handler.py @@ -36,12 +36,13 @@ def make_file_io_non_caching(io: IO[str]) -> IO[str]: class NonCachingFileHandler(FileHandler): """ - This is an extension of the python FileHandler that advises the Kernel to not cache the file - in PageCache when it is written. While there is nothing wrong with such cache (it will be cleaned - when memory is needed), it causes ever-growing memory usage when scheduler is running as it keeps - on writing new log files and the files are not rotated later on. This might lead to confusion - for our users, who are monitoring memory usage of Scheduler - without realising that it is - harmless and expected in this case. + An extension of FileHandler, advises the Kernel to not cache the file in PageCache when it is written. + + While there is nothing wrong with such cache (it will be cleaned when memory is needed), it + causes ever-growing memory usage when scheduler is running as it keeps on writing new log + files and the files are not rotated later on. This might lead to confusion for our users, + who are monitoring memory usage of Scheduler - without realising that it is harmless and + expected in this case. See https://github.com/apache/airflow/issues/14924 @@ -54,12 +55,13 @@ def _open(self): class NonCachingRotatingFileHandler(RotatingFileHandler): """ - This is an extension of the python RotatingFileHandler that advises the Kernel to not cache the file - in PageCache when it is written. While there is nothing wrong with such cache (it will be cleaned - when memory is needed), it causes ever-growing memory usage when scheduler is running as it keeps - on writing new log files and the files are not rotated later on. This might lead to confusion - for our users, who are monitoring memory usage of Scheduler - without realising that it is - harmless and expected in this case. + An extension of RotatingFileHandler, advises the Kernel to not cache the file in PageCache when written. + + While there is nothing wrong with such cache (it will be cleaned when memory is needed), it + causes ever-growing memory usage when scheduler is running as it keeps on writing new log + files and the files are not rotated later on. This might lead to confusion for our users, + who are monitoring memory usage of Scheduler - without realising that it is harmless and + expected in this case. See https://github.com/apache/airflow/issues/27065 diff --git a/airflow/utils/log/trigger_handler.py b/airflow/utils/log/trigger_handler.py index 50756cb04837..54f8ace65e74 100644 --- a/airflow/utils/log/trigger_handler.py +++ b/airflow/utils/log/trigger_handler.py @@ -67,8 +67,7 @@ def filter(self, record): class TriggererHandlerWrapper(logging.Handler): """ - Wrap inheritors of FileTaskHandler and direct log messages - to them based on trigger_id. + Wrap inheritors of FileTaskHandler and direct log messages to them based on trigger_id. :meta private: """ diff --git a/airflow/utils/mixins.py b/airflow/utils/mixins.py index d157c7f078ff..e3c6a8efec6e 100644 --- a/airflow/utils/mixins.py +++ b/airflow/utils/mixins.py @@ -35,8 +35,9 @@ class MultiprocessingStartMethodMixin: def _get_multiprocessing_start_method(self) -> str: """ - Determine method of creating new processes by checking if the - mp_start_method is set in configs, else, it uses the OS default. + Determine method of creating new processes. + + Checks if the mp_start_method is set in configs, else, it uses the OS default. """ if conf.has_option("core", "mp_start_method"): return conf.get_mandatory_value("core", "mp_start_method") diff --git a/airflow/utils/module_loading.py b/airflow/utils/module_loading.py index 8decbc6a61cb..d81ab65e8589 100644 --- a/airflow/utils/module_loading.py +++ b/airflow/utils/module_loading.py @@ -25,8 +25,9 @@ def import_string(dotted_path: str): """ - Import a dotted module path and return the attribute/class designated by the - last name in the path. Raise ImportError if the import failed. + Import a dotted module path and return the attribute/class designated by the last name in the path. + + Raise ImportError if the import failed. """ try: module_path, class_name = dotted_path.rsplit(".", 1) diff --git a/airflow/utils/net.py b/airflow/utils/net.py index 57bd9008b95d..992aee67e800 100644 --- a/airflow/utils/net.py +++ b/airflow/utils/net.py @@ -26,7 +26,9 @@ # patched version of socket.getfqdn() - see https://github.com/python/cpython/issues/49254 @lru_cache(maxsize=None) def getfqdn(name=""): - """Get fully qualified domain name from name. + """ + Get fully qualified domain name from name. + An empty argument is interpreted as meaning the local host. """ name = name.strip() @@ -50,8 +52,5 @@ def get_host_ip_address(): def get_hostname(): - """ - Fetch the hostname using the callable from the config or using - `airflow.utils.net.getfqdn` as a fallback. - """ + """Fetch the hostname using the callable from config or use `airflow.utils.net.getfqdn` as a fallback.""" return conf.getimport("core", "hostname_callable", fallback="airflow.utils.net.getfqdn")() diff --git a/airflow/utils/operator_helpers.py b/airflow/utils/operator_helpers.py index 0390edc81cde..20f272f4f36d 100644 --- a/airflow/utils/operator_helpers.py +++ b/airflow/utils/operator_helpers.py @@ -62,6 +62,8 @@ def context_to_airflow_vars(context: Mapping[str, Any], in_env_var_format: bool = False) -> dict[str, str]: """ + Return values used to externally reconstruct relations between dags, dag_runs, tasks and task_instances. + Given a context, this function provides a dictionary of values that can be used to externally reconstruct relations between dags, dag_runs, tasks and task_instances. Default to abc.def.ghi format and can be made to ABC_DEF_GHI format if @@ -185,12 +187,10 @@ def determine_kwargs( kwargs: Mapping[str, Any], ) -> Mapping[str, Any]: """ - Inspect the signature of a given callable to determine which arguments in kwargs need - to be passed to the callable. + Inspect the signature of a callable to determine which kwargs need to be passed to the callable. :param func: The callable that you want to invoke - :param args: The positional arguments that needs to be passed to the callable, so we - know how many to skip. + :param args: The positional arguments that need to be passed to the callable, so we know how many to skip. :param kwargs: The keyword arguments that need to be filtered before passing to the callable. :return: A dictionary which contains the keyword arguments that are compatible with the callable. """ @@ -199,6 +199,8 @@ def determine_kwargs( def make_kwargs_callable(func: Callable[..., R]) -> Callable[..., R]: """ + Creates a new callable that only forwards necessary arguments from any provided input. + Make a new callable that can accept any number of positional or keyword arguments but only forwards those required by the given callable func. """ diff --git a/airflow/utils/operator_resources.py b/airflow/utils/operator_resources.py index 638034a81a92..cd6a22524412 100644 --- a/airflow/utils/operator_resources.py +++ b/airflow/utils/operator_resources.py @@ -70,10 +70,7 @@ def units_str(self): @property def qty(self): - """ - The number of units of the specified resource that are required for - execution of the operator. - """ + """The number of units of the specified resource that are required for execution of the operator.""" return self._qty def to_dict(self): @@ -114,8 +111,9 @@ def __init__(self, qty): class Resources: """ - The resources required by an operator. Resources that are not specified will use the - default values from the airflow config. + The resources required by an operator. + + Resources that are not specified will use the default values from the airflow config. :param cpus: The number of cpu cores that are required :param ram: The amount of RAM required diff --git a/airflow/utils/platform.py b/airflow/utils/platform.py index 691aac033b3c..72906683f63e 100644 --- a/airflow/utils/platform.py +++ b/airflow/utils/platform.py @@ -32,10 +32,7 @@ def is_tty(): - """ - Checks if the standard output is connected (is associated with a terminal device) to a tty(-like) - device. - """ + """Check if stdout is connected (is associated with a terminal device) to a tty(-like) device.""" if not hasattr(sys.stdout, "isatty"): return False return sys.stdout.isatty() @@ -69,8 +66,7 @@ def get_airflow_git_version(): @cache def getuser() -> str: """ - Gets the username associated with the current user, or error with a nice - error message if there's no current user. + Get the username of the current user, or error with a nice error message if there's no current user. We don't want to fall back to os.getuid() because not having a username probably means the rest of the user environment is wrong (e.g. no $HOME). diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py index 663f0f496c7b..74ad50bf7404 100644 --- a/airflow/utils/process_utils.py +++ b/airflow/utils/process_utils.py @@ -300,6 +300,7 @@ def patch_environ(new_env_variables: dict[str, str]) -> Generator[None, None, No def check_if_pidfile_process_is_running(pid_file: str, process_name: str): """ Checks if a pidfile already exists and process is still running. + If process is dead then pidfile is removed. :param pid_file: path to the pidfile diff --git a/airflow/utils/session.py b/airflow/utils/session.py index 0f8bdb710367..5c7e9eef505c 100644 --- a/airflow/utils/session.py +++ b/airflow/utils/session.py @@ -61,6 +61,7 @@ def find_session_idx(func: Callable[PS, RT]) -> int: def provide_session(func: Callable[PS, RT]) -> Callable[PS, RT]: """ Function decorator that provides a session if it isn't provided. + If you want to reuse a session or run the function as part of a database transaction, you pass it to the function, if not this wrapper will create one and close it for you. diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 04af5bddc31c..499aa31c3497 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -50,8 +50,8 @@ class UtcDateTime(TypeDecorator): """ - Almost equivalent to :class:`~sqlalchemy.types.TIMESTAMP` with - ``timezone=True`` option, but it differs from that by: + Similar to :class:`~sqlalchemy.types.TIMESTAMP` with ``timezone=True`` option, with some differences. + - Never silently take naive :class:`~datetime.datetime`, instead it always raise :exc:`ValueError` unless time zone aware value. - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` @@ -86,11 +86,11 @@ def process_bind_param(self, value, dialect): def process_result_value(self, value, dialect): """ - Processes DateTimes from the DB making sure it is always - returning UTC. Not using timezone.convert_to_utc as that - converts to configured TIMEZONE while the DB might be - running with some other setting. We assume UTC datetimes - in the database. + Processes DateTimes from the DB making sure it is always returning UTC. + + Not using timezone.convert_to_utc as that converts to configured TIMEZONE + while the DB might be running with some other setting. We assume UTC + datetimes in the database. """ if value is not None: if value.tzinfo is None: @@ -110,8 +110,9 @@ def load_dialect_impl(self, dialect): class ExtendedJSON(TypeDecorator): """ - A version of the JSON column that uses the Airflow extended JSON - serialization provided by airflow.serialization. + A version of the JSON column that uses the Airflow extended JSON serialization. + + See airflow.serialization. """ impl = Text @@ -244,10 +245,11 @@ def ensure_pod_is_valid_after_unpickling(pod: V1Pod) -> V1Pod | None: class ExecutorConfigType(PickleType): """ - Adds special handling for K8s executor config. If we unpickle a k8s object that was - pickled under an earlier k8s library version, then the unpickled object may throw an error - when to_dict is called. To be more tolerant of version changes we convert to JSON using - Airflow's serializer before pickling. + Adds special handling for K8s executor config. + + If we unpickle a k8s object that was pickled under an earlier k8s library version, then + the unpickled object may throw an error when to_dict is called. To be more tolerant of + version changes we convert to JSON using Airflow's serializer before pickling. """ cache_ok = True @@ -293,11 +295,11 @@ def process(value): def compare_values(self, x, y): """ - The TaskInstance.executor_config attribute is a pickled object that may contain - kubernetes objects. If the installed library version has changed since the - object was originally pickled, due to the underlying ``__eq__`` method on these - objects (which converts them to JSON), we may encounter attribute errors. In this - case we should replace the stored object. + The TaskInstance.executor_config attribute is a pickled object that may contain kubernetes objects. + + If the installed library version has changed since the object was originally pickled, + due to the underlying ``__eq__`` method on these objects (which converts them to JSON), + we may encounter attribute errors. In this case we should replace the stored object. From https://github.com/apache/airflow/pull/24356 we use our serializer to store k8s objects, but there could still be raw pickled k8s objects in the database, diff --git a/airflow/utils/state.py b/airflow/utils/state.py index f18fd48c82fb..a0c828ee0747 100644 --- a/airflow/utils/state.py +++ b/airflow/utils/state.py @@ -82,10 +82,7 @@ def __str__(self) -> str: class State: - """ - Static class with task instance state constants and color methods to - avoid hardcoding. - """ + """Static class with task instance state constants and color methods to avoid hardcoding.""" # Backwards-compat constants for code that does not yet use the enum # These first three are shared by DagState and TaskState diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index 50fa5ca19a1a..3c3a01bc7de9 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -15,10 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -A TaskGroup is a collection of closely related tasks on the same DAG that should be grouped -together when the DAG is displayed graphically. -""" +"""A collection of closely related tasks on the same DAG that should be grouped together visually.""" from __future__ import annotations import copy @@ -53,8 +50,10 @@ class TaskGroup(DAGNode): """ - A collection of tasks. When set_downstream() or set_upstream() are called on the - TaskGroup, it is applied across all tasks within the group if necessary. + A collection of tasks. + + When set_downstream() or set_upstream() are called on the TaskGroup, it is applied across + all tasks within the group if necessary. :param group_id: a unique, meaningful id for the TaskGroup. group_id must not conflict with group_id of TaskGroup or task_id of tasks in the DAG. Root TaskGroup has group_id @@ -318,6 +317,7 @@ def _set_relatives( ) -> None: """ Call set_upstream/set_downstream for all root/leaf tasks within this TaskGroup. + Update upstream_group_ids/downstream_group_ids/upstream_task_ids/downstream_task_ids. """ if not isinstance(task_or_task_list, Sequence): @@ -358,10 +358,7 @@ def leaves(self) -> list[BaseOperator]: return list(self.get_leaves()) def get_roots(self) -> Generator[BaseOperator, None, None]: - """ - Returns a generator of tasks that are root tasks, i.e. those with no upstream - dependencies within the TaskGroup. - """ + """Return a generator of tasks with no upstream dependencies within the TaskGroup.""" tasks = list(self) ids = {x.task_id for x in tasks} for task in tasks: @@ -369,10 +366,7 @@ def get_roots(self) -> Generator[BaseOperator, None, None]: yield task def get_leaves(self) -> Generator[BaseOperator, None, None]: - """ - Returns a generator of tasks that are leaf tasks, i.e. those with no downstream - dependencies within the TaskGroup. - """ + """Return a generator of tasks with no downstream dependencies within the TaskGroup.""" tasks = list(self) ids = {x.task_id for x in tasks} @@ -393,10 +387,7 @@ def recurse_for_first_non_setup_teardown(task): yield from recurse_for_first_non_setup_teardown(task) def child_id(self, label): - """ - Prefix label with group_id if prefix_group_id is True. Otherwise return the label - as-is. - """ + """Prefix label with group_id if prefix_group_id is True. Otherwise return the label as-is.""" if self.prefix_group_id: group_id = self.group_id if group_id: @@ -407,6 +398,8 @@ def child_id(self, label): @property def upstream_join_id(self) -> str: """ + Creates a unique ID for upstream dependencies of this TaskGroup. + If this TaskGroup has immediate upstream TaskGroups or tasks, a proxy node called upstream_join_id will be created in Graph view to join the outgoing edges from this TaskGroup to reduce the total number of edges needed to be displayed. @@ -416,6 +409,8 @@ def upstream_join_id(self) -> str: @property def downstream_join_id(self) -> str: """ + Creates a unique ID for downstream dependencies of this TaskGroup. + If this TaskGroup has immediate downstream TaskGroups or tasks, a proxy node called downstream_join_id will be created in Graph view to join the outgoing edges from this TaskGroup to reduce the total number of edges needed to be displayed. @@ -450,7 +445,8 @@ def serialize_for_task_group(self) -> tuple[DagAttributeTypes, Any]: def hierarchical_alphabetical_sort(self): """ - Sorts children in hierarchical alphabetical order: + Sorts children in hierarchical alphabetical order. + - groups in alphabetical order first - tasks in alphabetical order after them. @@ -462,8 +458,7 @@ def hierarchical_alphabetical_sort(self): def topological_sort(self, _include_subdag_tasks: bool = False): """ - Sorts children in topographical order, such that a task comes after any of its - upstream dependencies. + Sorts children in topographical order, such that a task comes after any of its upstream dependencies. :return: list of tasks in topological order """ @@ -680,10 +675,7 @@ def get_current_task_group(cls, dag: DAG | None) -> TaskGroup | None: def task_group_to_dict(task_item_or_group): - """ - Create a nested dict representation of this TaskGroup and its children used to construct - the Graph. - """ + """Create a nested dict representation of this TaskGroup and its children used to construct the Graph.""" from airflow.models.abstractoperator import AbstractOperator if isinstance(task := task_item_or_group, AbstractOperator): diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 2b65055f38d5..1db15ac61219 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -241,6 +241,7 @@ def coerce_datetime(v: dt.datetime | None, tz: dt.tzinfo | None = None) -> DateT def td_format(td_object: None | dt.timedelta | float | int) -> str | None: """ Format a timedelta object or float/int into a readable string for time duration. + For example timedelta(seconds=3752) would become `1h:2M:32s`. If the time is less than a second, the return will be `<1s`. """ diff --git a/airflow/utils/types.py b/airflow/utils/types.py index 788b072f3984..0eab9b3b8785 100644 --- a/airflow/utils/types.py +++ b/airflow/utils/types.py @@ -68,9 +68,6 @@ def from_run_id(run_id: str) -> DagRunType: class EdgeInfoType(TypedDict): - """ - Represents extra metadata that the DAG can store about an edge, - usually generated from an EdgeModifier. - """ + """Extra metadata that the DAG can store about an edge, usually generated from an EdgeModifier.""" label: str | None From 83e98c361e18ad5ee2ab4dcf66ca3afb82290c72 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Tue, 18 Jul 2023 09:51:56 +0530 Subject: [PATCH 215/533] Update providers count (#32653) I was going through the document and realised that we now have almost 80+ providers. So felt a bit odd to read that it said only 60+. So quickly updating these values to 80 instead of 60. --- docs/apache-airflow-providers/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow-providers/index.rst b/docs/apache-airflow-providers/index.rst index bb7b9884020b..0c960a8eeef4 100644 --- a/docs/apache-airflow-providers/index.rst +++ b/docs/apache-airflow-providers/index.rst @@ -28,7 +28,7 @@ multitude of external systems, but they can also extend Airflow core with new ca You can install those provider packages separately in order to interface with a given service. The providers for ``Apache Airflow`` are designed in the way that you can write your own providers easily. The -``Apache Airflow Community`` develops and maintain more than 60 provider packages, but you are free to +``Apache Airflow Community`` develops and maintain more than 80 provider packages, but you are free to develop your own providers - the providers you build have exactly the same capability as the providers written by the community, so you can release and share those providers with others. @@ -137,7 +137,7 @@ Community maintained providers From the point of view of the community, Airflow is delivered in multiple, separate packages. The core of Airflow scheduling system is delivered as ``apache-airflow`` package and there are more than -60 provider packages which can be installed separately as so called ``Airflow Provider packages``. +80 provider packages which can be installed separately as so called ``Airflow Provider packages``. Those packages are available as ``apache-airflow-providers`` packages - for example there is an ``apache-airflow-providers-amazon`` or ``apache-airflow-providers-google`` package). @@ -338,7 +338,7 @@ If you have done that, airflow does the following at runtime: **Should I name my provider specifically or should it be created in ``airflow.providers`` package?** -We have quite a number (>60) of providers managed by the community and we are going to maintain them +We have quite a number (>80) of providers managed by the community and we are going to maintain them together with Apache Airflow. All those providers have well-defined structured and follow the naming conventions we defined and they are all in ``airflow.providers`` package. If your intention is to contribute your provider, then you should follow those conventions and make a PR to Apache Airflow From f6bff828af28a9f7f25ef35ec77da4ca26388258 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Tue, 18 Jul 2023 11:53:31 +0530 Subject: [PATCH 216/533] Fixing breeze add-back-references command with doc (#32649) --- dev/README_RELEASE_AIRFLOW.md | 2 +- dev/README_RELEASE_HELM_CHART.md | 4 +--- dev/README_RELEASE_PROVIDER_PACKAGES.md | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 08ec16a68bfd..06d98041fd14 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -805,7 +805,7 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo ```shell script breeze release-management publish-docs --package-filter apache-airflow --package-filter docker-stack - breeze release-management add-back-references --airflow-site-directory --gen-type airflow + breeze release-management add-back-references --gen-type airflow git add . git commit -m "Add documentation for Apache Airflow ${VERSION}" git push diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index 087c56b3f83d..01994e6241a0 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -633,9 +633,7 @@ between the two repositories to be able to build the documentation. Regenerate `index.yaml` so it can be added to the Airflow website to allow: `helm repo add https://airflow.apache.org`. ```shell - cd "${AIRFLOW_SITE_DIRECTORY}" - breeze release-management add-back-references --airflow-site-directory --gen-type helm - cd .. + breeze release-management add-back-references --gen-type helm curl https://dist.apache.org/repos/dist/dev/airflow/helm-chart/$RC/index.yaml -o index.yaml cp ${AIRFLOW_SVN_RELEASE_HELM}/${VERSION}/airflow-${VERSION}.tgz . helm repo index --merge ./index.yaml . --url "https://downloads.apache.org/airflow/helm-chart/$VERSION" diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 323a83a9666c..3e13d7de9e75 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -392,10 +392,7 @@ breeze release-management publish-docs \ --package-filter 'apache-airflow-providers-*' \ --override-versioned -cd "${AIRFLOW_SITE_DIRECTORY}" -breeze release-management add-back-references --airflow-site-directory --gen-type providers -cd .. - +breeze release-management add-back-references --gen-type providers ``` If you see `ModuleNotFoundError: No module named 'docs'`, set: From 52d932f659d881a0b17bc1c1ba7e7bfd87d45848 Mon Sep 17 00:00:00 2001 From: Alan Ma Date: Tue, 18 Jul 2023 04:22:58 -0700 Subject: [PATCH 217/533] docs: Note Apache Beam requires `gcloud` to execute Dataflow jobs (#32663) * Update operators.rst Include warning that gcloud is required to submit Apache Beam pipelines to Dataflow * Update docs/apache-airflow-providers-apache-beam/operators.rst --------- Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- docs/apache-airflow-providers-apache-beam/operators.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/apache-airflow-providers-apache-beam/operators.rst b/docs/apache-airflow-providers-apache-beam/operators.rst index f7487fb26460..3ec91a050efd 100644 --- a/docs/apache-airflow-providers-apache-beam/operators.rst +++ b/docs/apache-airflow-providers-apache-beam/operators.rst @@ -24,6 +24,11 @@ streaming data-parallel processing pipelines. Using one of the open source Beam that defines the pipeline. The pipeline is then executed by one of Beam's supported distributed processing back-ends, which include Apache Flink, Apache Spark, and Google Cloud Dataflow. +.. note:: + This operator requires ``gcloud`` command (Google Cloud SDK) to be installed on the Airflow worker + when the Apache Beam pipeline runs on the + [Dataflow service](https://cloud.google.com/dataflow/docs). + .. _howto/operator:BeamRunPythonPipelineOperator: From 375d2fabcc2cef4b17d2129b557ac6fcb6c65067 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Tue, 18 Jul 2023 11:17:18 -0700 Subject: [PATCH 218/533] Remove system test changes accidentally added by #32355 (#32677) --- .../amazon/aws/example_eks_with_fargate_in_one_step.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py index 6b9907d83610..ae67a26588bd 100644 --- a/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/tests/system/providers/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -81,9 +81,6 @@ # Opting to use the same ARN for the cluster and the pod here, # but a different ARN could be configured and passed if desired. fargate_pod_execution_role_arn=fargate_pod_role_arn, - deferrable=True, - waiter_delay=30, - wait_for_completion=399, ) # [END howto_operator_eks_create_cluster_with_fargate_profile] From 4afcef919ba76f57fb5dbd97a24e65836be5b0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Ducharme?= Date: Tue, 18 Jul 2023 17:16:27 -0400 Subject: [PATCH 219/533] Chart: Add pod security context to pgbouncer (#32662) --- .../pgbouncer/pgbouncer-deployment.yaml | 8 ++-- chart/values.schema.json | 16 +++++++- chart/values.yaml | 1 + helm_tests/security/test_security_context.py | 39 +++++++++++++------ 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/chart/templates/pgbouncer/pgbouncer-deployment.yaml b/chart/templates/pgbouncer/pgbouncer-deployment.yaml index fde07807c4e6..14665692fcf6 100644 --- a/chart/templates/pgbouncer/pgbouncer-deployment.yaml +++ b/chart/templates/pgbouncer/pgbouncer-deployment.yaml @@ -26,8 +26,9 @@ {{- $tolerations := or .Values.pgbouncer.tolerations .Values.tolerations }} {{- $topologySpreadConstraints := or .Values.pgbouncer.topologySpreadConstraints .Values.topologySpreadConstraints }} {{- $revisionHistoryLimit := or .Values.pgbouncer.revisionHistoryLimit .Values.revisionHistoryLimit }} -{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.pgbouncer) }} -{{- $containerSecurityContextMetricsExporter := include "containerSecurityContext" (list . .Values.pgbouncer.metricsExporterSidecar) }} +{{- $securityContext := include "localPodSecurityContext" .Values.pgbouncer }} +{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.pgbouncer }} +{{- $containerSecurityContextMetricsExporter := include "externalContainerSecurityContext" .Values.pgbouncer.metricsExporterSidecar }} apiVersion: apps/v1 kind: Deployment metadata: @@ -82,8 +83,7 @@ spec: tolerations: {{- toYaml $tolerations | nindent 8 }} topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }} serviceAccountName: {{ include "pgbouncer.serviceAccountName" . }} - securityContext: - runAsUser: {{ .Values.pgbouncer.uid }} + securityContext: {{ $securityContext | nindent 8 }} restartPolicy: Always {{- if or .Values.registry.secretName .Values.registry.connection }} imagePullSecrets: diff --git a/chart/values.schema.json b/chart/values.schema.json index a4a329321a2f..ce116a2ea77a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -5080,10 +5080,24 @@ "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" }, "securityContexts": { - "description": "Security context definition for the PgBouncer. If not set, the values from global `securityContexts` will be used.", + "description": "Security context definition for the PgBouncer.", "type": "object", "x-docsSection": "Kubernetes", "properties": { + "pod": { + "description": "Pod security context definition for the PgBouncer.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "runAsUser": 65534, + "runAsGroup": 0, + "fsGroup": 0 + } + ] + }, "container": { "description": "Container security context definition for the PgBouncer.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 53d963000797..9569ec2d82f7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1833,6 +1833,7 @@ pgbouncer: # Detailed default security context for pgbouncer for container level securityContexts: + pod: {} container: {} metricsExporterSidecar: diff --git a/helm_tests/security/test_security_context.py b/helm_tests/security/test_security_context.py index f7693af19805..72fba4a4fbe8 100644 --- a/helm_tests/security/test_security_context.py +++ b/helm_tests/security/test_security_context.py @@ -57,6 +57,14 @@ def test_check_statsd_uid(self): assert 3000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0]) + def test_check_pgbouncer_uid(self): + docs = render_chart( + values={"pgbouncer": {"enabled": True, "uid": 3000}}, + show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"], + ) + + assert 3000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0]) + def test_check_cleanup_job(self): docs = render_chart( values={"uid": 3000, "gid": 30, "cleanup": {"enabled": True}}, @@ -219,7 +227,10 @@ def test_global_security_context(self): ctx_value_pod = {"runAsUser": 7000} ctx_value_container = {"allowPrivilegeEscalation": False} docs = render_chart( - values={"securityContexts": {"containers": ctx_value_container, "pod": ctx_value_pod}}, + values={ + "securityContexts": {"containers": ctx_value_container, "pod": ctx_value_pod}, + "pgbouncer": {"enabled": True}, + }, show_only=[ "templates/flower/flower-deployment.yaml", "templates/scheduler/scheduler-deployment.yaml", @@ -228,31 +239,35 @@ def test_global_security_context(self): "templates/jobs/create-user-job.yaml", "templates/jobs/migrate-database-job.yaml", "templates/triggerer/triggerer-deployment.yaml", + "templates/pgbouncer/pgbouncer-deployment.yaml", "templates/statsd/statsd-deployment.yaml", "templates/redis/redis-statefulset.yaml", ], ) - - for index in range(len(docs) - 2): + for doc in docs[:-3]: assert ctx_value_container == jmespath.search( - "spec.template.spec.containers[0].securityContext", docs[index] + "spec.template.spec.containers[0].securityContext", doc ) - assert ctx_value_pod == jmespath.search("spec.template.spec.securityContext", docs[index]) + assert ctx_value_pod == jmespath.search("spec.template.spec.securityContext", doc) - # Global security context is not propagated to redis and statsd, so we test default value + # Global security context is not propagated to pgbouncer, redis and statsd, so we test default value default_ctx_value_container = {"allowPrivilegeEscalation": False, "capabilities": {"drop": ["ALL"]}} + default_ctx_value_pod_pgbouncer = {"runAsUser": 65534} default_ctx_value_pod_statsd = {"runAsUser": 65534} default_ctx_value_pod_redis = {"runAsUser": 0} - for index in range(len(docs) - 2, len(docs)): + for doc in docs[-3:]: assert default_ctx_value_container == jmespath.search( - "spec.template.spec.containers[0].securityContext", docs[index] + "spec.template.spec.containers[0].securityContext", doc ) - assert default_ctx_value_pod_statsd == jmespath.search( - "spec.template.spec.securityContext", docs[len(docs) - 2] + # Test pgbouncer metrics-exporter container + assert default_ctx_value_container == jmespath.search( + "spec.template.spec.containers[1].securityContext", docs[-3] ) - assert default_ctx_value_pod_redis == jmespath.search( - "spec.template.spec.securityContext", docs[len(docs) - 1] + assert default_ctx_value_pod_pgbouncer == jmespath.search( + "spec.template.spec.securityContext", docs[-3] ) + assert default_ctx_value_pod_statsd == jmespath.search("spec.template.spec.securityContext", docs[-2]) + assert default_ctx_value_pod_redis == jmespath.search("spec.template.spec.securityContext", docs[-1]) # Test securityContexts for main containers def test_main_container_setting(self): From 62044f173b060a43db4eecc3321b815263a59c1a Mon Sep 17 00:00:00 2001 From: Vijayasarathi Balasubramanian Date: Tue, 18 Jul 2023 17:37:21 -0400 Subject: [PATCH 220/533] Documentation update to increase readability (#32658) Co-authored-by: Niko Oliveira Co-authored-by: Tzu-ping Chung --- .../administration-and-deployment/modules_management.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/administration-and-deployment/modules_management.rst b/docs/apache-airflow/administration-and-deployment/modules_management.rst index ec1a9b11e419..6ee9f5b65029 100644 --- a/docs/apache-airflow/administration-and-deployment/modules_management.rst +++ b/docs/apache-airflow/administration-and-deployment/modules_management.rst @@ -44,7 +44,7 @@ How package/modules loading in Python works The list of directories from which Python tries to load the module is given by the variable ``sys.path``. Python really tries to `intelligently determine the contents `_ -of this variable, including depending on the operating system and how Python +of this variable, depending on the operating system and how Python is installed and which Python version is used. You can check the contents of this variable for the current Python environment From 7ed791dab72709fbc5c9c27687a8b014c3e9906d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Tue, 18 Jul 2023 14:45:00 -0700 Subject: [PATCH 221/533] Do not return success from AWS ECS trigger after max_attempts (#32589) --- airflow/providers/amazon/aws/triggers/ecs.py | 7 ++++--- .../providers/amazon/aws/triggers/test_ecs.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/airflow/providers/amazon/aws/triggers/ecs.py b/airflow/providers/amazon/aws/triggers/ecs.py index 29ad22e13e79..af6f72e7714e 100644 --- a/airflow/providers/amazon/aws/triggers/ecs.py +++ b/airflow/providers/amazon/aws/triggers/ecs.py @@ -22,6 +22,7 @@ from botocore.exceptions import ClientError, WaiterError +from airflow import AirflowException from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.ecs import EcsHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook @@ -170,7 +171,8 @@ async def run(self) -> AsyncIterator[TriggerEvent]: await waiter.wait( cluster=self.cluster, tasks=[self.task_arn], WaiterConfig={"MaxAttempts": 1} ) - break # we reach this point only if the waiter met a success criteria + # we reach this point only if the waiter met a success criteria + yield TriggerEvent({"status": "success", "task_arn": self.task_arn}) except WaiterError as error: if "terminal failure" in str(error): raise @@ -179,8 +181,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: finally: if self.log_group and self.log_stream: logs_token = await self._forward_logs(logs_client, logs_token) - - yield TriggerEvent({"status": "success", "task_arn": self.task_arn}) + raise AirflowException("Waiter error: max attempts reached") async def _forward_logs(self, logs_client, next_token: str | None = None) -> str | None: """ diff --git a/tests/providers/amazon/aws/triggers/test_ecs.py b/tests/providers/amazon/aws/triggers/test_ecs.py index 551ab39a4486..e897bce74007 100644 --- a/tests/providers/amazon/aws/triggers/test_ecs.py +++ b/tests/providers/amazon/aws/triggers/test_ecs.py @@ -22,6 +22,7 @@ import pytest from botocore.exceptions import WaiterError +from airflow import AirflowException from airflow.providers.amazon.aws.hooks.ecs import EcsHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook from airflow.providers.amazon.aws.triggers.ecs import ( @@ -56,6 +57,26 @@ async def test_run_until_error(self, _, client_mock): assert wait_mock.call_count == 3 + @pytest.mark.asyncio + @mock.patch.object(EcsHook, "async_conn") + # this mock is only necessary to avoid a "No module named 'aiobotocore'" error in the LatestBoto CI step + @mock.patch.object(AwsLogsHook, "async_conn") + async def test_run_until_timeout(self, _, client_mock): + a_mock = mock.MagicMock() + client_mock.__aenter__.return_value = a_mock + wait_mock = AsyncMock() + wait_mock.side_effect = WaiterError("name", "reason", {"tasks": [{"lastStatus": "my_status"}]}) + a_mock.get_waiter().wait = wait_mock + + trigger = TaskDoneTrigger("cluster", "task_arn", 0, 10, None, None) + + with pytest.raises(AirflowException) as err: + generator = trigger.run() + await generator.asend(None) + + assert wait_mock.call_count == 10 + assert "max attempts" in str(err.value) + @pytest.mark.asyncio @mock.patch.object(EcsHook, "async_conn") # this mock is only necessary to avoid a "No module named 'aiobotocore'" error in the LatestBoto CI step From 60c49ab2dfabaf450b80a5c7569743dd383500a6 Mon Sep 17 00:00:00 2001 From: Daniel Reeves <31971762+dwreeves@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:46:20 -0400 Subject: [PATCH 222/533] Add more accurate typing for DbApiHook.run method (#31846) Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- airflow/providers/apache/hive/hooks/hive.py | 2 +- airflow/providers/apache/pinot/hooks/pinot.py | 4 +- airflow/providers/common/sql/hooks/sql.py | 56 ++++++++++++++++--- airflow/providers/common/sql/operators/sql.py | 4 +- .../databricks/hooks/databricks_sql.py | 35 ++++++++++-- airflow/providers/exasol/hooks/exasol.py | 42 +++++++++++--- .../providers/google/cloud/hooks/bigquery.py | 2 +- .../google/cloud/operators/cloud_sql.py | 4 +- .../google/suite/transfers/sql_to_sheets.py | 2 +- airflow/providers/neo4j/operators/neo4j.py | 4 +- airflow/providers/presto/hooks/presto.py | 28 +++------- .../providers/slack/transfers/sql_to_slack.py | 8 +-- .../providers/snowflake/hooks/snowflake.py | 36 ++++++++++-- .../snowflake/operators/snowflake.py | 6 +- .../snowflake/transfers/snowflake_to_slack.py | 4 +- airflow/providers/trino/hooks/trino.py | 30 +++------- 16 files changed, 181 insertions(+), 86 deletions(-) diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index 89fcda466ffc..c67901445a4d 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -1014,7 +1014,7 @@ def to_csv( self.log.info("Done. Loaded a total of %s rows.", i) def get_records( - self, sql: str | list[str], parameters: Iterable | Mapping | None = None, **kwargs + self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None, **kwargs ) -> Any: """ Get a set of records from a Hive query; optionally pass a 'schema' kwarg to specify target schema. diff --git a/airflow/providers/apache/pinot/hooks/pinot.py b/airflow/providers/apache/pinot/hooks/pinot.py index 1053e220af00..d9fd9044a21e 100644 --- a/airflow/providers/apache/pinot/hooks/pinot.py +++ b/airflow/providers/apache/pinot/hooks/pinot.py @@ -288,7 +288,7 @@ def get_uri(self) -> str: return f"{conn_type}://{host}/{endpoint}" def get_records( - self, sql: str | list[str], parameters: Iterable | Mapping | None = None, **kwargs + self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None, **kwargs ) -> Any: """ Executes the sql and returns a set of records. @@ -301,7 +301,7 @@ def get_records( cur.execute(sql) return cur.fetchall() - def get_first(self, sql: str | list[str], parameters: Iterable | Mapping | None = None) -> Any: + def get_first(self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None) -> Any: """ Executes the sql and returns the first resulting row. diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index 3f2f964a0bb6..d444a952871c 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -18,7 +18,18 @@ from contextlib import closing from datetime import datetime -from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, Protocol, Sequence, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Iterable, + Mapping, + Protocol, + Sequence, + TypeVar, + cast, + overload, +) from urllib.parse import urlparse import sqlparse @@ -34,6 +45,9 @@ from airflow.providers.openlineage.sqlparser import DatabaseInfo +T = TypeVar("T") + + def return_single_query_results(sql: str | Iterable[str], return_last: bool, split_statements: bool): """ Determines when results of single query only should be returned. @@ -184,7 +198,7 @@ def get_sqlalchemy_engine(self, engine_kwargs=None): engine_kwargs = {} return create_engine(self.get_uri(), **engine_kwargs) - def get_pandas_df(self, sql, parameters=None, **kwargs): + def get_pandas_df(self, sql, parameters: Iterable | Mapping[str, Any] | None = None, **kwargs): """ Executes the sql and returns a pandas dataframe. @@ -204,7 +218,9 @@ def get_pandas_df(self, sql, parameters=None, **kwargs): with closing(self.get_conn()) as conn: return psql.read_sql(sql, con=conn, params=parameters, **kwargs) - def get_pandas_df_by_chunks(self, sql, parameters=None, *, chunksize, **kwargs): + def get_pandas_df_by_chunks( + self, sql, parameters: Iterable | Mapping[str, Any] | None = None, *, chunksize: int | None, **kwargs + ): """ Executes the sql and returns a generator. @@ -228,7 +244,7 @@ def get_pandas_df_by_chunks(self, sql, parameters=None, *, chunksize, **kwargs): def get_records( self, sql: str | list[str], - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, ) -> Any: """ Executes the sql and returns a set of records. @@ -238,7 +254,7 @@ def get_records( """ return self.run(sql=sql, parameters=parameters, handler=fetch_all_handler) - def get_first(self, sql: str | list[str], parameters: Iterable | Mapping | None = None) -> Any: + def get_first(self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None) -> Any: """ Executes the sql and returns the first resulting row. @@ -268,15 +284,39 @@ def last_description(self) -> Sequence[Sequence] | None: return None return self.descriptions[-1] + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: None = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> None: + ... + + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: Callable[[Any], T] = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> T | list[T]: + ... + def run( self, sql: str | Iterable[str], autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, + handler: Callable[[Any], T] | None = None, split_statements: bool = False, return_last: bool = True, - ) -> Any | list[Any] | None: + ) -> T | list[T] | None: """Run a command or a list of commands. Pass a list of SQL statements to the sql parameter to get them to diff --git a/airflow/providers/common/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py index f1784f618e3e..bf2b38f05544 100644 --- a/airflow/providers/common/sql/operators/sql.py +++ b/airflow/providers/common/sql/operators/sql.py @@ -762,7 +762,7 @@ def __init__( sql: str, conn_id: str | None = None, database: str | None = None, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, **kwargs, ) -> None: super().__init__(conn_id=conn_id, database=database, **kwargs) @@ -1129,7 +1129,7 @@ def __init__( follow_task_ids_if_false: list[str], conn_id: str = "default_conn_id", database: str | None = None, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, **kwargs, ) -> None: super().__init__(conn_id=conn_id, database=database, **kwargs) diff --git a/airflow/providers/databricks/hooks/databricks_sql.py b/airflow/providers/databricks/hooks/databricks_sql.py index 674219e1c8a4..31c816f39bfe 100644 --- a/airflow/providers/databricks/hooks/databricks_sql.py +++ b/airflow/providers/databricks/hooks/databricks_sql.py @@ -18,7 +18,7 @@ from contextlib import closing from copy import copy -from typing import Any, Callable, Iterable, Mapping +from typing import Any, Callable, Iterable, Mapping, TypeVar, overload from databricks import sql # type: ignore[attr-defined] from databricks.sql.client import Connection # type: ignore[attr-defined] @@ -30,6 +30,9 @@ LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "api/2.0/sql/endpoints") +T = TypeVar("T") + + class DatabricksSqlHook(BaseDatabricksHook, DbApiHook): """Hook to interact with Databricks SQL. @@ -138,15 +141,39 @@ def get_conn(self) -> Connection: ) return self._sql_conn + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: None = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> None: + ... + + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: Callable[[Any], T] = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> T | list[T]: + ... + def run( self, sql: str | Iterable[str], autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, + handler: Callable[[Any], T] | None = None, split_statements: bool = True, return_last: bool = True, - ) -> Any | list[Any] | None: + ) -> T | list[T] | None: """Runs a command or a list of commands. Pass a list of SQL statements to the SQL parameter to get them to diff --git a/airflow/providers/exasol/hooks/exasol.py b/airflow/providers/exasol/hooks/exasol.py index ac45a7915f5c..0911e396041b 100644 --- a/airflow/providers/exasol/hooks/exasol.py +++ b/airflow/providers/exasol/hooks/exasol.py @@ -18,7 +18,7 @@ from __future__ import annotations from contextlib import closing -from typing import Any, Callable, Iterable, Mapping, Sequence +from typing import Any, Callable, Iterable, Mapping, Sequence, TypeVar, overload import pandas as pd import pyexasol @@ -26,6 +26,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results +T = TypeVar("T") + class ExasolHook(DbApiHook): """Interact with Exasol. @@ -66,7 +68,9 @@ def get_conn(self) -> ExaConnection: conn = pyexasol.connect(**conn_args) return conn - def get_pandas_df(self, sql: str, parameters: dict | None = None, **kwargs) -> pd.DataFrame: + def get_pandas_df( + self, sql, parameters: Iterable | Mapping[str, Any] | None = None, **kwargs + ) -> pd.DataFrame: """Execute the SQL and return a Pandas dataframe. :param sql: The sql statement to be executed (str) or a list of @@ -83,7 +87,7 @@ def get_pandas_df(self, sql: str, parameters: dict | None = None, **kwargs) -> p def get_records( self, sql: str | list[str], - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, ) -> list[dict | tuple[Any, ...]]: """Execute the SQL and return a set of records. @@ -95,7 +99,7 @@ def get_records( with closing(conn.execute(sql, parameters)) as cur: return cur.fetchall() - def get_first(self, sql: str | list[str], parameters: Iterable | Mapping | None = None) -> Any: + def get_first(self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None) -> Any: """Execute the SQL and return the first resulting row. :param sql: the sql statement to be executed (str) or a list of @@ -157,15 +161,39 @@ def get_description(statement: ExaStatement) -> Sequence[Sequence]: ) return cols + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: None = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> None: + ... + + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: Callable[[Any], T] = ..., + split_statements: bool = ..., + return_last: bool = ..., + ) -> T | list[T]: + ... + def run( self, sql: str | Iterable[str], autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, + handler: Callable[[Any], T] | None = None, split_statements: bool = False, return_last: bool = True, - ) -> Any | list[Any] | None: + ) -> T | list[T] | None: """Run a command or a list of commands. Pass a list of SQL statements to the SQL parameter to get them to diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index f76fd1b1e1a8..01c00046d1a8 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -241,7 +241,7 @@ def insert_rows( def get_pandas_df( self, sql: str, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, dialect: str | None = None, **kwargs, ) -> DataFrame: diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index c60a5bc09a33..61275dbc2926 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -18,7 +18,7 @@ """This module contains Google Cloud SQL operators.""" from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence from googleapiclient.errors import HttpError @@ -1189,7 +1189,7 @@ def __init__( *, sql: str | Iterable[str], autocommit: bool = False, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, gcp_conn_id: str = "google_cloud_default", gcp_cloudsql_conn_id: str = "google_cloud_sql_default", sql_proxy_binary_path: str | None = None, diff --git a/airflow/providers/google/suite/transfers/sql_to_sheets.py b/airflow/providers/google/suite/transfers/sql_to_sheets.py index f8ee69440828..f38d9b230a1e 100644 --- a/airflow/providers/google/suite/transfers/sql_to_sheets.py +++ b/airflow/providers/google/suite/transfers/sql_to_sheets.py @@ -68,7 +68,7 @@ def __init__( sql: str, spreadsheet_id: str, sql_conn_id: str, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, database: str | None = None, spreadsheet_range: str = "Sheet1", gcp_conn_id: str = "google_cloud_default", diff --git a/airflow/providers/neo4j/operators/neo4j.py b/airflow/providers/neo4j/operators/neo4j.py index 3c5278423004..6cd36f9472f8 100644 --- a/airflow/providers/neo4j/operators/neo4j.py +++ b/airflow/providers/neo4j/operators/neo4j.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence from airflow.models import BaseOperator from airflow.providers.neo4j.hooks.neo4j import Neo4jHook @@ -46,7 +46,7 @@ def __init__( *, sql: str, neo4j_conn_id: str = "neo4j_default", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/airflow/providers/presto/hooks/presto.py b/airflow/providers/presto/hooks/presto.py index 477b4cd797de..7ce902180789 100644 --- a/airflow/providers/presto/hooks/presto.py +++ b/airflow/providers/presto/hooks/presto.py @@ -19,7 +19,7 @@ import json import os -from typing import Any, Callable, Iterable, Mapping +from typing import Any, Iterable, Mapping, TypeVar import prestodb from prestodb.exceptions import DatabaseError @@ -31,6 +31,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING, DEFAULT_FORMAT_PREFIX +T = TypeVar("T") + def generate_presto_client_info() -> str: """Return json string with dag_id, task_id, execution_date and try_number.""" @@ -136,7 +138,7 @@ def get_isolation_level(self) -> Any: def get_records( self, sql: str | list[str] = "", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, ) -> Any: if not isinstance(sql, str): raise ValueError(f"The sql in Presto Hook must be a string and is {sql}!") @@ -145,7 +147,9 @@ def get_records( except DatabaseError as e: raise PrestoException(e) - def get_first(self, sql: str | list[str] = "", parameters: Iterable | Mapping | None = None) -> Any: + def get_first( + self, sql: str | list[str] = "", parameters: Iterable | Mapping[str, Any] | None = None + ) -> Any: if not isinstance(sql, str): raise ValueError(f"The sql in Presto Hook must be a string and is {sql}!") try: @@ -170,24 +174,6 @@ def get_pandas_df(self, sql: str = "", parameters=None, **kwargs): df = pandas.DataFrame(**kwargs) return df - def run( - self, - sql: str | Iterable[str], - autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, - split_statements: bool = False, - return_last: bool = True, - ) -> Any | list[Any] | None: - return super().run( - sql=sql, - autocommit=autocommit, - parameters=parameters, - handler=handler, - split_statements=split_statements, - return_last=return_last, - ) - def insert_rows( self, table: str, diff --git a/airflow/providers/slack/transfers/sql_to_slack.py b/airflow/providers/slack/transfers/sql_to_slack.py index caac3eb7d1e2..97017c80d7e9 100644 --- a/airflow/providers/slack/transfers/sql_to_slack.py +++ b/airflow/providers/slack/transfers/sql_to_slack.py @@ -17,7 +17,7 @@ from __future__ import annotations from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence from pandas import DataFrame from tabulate import tabulate @@ -51,7 +51,7 @@ def __init__( sql: str, sql_conn_id: str, sql_hook_params: dict | None = None, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, **kwargs, ): super().__init__(**kwargs) @@ -125,7 +125,7 @@ def __init__( slack_channel: str | None = None, slack_message: str, results_df_name: str = "results_df", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, **kwargs, ) -> None: @@ -246,7 +246,7 @@ def __init__( sql: str, sql_conn_id: str, sql_hook_params: dict | None = None, - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, slack_conn_id: str, slack_filename: str, slack_channels: str | Sequence[str] | None = None, diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index 59199cf8cd2e..76b39be6acc3 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -22,7 +22,7 @@ from functools import wraps from io import StringIO from pathlib import Path -from typing import Any, Callable, Iterable, Mapping +from typing import Any, Callable, Iterable, Mapping, TypeVar, overload from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -35,6 +35,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results from airflow.utils.strings import to_boolean +T = TypeVar("T") + def _try_to_boolean(value: Any): if isinstance(value, (str, type(None))): @@ -321,16 +323,42 @@ def set_autocommit(self, conn, autocommit: Any) -> None: def get_autocommit(self, conn): return getattr(conn, "autocommit_mode", False) + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: None = ..., + split_statements: bool = ..., + return_last: bool = ..., + return_dictionaries: bool = ..., + ) -> None: + ... + + @overload + def run( + self, + sql: str | Iterable[str], + autocommit: bool = ..., + parameters: Iterable | Mapping[str, Any] | None = ..., + handler: Callable[[Any], T] = ..., + split_statements: bool = ..., + return_last: bool = ..., + return_dictionaries: bool = ..., + ) -> T | list[T]: + ... + def run( self, sql: str | Iterable[str], autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, + handler: Callable[[Any], T] | None = None, split_statements: bool = True, return_last: bool = True, return_dictionaries: bool = False, - ) -> Any | list[Any] | None: + ) -> T | list[T] | None: """Runs a command or a list of commands. Pass a list of SQL statements to the SQL parameter to get them to diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 29b57e54eb97..1de82218ff34 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -201,7 +201,7 @@ def __init__( *, sql: str, snowflake_conn_id: str = "snowflake_default", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, autocommit: bool = True, do_xcom_push: bool = True, warehouse: str | None = None, @@ -266,7 +266,7 @@ def __init__( pass_value: Any, tolerance: Any = None, snowflake_conn_id: str = "snowflake_default", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, autocommit: bool = True, do_xcom_push: bool = True, warehouse: str | None = None, @@ -341,7 +341,7 @@ def __init__( date_filter_column: str = "ds", days_back: SupportsAbs[int] = -7, snowflake_conn_id: str = "snowflake_default", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, autocommit: bool = True, do_xcom_push: bool = True, warehouse: str | None = None, diff --git a/airflow/providers/snowflake/transfers/snowflake_to_slack.py b/airflow/providers/snowflake/transfers/snowflake_to_slack.py index 50d38b947170..8e818f4a4b16 100644 --- a/airflow/providers/snowflake/transfers/snowflake_to_slack.py +++ b/airflow/providers/snowflake/transfers/snowflake_to_slack.py @@ -17,7 +17,7 @@ from __future__ import annotations import warnings -from typing import Iterable, Mapping, Sequence +from typing import Any, Iterable, Mapping, Sequence from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.slack.transfers.sql_to_slack import SqlToSlackOperator @@ -68,7 +68,7 @@ def __init__( snowflake_conn_id: str = "snowflake_default", slack_conn_id: str = "slack_default", results_df_name: str = "results_df", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, warehouse: str | None = None, database: str | None = None, schema: str | None = None, diff --git a/airflow/providers/trino/hooks/trino.py b/airflow/providers/trino/hooks/trino.py index 67ee432ca386..5144978dab33 100644 --- a/airflow/providers/trino/hooks/trino.py +++ b/airflow/providers/trino/hooks/trino.py @@ -19,7 +19,7 @@ import json import os -from typing import Any, Callable, Iterable, Mapping +from typing import Any, Iterable, Mapping, TypeVar import trino from trino.exceptions import DatabaseError @@ -31,6 +31,8 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING, DEFAULT_FORMAT_PREFIX +T = TypeVar("T") + def generate_trino_client_info() -> str: """Return json string with dag_id, task_id, execution_date and try_number.""" @@ -154,7 +156,7 @@ def get_isolation_level(self) -> Any: def get_records( self, sql: str | list[str] = "", - parameters: Iterable | Mapping | None = None, + parameters: Iterable | Mapping[str, Any] | None = None, ) -> Any: if not isinstance(sql, str): raise ValueError(f"The sql in Trino Hook must be a string and is {sql}!") @@ -163,7 +165,9 @@ def get_records( except DatabaseError as e: raise TrinoException(e) - def get_first(self, sql: str | list[str] = "", parameters: Iterable | Mapping | None = None) -> Any: + def get_first( + self, sql: str | list[str] = "", parameters: Iterable | Mapping[str, Any] | None = None + ) -> Any: if not isinstance(sql, str): raise ValueError(f"The sql in Trino Hook must be a string and is {sql}!") try: @@ -172,7 +176,7 @@ def get_first(self, sql: str | list[str] = "", parameters: Iterable | Mapping | raise TrinoException(e) def get_pandas_df( - self, sql: str = "", parameters: Iterable | Mapping | None = None, **kwargs + self, sql: str = "", parameters: Iterable | Mapping[str, Any] | None = None, **kwargs ): # type: ignore[override] import pandas @@ -190,24 +194,6 @@ def get_pandas_df( df = pandas.DataFrame(**kwargs) return df - def run( - self, - sql: str | Iterable[str], - autocommit: bool = False, - parameters: Iterable | Mapping | None = None, - handler: Callable | None = None, - split_statements: bool = False, - return_last: bool = True, - ) -> Any | list[Any] | None: - return super().run( - sql=sql, - autocommit=autocommit, - parameters=parameters, - handler=handler, - split_statements=split_statements, - return_last=return_last, - ) - def insert_rows( self, table: str, From 68e9e0e440c19479996f79b81395289f402b7b25 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Tue, 18 Jul 2023 23:54:48 +0200 Subject: [PATCH 223/533] Add deprecation info to the Airflow modules and classes docstring (#32635) --- airflow/api/common/experimental/delete_dag.py | 1 + airflow/api/common/experimental/mark_tasks.py | 2 +- airflow/api/common/experimental/trigger_dag.py | 1 + airflow/contrib/secrets/__init__.py | 2 +- airflow/contrib/task_runner/__init__.py | 2 +- airflow/contrib/utils/__init__.py | 2 +- airflow/contrib/utils/log/__init__.py | 2 +- airflow/hooks/dbapi.py | 1 + airflow/operators/subdag.py | 4 ++-- 9 files changed, 10 insertions(+), 7 deletions(-) diff --git a/airflow/api/common/experimental/delete_dag.py b/airflow/api/common/experimental/delete_dag.py index 821b80aa9af5..42dd8a9048a4 100644 --- a/airflow/api/common/experimental/delete_dag.py +++ b/airflow/api/common/experimental/delete_dag.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.api.common.delete_dag` instead.""" from __future__ import annotations import warnings diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py index 303c9f98ee59..7341e00934b3 100644 --- a/airflow/api/common/experimental/mark_tasks.py +++ b/airflow/api/common/experimental/mark_tasks.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Task Instance APIs.""" +"""Task Instance APIs. This module is deprecated. Please use :mod:`airflow.api.common.mark_tasks` instead.""" from __future__ import annotations import warnings diff --git a/airflow/api/common/experimental/trigger_dag.py b/airflow/api/common/experimental/trigger_dag.py index 123b09cb1c02..4bcbf11fcbce 100644 --- a/airflow/api/common/experimental/trigger_dag.py +++ b/airflow/api/common/experimental/trigger_dag.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.api.common.trigger_dag` instead.""" from __future__ import annotations import warnings diff --git a/airflow/contrib/secrets/__init__.py b/airflow/contrib/secrets/__init__.py index 9e2b4c74abdd..72df9f65433f 100644 --- a/airflow/contrib/secrets/__init__.py +++ b/airflow/contrib/secrets/__init__.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This package is deprecated. Please use `airflow.secrets` or `airflow.providers.*.secrets`.""" +"""This package is deprecated. Please use :mod:`airflow.secrets` or `airflow.providers.*.secrets`.""" from __future__ import annotations import warnings diff --git a/airflow/contrib/task_runner/__init__.py b/airflow/contrib/task_runner/__init__.py index 82bd6bf04148..d956469e0cea 100644 --- a/airflow/contrib/task_runner/__init__.py +++ b/airflow/contrib/task_runner/__init__.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This package is deprecated. Please use `airflow.task.task_runner`.""" +"""This package is deprecated. Please use :mod:`airflow.task.task_runner`.""" from __future__ import annotations import warnings diff --git a/airflow/contrib/utils/__init__.py b/airflow/contrib/utils/__init__.py index 4057f36868d7..0facd9b77c0f 100644 --- a/airflow/contrib/utils/__init__.py +++ b/airflow/contrib/utils/__init__.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This package is deprecated. Please use `airflow.utils`.""" +"""This package is deprecated. Please use :mod:`airflow.utils`.""" from __future__ import annotations import warnings diff --git a/airflow/contrib/utils/log/__init__.py b/airflow/contrib/utils/log/__init__.py index baf3a481858c..e4a188cf0f5a 100644 --- a/airflow/contrib/utils/log/__init__.py +++ b/airflow/contrib/utils/log/__init__.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This package is deprecated. Please use `airflow.utils.log`.""" +"""This package is deprecated. Please use :mod:`airflow.utils.log`.""" from __future__ import annotations import warnings diff --git a/airflow/hooks/dbapi.py b/airflow/hooks/dbapi.py index b4cd1be66712..f3f28c40e021 100644 --- a/airflow/hooks/dbapi.py +++ b/airflow/hooks/dbapi.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.providers.common.sql.hooks.sql`.""" from __future__ import annotations import warnings diff --git a/airflow/operators/subdag.py b/airflow/operators/subdag.py index 0f242c09f374..680497217d51 100644 --- a/airflow/operators/subdag.py +++ b/airflow/operators/subdag.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This module is deprecated, please use :mod:`airflow.utils.task_group`. +This module is deprecated. Please use :mod:`airflow.utils.task_group`. The module which provides a way to nest your DAGs and so your levels of complexity. """ @@ -50,7 +50,7 @@ class SkippedStatePropagationOptions(Enum): class SubDagOperator(BaseSensorOperator): """ - This class is deprecated, please use `airflow.utils.task_group.TaskGroup`. + This class is deprecated, please use :class:`airflow.utils.task_group.TaskGroup`. This runs a sub dag. By convention, a sub dag's dag_id should be prefixed by its parent and a dot. As in `parent.child`. From 9d87110485789e6777d47e7720930a12be8dab00 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Tue, 18 Jul 2023 22:18:14 -0700 Subject: [PATCH 224/533] Fix a typo in breeze testing commands descriptions (#32685) --- dev/breeze/src/airflow_breeze/commands/testing_commands.py | 4 ++-- images/breeze/output-commands-hash.txt | 6 +++--- images/breeze/output_testing.svg | 2 +- images/breeze/output_testing_docker-compose-tests.svg | 4 ++-- images/breeze/output_testing_integration-tests.svg | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 10f2fdbfc6b5..12e7ad98b02a 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -99,7 +99,7 @@ def group_for_testing(): ) @click.option( "--wait-for-containers-timeout", - help="Timeout in seconds to wait for all containers to start", + help="Time to wait (in seconds) for all containers to start", envvar="WAIT_FOR_CONTAINERS_TIMEOUT", show_default=True, type=IntRange(0, 600), @@ -457,7 +457,7 @@ def command_for_tests( @group_for_testing.command( name="integration-tests", - help="Run the specified integratio tests.", + help="Run the specified integration tests.", context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 1889686688c7..a0bd58e32e2c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -63,8 +63,8 @@ setup:cdf58a9c27af2877dc5a658ed0a1d99c shell:87e7bdcebe1180395adfec86b2a065f5 start-airflow:f7216a8126ecf14b033e2ea677e1a105 static-checks:bb1248670000271a9e011c12747079c1 -testing:docker-compose-tests:e8b18bda81e2fa0d48866805469310dc +testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 -testing:integration-tests:48707c24aa076b70fccf6dbfb18b65a2 +testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 testing:tests:e18fbd845ca5783879244bf0f9f9c51e -testing:0d1b776df1e8b3915aa219e0c15a0917 +testing:f360a66d2b1b4c39880117ae6166007f diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index db5690e0ad3e..82c6f8e344e3 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -105,7 +105,7 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integratio tests.                                      │ +│integration-tests                        Run the specified integration tests.                                     │ │helm-tests                               Run Helm chart tests.                                                    │ │docker-compose-tests                     Run docker-compose tests.                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_docker-compose-tests.svg b/images/breeze/output_testing_docker-compose-tests.svg index ce83b2e6d3a9..2b687b4e1ea1 100644 --- a/images/breeze/output_testing_docker-compose-tests.svg +++ b/images/breeze/output_testing_docker-compose-tests.svg @@ -140,8 +140,8 @@ │(>3.8< | 3.9 | 3.10 | 3.11)                                 │ │[default: 3.8]                                              │ │--skip-docker-compose-deletionSkip deletion of docker-compose instance after the test│ -│--wait-for-containers-timeoutTimeout in seconds to wait for all containers to start(INTEGER RANGE)│ -│[default: 300; 0<=x<=600]                             │ +│--wait-for-containers-timeoutTime to wait (in seconds) for all containers to start(INTEGER RANGE)│ +│[default: 300; 0<=x<=600]                            │ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ │[default: apache/airflow]                       │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_integration-tests.svg b/images/breeze/output_testing_integration-tests.svg index 8d5c3478d8f7..be4d37e6e6c2 100644 --- a/images/breeze/output_testing_integration-tests.svg +++ b/images/breeze/output_testing_integration-tests.svg @@ -162,7 +162,7 @@ Usage: breeze testing integration-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... -Run the specified integratio tests. +Run the specified integration tests. ╭─ Basic flag for integration tests command ───────────────────────────────────────────────────────────────────────────╮ │--integrationIntegration(s) to enable when running (can be more than one).                              │ From 0fbef49b8df19f0b4d367bc3a58e8d083330faea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 09:44:02 +0300 Subject: [PATCH 225/533] Bump word-wrap from 1.2.3 to 1.2.4 in /airflow/www (#32680) Bumps [word-wrap](https://github.com/jonschlinkert/word-wrap) from 1.2.3 to 1.2.4. - [Release notes](https://github.com/jonschlinkert/word-wrap/releases) - [Commits](https://github.com/jonschlinkert/word-wrap/compare/1.2.3...1.2.4) --- updated-dependencies: - dependency-name: word-wrap dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- airflow/www/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 601cd22d344f..a3e6e864b7bd 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -11734,9 +11734,9 @@ wildcard@^2.0.0: integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== word-wrap@^1.2.3, word-wrap@~1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + version "1.2.4" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" + integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== wrap-ansi@^7.0.0: version "7.0.0" From 34edb6508d0e5d1f185abf400b9572139fb12b16 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 19 Jul 2023 18:13:38 +0200 Subject: [PATCH 226/533] Add protection against accidental Providers Manager initialization (#32694) ProvidersManager might get accidentally initialized during CLI argument parsing, and if it does, then CLI and arg completion takes huge performance hit - it takes seconds to intialize ProvidersManager when you have many providers packages installed, so initializing ProvidersManager should not happen just when you parse arguments of the CLIs but only when you execute commands. It's rather easy to trigger ProvidersManager by just importing some package (especially as we move executors to providers). This PR adds a new CLI command in providers to check status of initialization - this command will fail when provider's initlalization is executed during CLI parsing. It will also show the stack trace of where ProvidersManager has been initialized from. Test is added to detect such situation by running the new command in a separate subprocess, thus making sure ProvidersManager has not been initialized in another test. --- airflow/cli/cli_config.py | 7 +++++ airflow/cli/commands/celery_command.py | 7 ++++- airflow/cli/commands/provider_command.py | 27 +++++++++++++++++++ airflow/providers_manager.py | 14 ++++++++++ tests/cli/commands/test_celery_command.py | 12 ++++----- tests/cli/test_cli_parser.py | 10 +++++++ .../cli/commands/test_celery_command.py | 14 +++++----- 7 files changed, 78 insertions(+), 13 deletions(-) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 97aa83c10041..66b959051363 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -1820,8 +1820,15 @@ class GroupCommand(NamedTuple): func=lazy_load_command("airflow.cli.commands.provider_command.executors_list"), args=(ARG_OUTPUT, ARG_VERBOSE), ), + ActionCommand( + name="status", + help="Get information about provider initialization status", + func=lazy_load_command("airflow.cli.commands.provider_command.status"), + args=(ARG_VERBOSE,), + ), ) + USERS_COMMANDS = ( ActionCommand( name="list", diff --git a/airflow/cli/commands/celery_command.py b/airflow/cli/commands/celery_command.py index 6d3dbe64bbba..7adbea36e685 100644 --- a/airflow/cli/commands/celery_command.py +++ b/airflow/cli/commands/celery_command.py @@ -34,7 +34,6 @@ from airflow import settings from airflow.configuration import conf -from airflow.providers.celery.executors.celery_executor import app as celery_app from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging from airflow.utils.serve_logs import serve_logs @@ -45,6 +44,9 @@ @cli_utils.action_cli def flower(args): """Starts Flower, Celery monitoring tool.""" + # This needs to be imported locally to not trigger Providers Manager initialization + from airflow.providers.celery.executors.celery_executor import app as celery_app + options = [ "flower", conf.get("celery", "BROKER_URL"), @@ -132,6 +134,9 @@ def filter(self, record): @cli_utils.action_cli def worker(args): """Starts Airflow Celery worker.""" + # This needs to be imported locally to not trigger Providers Manager initialization + from airflow.providers.celery.executors.celery_executor import app as celery_app + # Disable connection pool so that celery worker does not hold an unnecessary db connection settings.reconfigure_orm(disable_connection_pool=True) if not settings.validate_session(): diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py index 67c78c57fe6e..0256a74089c0 100644 --- a/airflow/cli/commands/provider_command.py +++ b/airflow/cli/commands/provider_command.py @@ -17,6 +17,8 @@ """Providers sub-commands.""" from __future__ import annotations +import sys + import re2 from airflow.cli.simple_table import AirflowConsole @@ -179,3 +181,28 @@ def executors_list(args): "executor_class_names": x, }, ) + + +@suppress_logs_and_warning +def status(args): + """Informs if providers manager has been initialized. + + If provider is initialized, shows the stack trace and exit with error code 1. + """ + import rich + + if ProvidersManager.initialized(): + rich.print( + "\n[red]ProvidersManager was initialized during CLI parsing. This should not happen.\n", + file=sys.stderr, + ) + rich.print( + "\n[yellow]Please make sure no Providers Manager initialization happens during CLI parsing.\n", + file=sys.stderr, + ) + rich.print("Stack trace where it has been initialized:\n", file=sys.stderr) + rich.print(ProvidersManager.initialization_stack_trace(), file=sys.stderr) + sys.exit(1) + else: + rich.print("[green]All ok. Providers Manager was not initialized during the CLI parsing.") + sys.exit(0) diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 4f2f063599f9..108712d484ad 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -20,10 +20,12 @@ import fnmatch import functools +import inspect import json import logging import os import sys +import traceback import warnings from collections import OrderedDict from dataclasses import dataclass @@ -377,10 +379,22 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton): """ resource_version = "0" + _initialized: bool = False + _initialization_stack_trace = None + + @staticmethod + def initialized() -> bool: + return ProvidersManager._initialized + + @staticmethod + def initialization_stack_trace() -> str: + return ProvidersManager._initialization_stack_trace def __init__(self): """Initializes the manager.""" super().__init__() + ProvidersManager._initialized = True + ProvidersManager._initialization_stack_trace = "".join(traceback.format_stack(inspect.currentframe())) self._initialized_cache: dict[str, bool] = {} # Keeps dict of providers keyed by module name self._provider_dict: dict[str, ProviderInfo] = {} diff --git a/tests/cli/commands/test_celery_command.py b/tests/cli/commands/test_celery_command.py index 9acea81c6019..0e8dee24570b 100644 --- a/tests/cli/commands/test_celery_command.py +++ b/tests/cli/commands/test_celery_command.py @@ -88,7 +88,7 @@ def test_if_right_pid_is_read(self, mock_process, mock_setup_locations): mock_process.return_value.terminate.assert_called_once_with() @mock.patch("airflow.cli.commands.celery_command.read_pid_from_pidfile") - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @mock.patch("airflow.cli.commands.celery_command.setup_locations") @conf_vars({("core", "executor"): "CeleryExecutor"}) def test_same_pid_file_is_used_in_start_and_stop( @@ -113,7 +113,7 @@ def test_same_pid_file_is_used_in_start_and_stop( @mock.patch("airflow.cli.commands.celery_command.remove_existing_pidfile") @mock.patch("airflow.cli.commands.celery_command.read_pid_from_pidfile") - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @mock.patch("airflow.cli.commands.celery_command.psutil.Process") @mock.patch("airflow.cli.commands.celery_command.setup_locations") @conf_vars({("core", "executor"): "CeleryExecutor"}) @@ -151,7 +151,7 @@ def setup_class(cls): @mock.patch("airflow.cli.commands.celery_command.setup_locations") @mock.patch("airflow.cli.commands.celery_command.Process") - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @conf_vars({("core", "executor"): "CeleryExecutor"}) def test_worker_started_with_required_arguments(self, mock_celery_app, mock_popen, mock_locations): pid_file = "pid_file" @@ -211,7 +211,7 @@ def setup_class(cls): cls.parser = cli_parser.get_parser() @mock.patch("airflow.cli.commands.celery_command.Process") - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @conf_vars({("core", "executor"): "CeleryExecutor"}) def test_worker_failure_gracefull_shutdown(self, mock_celery_app, mock_popen): args = self.parser.parse_args(["celery", "worker"]) @@ -228,7 +228,7 @@ class TestFlowerCommand: def setup_class(cls): cls.parser = cli_parser.get_parser() - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @conf_vars({("core", "executor"): "CeleryExecutor"}) def test_run_command(self, mock_celery_app): args = self.parser.parse_args( @@ -267,7 +267,7 @@ def test_run_command(self, mock_celery_app): @mock.patch("airflow.cli.commands.celery_command.TimeoutPIDLockFile") @mock.patch("airflow.cli.commands.celery_command.setup_locations") @mock.patch("airflow.cli.commands.celery_command.daemon") - @mock.patch("airflow.cli.commands.celery_command.celery_app") + @mock.patch("airflow.providers.celery.executors.celery_executor.app") @conf_vars({("core", "executor"): "CeleryExecutor"}) def test_run_command_daemon(self, mock_celery_app, mock_daemon, mock_setup_locations, mock_pid_file): mock_setup_locations.return_value = ( diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 935417b4b619..378ebd4e715e 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -22,6 +22,7 @@ import contextlib import io import re +import subprocess import timeit from collections import Counter from unittest.mock import patch @@ -279,3 +280,12 @@ def test_cli_run_time(self): timing_result = timeit.timeit(stmt=timing_code, number=num_samples, setup=setup_code) / num_samples # Average run time of Airflow CLI should at least be within 3.5s assert timing_result < threshold + + def test_cli_parsing_does_not_initialize_providers_manager(self): + """Test that CLI parsing does not initialize providers manager. + + This test is here to make sure that we do not initialize providers manager - it is run as a + separate subprocess, to make sure we do not have providers manager initialized in the main + process from other tests. + """ + subprocess.check_call(["airflow", "providers", "status"]) diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 306d5fcfd88d..b421c949f0e1 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -33,10 +33,11 @@ class TestWorkerServeLogs: def setup_class(cls): cls.parser = cli_parser.get_parser() - @mock.patch("airflow.cli.commands.celery_command.celery_app") @conf_vars({("core", "executor"): "CeleryExecutor"}) - def test_serve_logs_on_worker_start(self, mock_celery_app): - with mock.patch("airflow.cli.commands.celery_command.Process") as mock_process: + def test_serve_logs_on_worker_start(self): + with mock.patch("airflow.cli.commands.celery_command.Process") as mock_process, mock.patch( + "airflow.providers.celery.executors.celery_executor.app" + ): args = self.parser.parse_args(["celery", "worker", "--concurrency", "1"]) with mock.patch("celery.platforms.check_privileges") as mock_privil: @@ -44,10 +45,11 @@ def test_serve_logs_on_worker_start(self, mock_celery_app): celery_command.worker(args) mock_process.assert_called() - @mock.patch("airflow.cli.commands.celery_command.celery_app") @conf_vars({("core", "executor"): "CeleryExecutor"}) - def test_skip_serve_logs_on_worker_start(self, mock_celery_app): - with mock.patch("airflow.cli.commands.celery_command.Process") as mock_popen: + def test_skip_serve_logs_on_worker_start(self): + with mock.patch("airflow.cli.commands.celery_command.Process") as mock_popen, mock.patch( + "airflow.providers.celery.executors.celery_executor.app" + ): args = self.parser.parse_args(["celery", "worker", "--concurrency", "1", "--skip-serve-logs"]) with mock.patch("celery.platforms.check_privileges") as mock_privil: From b9c84eb6639e825ed951c08e477411bf52dfc437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Wed, 19 Jul 2023 10:37:32 -0700 Subject: [PATCH 227/533] add deferrable mode to rds start & stop DB (#32437) * remove tests on obsolete trigger * add delay & attempts to params --------- Co-authored-by: Wei Lee --- airflow/providers/amazon/aws/operators/rds.py | 168 ++++++++++----- airflow/providers/amazon/aws/triggers/rds.py | 166 ++++++++++++++- airflow/providers/amazon/aws/utils/rds.py | 4 +- docs/spelling_wordlist.txt | 1 + .../amazon/aws/operators/test_rds.py | 29 +++ .../providers/amazon/aws/triggers/test_rds.py | 201 +++++------------- 6 files changed, 363 insertions(+), 206 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/rds.py b/airflow/providers/amazon/aws/operators/rds.py index c03725164309..7a544a8c0045 100644 --- a/airflow/providers/amazon/aws/operators/rds.py +++ b/airflow/providers/amazon/aws/operators/rds.py @@ -20,15 +20,20 @@ import json import warnings from datetime import timedelta -from typing import TYPE_CHECKING, Sequence +from typing import TYPE_CHECKING, Any, Sequence +from astroid.decorators import cachedproperty from mypy_boto3_rds.type_defs import TagTypeDef from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.rds import RdsHook -from airflow.providers.amazon.aws.triggers.rds import RdsDbInstanceTrigger +from airflow.providers.amazon.aws.triggers.rds import ( + RdsDbAvailableTrigger, + RdsDbDeletedTrigger, + RdsDbStoppedTrigger, +) from airflow.providers.amazon.aws.utils.rds import RdsDbType from airflow.providers.amazon.aws.utils.tags import format_tags from airflow.providers.amazon.aws.utils.waiter_with_logging import wait @@ -62,13 +67,17 @@ def __init__( AirflowProviderDeprecationWarning, stacklevel=3, # 2 is in the operator's init, 3 is in the user code creating the operator ) - hook_params = hook_params or {} - self.region_name = region_name or hook_params.pop("region_name", None) - self.hook = RdsHook(aws_conn_id=aws_conn_id, region_name=self.region_name, **(hook_params)) + self.hook_params = hook_params or {} + self.aws_conn_id = aws_conn_id + self.region_name = region_name or self.hook_params.pop("region_name", None) super().__init__(*args, **kwargs) self._await_interval = 60 # seconds + @cachedproperty + def hook(self) -> RdsHook: + return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name, **self.hook_params) + def execute(self, context: Context) -> str: """Different implementations for snapshots, tasks and events.""" raise NotImplementedError @@ -106,10 +115,9 @@ def __init__( db_snapshot_identifier: str, tags: Sequence[TagTypeDef] | dict | None = None, wait_for_completion: bool = True, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_type = RdsDbType(db_type) self.db_identifier = db_identifier self.db_snapshot_identifier = db_snapshot_identifier @@ -194,10 +202,9 @@ def __init__( target_custom_availability_zone: str = "", source_region: str = "", wait_for_completion: bool = True, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_type = RdsDbType(db_type) self.source_db_snapshot_identifier = source_db_snapshot_identifier @@ -274,10 +281,9 @@ def __init__( db_type: str, db_snapshot_identifier: str, wait_for_completion: bool = True, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_type = RdsDbType(db_type) self.db_snapshot_identifier = db_snapshot_identifier @@ -345,10 +351,9 @@ def __init__( s3_prefix: str = "", export_only: list[str] | None = None, wait_for_completion: bool = True, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.export_task_identifier = export_task_identifier self.source_arn = source_arn @@ -397,10 +402,9 @@ def __init__( export_task_identifier: str, wait_for_completion: bool = True, check_interval: int = 30, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.export_task_identifier = export_task_identifier self.wait_for_completion = wait_for_completion @@ -461,10 +465,9 @@ def __init__( enabled: bool = True, tags: Sequence[TagTypeDef] | dict | None = None, wait_for_completion: bool = True, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.subscription_name = subscription_name self.sns_topic_arn = sns_topic_arn @@ -511,10 +514,9 @@ def __init__( self, *, subscription_name: str, - aws_conn_id: str = "aws_default", **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.subscription_name = subscription_name @@ -545,7 +547,6 @@ class RdsCreateDbInstanceOperator(RdsBaseOperator): :param engine: The name of the database engine to be used for this instance :param rds_kwargs: Named arguments to pass to boto3 RDS client function ``create_db_instance`` https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/rds.html#RDS.Client.create_db_instance - :param aws_conn_id: The Airflow connection used for AWS credentials. :param wait_for_completion: If True, waits for creation of the DB instance to complete. (default: True) :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check DB instance state :param waiter_max_attempts: The maximum number of attempts to check DB instance state @@ -563,14 +564,13 @@ def __init__( db_instance_class: str, engine: str, rds_kwargs: dict | None = None, - aws_conn_id: str = "aws_default", wait_for_completion: bool = True, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), waiter_delay: int = 30, waiter_max_attempts: int = 60, **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_instance_identifier = db_instance_identifier self.db_instance_class = db_instance_class @@ -580,7 +580,6 @@ def __init__( self.deferrable = deferrable self.waiter_delay = waiter_delay self.waiter_max_attempts = waiter_max_attempts - self.aws_conn_id = aws_conn_id def execute(self, context: Context) -> str: self.log.info("Creating new DB instance %s", self.db_instance_identifier) @@ -593,15 +592,15 @@ def execute(self, context: Context) -> str: ) if self.deferrable: self.defer( - trigger=RdsDbInstanceTrigger( - db_instance_identifier=self.db_instance_identifier, + trigger=RdsDbAvailableTrigger( + db_identifier=self.db_instance_identifier, waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, region_name=self.region_name, - waiter_name="db_instance_available", # ignoring type because create_db_instance is a dict response=create_db_instance, # type: ignore[arg-type] + db_type=RdsDbType.INSTANCE, ), method_name="execute_complete", timeout=timedelta(seconds=self.waiter_delay * self.waiter_max_attempts), @@ -638,7 +637,6 @@ class RdsDeleteDbInstanceOperator(RdsBaseOperator): :param db_instance_identifier: The DB instance identifier for the DB instance to be deleted :param rds_kwargs: Named arguments to pass to boto3 RDS client function ``delete_db_instance`` https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/rds.html#RDS.Client.delete_db_instance - :param aws_conn_id: The Airflow connection used for AWS credentials. :param wait_for_completion: If True, waits for deletion of the DB instance to complete. (default: True) :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check DB instance state :param waiter_max_attempts: The maximum number of attempts to check DB instance state @@ -654,21 +652,19 @@ def __init__( *, db_instance_identifier: str, rds_kwargs: dict | None = None, - aws_conn_id: str = "aws_default", wait_for_completion: bool = True, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), waiter_delay: int = 30, waiter_max_attempts: int = 60, **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_instance_identifier = db_instance_identifier self.rds_kwargs = rds_kwargs or {} self.wait_for_completion = False if deferrable else wait_for_completion self.deferrable = deferrable self.waiter_delay = waiter_delay self.waiter_max_attempts = waiter_max_attempts - self.aws_conn_id = aws_conn_id def execute(self, context: Context) -> str: self.log.info("Deleting DB instance %s", self.db_instance_identifier) @@ -679,15 +675,15 @@ def execute(self, context: Context) -> str: ) if self.deferrable: self.defer( - trigger=RdsDbInstanceTrigger( - db_instance_identifier=self.db_instance_identifier, + trigger=RdsDbDeletedTrigger( + db_identifier=self.db_instance_identifier, waiter_delay=self.waiter_delay, waiter_max_attempts=self.waiter_max_attempts, aws_conn_id=self.aws_conn_id, region_name=self.region_name, - waiter_name="db_instance_deleted", # ignoring type because delete_db_instance is a dict response=delete_db_instance, # type: ignore[arg-type] + db_type=RdsDbType.INSTANCE, ), method_name="execute_complete", timeout=timedelta(seconds=self.waiter_delay * self.waiter_max_attempts), @@ -723,8 +719,11 @@ class RdsStartDbOperator(RdsBaseOperator): :param db_identifier: The AWS identifier of the DB to start :param db_type: Type of the DB - either "instance" or "cluster" (default: "instance") - :param aws_conn_id: The Airflow connection used for AWS credentials. (default: "aws_default") :param wait_for_completion: If True, waits for DB to start. (default: True) + :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check DB instance state + :param waiter_max_attempts: The maximum number of attempts to check DB instance state + :param deferrable: If True, the operator will wait asynchronously for the DB instance to be created. + This implies waiting for completion. This mode requires aiobotocore module to be installed. """ template_fields = ("db_identifier", "db_type") @@ -734,26 +733,52 @@ def __init__( *, db_identifier: str, db_type: RdsDbType | str = RdsDbType.INSTANCE, - aws_conn_id: str = "aws_default", wait_for_completion: bool = True, + waiter_delay: int = 30, + waiter_max_attempts: int = 40, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_identifier = db_identifier self.db_type = db_type self.wait_for_completion = wait_for_completion + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts + self.deferrable = deferrable def execute(self, context: Context) -> str: self.db_type = RdsDbType(self.db_type) - start_db_response = self._start_db() - if self.wait_for_completion: + start_db_response: dict[str, Any] = self._start_db() + if self.deferrable: + self.defer( + trigger=RdsDbAvailableTrigger( + db_identifier=self.db_identifier, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region_name=self.region_name, + response=start_db_response, + db_type=RdsDbType.INSTANCE, + ), + method_name="execute_complete", + ) + elif self.wait_for_completion: self._wait_until_db_available() return json.dumps(start_db_response, default=str) + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: + if event is None or event["status"] != "success": + raise AirflowException(f"Failed to start DB: {event}") + else: + return json.dumps(event["response"], default=str) + def _start_db(self): self.log.info("Starting DB %s '%s'", self.db_type.value, self.db_identifier) if self.db_type == RdsDbType.INSTANCE: - response = self.hook.conn.start_db_instance(DBInstanceIdentifier=self.db_identifier) + response = self.hook.conn.start_db_instance( + DBInstanceIdentifier=self.db_identifier, + ) else: response = self.hook.conn.start_db_cluster(DBClusterIdentifier=self.db_identifier) return response @@ -761,9 +786,19 @@ def _start_db(self): def _wait_until_db_available(self): self.log.info("Waiting for DB %s to reach 'available' state", self.db_type.value) if self.db_type == RdsDbType.INSTANCE: - self.hook.wait_for_db_instance_state(self.db_identifier, target_state="available") + self.hook.wait_for_db_instance_state( + self.db_identifier, + target_state="available", + check_interval=self.waiter_delay, + max_attempts=self.waiter_max_attempts, + ) else: - self.hook.wait_for_db_cluster_state(self.db_identifier, target_state="available") + self.hook.wait_for_db_cluster_state( + self.db_identifier, + target_state="available", + check_interval=self.waiter_delay, + max_attempts=self.waiter_max_attempts, + ) class RdsStopDbOperator(RdsBaseOperator): @@ -779,8 +814,11 @@ class RdsStopDbOperator(RdsBaseOperator): :param db_snapshot_identifier: The instance identifier of the DB Snapshot to create before stopping the DB instance. The default value (None) skips snapshot creation. This parameter is ignored when ``db_type`` is "cluster" - :param aws_conn_id: The Airflow connection used for AWS credentials. (default: "aws_default") :param wait_for_completion: If True, waits for DB to stop. (default: True) + :param waiter_delay: Time (in seconds) to wait between two consecutive calls to check DB instance state + :param waiter_max_attempts: The maximum number of attempts to check DB instance state + :param deferrable: If True, the operator will wait asynchronously for the DB instance to be created. + This implies waiting for completion. This mode requires aiobotocore module to be installed. """ template_fields = ("db_identifier", "db_snapshot_identifier", "db_type") @@ -791,23 +829,47 @@ def __init__( db_identifier: str, db_type: RdsDbType | str = RdsDbType.INSTANCE, db_snapshot_identifier: str | None = None, - aws_conn_id: str = "aws_default", wait_for_completion: bool = True, + waiter_delay: int = 30, + waiter_max_attempts: int = 40, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): - super().__init__(aws_conn_id=aws_conn_id, **kwargs) + super().__init__(**kwargs) self.db_identifier = db_identifier self.db_type = db_type self.db_snapshot_identifier = db_snapshot_identifier self.wait_for_completion = wait_for_completion + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts + self.deferrable = deferrable def execute(self, context: Context) -> str: self.db_type = RdsDbType(self.db_type) - stop_db_response = self._stop_db() - if self.wait_for_completion: + stop_db_response: dict[str, Any] = self._stop_db() + if self.deferrable: + self.defer( + trigger=RdsDbStoppedTrigger( + db_identifier=self.db_identifier, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region_name=self.region_name, + response=stop_db_response, + db_type=RdsDbType.INSTANCE, + ), + method_name="execute_complete", + ) + elif self.wait_for_completion: self._wait_until_db_stopped() return json.dumps(stop_db_response, default=str) + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: + if event is None or event["status"] != "success": + raise AirflowException(f"Failed to start DB: {event}") + else: + return json.dumps(event["response"], default=str) + def _stop_db(self): self.log.info("Stopping DB %s '%s'", self.db_type.value, self.db_identifier) if self.db_type == RdsDbType.INSTANCE: @@ -829,9 +891,19 @@ def _stop_db(self): def _wait_until_db_stopped(self): self.log.info("Waiting for DB %s to reach 'stopped' state", self.db_type.value) if self.db_type == RdsDbType.INSTANCE: - self.hook.wait_for_db_instance_state(self.db_identifier, target_state="stopped") + self.hook.wait_for_db_instance_state( + self.db_identifier, + target_state="stopped", + check_interval=self.waiter_delay, + max_attempts=self.waiter_max_attempts, + ) else: - self.hook.wait_for_db_cluster_state(self.db_identifier, target_state="stopped") + self.hook.wait_for_db_cluster_state( + self.db_identifier, + target_state="stopped", + check_interval=self.waiter_delay, + max_attempts=self.waiter_max_attempts, + ) __all__ = [ diff --git a/airflow/providers/amazon/aws/triggers/rds.py b/airflow/providers/amazon/aws/triggers/rds.py index 0551d67591df..b8f20b9c9bb7 100644 --- a/airflow/providers/amazon/aws/triggers/rds.py +++ b/airflow/providers/amazon/aws/triggers/rds.py @@ -16,19 +16,21 @@ # under the License. from __future__ import annotations +import warnings from typing import Any +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.rds import RdsHook +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger +from airflow.providers.amazon.aws.utils.rds import RdsDbType from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait from airflow.triggers.base import BaseTrigger, TriggerEvent class RdsDbInstanceTrigger(BaseTrigger): """ - Trigger for RdsCreateDbInstanceOperator and RdsDeleteDbInstanceOperator. - - The trigger will asynchronously poll the boto3 API and wait for the - DB instance to be in the state specified by the waiter. + Deprecated Trigger for RDS operations. Do not use. :param waiter_name: Name of the waiter to use, for instance 'db_instance_available' or 'db_instance_deleted'. @@ -36,7 +38,7 @@ class RdsDbInstanceTrigger(BaseTrigger): :param waiter_delay: The amount of time in seconds to wait between attempts. :param waiter_max_attempts: The maximum number of attempts to be made. :param aws_conn_id: The Airflow connection used for AWS credentials. - :param hook_params: The parameters to pass to the RdsHook. + :param region_name: AWS region where the DB is located, if different from the default one. :param response: The response from the RdsHook, to be passed back to the operator. """ @@ -50,6 +52,12 @@ def __init__( region_name: str | None, response: dict[str, Any], ): + warnings.warn( + "This trigger is deprecated, please use the other RDS triggers " + "such as RdsDbDeletedTrigger, RdsDbStoppedTrigger or RdsDbAvailableTrigger", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) self.db_instance_identifier = db_instance_identifier self.waiter_delay = waiter_delay self.waiter_max_attempts = waiter_max_attempts @@ -87,3 +95,151 @@ async def run(self): status_args=["DBInstances[0].DBInstanceStatus"], ) yield TriggerEvent({"status": "success", "response": self.response}) + + +_waiter_arg = { + RdsDbType.INSTANCE: "DBInstanceIdentifier", + RdsDbType.CLUSTER: "DBClusterIdentifier", +} +_status_paths = { + RdsDbType.INSTANCE: ["DBInstances[].DBInstanceStatus", "DBInstances[].StatusInfos"], + RdsDbType.CLUSTER: ["DBClusters[].Status"], +} + + +class RdsDbAvailableTrigger(AwsBaseWaiterTrigger): + """ + Trigger to wait asynchronously for a DB instance or cluster to be available. + + :param db_identifier: The DB identifier for the DB instance or cluster to be polled. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: AWS region where the DB is located, if different from the default one. + :param response: The response from the RdsHook, to be passed back to the operator. + :param db_type: The type of DB: instance or cluster. + """ + + def __init__( + self, + db_identifier: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + response: dict[str, Any], + db_type: RdsDbType, + ) -> None: + super().__init__( + serialized_fields={ + "db_identifier": db_identifier, + "response": response, + "db_type": db_type, + }, + waiter_name=f"db_{db_type.value}_available", + waiter_args={_waiter_arg[db_type]: db_identifier}, + failure_message="Error while waiting for DB to be available", + status_message="DB initialization in progress", + status_queries=_status_paths[db_type], + return_key="response", + return_value=response, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) + + def hook(self) -> AwsGenericHook: + return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class RdsDbDeletedTrigger(AwsBaseWaiterTrigger): + """ + Trigger to wait asynchronously for a DB instance or cluster to be deleted. + + :param db_identifier: The DB identifier for the DB instance or cluster to be polled. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: AWS region where the DB is located, if different from the default one. + :param response: The response from the RdsHook, to be passed back to the operator. + :param db_type: The type of DB: instance or cluster. + """ + + def __init__( + self, + db_identifier: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + response: dict[str, Any], + db_type: RdsDbType, + ) -> None: + super().__init__( + serialized_fields={ + "db_identifier": db_identifier, + "response": response, + "db_type": db_type, + }, + waiter_name=f"db_{db_type.value}_deleted", + waiter_args={_waiter_arg[db_type]: db_identifier}, + failure_message="Error while deleting DB", + status_message="DB deletion in progress", + status_queries=_status_paths[db_type], + return_key="response", + return_value=response, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) + + def hook(self) -> AwsGenericHook: + return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + +class RdsDbStoppedTrigger(AwsBaseWaiterTrigger): + """ + Trigger to wait asynchronously for a DB instance or cluster to be stopped. + + :param db_identifier: The DB identifier for the DB instance or cluster to be polled. + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region_name: AWS region where the DB is located, if different from the default one. + :param response: The response from the RdsHook, to be passed back to the operator. + :param db_type: The type of DB: instance or cluster. + """ + + def __init__( + self, + db_identifier: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + region_name: str | None, + response: dict[str, Any], + db_type: RdsDbType, + ) -> None: + super().__init__( + serialized_fields={ + "db_identifier": db_identifier, + "response": response, + "db_type": db_type, + }, + waiter_name=f"db_{db_type.value}_stopped", + waiter_args={_waiter_arg[db_type]: db_identifier}, + failure_message="Error while stopping DB", + status_message="DB is being stopped", + status_queries=_status_paths[db_type], + return_key="response", + return_value=response, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + region_name=region_name, + ) + + def hook(self) -> AwsGenericHook: + return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) diff --git a/airflow/providers/amazon/aws/utils/rds.py b/airflow/providers/amazon/aws/utils/rds.py index 1ba511c4b4e9..eacbf0b63a9a 100644 --- a/airflow/providers/amazon/aws/utils/rds.py +++ b/airflow/providers/amazon/aws/utils/rds.py @@ -22,5 +22,5 @@ class RdsDbType(Enum): """Only available types for the RDS.""" - INSTANCE: str = "instance" - CLUSTER: str = "cluster" + INSTANCE = "instance" + CLUSTER = "cluster" diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 0975656c75bf..ad3b7cd8f8fe 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1208,6 +1208,7 @@ RBAC rbac rc rdbms +RDS readfp Readme readme diff --git a/tests/providers/amazon/aws/operators/test_rds.py b/tests/providers/amazon/aws/operators/test_rds.py index d2408fa47dd7..47122d55c0ae 100644 --- a/tests/providers/amazon/aws/operators/test_rds.py +++ b/tests/providers/amazon/aws/operators/test_rds.py @@ -18,11 +18,13 @@ from __future__ import annotations import logging +from unittest import mock from unittest.mock import patch import pytest from moto import mock_rds +from airflow.exceptions import TaskDeferred from airflow.models import DAG from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook from airflow.providers.amazon.aws.hooks.rds import RdsHook @@ -40,6 +42,7 @@ RdsStartExportTaskOperator, RdsStopDbOperator, ) +from airflow.providers.amazon.aws.triggers.rds import RdsDbAvailableTrigger, RdsDbStoppedTrigger from airflow.utils import timezone DEFAULT_DATE = timezone.datetime(2019, 1, 1) @@ -832,6 +835,19 @@ def test_stop_db_instance_no_wait(self, mock_await_status): assert status == "stopped" mock_await_status.assert_not_called() + @mock.patch.object(RdsHook, "conn") + def test_deferred(self, conn_mock): + op = RdsStopDbOperator( + task_id="test_stop_db_instance_no_wait", + db_identifier=DB_INSTANCE_NAME, + deferrable=True, + ) + + with pytest.raises(TaskDeferred) as defer: + op.execute({}) + + assert isinstance(defer.value.trigger, RdsDbStoppedTrigger) + @mock_rds def test_stop_db_instance_create_snapshot(self): _create_db_instance(self.hook) @@ -932,3 +948,16 @@ def test_start_db_cluster(self): result_after = self.hook.conn.describe_db_clusters(DBClusterIdentifier=DB_CLUSTER_NAME) status_after = result_after["DBClusters"][0]["Status"] assert status_after == "available" + + @mock.patch.object(RdsHook, "conn") + def test_deferred(self, conn_mock): + op = RdsStartDbOperator( + task_id="test_stop_db_instance_no_wait", + db_identifier=DB_INSTANCE_NAME, + deferrable=True, + ) + + with pytest.raises(TaskDeferred) as defer: + op.execute({}) + + assert isinstance(defer.value.trigger, RdsDbAvailableTrigger) diff --git a/tests/providers/amazon/aws/triggers/test_rds.py b/tests/providers/amazon/aws/triggers/test_rds.py index 9c518c8eee5f..57db41a5e02a 100644 --- a/tests/providers/amazon/aws/triggers/test_rds.py +++ b/tests/providers/amazon/aws/triggers/test_rds.py @@ -16,16 +16,14 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import AsyncMock - import pytest -from botocore.exceptions import WaiterError -from airflow.exceptions import AirflowException -from airflow.providers.amazon.aws.hooks.rds import RdsHook -from airflow.providers.amazon.aws.triggers.rds import RdsDbInstanceTrigger -from airflow.triggers.base import TriggerEvent +from airflow.providers.amazon.aws.triggers.rds import ( + RdsDbAvailableTrigger, + RdsDbDeletedTrigger, + RdsDbStoppedTrigger, +) +from airflow.providers.amazon.aws.utils.rds import RdsDbType TEST_DB_INSTANCE_IDENTIFIER = "test-db-instance-identifier" TEST_WAITER_DELAY = 10 @@ -40,146 +38,47 @@ } -class TestRdsDbInstanceTrigger: - def test_rds_db_instance_trigger_serialize(self): - rds_db_instance_trigger = RdsDbInstanceTrigger( - waiter_name="test-waiter", - db_instance_identifier=TEST_DB_INSTANCE_IDENTIFIER, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - aws_conn_id=TEST_AWS_CONN_ID, - region_name=TEST_REGION, - response=TEST_RESPONSE, - ) - class_path, args = rds_db_instance_trigger.serialize() - - assert class_path == "airflow.providers.amazon.aws.triggers.rds.RdsDbInstanceTrigger" - assert args["waiter_name"] == "test-waiter" - assert args["db_instance_identifier"] == TEST_DB_INSTANCE_IDENTIFIER - assert args["waiter_delay"] == str(TEST_WAITER_DELAY) - assert args["waiter_max_attempts"] == str(TEST_WAITER_MAX_ATTEMPTS) - assert args["aws_conn_id"] == TEST_AWS_CONN_ID - assert args["region_name"] == TEST_REGION - assert args["response"] == TEST_RESPONSE - - @pytest.mark.asyncio - @mock.patch.object(RdsHook, "async_conn") - async def test_rds_db_instance_trigger_run(self, mock_async_conn): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - a_mock.get_waiter().wait = AsyncMock() - - rds_db_instance_trigger = RdsDbInstanceTrigger( - waiter_name="test-waiter", - db_instance_identifier=TEST_DB_INSTANCE_IDENTIFIER, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - aws_conn_id=TEST_AWS_CONN_ID, - region_name=TEST_REGION, - response=TEST_RESPONSE, - ) - - generator = rds_db_instance_trigger.run() - response = await generator.asend(None) - - assert response == TriggerEvent({"status": "success", "response": TEST_RESPONSE}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RdsHook, "async_conn") - async def test_rds_db_instance_trigger_run_multiple_attempts(self, mock_async_conn, mock_sleep): - mock_sleep.return_value = True - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"DBInstances": [{"DBInstanceStatus": "CREATING"}]}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - - rds_db_instance_trigger = RdsDbInstanceTrigger( - waiter_name="test-waiter", - db_instance_identifier=TEST_DB_INSTANCE_IDENTIFIER, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - aws_conn_id=TEST_AWS_CONN_ID, - region_name=TEST_REGION, - response=TEST_RESPONSE, - ) - - generator = rds_db_instance_trigger.run() - response = await generator.asend(None) - assert a_mock.get_waiter().wait.call_count == 4 - - assert response == TriggerEvent({"status": "success", "response": TEST_RESPONSE}) - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RdsHook, "async_conn") - async def test_rds_db_instance_trigger_run_attempts_exceeded(self, mock_async_conn, mock_sleep): - mock_sleep.return_value = True - - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - error = WaiterError( - name="test_name", - reason="test_reason", - last_response={"DBInstances": [{"DBInstanceStatus": "CREATING"}]}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error, error, error, True]) - - rds_db_instance_trigger = RdsDbInstanceTrigger( - waiter_name="test-waiter", - db_instance_identifier=TEST_DB_INSTANCE_IDENTIFIER, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=2, - aws_conn_id=TEST_AWS_CONN_ID, - region_name=TEST_REGION, - response=TEST_RESPONSE, - ) - - with pytest.raises(AirflowException) as exc: - generator = rds_db_instance_trigger.run() - await generator.asend(None) - - assert "Waiter error: max attempts reached" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 2 - - @pytest.mark.asyncio - @mock.patch("asyncio.sleep") - @mock.patch.object(RdsHook, "async_conn") - async def test_rds_db_instance_trigger_run_attempts_failed(self, mock_async_conn, mock_sleep): - a_mock = mock.MagicMock() - mock_async_conn.__aenter__.return_value = a_mock - - error_creating = WaiterError( - name="test_name", - reason="test_reason", - last_response={"DBInstances": [{"DBInstanceStatus": "CREATING"}]}, - ) - - error_failed = WaiterError( - name="test_name", - reason="Waiter encountered a terminal failure state:", - last_response={"DBInstances": [{"DBInstanceStatus": "FAILED"}]}, - ) - a_mock.get_waiter().wait = AsyncMock(side_effect=[error_creating, error_creating, error_failed]) - mock_sleep.return_value = True - - rds_db_instance_trigger = RdsDbInstanceTrigger( - waiter_name="test-waiter", - db_instance_identifier=TEST_DB_INSTANCE_IDENTIFIER, - waiter_delay=TEST_WAITER_DELAY, - waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, - aws_conn_id=TEST_AWS_CONN_ID, - region_name=TEST_REGION, - response=TEST_RESPONSE, - ) - - with pytest.raises(AirflowException) as exc: - generator = rds_db_instance_trigger.run() - await generator.asend(None) - assert "Error checking DB Instance status" in str(exc.value) - assert a_mock.get_waiter().wait.call_count == 3 +class TestRdsTriggers: + @pytest.mark.parametrize( + "trigger", + [ + RdsDbAvailableTrigger( + db_identifier=TEST_DB_INSTANCE_IDENTIFIER, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + aws_conn_id=TEST_AWS_CONN_ID, + region_name=TEST_REGION, + response=TEST_RESPONSE, + db_type=RdsDbType.INSTANCE, + ), + RdsDbDeletedTrigger( + db_identifier=TEST_DB_INSTANCE_IDENTIFIER, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + aws_conn_id=TEST_AWS_CONN_ID, + region_name=TEST_REGION, + response=TEST_RESPONSE, + db_type=RdsDbType.INSTANCE, + ), + RdsDbStoppedTrigger( + db_identifier=TEST_DB_INSTANCE_IDENTIFIER, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + aws_conn_id=TEST_AWS_CONN_ID, + region_name=TEST_REGION, + response=TEST_RESPONSE, + db_type=RdsDbType.INSTANCE, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 From a2eaca897734ac43d5263832b319c99b3510b7b5 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Wed, 19 Jul 2023 11:54:58 -0700 Subject: [PATCH 228/533] Allow setup without teardown (#32679) --- airflow/models/abstractoperator.py | 7 ++- airflow/models/dag.py | 4 -- tests/models/test_dag.py | 86 +++++++++--------------------- 3 files changed, 29 insertions(+), 68 deletions(-) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 9a8f88ce7df2..0c6d89fff3b5 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -302,7 +302,8 @@ def get_upstreams_only_setups_and_teardowns(self) -> Iterable[Operator]: This method is meant to be used when we are clearing the task (non-upstream) and we need to add in the *relevant* setups and their teardowns. - Relevant in this case means, the setup has a teardown that is downstream of ``self``. + Relevant in this case means, the setup has a teardown that is downstream of ``self``, + or the setup has no teardowns. """ downstream_teardown_ids = { x.task_id for x in self.get_flat_relatives(upstream=False) if x.is_teardown @@ -310,7 +311,9 @@ def get_upstreams_only_setups_and_teardowns(self) -> Iterable[Operator]: for task in self.get_flat_relatives(upstream=True): if not task.is_setup: continue - if not task.downstream_task_ids.isdisjoint(downstream_teardown_ids): + has_no_teardowns = not any(True for x in task.downstream_list if x.is_teardown) + # if task has no teardowns or has teardowns downstream of self + if has_no_teardowns or task.downstream_task_ids.intersection(downstream_teardown_ids): yield task for t in task.downstream_list: if t.is_teardown and not t == self: diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 391be9e58236..1d380274f63a 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -714,10 +714,6 @@ def validate_setup_teardown(self): :meta private: """ for task in self.tasks: - if task.is_setup and not any(x.is_teardown for x in task.downstream_list): - raise AirflowDagInconsistent( - f"Dag has setup without teardown: dag='{self.dag_id}', task='{task.task_id}'" - ) if task.is_teardown and all(x.is_setup for x in task.upstream_list): raise AirflowDagInconsistent( f"Dag has teardown task without an upstream work task: dag='{self.dag_id}'," diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 14527fd7facc..494c7c7c6612 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -3623,39 +3623,46 @@ def cleared_neither(task): def test_get_flat_relative_ids_with_setup(self): with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: - s1, w1, w2, w3, t1 = self.make_tasks(dag, "s1, w1, w2, w3, t1") + s1, w1, w2, w3, w4, t1 = self.make_tasks(dag, "s1, w1, w2, w3, w4, t1") s1 >> w1 >> w2 >> w3 - # there is no teardown downstream of w1, so we assume w1 does not need s1 - assert set(w1.get_upstreams_only_setups_and_teardowns()) == set() + # w1 is downstream of s1, and s1 has no teardown, so clearing w1 clears s1 + assert set(w1.get_upstreams_only_setups_and_teardowns()) == {s1} # same with w2 and w3 - assert set(w2.get_upstreams_only_setups_and_teardowns()) == set() - assert set(w3.get_upstreams_only_setups_and_teardowns()) == set() - assert self.cleared_downstream(w2) == {w2, w3} + assert set(w2.get_upstreams_only_setups_and_teardowns()) == {s1} + assert set(w3.get_upstreams_only_setups_and_teardowns()) == {s1} + # so if we clear w2, we should also get s1, and w3, but not w1 + assert self.cleared_downstream(w2) == {s1, w2, w3} w3 >> t1 # now, w2 has a downstream teardown, but it's not connected directly to s1 - # (this is how we signal "this is the teardown for this setup") - # so still, we don't regard s1 as a setup for w2 - assert set(w2.get_upstreams_only_setups_and_teardowns()) == set() - assert self.cleared_downstream(w2) == {w2, w3, t1} + assert set(w2.get_upstreams_only_setups_and_teardowns()) == {s1} + # so if we clear downstream then s1 will be cleared, and t1 will be cleared but only by virtue of + # being downstream of w2 -- not as a result of being the teardown for s1, which it ain't + assert self.cleared_downstream(w2) == {s1, w2, w3, t1} + # and, another consequence of not linking s1 and t1 is that when we clear upstream, note that + # t1 doesn't get cleared -- cus it's not upstream and it's not linked to s1 + assert self.cleared_upstream(w2) == {s1, w1, w2} + # note also that if we add a 4th work task after t1, it will still be "in scope" for s1 + t1 >> w4 + assert self.cleared_downstream(w4) == {s1, w4} s1 >> t1 - # now, we know that t1 is the teardown for s1, and it's downstream of - # w2, so we can infer that w2 requires it, so now when we clear w2, - # we will get s1 (because it's a setup for w2) and t1 (because - # it is a teardown for s1) + # now, we know that t1 is the teardown for s1, so now we know that s1 will be "torn down" + # by the time w4 runs, so we now know that w4 no longer requires s1, so when we clear w4, + # s1 will not also be cleared + self.cleared_downstream(w4) == {w4} assert set(w1.get_upstreams_only_setups_and_teardowns()) == {s1, t1} - assert self.cleared_downstream(w1) == {s1, w1, w2, w3, t1} + assert self.cleared_downstream(w1) == {s1, w1, w2, w3, t1, w4} assert self.cleared_upstream(w1) == {s1, w1, t1} assert set(w2.get_upstreams_only_setups_and_teardowns()) == {s1, t1} assert set(w2.get_upstreams_follow_setups()) == {s1, w1, t1} - assert self.cleared_downstream(w2) == {s1, w2, w3, t1} + assert self.cleared_downstream(w2) == {s1, w2, w3, t1, w4} assert self.cleared_upstream(w2) == {s1, w1, w2, t1} - assert self.cleared_downstream(w3) == {s1, w3, t1} + assert self.cleared_downstream(w3) == {s1, w3, t1, w4} assert self.cleared_upstream(w3) == {s1, w1, w2, w3, t1} def test_get_flat_relative_ids_with_setup_nested_ctx_mgr(self): @@ -3695,32 +3702,6 @@ def test_get_flat_relative_ids_with_setup_nested_no_ctx_mgr(self): # t1 not included because t1 is not downstream assert self.cleared_downstream(w3) == {s2, w3, t2} - def test_setup_without_teardown(self): - """A setup needs a teardown to define its scope.""" - with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: - s1, w1, t1 = self.make_tasks(dag, "s1, w1, t1") - # s1 has no teardown: fail - with pytest.raises(AirflowDagInconsistent): - dag.validate_setup_teardown() - - s1 >> w1 - # w1 depends on s1 but not as a "setup" per se, since s1 doesn't have a teardown to define - # its scope - with pytest.raises(AirflowDagInconsistent): - dag.validate_setup_teardown() - - w1 >> t1 - # now t1 is technically downstream of s1, but we still must wire it up explicitly - # to define the setup/teardown relationship - with pytest.raises(AirflowDagInconsistent): - dag.validate_setup_teardown() - - s1 >> t1 - # now, s1 and t1 are linked as setups and teardowns - # anything upstream of t1 and downstream of s1 is in the scope for s1 - # so now this passes validation - dag.validate_setup_teardown() - def test_get_flat_relative_ids_follows_teardowns(self): with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") @@ -3918,25 +3899,6 @@ def my_teardown(): def test_validate_setup_teardown_dag(self, dag_maker): """Test some invalid setups and teardowns in a dag""" - with dag_maker("test_dag") as dag: - s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") - - with s1: - w1 >> t1 - w2 >> t1 - with pytest.raises( - AirflowDagInconsistent, match="Dag has setup without teardown: dag='test_dag', task='s1'" - ): - dag.validate() - - with dag_maker("test_dag") as dag: - s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") - s1 >> w1 >> w2 >> t1 - with pytest.raises( - AirflowDagInconsistent, match="Dag has setup without teardown: dag='test_dag', task='s1'" - ): - dag.validate() - with dag_maker("test_dag") as dag: s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") w1 >> w2 From 9d7c2246815251727af6d9119d7ef7660338e8c8 Mon Sep 17 00:00:00 2001 From: "D. Ferruzzi" Date: Wed, 19 Jul 2023 12:10:11 -0700 Subject: [PATCH 229/533] D205 Support - Models (#32575) * D205 Support - Models Co-authored-by: Tzu-ping Chung --------- Co-authored-by: Tzu-ping Chung --- airflow/models/baseoperator.py | 73 ++++++++--------- airflow/models/connection.py | 23 +++--- airflow/models/crypto.py | 3 +- airflow/models/dag.py | 124 ++++++++++++++--------------- airflow/models/dagbag.py | 22 +++-- airflow/models/dagpickle.py | 11 ++- airflow/models/dagrun.py | 8 +- airflow/models/errors.py | 5 +- airflow/models/param.py | 28 ++++--- airflow/models/renderedtifields.py | 6 +- airflow/models/serialized_dag.py | 23 +++--- airflow/models/skipmixin.py | 5 +- airflow/models/slamiss.py | 4 +- airflow/models/taskinstance.py | 99 ++++++++++++----------- airflow/models/taskmixin.py | 16 ++-- airflow/models/taskreschedule.py | 3 +- airflow/models/trigger.py | 29 +++---- airflow/models/variable.py | 10 +-- airflow/models/xcom.py | 3 +- 19 files changed, 236 insertions(+), 259 deletions(-) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 40fe3b8f0bd6..e389a8df8ab4 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -355,8 +355,7 @@ class BaseOperatorMeta(abc.ABCMeta): @classmethod def _apply_defaults(cls, func: T) -> T: """ - Function decorator that Looks for an argument named "default_args", and - fills the unspecified arguments from it. + Look for an argument named "default_args", and fill the unspecified arguments from it. Since python2.* isn't clear about which arguments are missing when calling a function, and that this can be quite confusing with multi-level @@ -463,10 +462,12 @@ def __new__(cls, name, bases, namespace, **kwargs): @functools.total_ordering class BaseOperator(AbstractOperator, metaclass=BaseOperatorMeta): """ - Abstract base class for all operators. Since operators create objects that - become nodes in the dag, BaseOperator contains many recursive methods for - dag crawling behavior. To derive this class, you are expected to override - the constructor as well as the 'execute' method. + Abstract base class for all operators. + + Since operators create objects that become nodes in the DAG, BaseOperator + contains many recursive methods for DAG crawling behavior. To derive from + this class, you are expected to override the constructor and the 'execute' + method. Operators derived from this class should perform or trigger certain tasks synchronously (wait for completion). Example of operators could be an @@ -992,9 +993,10 @@ def __hash__(self): # including lineage information def __or__(self, other): """ - Called for [This Operator] | [Operator], The inlets of other - will be set to pick up the outlets from this operator. Other will - be set as a downstream task of this operator. + Called for [This Operator] | [Operator]. + + The inlets of other will be set to pick up the outlets from this operator. + Other will be set as a downstream task of this operator. """ if isinstance(other, BaseOperator): if not self.outlets and not self.supports_lineage: @@ -1010,8 +1012,9 @@ def __or__(self, other): def __gt__(self, other): """ - Called for [Operator] > [Outlet], so that if other is an attr annotated object - it is set as an outlet of this Operator. + Called for [Operator] > [Outlet]. + + If other is an attr annotated object it is set as an outlet of this Operator. """ if not isinstance(other, Iterable): other = [other] @@ -1025,8 +1028,9 @@ def __gt__(self, other): def __lt__(self, other): """ - Called for [Inlet] > [Operator] or [Operator] < [Inlet], so that if other is - an attr annotated object it is set as an inlet to this operator. + Called for [Inlet] > [Operator] or [Operator] < [Inlet]. + + If other is an attr annotated object it is set as an inlet to this operator. """ if not isinstance(other, Iterable): other = [other] @@ -1086,10 +1090,7 @@ def dag(self) -> DAG: # type: ignore[override] @dag.setter def dag(self, dag: DAG | None): - """ - Operators can be assigned to one DAG, one time. Repeat assignments to - that same DAG are ok. - """ + """Operators can be assigned to one DAG, one time. Repeat assignments to that same DAG are ok.""" from airflow.models.dag import DAG if dag is None: @@ -1128,19 +1129,17 @@ def has_dag(self): """ def prepare_for_execution(self) -> BaseOperator: - """ - Lock task for execution to disable custom action in __setattr__ and - returns a copy of the task. - """ + """Lock task for execution to disable custom action in ``__setattr__`` and return a copy.""" other = copy.copy(self) other._lock_for_execution = True return other def set_xcomargs_dependencies(self) -> None: """ - Resolves upstream dependencies of a task. In this way passing an ``XComArg`` - as value for a template field will result in creating upstream relation between - two tasks. + Resolves upstream dependencies of a task. + + In this way passing an ``XComArg`` as value for a template field + will result in creating upstream relation between two tasks. **Example**: :: @@ -1173,6 +1172,7 @@ def pre_execute(self, context: Any): def execute(self, context: Context) -> Any: """ This is the main method to derive when creating an operator. + Context is the same dictionary used as when rendering jinja templates. Refer to get_template_context for more context. @@ -1183,18 +1183,18 @@ def execute(self, context: Context) -> Any: def post_execute(self, context: Any, result: Any = None): """ This hook is triggered right after self.execute() is called. - It is passed the execution context and any results returned by the - operator. + + It is passed the execution context and any results returned by the operator. """ if self._post_execute_hook is not None: self._post_execute_hook(context, result) def on_kill(self) -> None: """ - Override this method to clean up subprocesses when a task instance - gets killed. Any use of the threading, subprocess or multiprocessing - module within an operator needs to be cleaned up, or it will leave - ghost processes behind. + Override this method to clean up subprocesses when a task instance gets killed. + + Any use of the threading, subprocess or multiprocessing module within an + operator needs to be cleaned up, or it will leave ghost processes behind. """ def __deepcopy__(self, memo): @@ -1254,10 +1254,7 @@ def clear( downstream: bool = False, session: Session = NEW_SESSION, ): - """ - Clears the state of task instances associated with the task, following - the parameters specified. - """ + """Clears the state of task instances associated with the task, following the parameters specified.""" qry = select(TaskInstance).where(TaskInstance.dag_id == self.dag_id) if start_date: @@ -1375,10 +1372,7 @@ def dry_run(self) -> None: self.log.info(content) def get_direct_relatives(self, upstream: bool = False) -> Iterable[Operator]: - """ - Get list of the direct relatives to the current task, upstream or - downstream. - """ + """Get list of the direct relatives to the current task, upstream or downstream.""" if upstream: return self.upstream_list else: @@ -1550,8 +1544,7 @@ def defer( timeout: timedelta | None = None, ): """ - Marks this Operator as being "deferred" - that is, suspending its - execution until the provided trigger fires an event. + Mark this Operator "deferred", suspending its execution until the provided trigger fires an event. This is achieved by raising a special exception (TaskDeferred) which is caught in the main _execute_task wrapper. diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 0bc3ca38d48f..18c6bcce59b7 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -60,10 +60,10 @@ def _parse_netloc_to_hostname(uri_parts): class Connection(Base, LoggingMixin): """ - Placeholder to store information about different database instances - connection information. The idea here is that scripts use references to - database instances (conn_id) instead of hard coding hostname, logins and - passwords when using operators or hooks. + Placeholder to store information about different database instances connection information. + + The idea here is that scripts use references to database instances (conn_id) + instead of hard coding hostname, logins and passwords when using operators or hooks. .. seealso:: For more information on how to use this class, see: :doc:`/howto/connection` @@ -141,8 +141,9 @@ def __init__( @staticmethod def _validate_extra(extra, conn_id) -> None: """ - Here we verify that ``extra`` is a JSON-encoded Python dict. From Airflow 3.0, we should no - longer suppress these errors but raise instead. + Here we verify that ``extra`` is a JSON-encoded Python dict. + + From Airflow 3.0, we should no longer suppress these errors but raise instead. """ if extra is None: return None @@ -376,8 +377,9 @@ def __repr__(self): def log_info(self): """ - This method is deprecated. You can read each field individually or use the - default representation (`__repr__`). + This method is deprecated. + + You can read each field individually or use the default representation (`__repr__`). """ warnings.warn( "This method is deprecated. You can read each field individually or " @@ -393,8 +395,9 @@ def log_info(self): def debug_info(self): """ - This method is deprecated. You can read each field individually or use the - default representation (`__repr__`). + This method is deprecated. + + You can read each field individually or use the default representation (`__repr__`). """ warnings.warn( "This method is deprecated. You can read each field individually or " diff --git a/airflow/models/crypto.py b/airflow/models/crypto.py index df072665fcbd..97495ea90e2d 100644 --- a/airflow/models/crypto.py +++ b/airflow/models/crypto.py @@ -38,8 +38,7 @@ def encrypt(self, b): class NullFernet: """ - A "Null" encryptor class that doesn't encrypt or decrypt but that presents - a similar interface to Fernet. + A "Null" encryptor class that doesn't encrypt or decrypt but that presents a similar interface to Fernet. The purpose of this is to make the rest of the code not have to know the difference, and to only display the message once, not 20 times when diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 1d380274f63a..ae8ed2b6b615 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -220,6 +220,7 @@ def create_timetable(interval: ScheduleIntervalArg, timezone: Timezone) -> Timet def get_last_dagrun(dag_id, session, include_externally_triggered=False): """ Returns the last dag run for a dag, None if there was none. + Last dag run can be any type of run e.g. scheduled or backfilled. Overridden DagRuns are ignored. """ @@ -235,6 +236,8 @@ def get_dataset_triggered_next_run_info( dag_ids: list[str], *, session: Session ) -> dict[str, dict[str, int | str]]: """ + Get next run info for a list of dag_ids. + Given a list of dag_ids, get string representing how close any that are dataset triggered are their next run, e.g. "1 of 2 datasets updated". """ @@ -273,12 +276,12 @@ def get_dataset_triggered_next_run_info( @functools.total_ordering class DAG(LoggingMixin): """ - A dag (directed acyclic graph) is a collection of tasks with directional - dependencies. A dag also has a schedule, a start date and an end date - (optional). For each schedule, (say daily or hourly), the DAG needs to run - each individual tasks as their dependencies are met. Certain tasks have - the property of depending on their own past, meaning that they can't run - until their previous schedule (and upstream tasks) are completed. + A dag (directed acyclic graph) is a collection of tasks with directional dependencies. + + A dag also has a schedule, a start date and an end date (optional). For each schedule, + (say daily or hourly), the DAG needs to run each individual tasks as their dependencies + are met. Certain tasks have the property of depending on their own past, meaning that + they can't run until their previous schedule (and upstream tasks) are completed. DAGs essentially act as namespaces for tasks. A task_id can only be added once to a DAG. @@ -764,9 +767,11 @@ def __exit__(self, _type, _value, _tb): @staticmethod def _upgrade_outdated_dag_access_control(access_control=None): """ - Looks for outdated dag level actions (can_dag_read and can_dag_edit) in DAG - access_controls (for example, {'role1': {'can_dag_read'}, 'role2': {'can_dag_read', 'can_dag_edit'}}) - and replaces them with updated actions (can_read and can_edit). + Look for outdated dag level actions in DAG access_controls and replace them with updated actions. + + For example, in DAG access_control {'role1': {'can_dag_read'}} 'can_dag_read' + will be replaced with 'can_read', in {'role2': {'can_dag_read', 'can_dag_edit'}} + 'can_dag_edit' will be replaced with 'can_edit', etc. """ if not access_control: return None @@ -1102,8 +1107,9 @@ def iter_dagrun_infos_between( def get_run_dates(self, start_date, end_date=None) -> list: """ - Returns a list of dates between the interval received as parameter using this - dag's schedule interval. Returned dates can be used for execution dates. + Returns a list of dates between the interval received as parameter using this dag's schedule interval. + + Returned dates can be used for execution dates. :param start_date: The start date of the interval. :param end_date: The end date of the interval. Defaults to ``timezone.utcnow()``. @@ -1316,10 +1322,7 @@ def allow_future_exec_dates(self) -> bool: @provide_session def get_concurrency_reached(self, session=NEW_SESSION) -> bool: - """ - Returns a boolean indicating whether the max_active_tasks limit for this DAG - has been reached. - """ + """Returns a boolean indicating whether the max_active_tasks limit for this DAG has been reached.""" TI = TaskInstance total_tasks = session.scalar( select(func.count(TI.task_id)).where( @@ -1377,10 +1380,11 @@ def normalized_schedule_interval(self) -> ScheduleInterval: @provide_session def handle_callback(self, dagrun, success=True, reason=None, session=NEW_SESSION): """ - Triggers the appropriate callback depending on the value of success, namely the - on_failure_callback or on_success_callback. This method gets the context of a - single TaskInstance part of this DagRun and passes that to the callable along - with a 'reason', primarily to differentiate DagRun failures. + Triggers on_failure_callback or on_success_callback as appropriate. + + This method gets the context of a single TaskInstance part of this DagRun + and passes that to the callable along with a 'reason', primarily to + differentiate DagRun failures. .. note: The logs end up in ``$AIRFLOW_HOME/logs/scheduler/latest/PROJECT/DAG_FILE.py.log`` @@ -1450,8 +1454,7 @@ def get_dagrun( session: Session = NEW_SESSION, ): """ - Returns the dag run for a given execution date or run_id if it exists, otherwise - none. + Returns the dag run for a given execution date or run_id if it exists, otherwise none. :param execution_date: The execution date of the DagRun to find. :param run_id: The run_id of the DagRun to find. @@ -1557,10 +1560,7 @@ def get_template_env(self, *, force_sandboxed: bool = False) -> jinja2.Environme return env def set_dependency(self, upstream_task_id, downstream_task_id): - """ - Simple utility method to set dependency between two tasks that - already have been added to the DAG using add_task(). - """ + """Set dependency between two tasks that already have been added to the DAG using add_task().""" self.get_task(upstream_task_id).set_downstream(self.get_task(downstream_task_id)) @provide_session @@ -1902,8 +1902,7 @@ def set_task_instance_state( session=NEW_SESSION, ) -> list[TaskInstance]: """ - Set the state of a TaskInstance to the given state, and clear its downstream tasks that are - in failed or upstream_failed state. + Set the state of a TaskInstance and clear downstream tasks in failed or upstream_failed state. :param task_id: Task ID of the TaskInstance :param map_indexes: Only set TaskInstance if its map_index matches. @@ -1996,8 +1995,7 @@ def set_task_group_state( session: Session = NEW_SESSION, ) -> list[TaskInstance]: """ - Set the state of the TaskGroup to the given state, and clear its downstream tasks that are - in failed or upstream_failed state. + Set TaskGroup to the given state and clear downstream tasks in failed or upstream_failed state. :param group_id: The group_id of the TaskGroup :param execution_date: Execution date of the TaskInstance @@ -2095,8 +2093,7 @@ def leaves(self) -> list[Operator]: def topological_sort(self, include_subdag_tasks: bool = False): """ - Sorts tasks in topographical order, such that a task comes after any of its - upstream dependencies. + Sorts tasks in topographical order, such that a task comes after any of its upstream dependencies. Deprecated in place of ``task_group.topological_sort`` """ @@ -2154,8 +2151,7 @@ def clear( exclude_task_ids: frozenset[str] | frozenset[tuple[str, int]] | None = frozenset(), ) -> int | Iterable[TaskInstance]: """ - Clears a set of task instances associated with the current dag for - a specified date range. + Clears a set of task instances associated with the current dag for a specified date range. :param task_ids: List of task ids or (``task_id``, ``map_index``) tuples to clear :param start_date: The minimum execution_date to clear @@ -2338,6 +2334,8 @@ def partial_subset( include_direct_upstream=False, ): """ + Return a subset of the current dag based on regex matching one or more tasks. + Returns a subset of the current dag as a deep copy of the current dag based on a regex that should match one or many tasks, and includes upstream and downstream neighbours based on the flag passed. @@ -2771,6 +2769,7 @@ def create_dagrun( ): """ Creates a dag run from this dag including the tasks associated with this dag. + Returns the dag run. :param run_id: defines the run id for this dag run @@ -2892,8 +2891,7 @@ def bulk_write_to_db( session=NEW_SESSION, ): """ - Ensure the DagModel rows for the given dags are up-to-date in the dag table in the DB, including - calculated fields. + Ensure the DagModel rows for the given dags are up-to-date in the dag table in the DB. Note that this method can be called for both DAGs and SubDAGs. A SubDag is actually a SubDagOperator. @@ -3122,9 +3120,9 @@ def bulk_write_to_db( @provide_session def sync_to_db(self, processor_subdir: str | None = None, session=NEW_SESSION): """ - Save attributes about this DAG to the DB. Note that this method - can be called for both DAGs and SubDAGs. A SubDag is actually a - SubDagOperator. + Save attributes about this DAG to the DB. + + Note that this method can be called for both DAGs and SubDAGs. A SubDag is actually a SubDagOperator. :return: None """ @@ -3141,8 +3139,7 @@ def get_default_view(self): @provide_session def deactivate_unknown_dags(active_dag_ids, session=NEW_SESSION): """ - Given a list of known DAGs, deactivate any other DAGs that are - marked as active in the ORM. + Given a list of known DAGs, deactivate any other DAGs that are marked as active in the ORM. :param active_dag_ids: list of DAG IDs that are active :return: None @@ -3158,11 +3155,11 @@ def deactivate_unknown_dags(active_dag_ids, session=NEW_SESSION): @provide_session def deactivate_stale_dags(expiration_date, session=NEW_SESSION): """ - Deactivate any DAGs that were last touched by the scheduler before - the expiration date. These DAGs were likely deleted. + Deactivate any DAGs that were last touched by the scheduler before the expiration date. + + These DAGs were likely deleted. - :param expiration_date: set inactive DAGs that were touched before this - time + :param expiration_date: set inactive DAGs that were touched before this time :return: None """ for dag in session.scalars( @@ -3249,10 +3246,7 @@ def get_serialized_fields(cls): return cls.__serialized_fields def get_edge_info(self, upstream_task_id: str, downstream_task_id: str) -> EdgeInfoType: - """ - Returns edge information for the given pair of tasks if present, and - an empty edge if there is no information. - """ + """Return edge information for the given pair of tasks or an empty edge if there is no information.""" # Note - older serialized DAGs may not have edge_info being a dict at all empty = cast(EdgeInfoType, {}) if self.edge_info: @@ -3262,14 +3256,17 @@ def get_edge_info(self, upstream_task_id: str, downstream_task_id: str) -> EdgeI def set_edge_info(self, upstream_task_id: str, downstream_task_id: str, info: EdgeInfoType): """ - Sets the given edge information on the DAG. Note that this will overwrite, - rather than merge with, existing info. + Sets the given edge information on the DAG. + + Note that this will overwrite, rather than merge with, existing info. """ self.edge_info.setdefault(upstream_task_id, {})[downstream_task_id] = info def validate_schedule_and_params(self): """ - Validates & raise exception if there are any Params in the DAG which neither have a default value nor + Validates Param values when the schedule_interval is not None. + + Raise exception if there are any Params in the DAG which neither have a default value nor have the null in schema['type'] list, but the DAG have a schedule_interval which is not None. """ if not self.timetable.can_be_scheduled: @@ -3283,7 +3280,9 @@ def validate_schedule_and_params(self): ) def iter_invalid_owner_links(self) -> Iterator[tuple[str, str]]: - """Parses a given link, and verifies if it's a valid URL, or a 'mailto' link. + """ + Parses a given link, and verifies if it's a valid URL, or a 'mailto' link. + Returns an iterator of invalid (owner, link) pairs. """ for owner, link in self.owner_links.items(): @@ -3312,10 +3311,10 @@ def __repr__(self): class DagOwnerAttributes(Base): - """Table defining different owner attributes. + """ + Table defining different owner attributes. - For example, a link for an owner that will be passed as a hyperlink to the - "DAGs" view. + For example, a link for an owner that will be passed as a hyperlink to the "DAGs" view. """ __tablename__ = "dag_owner_attributes" @@ -3515,10 +3514,7 @@ def get_paused_dag_ids(dag_ids: list[str], session: Session = NEW_SESSION) -> se return paused_dag_ids def get_default_view(self) -> str: - """ - Get the Default DAG View, returns the default config value if DagModel does not - have a value. - """ + """Get the Default DAG View, returns the default config value if DagModel does not have a value.""" # This is for backwards-compatibility with old dags that don't have None as default_view return self.default_view or airflow_conf.get_mandatory_value("webserver", "dag_default_view").lower() @@ -3730,7 +3726,8 @@ def dag( fail_stop: bool = False, ) -> Callable[[Callable], Callable[..., DAG]]: """ - Python dag decorator. Wraps a function into an Airflow DAG. + Python dag decorator which wraps a function into an Airflow DAG. + Accepts kwargs for operator kwarg. Can be used to parameterize DAGs. :param dag_args: Arguments for DAG object @@ -3869,9 +3866,10 @@ def get_current_dag(cls) -> DAG | None: def _run_task(ti: TaskInstance, session): """ - Run a single task instance, and push result to Xcom for downstream tasks. Bypasses a lot of - extra steps used in `task.run` to keep our local running as fast as possible - This function is only meant for the `dag.test` function as a helper function. + Run a single task instance, and push result to Xcom for downstream tasks. + + Bypasses a lot of extra steps used in `task.run` to keep our local running as fast as + possible. This function is only meant for the `dag.test` function as a helper function. Args: ti: TaskInstance to run diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index 1b80cb01582a..e53c2ce3bd4f 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -73,13 +73,13 @@ class FileLoadStat(NamedTuple): class DagBag(LoggingMixin): """ - A dagbag is a collection of dags, parsed out of a folder tree and has high - level configuration settings, like what database to use as a backend and - what executor to use to fire off tasks. This makes it easier to run - distinct environments for say production and development, tests, or for - different teams or security profiles. What would have been system level - settings are now dagbag level so that one system can run multiple, - independent settings sets. + A dagbag is a collection of dags, parsed out of a folder tree and has high level configuration settings. + + Some possible setting are database to use as a backend and what executor + to use to fire off tasks. This makes it easier to run distinct environments + for say production and development, tests, or for different teams or security + profiles. What would have been system level settings are now dagbag level so + that one system can run multiple, independent settings sets. :param dag_folder: the folder to scan to find DAGs :param include_examples: whether to include the examples that ship @@ -277,10 +277,7 @@ def _add_dag_from_db(self, dag_id: str, session: Session): self.dags_hash[dag.dag_id] = row.dag_hash def process_file(self, filepath, only_if_updated=True, safe_mode=True): - """ - Given a path to a python module or zip file, this method imports - the module and look for dag objects within it. - """ + """Given a path to a python module or zip file, import the module and look for dag objects within.""" from airflow.models.dag import DagContext # if the source file no longer exists in the DB or in the filesystem, @@ -525,8 +522,7 @@ def collect_dags( safe_mode: bool = conf.getboolean("core", "DAG_DISCOVERY_SAFE_MODE"), ): """ - Given a file path or a folder, this method looks for python modules, - imports them and adds them to the dagbag collection. + Look for python modules in a given path, import them, and add them to the dagbag collection. Note that if a ``.airflowignore`` file is found while processing the directory, it will behave much like a ``.gitignore``, diff --git a/airflow/models/dagpickle.py b/airflow/models/dagpickle.py index ae2c04ccaf51..e6f4561d8e1b 100644 --- a/airflow/models/dagpickle.py +++ b/airflow/models/dagpickle.py @@ -32,14 +32,13 @@ class DagPickle(Base): """ - Dags can originate from different places (user repos, main repo, ...) - and also get executed in different places (different executors). This - object represents a version of a DAG and becomes a source of truth for - a BackfillJob execution. A pickle is a native python serialized object, + Represents a version of a DAG and becomes a source of truth for a BackfillJob execution. + + Dags can originate from different places (user repos, main repo, ...) and also get executed + in different places (different executors). A pickle is a native python serialized object, and in this case gets stored in the database for the duration of the job. - The executors pick up the DagPickle id and read the dag definition from - the database. + The executors pick up the DagPickle id and read the dag definition from the database. """ id = Column(Integer, primary_key=True) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 2d3ff26d7f19..baca09602473 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -575,8 +575,7 @@ def update_state( self, session: Session = NEW_SESSION, execute_callbacks: bool = True ) -> tuple[list[TI], DagCallbackRequest | None]: """ - Determines the overall state of the DagRun based on the state - of its TaskInstances. + Determines the overall state of the DagRun based on the state of its TaskInstances. :param session: Sqlalchemy ORM Session :param execute_callbacks: Should dag callbacks (success/failure, SLA etc.) be invoked @@ -974,8 +973,9 @@ def _emit_duration_stats_for_finished_state(self): @provide_session def verify_integrity(self, *, session: Session = NEW_SESSION) -> None: """ - Verifies the DagRun by checking for removed tasks or tasks that are not in the - database yet. It will set state to removed or add the task if required. + Verifies the DagRun by checking for removed tasks or tasks that are not in the database yet. + + It will set state to removed or add the task if required. :missing_indexes: A dictionary of task vs indexes that are missing. :param session: Sqlalchemy ORM Session diff --git a/airflow/models/errors.py b/airflow/models/errors.py index 974d9b8eebbe..9d2da0917d98 100644 --- a/airflow/models/errors.py +++ b/airflow/models/errors.py @@ -24,10 +24,7 @@ class ImportError(Base): - """ - A table to store all Import Errors. The ImportErrors are recorded when parsing DAGs. - This errors are displayed on the Webserver. - """ + """Stores all Import Errors which are recorded when parsing DAGs and displayed on the Webserver.""" __tablename__ = "import_error" id = Column(Integer, primary_key=True) diff --git a/airflow/models/param.py b/airflow/models/param.py index d9a547b9ad86..5bb1db3d4412 100644 --- a/airflow/models/param.py +++ b/airflow/models/param.py @@ -42,8 +42,9 @@ class Param: """ - Class to hold the default value of a Param and rule set to do the validations. Without the rule set - it always validates and returns the default value. + Class to hold the default value of a Param and rule set to do the validations. + + Without the rule set it always validates and returns the default value. :param default: The value this Param object holds :param description: Optional help text for the Param @@ -100,6 +101,7 @@ def _warn_if_not_rfc3339_dt(value): def resolve(self, value: Any = NOTSET, suppress_exception: bool = False) -> Any: """ Runs the validations and returns the Param's final value. + May raise ValueError on failed validations, or TypeError if no value is passed and no value already exists. We first check that value is json-serializable; if not, warn. @@ -157,19 +159,21 @@ def deserialize(data: dict[str, Any], version: int) -> Param: class ParamsDict(MutableMapping[str, Any]): """ - Class to hold all params for dags or tasks. All the keys are strictly string and values - are converted into Param's object if they are not already. This class is to replace param's - dictionary implicitly and ideally not needed to be used directly. + Class to hold all params for dags or tasks. + + All the keys are strictly string and values are converted into Param's object + if they are not already. This class is to replace param's dictionary implicitly + and ideally not needed to be used directly. + + + :param dict_obj: A dict or dict like object to init ParamsDict + :param suppress_exception: Flag to suppress value exceptions while initializing the ParamsDict """ __version__: ClassVar[int] = 1 __slots__ = ["__dict", "suppress_exception"] def __init__(self, dict_obj: MutableMapping | None = None, suppress_exception: bool = False): - """ - :param dict_obj: A dict or dict like object to init ParamsDict - :param suppress_exception: Flag to suppress value exceptions while initializing the ParamsDict - """ params_dict: dict[str, Param] = {} dict_obj = dict_obj or {} for k, v in dict_obj.items(): @@ -213,8 +217,7 @@ def __repr__(self): def __setitem__(self, key: str, value: Any) -> None: """ - Override for dictionary's ``setitem`` method. This method make sure that all values are of - Param's type only. + Override for dictionary's ``setitem`` method to ensure all values are of Param's type only. :param key: A key which needs to be inserted or updated in the dict :param value: A value which needs to be set against the key. It could be of any @@ -236,8 +239,7 @@ def __setitem__(self, key: str, value: Any) -> None: def __getitem__(self, key: str) -> Any: """ - Override for dictionary's ``getitem`` method. After fetching the key, it would call the - resolve method as well on the Param object. + Override for dictionary's ``getitem`` method to call the resolve method after fetching the key. :param key: The key to fetch """ diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 37a5f08b27e4..4b2f80e47d1b 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -127,8 +127,7 @@ def _redact(self): @provide_session def get_templated_fields(cls, ti: TaskInstance, session: Session = NEW_SESSION) -> dict | None: """ - Get templated field for a TaskInstance from the RenderedTaskInstanceFields - table. + Get templated field for a TaskInstance from the RenderedTaskInstanceFields table. :param ti: Task Instance :param session: SqlAlchemy Session @@ -153,8 +152,7 @@ def get_templated_fields(cls, ti: TaskInstance, session: Session = NEW_SESSION) @provide_session def get_k8s_pod_yaml(cls, ti: TaskInstance, session: Session = NEW_SESSION) -> dict | None: """ - Get rendered Kubernetes Pod Yaml for a TaskInstance from the RenderedTaskInstanceFields - table. + Get rendered Kubernetes Pod Yaml for a TaskInstance from the RenderedTaskInstanceFields table. :param ti: Task Instance :param session: SqlAlchemy Session diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index 6ee430bca014..90480d8a3343 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -128,7 +128,9 @@ def write_dag( processor_subdir: str | None = None, session: Session = NEW_SESSION, ) -> bool: - """Serializes a DAG and writes it into database. + """ + Serializes a DAG and writes it into database. + If the record already exists, it checks if the Serialized DAG changed or not. If it is changed, it updates the record, ignores otherwise. @@ -225,7 +227,9 @@ def dag(self) -> SerializedDAG: @classmethod @provide_session def remove_dag(cls, dag_id: str, session: Session = NEW_SESSION) -> None: - """Deletes a DAG with given dag_id. + """ + Deletes a DAG with given dag_id. + :param dag_id: dag_id to be deleted :param session: ORM Session. """ @@ -286,8 +290,8 @@ def get_dag(cls, dag_id: str, session: Session = NEW_SESSION) -> SerializedDAG | def get(cls, dag_id: str, session: Session = NEW_SESSION) -> SerializedDagModel | None: """ Get the SerializedDAG for the given dag ID. - It will cope with being passed the ID of a subdag by looking up the - root dag_id from the DAG table. + + It will cope with being passed the ID of a subdag by looking up the root dag_id from the DAG table. :param dag_id: the DAG to fetch :param session: ORM Session @@ -310,8 +314,9 @@ def bulk_sync_to_db( session: Session = NEW_SESSION, ) -> None: """ - Saves DAGs as Serialized DAG objects in the database. Each - DAG is saved in a separate database query. + Saves DAGs as Serialized DAG objects in the database. + + Each DAG is saved in a separate database query. :param dags: the DAG objects to save to the DB :param session: ORM Session @@ -330,8 +335,7 @@ def bulk_sync_to_db( @provide_session def get_last_updated_datetime(cls, dag_id: str, session: Session = NEW_SESSION) -> datetime | None: """ - Get the date when the Serialized DAG associated to DAG was last updated - in serialized_dag table. + Get the date when the Serialized DAG associated to DAG was last updated in serialized_dag table. :param dag_id: DAG ID :param session: ORM Session @@ -368,8 +372,7 @@ def get_latest_version_hash_and_updated_datetime( session: Session, ) -> tuple[str, datetime] | None: """ - Get the latest DAG version for a given DAG ID, as well as the date when the Serialized DAG associated - to DAG was last updated in serialized_dag table. + Get the latest version for a DAG ID and the date it was last updated in serialized_dag table. :meta private: :param dag_id: DAG ID diff --git a/airflow/models/skipmixin.py b/airflow/models/skipmixin.py index 10991cadc76a..5ddc80308a38 100644 --- a/airflow/models/skipmixin.py +++ b/airflow/models/skipmixin.py @@ -162,8 +162,9 @@ def skip_all_except( branch_task_ids: None | str | Iterable[str], ): """ - This method implements the logic for a branching operator; given a single - task ID or list of task IDs to follow, this skips all other tasks + This method implements the logic for a branching operator. + + Given a single task ID or list of task IDs to follow, this skips all other tasks immediately downstream of this operator. branch_task_ids is stored to XCom so that NotPreviouslySkippedDep knows skipped tasks or diff --git a/airflow/models/slamiss.py b/airflow/models/slamiss.py index cd2f0f1dd527..4fb7e53a17ba 100644 --- a/airflow/models/slamiss.py +++ b/airflow/models/slamiss.py @@ -26,8 +26,8 @@ class SlaMiss(Base): """ Model that stores a history of the SLA that have been missed. - It is used to keep track of SLA failures over time and to avoid double - triggering alert emails. + + It is used to keep track of SLA failures over time and to avoid double triggering alert emails. """ __tablename__ = "sla_miss" diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 3e328ec20858..4abfd94cd085 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -170,6 +170,7 @@ class TaskReturnCode(Enum): def set_current_context(context: Context) -> Generator[Context, None, None]: """ Sets the current execution context to the provided context object. + This method should be called once per Task execution, before calling operator.execute. """ _CURRENT_CONTEXT.append(context) @@ -208,9 +209,10 @@ def clear_task_instances( dag_run_state: DagRunState | Literal[False] = DagRunState.QUEUED, ) -> None: """ - Clears a set of task instances, but makes sure the running ones - get killed. Also sets Dagrun's `state` to QUEUED and `start_date` - to the time of execution. But only for finished DRs (SUCCESS and FAILED). + Clears a set of task instances, but makes sure the running ones get killed. + + Also sets Dagrun's `state` to QUEUED and `start_date` to the time of execution. + But only for finished DRs (SUCCESS and FAILED). Doesn't clear DR's `state` and `start_date`for running DRs (QUEUED and RUNNING) because clearing the state for already running DR is redundant and clearing `start_date` affects DR's duration. @@ -358,9 +360,10 @@ def _creator_note(val): class TaskInstance(Base, LoggingMixin): """ - Task instances store the state of a task instance. This table is the - authority and single source of truth around what tasks have run and the - state they are in. + Task instances store the state of a task instance. + + This table is the authority and single source of truth around what tasks + have run and the state they are in. The SqlAlchemy model doesn't have a SqlAlchemy foreign key to the task or dag model deliberately to have more control over transactions. @@ -586,8 +589,7 @@ def init_on_load(self) -> None: @hybrid_property def try_number(self): """ - Return the try number that this task number will be when it is actually - run. + Return the try number that this task number will be when it is actually run. If the TaskInstance is currently running, this will match the column in the database, in all other cases this will be incremented. @@ -604,15 +606,13 @@ def try_number(self, value: int) -> None: @property def prev_attempted_tries(self) -> int: """ - Based on this instance's try_number, this will calculate - the number of previously attempted tries, defaulting to 0. - """ - # Expose this for the Task Tries and Gantt graph views. - # Using `try_number` throws off the counts for non-running tasks. - # Also useful in error logging contexts to get - # the try number for the last try that was attempted. - # https://issues.apache.org/jira/browse/AIRFLOW-2143 + Calculate the number of previously attempted tries, defaulting to 0. + Expose this for the Task Tries and Gantt graph views. + Using `try_number` throws off the counts for non-running tasks. + Also useful in error logging contexts to get the try number for the last try that was attempted. + https://issues.apache.org/jira/browse/AIRFLOW-2143 + """ return self._try_number @property @@ -640,9 +640,9 @@ def command_as_list( cfg_path=None, ) -> list[str]: """ - Returns a command that can be executed anywhere where airflow is - installed. This command is part of the message sent to executors by - the orchestrator. + Returns a command that can be executed anywhere where airflow is installed. + + This command is part of the message sent to executors by the orchestrator. """ dag: DAG | DagModel # Use the dag if we have it, else fallback to the ORM dag_model, which might not be loaded @@ -791,9 +791,10 @@ def mark_success_url(self) -> str: @provide_session def current_state(self, session: Session = NEW_SESSION) -> str: """ - Get the very latest state from the database, if a session is passed, - we use and looking up the state becomes part of the session, otherwise - a new session is used. + Get the very latest state from the database. + + If a session is passed, we use and looking up the state becomes part of the session, + otherwise a new session is used. sqlalchemy.inspect is used here to get the primary keys ensuring that if they change it will not regress @@ -948,10 +949,7 @@ def set_state(self, state: str | None, session: Session = NEW_SESSION) -> bool: @property def is_premature(self) -> bool: - """ - Returns whether a task is in UP_FOR_RETRY state and its retry interval - has elapsed. - """ + """Returns whether a task is in UP_FOR_RETRY state and its retry interval has elapsed.""" # is the task still in the retry waiting period? return self.state == TaskInstanceState.UP_FOR_RETRY and not self.ready_for_retry() @@ -959,6 +957,7 @@ def is_premature(self) -> bool: def are_dependents_done(self, session: Session = NEW_SESSION) -> bool: """ Checks whether the immediate dependents of this task instance have succeeded or have been skipped. + This is meant to be used by wait_for_downstream. This is useful when you do not want to start processing the next @@ -1035,7 +1034,8 @@ def get_previous_ti( def previous_ti(self) -> TaskInstance | None: """ This attribute is deprecated. - Please use `airflow.models.taskinstance.TaskInstance.get_previous_ti` method. + + Please use :class:`airflow.models.taskinstance.TaskInstance.get_previous_ti`. """ warnings.warn( """ @@ -1051,7 +1051,8 @@ def previous_ti(self) -> TaskInstance | None: def previous_ti_success(self) -> TaskInstance | None: """ This attribute is deprecated. - Please use `airflow.models.taskinstance.TaskInstance.get_previous_ti` method. + + Please use :class:`airflow.models.taskinstance.TaskInstance.get_previous_ti`. """ warnings.warn( """ @@ -1098,7 +1099,8 @@ def get_previous_start_date( def previous_start_date_success(self) -> pendulum.DateTime | None: """ This attribute is deprecated. - Please use `airflow.models.taskinstance.TaskInstance.get_previous_start_date` method. + + Please use :class:`airflow.models.taskinstance.TaskInstance.get_previous_start_date`. """ warnings.warn( """ @@ -1115,15 +1117,13 @@ def are_dependencies_met( self, dep_context: DepContext | None = None, session: Session = NEW_SESSION, verbose: bool = False ) -> bool: """ - Returns whether or not all the conditions are met for this task instance to be run - given the context for the dependencies (e.g. a task instance being force run from - the UI will ignore some dependencies). + Are all conditions met for this task instance to be run given the context for the dependencies. + + (e.g. a task instance being force run from the UI will ignore some dependencies). - :param dep_context: The execution context that determines the dependencies that - should be evaluated. + :param dep_context: The execution context that determines the dependencies that should be evaluated. :param session: database session - :param verbose: whether log details on failed dependencies on - info or debug log level + :param verbose: whether log details on failed dependencies on info or debug log level """ dep_context = dep_context or DepContext() failed = False @@ -1169,8 +1169,9 @@ def __repr__(self) -> str: def next_retry_datetime(self): """ - Get datetime of the next retry if the task instance fails. For exponential - backoff, retry_delay is used as base and will be converted to seconds. + Get datetime of the next retry if the task instance fails. + + For exponential backoff, retry_delay is used as base and will be converted to seconds. """ from airflow.models.abstractoperator import MAX_RETRY_DELAY @@ -1209,10 +1210,7 @@ def next_retry_datetime(self): return self.end_date + delay def ready_for_retry(self) -> bool: - """ - Checks on whether the task instance is in the right state and timeframe - to be retried. - """ + """Checks on whether the task instance is in the right state and timeframe to be retried.""" return self.state == TaskInstanceState.UP_FOR_RETRY and self.next_retry_datetime() < timezone.utcnow() @provide_session @@ -1256,8 +1254,9 @@ def check_and_change_state_before_execution( session: Session = NEW_SESSION, ) -> bool: """ - Checks dependencies and then sets state to RUNNING if they are met. Returns - True if and only if state is set to RUNNING, which implies that task should be + Checks dependencies and then sets state to RUNNING if they are met. + + Returns True if and only if state is set to RUNNING, which implies that task should be executed, in preparation for _run_raw_task. :param verbose: whether to turn on more verbose logging @@ -1394,6 +1393,7 @@ def _log_state(self, lead_msg: str = "") -> None: def emit_state_change_metric(self, new_state: TaskInstanceState) -> None: """ Sends a time metric representing how much time a given state transition took. + The previous state and metric name is deduced from the state the task was put in. :param new_state: The state that has just been set for this task. @@ -1456,6 +1456,8 @@ def _run_raw_task( session: Session = NEW_SESSION, ) -> TaskReturnCode | None: """ + Run a task, update the state upon completion, and run any appropriate callbacks. + Immediately runs the task (without checking or changing db state before execution) and then sets the appropriate final state after completion and runs any post-execute callbacks. Meant to be called @@ -1739,10 +1741,7 @@ def _execute_task(self, context, task_orig): @provide_session def _defer_task(self, session: Session, defer: TaskDeferred) -> None: - """ - Marks the task as deferred and sets up the trigger that is needed - to resume it. - """ + """Marks the task as deferred and sets up the trigger that is needed to resume it.""" from airflow.models.trigger import Trigger # First, make the trigger entry @@ -2222,6 +2221,7 @@ def get_triggering_events() -> dict[str, list[DatasetEvent]]: def get_rendered_template_fields(self, session: Session = NEW_SESSION) -> None: """ Update task with rendered template fields for presentation in UI. + If task has already run, will fetch from DB; otherwise will render. """ from airflow.models.renderedtifields import RenderedTaskInstanceFields @@ -2677,8 +2677,7 @@ def filter_for_tis(tis: Iterable[TaskInstance | TaskInstanceKey]) -> BooleanClau @classmethod def ti_selector_condition(cls, vals: Collection[str | tuple[str, int]]) -> ColumnOperators: """ - Build an SQLAlchemy filter for a list where each element can contain - whether a task_id, or a tuple of (task_id,map_index). + Build an SQLAlchemy filter for a list of task_ids or tuples of (task_id,map_index). :meta private: """ diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py index 3a4f17974b91..0dd013a57926 100644 --- a/airflow/models/taskmixin.py +++ b/airflow/models/taskmixin.py @@ -90,6 +90,7 @@ def update_relative( ) -> None: """ Update relationship information about another TaskMixin. Default is no-op. + Override if necessary. """ @@ -169,8 +170,9 @@ def __init_subclass__(cls) -> None: class DAGNode(DependencyMixin, metaclass=ABCMeta): """ - A base class for a node in the graph of a workflow -- an Operator or a Task Group, either mapped or - unmapped. + A base class for a node in the graph of a workflow. + + A node may be an Operator or a Task Group, either mapped or unmapped. """ dag: DAG | None = None @@ -307,20 +309,14 @@ def upstream_list(self) -> Iterable[Operator]: return [self.dag.get_task(tid) for tid in self.upstream_task_ids] def get_direct_relative_ids(self, upstream: bool = False) -> set[str]: - """ - Get set of the direct relative ids to the current task, upstream or - downstream. - """ + """Get set of the direct relative ids to the current task, upstream or downstream.""" if upstream: return self.upstream_task_ids else: return self.downstream_task_ids def get_direct_relatives(self, upstream: bool = False) -> Iterable[DAGNode]: - """ - Get list of the direct relatives to the current task, upstream or - downstream. - """ + """Get list of the direct relatives to the current task, upstream or downstream.""" if upstream: return self.upstream_list else: diff --git a/airflow/models/taskreschedule.py b/airflow/models/taskreschedule.py index d8fe2c3ff28c..7b642f0a266b 100644 --- a/airflow/models/taskreschedule.py +++ b/airflow/models/taskreschedule.py @@ -135,8 +135,7 @@ def find_for_task_instance( try_number: int | None = None, ) -> list[TaskReschedule]: """ - Returns all task reschedules for the task instance and try number, - in ascending order. + Returns all task reschedules for the task instance and try number, in ascending order. :param session: the database session object :param task_instance: the task instance to find task reschedules for diff --git a/airflow/models/trigger.py b/airflow/models/trigger.py index c0d749eb5905..4f50918a09d4 100644 --- a/airflow/models/trigger.py +++ b/airflow/models/trigger.py @@ -36,6 +36,8 @@ class Trigger(Base): """ + Base Trigger class. + Triggers are a workload that run in an asynchronous event loop shared with other Triggers, and fire off events that will unpause deferred Tasks, start linked DAGs, etc. @@ -82,10 +84,7 @@ def __init__( @classmethod @internal_api_call def from_object(cls, trigger: BaseTrigger) -> Trigger: - """ - Alternative constructor that creates a trigger row based directly - off of a Trigger object. - """ + """Alternative constructor that creates a trigger row based directly off of a Trigger object.""" classpath, kwargs = trigger.serialize() return cls(classpath=classpath, kwargs=kwargs) @@ -93,10 +92,7 @@ def from_object(cls, trigger: BaseTrigger) -> Trigger: @internal_api_call @provide_session def bulk_fetch(cls, ids: Iterable[int], session: Session = NEW_SESSION) -> dict[int, Trigger]: - """ - Fetches all the Triggers by ID and returns a dict mapping - ID -> Trigger instance. - """ + """Fetches all the Triggers by ID and returns a dict mapping ID -> Trigger instance.""" query = ( session.query(cls) .filter(cls.id.in_(ids)) @@ -143,10 +139,7 @@ def clean_unused(cls, session: Session = NEW_SESSION) -> None: @internal_api_call @provide_session def submit_event(cls, trigger_id, event, session: Session = NEW_SESSION) -> None: - """ - Takes an event from an instance of itself, and triggers all dependent - tasks to resume. - """ + """Takes an event from an instance of itself, and triggers all dependent tasks to resume.""" for task_instance in session.query(TaskInstance).filter( TaskInstance.trigger_id == trigger_id, TaskInstance.state == TaskInstanceState.DEFERRED ): @@ -164,11 +157,11 @@ def submit_event(cls, trigger_id, event, session: Session = NEW_SESSION) -> None @provide_session def submit_failure(cls, trigger_id, exc=None, session: Session = NEW_SESSION) -> None: """ - Called when a trigger has failed unexpectedly, and we need to mark - everything that depended on it as failed. Notably, we have to actually - run the failure code from a worker as it may have linked callbacks, so - hilariously we have to re-schedule the task instances to a worker just - so they can then fail. + When a trigger has failed unexpectedly, mark everything that depended on it as failed. + + Notably, we have to actually run the failure code from a worker as it may + have linked callbacks, so hilariously we have to re-schedule the task + instances to a worker just so they can then fail. We use a special __fail__ value for next_method to achieve this that the runtime code understands as immediate-fail, and pack the error into @@ -202,6 +195,8 @@ def ids_for_triggerer(cls, triggerer_id, session: Session = NEW_SESSION) -> list @provide_session def assign_unassigned(cls, triggerer_id, capacity, heartrate, session: Session = NEW_SESSION) -> None: """ + Assign unassigned triggers based on a number of conditions. + Takes a triggerer_id, the capacity for that triggerer and the Triggerer job heartrate, and assigns unassigned triggers until that capacity is reached, or there are no more unassigned triggers. diff --git a/airflow/models/variable.py b/airflow/models/variable.py index bba785ee0a49..edfe2dea9643 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -38,10 +38,7 @@ class Variable(Base, LoggingMixin): - """ - Variables are a generic way to store and retrieve arbitrary content or settings - as a simple key value store within Airflow. - """ + """A generic way to store and retrieve arbitrary content or settings as a simple key/value store.""" __tablename__ = "variable" __NO_DEFAULT_SENTINEL = object() @@ -99,8 +96,9 @@ def val(cls): @classmethod def setdefault(cls, key, default, description=None, deserialize_json=False): """ - Like a Python builtin dict object, setdefault returns the current value - for a key, and if it isn't there, stores the default value and returns it. + Return the current value for a key or store the default value and return it. + + Works the same as the Python builtin dict object. :param key: Dict key for this Variable :param default: Default value to set and return if the variable diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py index 42485a5830fb..50e62ea29ff6 100644 --- a/airflow/models/xcom.py +++ b/airflow/models/xcom.py @@ -122,7 +122,8 @@ class BaseXCom(Base, LoggingMixin): @reconstructor def init_on_load(self): """ - Called by the ORM after the instance has been loaded from the DB or otherwise reconstituted + Called by the ORM after the instance has been loaded from the DB or otherwise reconstituted. + i.e automatically deserialize Xcom value when loading from DB. """ self.value = self.orm_deserialize_value() From 94122d1a6fcca5115fc4a3f2659418878f16e63c Mon Sep 17 00:00:00 2001 From: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:40:02 -0600 Subject: [PATCH 230/533] Fix chart named template comments (#32681) Not very exciting, but I noticed a formatting issue and a typo in our named template comments. --- chart/templates/_helpers.yaml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 6d4d4bac8d43..e0c158c765b1 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -858,9 +858,8 @@ If no value is passed for securityContexts.container or .securityContexts. | .securityContexts.container | -> | securityContexts.containers | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]| +-----------------------------------+ +----------------------------+ +-----------------------------------------------------------+ - The template can be called like so: - include "containerSecurityContext" (list . .Values.statsd) + include "containerSecurityContext" (list . .Values.webserver) Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template. */}} @@ -880,18 +879,17 @@ capabilities: {{- end -}} {{- end -}} - {{/* - Set the default value for external container securityContext(redis and statsd). - If no value is passed for .securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities. - - +-----------------------------------+ +-----------------------------------------------------------+ - | .securityContexts.container | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]| - +-----------------------------------+ +-----------------------------------------------------------+ +{{/* +Set the default value for external container securityContext(redis and statsd). +If no value is passed for .securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities. + +-----------------------------------+ +-----------------------------------------------------------+ + | .securityContexts.container | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]| + +-----------------------------------+ +-----------------------------------------------------------+ - The template can be called like so: - include "externalContainerSecurityContext" .Values.statsd - */}} +The template can be called like so: + include "externalContainerSecurityContext" .Values.statsd +*/}} {{- define "externalContainerSecurityContext" -}} {{- if .securityContexts.container -}} {{ toYaml .securityContexts.container | print }} From bbd290266c0aba07a9e4c4fc24e1eb9b6b4c61d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Wed, 19 Jul 2023 12:43:21 -0700 Subject: [PATCH 231/533] Fix bug in prune_dict where empty dict and list would be removed even in strict mode (#32573) --- airflow/providers/amazon/aws/utils/__init__.py | 17 ++++++++++++++++- airflow/utils/helpers.py | 4 ++-- tests/providers/amazon/aws/utils/test_utils.py | 12 ------------ tests/utils/test_helpers.py | 4 +++- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/airflow/providers/amazon/aws/utils/__init__.py b/airflow/providers/amazon/aws/utils/__init__.py index 8418e204815b..312366df26ae 100644 --- a/airflow/providers/amazon/aws/utils/__init__.py +++ b/airflow/providers/amazon/aws/utils/__init__.py @@ -21,13 +21,28 @@ from datetime import datetime from enum import Enum +from airflow.utils.helpers import prune_dict from airflow.version import version log = logging.getLogger(__name__) def trim_none_values(obj: dict): - return {key: val for key, val in obj.items() if val is not None} + from packaging.version import Version + + from airflow.version import version + + if Version(version) < Version("2.7"): + # before version 2.7, the behavior is not the same. + # Empty dict and lists are removed from the given dict. + return {key: val for key, val in obj.items() if val is not None} + else: + # once airflow 2.6 rolls out of compatibility support for provider packages, + # we can replace usages of this method with the core one in our code, + # and uncomment this warning for users who may use it. + # warnings.warn("use airflow.utils.helpers.prune_dict() instead", + # AirflowProviderDeprecationWarning, stacklevel=2) + return prune_dict(obj) def datetime_to_epoch(date_time: datetime) -> int: diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index b900ca1033c7..24e2bc9e66be 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -341,7 +341,7 @@ def is_empty(x): continue elif isinstance(v, (list, dict)): new_val = prune_dict(v, mode=mode) - if new_val: + if not is_empty(new_val): new_dict[k] = new_val else: new_dict[k] = v @@ -353,7 +353,7 @@ def is_empty(x): continue elif isinstance(v, (list, dict)): new_val = prune_dict(v, mode=mode) - if new_val: + if not is_empty(new_val): new_list.append(new_val) else: new_list.append(v) diff --git a/tests/providers/amazon/aws/utils/test_utils.py b/tests/providers/amazon/aws/utils/test_utils.py index bf404aa4fc51..66d5f734dc7d 100644 --- a/tests/providers/amazon/aws/utils/test_utils.py +++ b/tests/providers/amazon/aws/utils/test_utils.py @@ -26,7 +26,6 @@ datetime_to_epoch_ms, datetime_to_epoch_us, get_airflow_version, - trim_none_values, ) DT = datetime(2000, 1, 1, tzinfo=pytz.UTC) @@ -37,17 +36,6 @@ class EnumTest(_StringCompareEnum): FOO = "FOO" -def test_trim_none_values(): - input_object = { - "test": "test", - "empty": None, - } - expected_output_object = { - "test": "test", - } - assert trim_none_values(input_object) == expected_output_object - - def test_datetime_to_epoch(): assert datetime_to_epoch(DT) == EPOCH diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index f7ca52190910..3b72768fc75f 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -309,6 +309,8 @@ def assert_at_most_one(true=0, truthy=0, false=0, falsy=0, notset=0): "c": {"b": "", "c": "hi", "d": ["", 0, "1"]}, "d": ["", 0, "1"], "e": ["", 0, {"b": "", "c": "hi", "d": ["", 0, "1"]}, ["", 0, "1"], [""]], + "f": {}, + "g": [""], }, ), ( @@ -324,7 +326,7 @@ def assert_at_most_one(true=0, truthy=0, false=0, falsy=0, notset=0): def test_prune_dict(self, mode, expected): l1 = ["", 0, "1", None] d1 = {"a": None, "b": "", "c": "hi", "d": l1} - d2 = {"a": None, "b": "", "c": d1, "d": l1, "e": [None, "", 0, d1, l1, [""]]} + d2 = {"a": None, "b": "", "c": d1, "d": l1, "e": [None, "", 0, d1, l1, [""]], "f": {}, "g": [""]} assert prune_dict(d2, mode=mode) == expected From 7092cfdbbfcfd3c03909229daa741a5bcd7ccc64 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:27:56 -0700 Subject: [PATCH 232/533] Fix bad delete logic for dagruns (#32684) Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- airflow/www/utils.py | 12 +++++++++-- tests/www/test_utils.py | 47 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 3624fbe84137..bcf368ea2048 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -50,6 +50,7 @@ from airflow.utils.code_utils import get_python_source from airflow.utils.helpers import alchemy_to_dict from airflow.utils.json import WebEncoder +from airflow.utils.sqlalchemy import tuple_in_condition from airflow.utils.state import State, TaskInstanceState from airflow.www.forms import DateTimeWithTimezoneField from airflow.www.widgets import AirflowDateTimePickerWidget @@ -60,6 +61,8 @@ from airflow.www.fab_security.sqla.manager import SecurityManager +TI = TaskInstance + def datetime_to_string(value: DateTime | None) -> str | None: if value is None: @@ -844,12 +847,17 @@ class DagRunCustomSQLAInterface(CustomSQLAInterface): """ def delete(self, item: Model, raise_exception: bool = False) -> bool: - self.session.execute(delete(TaskInstance).where(TaskInstance.run_id == item.run_id)) + self.session.execute(delete(TI).where(TI.dag_id == item.dag_id, TI.run_id == item.run_id)) return super().delete(item, raise_exception=raise_exception) def delete_all(self, items: list[Model]) -> bool: self.session.execute( - delete(TaskInstance).where(TaskInstance.run_id.in_(item.run_id for item in items)) + delete(TI).where( + tuple_in_condition( + (TI.dag_id, TI.run_id), + ((x.dag_id, x.run_id) for x in items), + ) + ) ) return super().delete_all(items) diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 12fe017c6270..1dd1665a9bc0 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -17,17 +17,20 @@ # under the License. from __future__ import annotations +import itertools import re from datetime import datetime from unittest.mock import Mock from urllib.parse import parse_qs +import pendulum from bs4 import BeautifulSoup from markupsafe import Markup +from airflow.models import DagRun from airflow.utils import json as utils_json from airflow.www import utils -from airflow.www.utils import json_f, wrapped_markdown +from airflow.www.utils import DagRunCustomSQLAInterface, json_f, wrapped_markdown class TestUtils: @@ -156,7 +159,6 @@ def test_state_token(self): assert "" not in html def test_task_instance_link(self): - from airflow.www.app import cached_app with cached_app(testing=True).test_request_context(): @@ -413,3 +415,44 @@ def test_wrapped_markdown_with_collapsible_section(self):
    """ == rendered ) + + +def test_dag_run_custom_sqla_interface_delete_no_collateral_damage(dag_maker, session): + interface = DagRunCustomSQLAInterface(obj=DagRun, session=session) + dag_ids = (f"test_dag_{x}" for x in range(1, 4)) + dates = (pendulum.datetime(2023, 1, x) for x in range(1, 4)) + for dag_id, date in itertools.product(dag_ids, dates): + with dag_maker(dag_id=dag_id) as dag: + dag.create_dagrun(execution_date=date, state="running", run_type="scheduled") + dag_runs = session.query(DagRun).all() + assert len(dag_runs) == 9 + assert len(set(x.run_id for x in dag_runs)) == 3 + run_id_for_single_delete = "scheduled__2023-01-01T00:00:00+00:00" + # we have 3 runs with this same run_id + assert len(list(x for x in dag_runs if x.run_id == run_id_for_single_delete)) == 3 + # each is a different dag + + # if we delete one, it shouldn't delete the others + one_run = [x for x in dag_runs if x.run_id == run_id_for_single_delete][0] + assert interface.delete(item=one_run) is True + session.commit() + dag_runs = session.query(DagRun).all() + # we should have one fewer dag run now + assert len(dag_runs) == 8 + + # now let's try multi delete + run_id_for_multi_delete = "scheduled__2023-01-02T00:00:00+00:00" + # verify we have 3 + runs_of_interest = [x for x in dag_runs if x.run_id == run_id_for_multi_delete] + assert len(runs_of_interest) == 3 + # and that each is different dag + assert len(set(x.dag_id for x in dag_runs)) == 3 + + to_delete = runs_of_interest[:2] + # now try multi delete + assert interface.delete_all(items=to_delete) is True + session.commit() + dag_runs = session.query(DagRun).all() + assert len(dag_runs) == 6 + assert len(set(x.dag_id for x in dag_runs)) == 3 + assert len(set(x.run_id for x in dag_runs)) == 3 From 7123dc162bb222fdee7e4c50ae8a448c43cdd7d3 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Thu, 20 Jul 2023 06:30:54 +0200 Subject: [PATCH 233/533] Remove click upper bound (#32634) * Remove click upper bound * Fix typo --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 756e3c8ac83d..44998d8747ce 100644 --- a/setup.py +++ b/setup.py @@ -191,7 +191,7 @@ def git_version() -> str: try: repo = git.Repo(str(AIRFLOW_SOURCES_ROOT / ".git")) - except (git.NoSuchPathError): + except git.NoSuchPathError: logger.warning(".git directory not found: Cannot compute the git version") return "" except git.InvalidGitRepositoryError: @@ -266,9 +266,9 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve doc = [ "astroid>=2.12.3", "checksumdir", - # Click 8.1.4 breaks our mypy checks. The upper limit can be lifted when the - # https://github.com/apache/airflow/issues/32412 issue is resolved - "click>=8.0,<8.1.4", + # click 8.1.4 and 8.1.5 generate mypy errors due to typing issue in the upstream package: + # https://github.com/pallets/click/issues/2558 + "click>=8.0,!=8.1.4,!=8.1.5", # Docutils 0.17.0 converts generated
    into
    and breaks our doc formatting # By adding a lot of whitespace separation. This limit can be lifted when we update our doc to handle #
    tags for sections @@ -515,12 +515,12 @@ def filter_out_excluded_extras() -> Iterable[tuple[str, list[str]]]: def add_extras_for_all_providers() -> None: - for (provider_name, provider_dict) in PROVIDER_DEPENDENCIES.items(): + for provider_name, provider_dict in PROVIDER_DEPENDENCIES.items(): EXTRAS_DEPENDENCIES[provider_name] = provider_dict[DEPS] def add_additional_extras() -> None: - for (extra_name, extra_dependencies) in ADDITIONAL_EXTRAS_DEPENDENCIES.items(): + for extra_name, extra_dependencies in ADDITIONAL_EXTRAS_DEPENDENCIES.items(): EXTRAS_DEPENDENCIES[extra_name] = extra_dependencies From faa8f543d5f7bf7113076dab5f579493ec2860ec Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 20 Jul 2023 08:20:23 +0200 Subject: [PATCH 234/533] Store config description in Airflow configuration object (#32669) We would like to use the config.yml approach as our main source of truth for airflow configuration. So far it has been split amongst multiple files: * confg.yml -> descroption * default_airflow.cfg -> JINJA template to keep cofiguration, examples and description (used to generate airflow.cfg when airflow starts) * default_test.cfg -> storing test-only configuration used in some test cases - used to generate unittest.cfg * scripts/in_container/airflow_ci.cfg where dCI-specific configuration overwrote the unittest.cfg This change consolidates it all into unified appraoch, where all configuration information is retrieved from .yml files stored in config_templates. No more additional template files processed by JINJA, no more special CI versions of it, no more unittestdb.cfg file where such configuration would be generated, no more unittestdb to be used separately for tests. * The default_*.cfg files were not real configuration files, becuase they were really JINJA templates and it got peoeple confused when copying the files. This change leaves the file empty with the comment that instructs the user how they can get the default configuration. * The default_airflow.cfg is gone and instead, we have a way to show defaults via `airflow config list --defaults` command * Unittest config is generated on-the-flight using defaults stored in confg_templates. constituing a single place where they need to be changed for the tests to use them * internally, description of the configuration is stored in AirflowConfigurationParser and used by the parser to generate the default configuration when needed. * we got rid of `{{{{` in templated config defaults by marking the templates with ``is_template`` and getting rid of processing those entries with regular formatting when generating the default values. This only concerns defaults from config.yml. Reading those configuration entries from file is unaffected. This change aims to be 100% backwards compatible with the previous implementation when it comes to functionality, even if internals changed. It also does not add provider-specific changes that are coming separately. The only changes visible to the user are: * generated airflow.cfg is slightly more readable and displays names of variables that can be used to override each configuration (which is very useful for copy&pasting) * user are advised, instead of copying the default_airflow.cfg to use `airflow config list --defaults` to generate production config. This configuration has all the entries commented out, so that they can selectively uncomment and change the values they want. This is now promoted as "best practice" in the documentation. --- .github/mergeable.yml | 5 - .gitignore | 2 - .pre-commit-config.yaml | 8 - .rat-excludes | 1 - Dockerfile.ci | 1 - STATIC_CODE_CHECKS.rst | 2 - TESTING.rst | 33 +- airflow/cli/cli_config.py | 37 +- airflow/cli/commands/config_command.py | 11 +- airflow/config_templates/config.yml | 34 +- .../config_templates/config.yml.schema.json | 5 + airflow/config_templates/default_airflow.cfg | 1513 +---------------- airflow/config_templates/default_test.cfg | 68 - airflow/config_templates/unit_tests.cfg | 102 ++ airflow/configuration.py | 703 +++++--- airflow/utils/db.py | 4 +- .../src/airflow_breeze/pre_commit_ids.py | 1 - docs/apache-airflow/howto/index.rst | 1 - docs/apache-airflow/howto/set-config.rst | 27 +- docs/apache-airflow/howto/use-test-config.rst | 36 - docs/apache-airflow/redirects.txt | 3 + docs/conf.py | 4 +- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-commands.svg | 108 +- images/breeze/output_static-checks.svg | 154 +- .../ci/pre_commit/pre_commit_yaml_to_cfg.py | 179 -- scripts/docker/entrypoint_ci.sh | 1 - scripts/in_container/airflow_ci.cfg | 59 - tests/cli/commands/test_config_command.py | 138 +- tests/cli/test_cli_parser.py | 17 +- tests/config_templates/deprecated.cfg | 9 - tests/config_templates/deprecated_cmd.cfg | 10 - tests/config_templates/deprecated_secret.cfg | 10 - tests/config_templates/empty.cfg | 9 - tests/conftest.py | 15 +- tests/core/test_config_templates.py | 98 -- tests/core/test_configuration.py | 151 +- 37 files changed, 1139 insertions(+), 2422 deletions(-) delete mode 100644 airflow/config_templates/default_test.cfg create mode 100644 airflow/config_templates/unit_tests.cfg delete mode 100644 docs/apache-airflow/howto/use-test-config.rst delete mode 100755 scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py delete mode 100644 scripts/in_container/airflow_ci.cfg delete mode 100644 tests/core/test_config_templates.py diff --git a/.github/mergeable.yml b/.github/mergeable.yml index e3101c6087c3..dc774f2b8159 100644 --- a/.github/mergeable.yml +++ b/.github/mergeable.yml @@ -39,8 +39,3 @@ mergeable: changed: file: 'package.json' files: ['yarn.lock'] - # If config.yml is updated, so should default_airflow.cfg - - do: dependent - changed: - file: 'config.yml' - files: ['default_airflow.cfg'] diff --git a/.gitignore b/.gitignore index a6528d887d24..6ee528b9ce88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Airflow configuration airflow.cfg -unittests.cfg airflow_login.py dbinit.py initdb.py @@ -8,7 +7,6 @@ secrets.py # Airflow sqlite databases airflow.db -unittests.db # Airflow temporary artifacts airflow/git_version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11ce7b2f2854..ff8085d03033 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -618,14 +618,6 @@ repos: pass_filenames: true additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml'] require_serial: true - - id: check-airflow-config-yaml-consistent - name: Check consistency between config.yml and default_config.cfg - language: python - entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py - files: config\.yml$|default_airflow\.cfg$|default\.cfg$ - pass_filenames: false - require_serial: true - additional_dependencies: ['pyyaml', 'packaging'] - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency language: python diff --git a/.rat-excludes b/.rat-excludes index 7d98d4e34865..cab48fa0eede 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -38,7 +38,6 @@ metastore_db .*md5 .*zip .*lock -unittests.cfg logs .bash_aliases venv diff --git a/Dockerfile.ci b/Dockerfile.ci index 88e431655985..23a24d26dd46 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -837,7 +837,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then unset AIRFLOW__CORE__UNIT_TEST_MODE mkdir -pv "${AIRFLOW_HOME}/logs/" - cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" # Change the default worker_concurrency for tests export AIRFLOW__CELERY__WORKER_CONCURRENCY=8 diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 4bcba4bb55e2..1729c8ba8e61 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -146,8 +146,6 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-aiobotocore-optional | Check if aiobotocore is an optional dependency only | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ -| check-airflow-config-yaml-consistent | Check consistency between config.yml and default_config.cfg | | -+-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-airflow-provider-compatibility | Check compatibility of Providers with Airflow | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-apache-license-rat | Check if licenses are OK for Apache | | diff --git a/TESTING.rst b/TESTING.rst index 94b4ff743859..09b7e9d31491 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -55,8 +55,37 @@ Follow the guidelines when writing unit tests: tests, so we run Pytest with ``--disable-warnings`` but instead we have ``pytest-capture-warnings`` plugin that overrides ``recwarn`` fixture behaviour. -**NOTE:** We plan to convert all unit tests to standard "asserts" semi-automatically, but this will be done later -in Airflow 2.0 development phase. That will include setUp/tearDown/context managers and decorators. + +.. note:: + + We are in the process of converting all unit tests to standard "asserts" and pytest fixtures + so if you find some tests that are still using classic setUp/tearDown approach or unittest asserts, feel + free to convert them to pytest. + +Airflow configuration for unit tests +------------------------------------ + +Some of the unit tests require special configuration set as the ``default``. This is done automatically by +adding ``AIRFLOW__CORE__UNIT_TEST_MODE=True`` to the environment variables in Pytest auto-used +fixture. This in turn makes Airflow load test configuration from the file +``airflow/config_templates/unit_tests.cfg``. Test configuration from there replaces the original +defaults from ``airflow/config_templates/config.yml``. If you want to add some test-only configuration, +as default for all tests you should add the value to this file. + +You can also of course override the values in individual test by patching environment variables following +the usual ``AIRFLOW__SECTION__KEY`` pattern or ``conf_vars`` context manager. + +.. note:: + + The test configuration for Airflow before July 2023 was automatically generated in a file named + ``AIRFLOW_HOME/unittest.cfg``. The template for it was stored in "config_templates" next to the yaml file. + However writing the file was only done for the first time you run airflow and you had to manually + maintain the file. It was pretty arcane knowledge, and this generated file in {AIRFLOW_HOME} + has been overwritten in the Breeze environment with another CI-specific file. Using ``unit_tests.cfg`` + as a single source of the configuration for tests - coming from Airflow sources + rather than from {AIRFLOW_HOME} is much more convenient and it is automatically used by pytest. + + The unittest.cfg file generated in {AIRFLOW_HOME} will no longer be used and can be removed. Airflow test types ------------------ diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 66b959051363..79327a4701ac 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -311,6 +311,31 @@ def string_lower_type(val): ARG_MARK_SUCCESS = Arg( ("-m", "--mark-success"), help="Mark jobs as succeeded without running them", action="store_true" ) +ARG_INCLUDE_DESCRIPTIONS = Arg( + ("-d", "--include-descriptions"), + help="Show descriptions for the configuration variables", + action="store_true", +) +ARG_INCLUDE_EXAMPLES = Arg( + ("-e", "--include-examples"), help="Show examples for the configuration variables", action="store_true" +) +ARG_INCLUDE_SOURCES = Arg( + ("-s", "--include-sources"), help="Show source of the configuration variable", action="store_true" +) +ARG_INCLUDE_ENV_VARS = Arg( + ("-V", "--include-env-vars"), help="Show environment variable for each option", action="store_true" +) +ARG_COMMENT_OUT_EVERYTHING = Arg( + ("-c", "--comment-out-everything"), + help="Comment out all configuration options. Useful as starting point for new installation", + action="store_true", +) +ARG_DEFAULTS = Arg( + ("-a", "--defaults"), + help="Show only defaults - do not include local configuration, sources," + " includes descriptions, examples, variables. Comment out everything.", + action="store_true", +) ARG_VERBOSE = Arg(("-v", "--verbose"), help="Make logging output more verbose", action="store_true") ARG_LOCAL = Arg(("-l", "--local"), help="Run the task using the LocalExecutor", action="store_true") ARG_DONOT_PICKLE = Arg( @@ -2002,7 +2027,17 @@ class GroupCommand(NamedTuple): name="list", help="List options for the configuration", func=lazy_load_command("airflow.cli.commands.config_command.show_config"), - args=(ARG_OPTIONAL_SECTION, ARG_COLOR, ARG_VERBOSE), + args=( + ARG_OPTIONAL_SECTION, + ARG_COLOR, + ARG_INCLUDE_DESCRIPTIONS, + ARG_INCLUDE_EXAMPLES, + ARG_INCLUDE_SOURCES, + ARG_INCLUDE_ENV_VARS, + ARG_COMMENT_OUT_EVERYTHING, + ARG_DEFAULTS, + ARG_VERBOSE, + ), ), ) diff --git a/airflow/cli/commands/config_command.py b/airflow/cli/commands/config_command.py index f8d8cac0cb86..8d664417237d 100644 --- a/airflow/cli/commands/config_command.py +++ b/airflow/cli/commands/config_command.py @@ -30,7 +30,16 @@ def show_config(args): """Show current application configuration.""" with io.StringIO() as output: - conf.write(output, section=args.section) + conf.write( + output, + section=args.section, + include_examples=args.include_examples or args.defaults, + include_descriptions=args.include_descriptions or args.defaults, + include_sources=args.include_sources and not args.defaults, + include_env_vars=args.include_env_vars or args.defaults, + comment_out_everything=args.comment_out_everything or args.defaults, + only_defaults=args.defaults, + ) code = output.getvalue() if should_use_colors(args): code = pygments.highlight(code=code, formatter=get_terminal_formatter(), lexer=IniLexer()) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 4dede4af281a..0629cb5d1195 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -44,10 +44,10 @@ core: default: "airflow.utils.net.getfqdn" might_contain_dag_callable: description: | - A callable to check if a python file has airflow dags defined or not - with argument as: `(file_path: str, zip_file: zipfile.ZipFile | None = None)` - return True if it has dags otherwise False - If this is not provided, Airflow uses its own heuristic rules. + A callable to check if a python file has airflow dags defined or not + with argument as: `(file_path: str, zip_file: zipfile.ZipFile | None = None)` + return True if it has dags otherwise False + If this is not provided, Airflow uses its own heuristic rules. version_added: 2.6.0 type: string example: ~ @@ -288,8 +288,8 @@ core: default: "0" default_task_retry_delay: description: | - The number of seconds each task is going to wait by default between retries. Can be overridden at - dag or task level. + The number of seconds each task is going to wait by default between retries. Can be overridden at + dag or task level. version_added: 2.4.0 type: integer example: ~ @@ -681,7 +681,7 @@ logging: description: | The remote_task_handler_kwargs param is loaded into a dictionary and passed to __init__ of remote task handler and it overrides the values provided by Airflow config. For example if you set - `delete_local_logs=False` and you provide ``{{"delete_local_copy": true}}``, then the local + `delete_local_logs=False` and you provide ``{"delete_local_copy": true}``, then the local log files will be deleted after they are uploaded to remote location. version_added: 2.6.0 type: string @@ -804,7 +804,8 @@ logging: Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter version_added: 2.0.0 type: string - example: "{{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}}" + example: "{ti.dag_id}-{ti.task_id}-{execution_date}-{try_number}" + is_template: true default: "" log_filename_template: description: | @@ -812,16 +813,18 @@ logging: version_added: 2.0.0 type: string example: ~ - default: "dag_id={{{{ ti.dag_id }}}}/run_id={{{{ ti.run_id }}}}/task_id={{{{ ti.task_id }}}}/\ - {{%% if ti.map_index >= 0 %%}}map_index={{{{ ti.map_index }}}}/{{%% endif %%}}\ - attempt={{{{ try_number }}}}.log" + is_template: true + default: "dag_id={{ ti.dag_id }}/run_id={{ ti.run_id }}/task_id={{ ti.task_id }}/\ + {%% if ti.map_index >= 0 %%}map_index={{ ti.map_index }}/{%% endif %%}\ + attempt={{ try_number }}.log" log_processor_filename_template: description: | Formatting for how airflow generates file names for log version_added: 2.0.0 type: string example: ~ - default: "{{{{ filename }}}}.log" + is_template: true + default: "{{ filename }}.log" dag_processor_manager_log_location: description: | Full path of dag_processor_manager logfile. @@ -1071,7 +1074,7 @@ secrets: The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. See documentation for the secrets backend you are using. JSON is expected. Example for AWS Systems Manager ParameterStore: - ``{{"connections_prefix": "/airflow/connections", "profile_name": "default"}}`` + ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` version_added: 1.10.10 type: string sensitive: true @@ -1825,7 +1828,7 @@ webserver: default: "True" auth_rate_limit: description: | - Rate limit for authentication endpoints. + Rate limit for authentication endpoints. version_added: 2.6.0 type: string example: ~ @@ -2688,7 +2691,8 @@ elasticsearch: version_added: 1.10.4 type: string example: ~ - default: "{{dag_id}}-{{task_id}}-{{run_id}}-{{map_index}}-{{try_number}}" + is_template: true + default: "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}" end_of_log_mark: description: | Used to mark the end of a log stream for a task diff --git a/airflow/config_templates/config.yml.schema.json b/airflow/config_templates/config.yml.schema.json index d800ade7d18b..9bc321acc150 100644 --- a/airflow/config_templates/config.yml.schema.json +++ b/airflow/config_templates/config.yml.schema.json @@ -72,6 +72,11 @@ "sensitive": { "type": "boolean", "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values" + }, + "is_template": { + "type": "boolean", + "description": "When true, the {VARS} have no special meaning there - they won't be expanded with env vars/local/global variables.", + "default": false } }, "required": [ diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 8046fbc18ad8..f0f9c1b60c82 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,1507 +15,31 @@ # specific language governing permissions and limitations # under the License. -# This is the template for Airflow's default configuration. When Airflow is -# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If -# it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# airflow.cfg instead. - - -# ----------------------- TEMPLATE BEGINS HERE ----------------------- - -[core] -# The folder where your airflow pipelines live, most likely a -# subfolder in a code repository. This path must be absolute. -dags_folder = {AIRFLOW_HOME}/dags - -# Hostname by providing a path to a callable, which will resolve the hostname. -# The format is "package.function". # -# For example, default value "airflow.utils.net.getfqdn" means that result from patched -# version of socket.getfqdn() - see https://github.com/python/cpython/issues/49254. +# NOTE: # -# No argument should be required in the function specified. -# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` -hostname_callable = airflow.utils.net.getfqdn - -# A callable to check if a python file has airflow dags defined or not -# with argument as: `(file_path: str, zip_file: zipfile.ZipFile | None = None)` -# return True if it has dags otherwise False -# If this is not provided, Airflow uses its own heuristic rules. -might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_heuristic - -# Default timezone in case supplied date times are naive -# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) -default_timezone = utc - -# The executor class that airflow should use. Choices include -# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, -# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the -# full import path to the class when using a custom executor. -executor = SequentialExecutor - -# The auth manager class that airflow should use. Full import path to the auth manager class. -auth_manager = airflow.auth.managers.fab.fab_auth_manager.FabAuthManager - -# This defines the maximum number of task instances that can run concurrently per scheduler in -# Airflow, regardless of the worker count. Generally this value, multiplied by the number of -# schedulers in your cluster, is the maximum number of task instances with the running -# state in the metadata database. -parallelism = 32 - -# The maximum number of task instances allowed to run concurrently in each DAG. To calculate -# the number of tasks that is running concurrently for a DAG, add up the number of running -# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``, -# which is defaulted as ``max_active_tasks_per_dag``. +# IF YOU ARE LOOKING FOR DEFAULT CONFIGURATION FILE HERE - LOOK NO MORE. READ EXPLANATION BELOW! # -# An example scenario when this would be useful is when you want to stop a new dag with an early -# start date from stealing all the executor slots in a cluster. -max_active_tasks_per_dag = 16 - -# Are DAGs paused by default at creation -dags_are_paused_at_creation = True - -# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs -# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, -# which is defaulted as ``max_active_runs_per_dag``. -max_active_runs_per_dag = 16 - -# The name of the method used in order to start Python processes via the multiprocessing module. -# This corresponds directly with the options available in the Python docs: -# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method. -# Must be one of the values returned by: -# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.get_all_start_methods. -# Example: mp_start_method = fork -# mp_start_method = - -# Whether to load the DAG examples that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_examples = True - -# Path to the folder containing Airflow plugins -plugins_folder = {AIRFLOW_HOME}/plugins - -# Should tasks be executed via forking of the parent process ("False", -# the speedier option) or by spawning a new python process ("True" slow, -# but means plugin changes picked up by tasks straight away) -execute_tasks_new_python_interpreter = False - -# Secret key to save connection passwords in the db -fernet_key = {FERNET_KEY} - -# Whether to disable pickling dags -donot_pickle = True - -# How long before timing out a python file import -dagbag_import_timeout = 30.0 - -# Should a traceback be shown in the UI for dagbag import errors, -# instead of just the exception message -dagbag_import_error_tracebacks = True - -# If tracebacks are shown, how many entries from the traceback should be shown -dagbag_import_error_traceback_depth = 2 - -# How long before timing out a DagFileProcessor, which processes a dag file -dag_file_processor_timeout = 50 - -# The class to use for running task instances in a subprocess. -# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class -# when using a custom task runner. -task_runner = StandardTaskRunner - -# If set, tasks without a ``run_as_user`` argument will be run with this user -# Can be used to de-elevate a sudo user running Airflow when executing tasks -default_impersonation = - -# What security module to use (for example kerberos) -security = - -# Turn unit test mode on (overwrites many configuration options with test -# values at runtime) -unit_test_mode = False - -# Whether to enable pickling for xcom (note that this is insecure and allows for -# RCE exploits). -enable_xcom_pickling = False - -# What classes can be imported during deserialization. This is a multi line value. -# The individual items will be parsed as regexp. Python built-in classes (like dict) -# are always allowed. Bare "." will be replaced so you can set airflow.* . -allowed_deserialization_classes = airflow\..* - -# When a task is killed forcefully, this is the amount of time in seconds that -# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED -killed_task_cleanup_time = 60 - -# Whether to override params with dag_run.conf. If you pass some key-value pairs -# through ``airflow dags backfill -c`` or -# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = True - -# If enabled, Airflow will only scan files containing both ``DAG`` and ``airflow`` (case-insensitive). -dag_discovery_safe_mode = True - -# The pattern syntax used in the ".airflowignore" files in the DAG directories. Valid values are -# ``regexp`` or ``glob``. -dag_ignore_file_syntax = regexp - -# The number of retries each task is going to have by default. Can be overridden at dag or task level. -default_task_retries = 0 - -# The number of seconds each task is going to wait by default between retries. Can be overridden at -# dag or task level. -default_task_retry_delay = 300 - -# The maximum delay (in seconds) each task is going to wait by default between retries. -# This is a global setting and cannot be overridden at task or DAG level. -max_task_retry_delay = 86400 - -# The weighting method used for the effective total priority weight of the task -default_task_weight_rule = downstream - -# The default task execution_timeout value for the operators. Expected an integer value to -# be passed into timedelta as seconds. If not specified, then the value is considered as None, -# meaning that the operators are never timed out by default. -default_task_execution_timeout = - -# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. -min_serialized_dag_update_interval = 30 - -# If True, serialized DAGs are compressed before writing to DB. -# Note: this will disable the DAG dependencies view -compress_serialized_dags = False - -# Fetching serialized DAG can not be faster than a minimum interval to reduce database -# read rate. This config controls when your DAGs are updated in the Webserver -min_serialized_dag_fetch_interval = 10 - -# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store -# in the Database. -# All the template_fields for each of Task Instance are stored in the Database. -# Keeping this number small may cause an error when you try to view ``Rendered`` tab in -# TaskInstance view for older tasks. -max_num_rendered_ti_fields_per_task = 30 - -# On each dagrun check against defined SLAs -check_slas = True - -# Path to custom XCom class that will be used to store and resolve operators results -# Example: xcom_backend = path.to.CustomXCom -xcom_backend = airflow.models.xcom.BaseXCom - -# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, -# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. -lazy_load_plugins = True - -# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). -# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or -# loaded from module. -lazy_discover_providers = True - -# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True +# This file used to have something that was similar to the default Airflow configuration but it was +# really just a template. It was used to generate the final configuration and it was confusing +# if you copied it to your configuration and some of values were wrong. # -# (Connection passwords are always hidden in logs) -hide_sensitive_var_conn_fields = True - -# A comma-separated list of extra sensitive keywords to look for in variables names or connection's -# extra JSON. -sensitive_var_conn_names = - -# Task Slot counts for ``default_pool``. This setting would not have any effect in an existing -# deployment where the ``default_pool`` is already created. For existing deployments, users can -# change the number of slots using Webserver, API or the CLI -default_pool_task_slot_count = 128 - -# The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a -# length exceeding this value, the task pushing the XCom will be failed automatically to prevent the -# mapped tasks from clogging the scheduler. -max_map_length = 1024 - -# The default umask to use for process when run in daemon mode (scheduler, worker, etc.) +# The first time you run Airflow, it will create a file called ``airflow.cfg`` in +# your ``$AIRFLOW_HOME`` directory (``~/airflow`` by default). This is in order to make it easy to +# "play" with airflow configuration. # -# This controls the file-creation mode mask which determines the initial value of file permission bits -# for newly created files. +# However, for production case you are advised to generate the configuration using command line: # -# This value is treated as an octal-integer. -daemon_umask = 0o077 - -# Class to use as dataset manager. -# Example: dataset_manager_class = airflow.datasets.manager.DatasetManager -# dataset_manager_class = - -# Kwargs to supply to dataset manager. -# Example: dataset_manager_kwargs = {{"some_param": "some_value"}} -# dataset_manager_kwargs = - -# (experimental) Whether components should use Airflow Internal API for DB connectivity. -database_access_isolation = False - -# (experimental) Airflow Internal API url. Only used if [core] database_access_isolation is True. -# Example: internal_api_url = http://localhost:8080 -# internal_api_url = - -# The ability to allow testing connections across Airflow UI, API and CLI. -# Supported options: Disabled, Enabled, Hidden. Default: Disabled -# Disabled - Disables the test connection functionality and disables the Test Connection button in UI. -# Enabled - Enables the test connection functionality and shows the Test Connection button in UI. -# Hidden - Disables the test connection functionality and hides the Test Connection button in UI. -# Before setting this to Enabled, make sure that you review the users who are able to add/edit -# connections and ensure they are trusted. Connection testing can be done maliciously leading to -# undesired and insecure outcomes. For more information on capabilities of users, see the documentation: -# https://airflow.apache.org/docs/apache-airflow/stable/security/index.html#capabilities-of-authenticated-ui-users -test_connection = Disabled - -[database] -# Path to the ``alembic.ini`` file. You can either provide the file path relative -# to the Airflow home directory or the absolute path if it is located elsewhere. -alembic_ini_file_path = alembic.ini - -# The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engines. -# More information here: -# http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri -sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/airflow.db - -# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value -# Example: sql_alchemy_engine_args = {{"arg1": True}} -# sql_alchemy_engine_args = - -# The encoding for the databases -sql_engine_encoding = utf-8 - -# Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id`` columns -# in case they have different encoding. -# By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb`` -# the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed -# the maximum size of allowed index when collation is set to ``utf8mb4`` variant -# (see https://github.com/apache/airflow/pull/17603#issuecomment-901121618). -# sql_engine_collation_for_ids = - -# If SqlAlchemy should pool database connections. -sql_alchemy_pool_enabled = True - -# The SqlAlchemy pool size is the maximum number of database connections -# in the pool. 0 indicates no limit. -sql_alchemy_pool_size = 5 - -# The maximum overflow size of the pool. -# When the number of checked-out connections reaches the size set in pool_size, -# additional connections will be returned up to this limit. -# When those additional connections are returned to the pool, they are disconnected and discarded. -# It follows then that the total number of simultaneous connections the pool will allow -# is pool_size + max_overflow, -# and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to ``-1`` to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. -sql_alchemy_max_overflow = 10 - -# The SqlAlchemy pool recycle is the number of seconds a connection -# can be idle in the pool before it is invalidated. This config does -# not apply to sqlite. If the number of DB connections is ever exceeded, -# a lower config value will allow the system to recover faster. -sql_alchemy_pool_recycle = 1800 - -# Check connection at the start of each connection pool checkout. -# Typically, this is a simple statement like "SELECT 1". -# More information here: -# https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic -sql_alchemy_pool_pre_ping = True - -# The schema to use for the metadata database. -# SqlAlchemy supports databases with the concept of multiple schemas. -sql_alchemy_schema = - -# Import path for connect args in SqlAlchemy. Defaults to an empty dict. -# This is useful when you want to configure db engine args that SqlAlchemy won't parse -# in connection string. -# See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.connect_args -# Example: sql_alchemy_connect_args = {{"timeout": 30}} -# sql_alchemy_connect_args = - -# Whether to load the default connections that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_default_connections = True - -# Number of times the code should be retried in case of DB Operational Errors. -# Not all transactions will be retried as it can cause undesired state. -# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. -max_db_retries = 3 - -# Whether to run alembic migrations during Airflow start up. Sometimes this operation can be expensive, -# and the users can assert the correct version through other means (e.g. through a Helm chart). -# Accepts "True" or "False". -check_migrations = True - -[logging] -# The folder where airflow should store its log files. -# This path must be absolute. -# There are a few existing configurations that assume this is set to the default. -# If you choose to override this you may need to update the dag_processor_manager_log_location and -# dag_processor_manager_log_location settings as well. -base_log_folder = {AIRFLOW_HOME}/logs - -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. Depending on your remote logging service, this may only be used for -# reading logs, not writing them. -remote_log_conn_id = - -# Whether the local log files for GCS, S3, WASB and OSS remote logging should be deleted after -# they are uploaded to the remote location. -delete_local_logs = False - -# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default -# Credentials -# `__ will -# be used. -google_key_path = - -# Storage bucket URL for remote logging -# S3 buckets should start with "s3://" -# Cloudwatch log groups should start with "cloudwatch://" -# GCS buckets should start with "gs://" -# WASB buckets should start with "wasb" just to help Airflow select correct handler -# Stackdriver logs should start with "stackdriver://" -remote_base_log_folder = - -# The remote_task_handler_kwargs param is loaded into a dictionary and passed to __init__ of remote -# task handler and it overrides the values provided by Airflow config. For example if you set -# `delete_local_logs=False` and you provide ``{{"delete_local_copy": true}}``, then the local -# log files will be deleted after they are uploaded to remote location. -# Example: remote_task_handler_kwargs = {{"delete_local_copy": true}} -remote_task_handler_kwargs = - -# Use server-side encryption for logs stored in S3 -encrypt_s3_logs = False - -# Logging level. +# airflow config list --defaults # -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -logging_level = INFO - -# Logging level for celery. If not set, it uses the value of logging_level +# This command will produce the output that you can copy to your configuration file and edit. +# It will contain all the default configuration options, with examples, nicely commented out +# so you need only un-comment and modify those that you want to change. +# This way you can easily keep track of all the configuration options that you changed from default +# and you can also easily upgrade your installation to new versions of Airflow when they come out and +# automatically use the defaults for existing options if they changed there. # -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -celery_logging_level = - -# Logging level for Flask-appbuilder UI. +# You can redirect it to your configuration file and edit it: # -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -fab_logging_level = WARNING - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Where to send dag parser logs. If "file", logs are sent to log files defined by child_process_log_directory. -dag_processor_log_target = file - -# Format of Dag Processor Log line -dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware - -# An import path to a function to add adaptations of each secret added with -# `airflow.utils.log.secrets_masker.mask_secret` to be masked in log messages. The given function -# is expected to require a single parameter: the secret to be adapted. It may return a -# single adaptation of the secret or an iterable of adaptations to each be masked as secrets. -# The original secret will be masked as well as any adaptations returned. -# Example: secret_mask_adapter = urllib.parse.quote -secret_mask_adapter = - -# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter -# Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}} -task_log_prefix_template = - -# Formatting for how airflow generates file names/paths for each task run. -log_filename_template = dag_id={{{{ ti.dag_id }}}}/run_id={{{{ ti.run_id }}}}/task_id={{{{ ti.task_id }}}}/{{%% if ti.map_index >= 0 %%}}map_index={{{{ ti.map_index }}}}/{{%% endif %%}}attempt={{{{ try_number }}}}.log - -# Formatting for how airflow generates file names for log -log_processor_filename_template = {{{{ filename }}}}.log - -# Full path of dag_processor_manager logfile. -dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Defaults to use ``task`` handler. -task_log_reader = task - -# A comma\-separated list of third-party logger names that will be configured to print messages to -# consoles\. -# Example: extra_logger_names = connexion,sqlalchemy -extra_logger_names = - -# When you start an airflow worker, airflow starts a tiny web server -# subprocess to serve the workers local log files to the airflow main -# web server, who then builds pages and sends them to users. This defines -# the port on which the logs are served. It needs to be unused, and open -# visible from the main web server to connect into the workers. -worker_log_server_port = 8793 - -# Port to serve logs from for triggerer. See worker_log_server_port description -# for more info. -trigger_log_server_port = 8794 - -# We must parse timestamps to interleave logs between trigger and task. To do so, -# we need to parse timestamps in log files. In case your log format is non-standard, -# you may provide import path to callable which takes a string log line and returns -# the timestamp (datetime.datetime compatible). -# Example: interleave_timestamp_parser = path.to.my_func -# interleave_timestamp_parser = - -# Permissions in the form or of octal string as understood by chmod. The permissions are important -# when you use impersonation, when logs are written by a different user than airflow. The most secure -# way of configuring it in this case is to add both users to the same group and make it the default -# group of both users. Group-writeable logs are default in airflow, but you might decide that you are -# OK with having the logs other-writeable, in which case you should set it to `0o777`. You might -# decide to add more security if you do not use impersonation and change it to `0o755` to make it -# only owner-writeable. You can also make it just readable only for owner by changing it to `0o700` if -# all the access (read/write) for your logs happens from the same user. -# Example: file_task_handler_new_folder_permissions = 0o775 -file_task_handler_new_folder_permissions = 0o775 - -# Permissions in the form or of octal string as understood by chmod. The permissions are important -# when you use impersonation, when logs are written by a different user than airflow. The most secure -# way of configuring it in this case is to add both users to the same group and make it the default -# group of both users. Group-writeable logs are default in airflow, but you might decide that you are -# OK with having the logs other-writeable, in which case you should set it to `0o666`. You might -# decide to add more security if you do not use impersonation and change it to `0o644` to make it -# only owner-writeable. You can also make it just readable only for owner by changing it to `0o600` if -# all the access (read/write) for your logs happens from the same user. -# Example: file_task_handler_new_file_permissions = 0o664 -file_task_handler_new_file_permissions = 0o664 - -# By default Celery sends all logs into stderr. -# If enabled any previous logging handlers will get *removed*. -# With this option AirFlow will create new handlers -# and send low level logs like INFO and WARNING to stdout, -# while sending higher severity logs to stderr. -celery_stdout_stderr_separation = False - -[metrics] - -# StatsD (https://github.com/etsy/statsd) integration settings. -# If you want to avoid emitting all the available metrics, you can configure an -# allow list of prefixes (comma separated) to send only the metrics that start -# with the elements of the list (e.g: "scheduler,executor,dagrun") -metrics_allow_list = - -# If you want to avoid emitting all the available metrics, you can configure a -# block list of prefixes (comma separated) to filter out metrics that start with -# the elements of the list (e.g: "scheduler,executor,dagrun"). -# If metrics_allow_list and metrics_block_list are both configured, metrics_block_list is ignored. -metrics_block_list = - -# Enables sending metrics to StatsD. -statsd_on = False -statsd_host = localhost -statsd_port = 8125 -statsd_prefix = airflow - -# A function that validate the StatsD stat name, apply changes to the stat name if necessary and return -# the transformed stat name. -# -# The function should have the following signature: -# def func_name(stat_name: str) -> str: -stat_name_handler = - -# To enable datadog integration to send airflow metrics. -statsd_datadog_enabled = False - -# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) -statsd_datadog_tags = - -# Set to False to disable metadata tags for some of the emitted metrics -statsd_datadog_metrics_tags = True - -# If you want to utilise your own custom StatsD client set the relevant -# module path below. -# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up -# statsd_custom_client_path = - -# If you want to avoid sending all the available metrics tags to StatsD, -# you can configure a block list of prefixes (comma separated) to filter out metric tags -# that start with the elements of the list (e.g: "job_id,run_id") -# Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id -statsd_disabled_tags = job_id,run_id - -# To enable sending Airflow metrics with StatsD-Influxdb tagging convention. -statsd_influxdb_enabled = False - -# Enables sending metrics to OpenTelemetry. -otel_on = False -otel_host = localhost -otel_port = 8889 -otel_prefix = airflow -otel_interval_milliseconds = 60000 - -# If True, all metrics are also emitted to the console. Defaults to False. -otel_debugging_on = False - -[secrets] -# Full class name of secrets backend to enable (will precede env vars and metastore in search path) -# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend -backend = - -# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. -# See documentation for the secrets backend you are using. JSON is expected. -# Example for AWS Systems Manager ParameterStore: -# ``{{"connections_prefix": "/airflow/connections", "profile_name": "default"}}`` -backend_kwargs = - -[cli] -# In what way should the cli access the API. The LocalClient will use the -# database directly, while the json_client will use the api running on the -# webserver -api_client = airflow.api.client.local_client - -# If you set web_server_url_prefix, do NOT forget to append it here, ex: -# ``endpoint_url = http://localhost:8080/myroot`` -# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` -endpoint_url = http://localhost:8080 - -[debug] -# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first -# failed task. Helpful for debugging purposes. -fail_fast = False - -[api] -# Enables the deprecated experimental API. Please note that these APIs do not have access control. -# The authenticated user has full access. -# -# .. warning:: -# -# This `Experimental REST API `__ is -# deprecated since version 2.0. Please consider using -# `the Stable REST API `__. -# For more information on migration, see -# `RELEASE_NOTES.rst `_ -enable_experimental_api = False - -# Comma separated list of auth backends to authenticate users of the API. See -# https://airflow.apache.org/docs/apache-airflow/stable/security/api.html for possible values. -# ("airflow.api.auth.backend.default" allows all requests for historic reasons) -auth_backends = airflow.api.auth.backend.session - -# Used to set the maximum page limit for API requests. If limit passed as param -# is greater than maximum page limit, it will be ignored and maximum page limit value -# will be set as the limit -maximum_page_limit = 100 - -# Used to set the default page limit when limit param is zero or not provided in API -# requests. Otherwise if positive integer is passed in the API requests as limit, the -# smallest number of user given limit or maximum page limit is taken as limit. -fallback_page_limit = 100 - -# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. -# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com -google_oauth2_audience = - -# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on -# `the Application Default Credentials -# `__ will -# be used. -# Example: google_key_path = /files/service-account-json -google_key_path = - -# Used in response to a preflight request to indicate which HTTP -# headers can be used when making the actual request. This header is -# the server side response to the browser's -# Access-Control-Request-Headers header. -access_control_allow_headers = - -# Specifies the method or methods allowed when accessing the resource. -access_control_allow_methods = - -# Indicates whether the response can be shared with requesting code from the given origins. -# Separate URLs with space. -access_control_allow_origins = - -# Indicates whether the *xcomEntries* endpoint supports the *deserialize* -# flag. If set to False, setting this flag in a request would result in a -# 400 Bad Request error. -enable_xcom_deserialize_support = False - -[lineage] -# what lineage backend to use -backend = - -[openlineage] - -# This section applies settings for OpenLineage integration. -# For backwards compatibility with `openlineage-python` one can still use -# `openlineage.yml` file or `OPENLINEAGE_` environment variables. However, below -# configuration takes precedence over those. -# More in documentation - https://openlineage.io/docs/client/python#configuration. -# Set this to true if you don't want OpenLineage to emit events. -disabled = False - -# OpenLineage namespace -# Example: namespace = food_delivery -# namespace = - -# Comma-separated paths to custom OpenLineage extractors. -# Example: extractors = full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass -extractors = - -# Path to YAML config. This provides backwards compatibility to pass config as -# `openlineage.yml` file. -config_path = - -# OpenLineage Client transport configuration. It should contain type -# and additional options per each type. -# -# Currently supported types are: -# -# * HTTP -# * Kafka -# * Console -# Example: transport = {{"type": "http", "url": "http://localhost:5000"}} -transport = - -# If disabled, OpenLineage events do not contain source code of particular -# operators, like PythonOperator. -# disable_source_code = - -[atlas] -sasl_enabled = False -host = -port = 21000 -username = -password = - -[operators] -# The default owner assigned to each new operator, unless -# provided explicitly or passed via ``default_args`` -default_owner = airflow - -# The default value of attribute "deferrable" in operators and sensors. -default_deferrable = false -default_cpus = 1 -default_ram = 512 -default_disk = 512 -default_gpus = 0 - -# Default queue that tasks get assigned to and that worker listen on. -default_queue = default - -# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. -# If set to False, an exception will be thrown, otherwise only the console message will be displayed. -allow_illegal_arguments = False - -[hive] -# Default mapreduce queue for HiveOperator tasks -default_hive_mapred_queue = - -# Template for mapred_job_name in HiveOperator, supports the following named parameters -# hostname, dag_id, task_id, execution_date -# mapred_job_name_template = - -[webserver] -# Path of webserver config file used for configuring the webserver parameters -config_file = {AIRFLOW_HOME}/webserver_config.py - -# The base url of your website as airflow cannot guess what domain or -# cname you are using. This is used in automated emails that -# airflow sends to point links to the right web server -base_url = http://localhost:8080 - -# Default timezone to display all dates in the UI, can be UTC, system, or -# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the -# default value of core/default_timezone will be used -# Example: default_ui_timezone = America/New_York -default_ui_timezone = UTC - -# The ip specified when starting the web server -web_server_host = 0.0.0.0 - -# The port on which to run the web server -web_server_port = 8080 - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_cert = - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_key = - -# The type of backend used to store web session data, can be 'database' or 'securecookie' -# Example: session_backend = securecookie -session_backend = database - -# Number of seconds the webserver waits before killing gunicorn master that doesn't respond -web_server_master_timeout = 120 - -# Number of seconds the gunicorn webserver waits before timing out on a worker -web_server_worker_timeout = 120 - -# Number of workers to refresh at a time. When set to 0, worker refresh is -# disabled. When nonzero, airflow periodically refreshes webserver workers by -# bringing up new ones and killing old ones. -worker_refresh_batch_size = 1 - -# Number of seconds to wait before refreshing a batch of workers. -worker_refresh_interval = 6000 - -# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, -# then reload the gunicorn. -reload_on_plugin_change = False - -# Secret key used to run your flask app. It should be as random as possible. However, when running -# more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise -# one of them will error with "CSRF session token is missing". -# The webserver key is also used to authorize requests to Celery workers when logs are retrieved. -# The token generated using the secret key has a short expiry time though - make sure that time on -# ALL the machines that you run airflow components on is synchronized (for example using ntpd) -# otherwise you might get "forbidden" errors when the logs are accessed. -secret_key = {SECRET_KEY} - -# Number of workers to run the Gunicorn web server -workers = 4 - -# The worker class gunicorn should use. Choices include -# sync (default), eventlet, gevent. Note when using gevent you might also want to set the -# "_AIRFLOW_PATCH_GEVENT" environment variable to "1" to make sure gevent patching is done as -# early as possible. -worker_class = sync - -# Log files for the gunicorn webserver. '-' means log to stderr. -access_logfile = - - -# Log files for the gunicorn webserver. '-' means log to stderr. -error_logfile = - - -# Access log format for gunicorn webserver. -# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" -# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format -access_logformat = - -# Expose the configuration file in the web server. Set to "non-sensitive-only" to show all values -# except those that have security implications. "True" shows all values. "False" hides the -# configuration completely. -expose_config = False - -# Expose hostname in the web server -expose_hostname = False - -# Expose stacktrace in the web server -expose_stacktrace = False - -# Default DAG view. Valid values are: ``grid``, ``graph``, ``duration``, ``gantt``, ``landing_times`` -dag_default_view = grid - -# Default DAG orientation. Valid values are: -# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) -dag_orientation = LR - -# Sorting order in grid view. Valid values are: ``topological``, ``hierarchical_alphabetical`` -grid_view_sorting_order = topological - -# The amount of time (in secs) webserver will wait for initial handshake -# while fetching logs from other worker machine -log_fetch_timeout_sec = 5 - -# Time interval (in secs) to wait before next log fetching. -log_fetch_delay_sec = 2 - -# Distance away from page bottom to enable auto tailing. -log_auto_tailing_offset = 30 - -# Animation speed for auto tailing log display. -log_animation_speed = 1000 - -# By default, the webserver shows paused DAGs. Flip this to hide paused -# DAGs by default -hide_paused_dags_by_default = False - -# Consistent page size across all listing views in the UI -page_size = 100 - -# Define the color of navigation bar -navbar_color = #fff - -# Default dagrun to show in UI -default_dag_run_display_number = 25 - -# Enable werkzeug ``ProxyFix`` middleware for reverse proxy -enable_proxy_fix = False - -# Number of values to trust for ``X-Forwarded-For``. -# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ -proxy_fix_x_for = 1 - -# Number of values to trust for ``X-Forwarded-Proto`` -proxy_fix_x_proto = 1 - -# Number of values to trust for ``X-Forwarded-Host`` -proxy_fix_x_host = 1 - -# Number of values to trust for ``X-Forwarded-Port`` -proxy_fix_x_port = 1 - -# Number of values to trust for ``X-Forwarded-Prefix`` -proxy_fix_x_prefix = 1 - -# Set secure flag on session cookie -cookie_secure = False - -# Set samesite policy on session cookie -cookie_samesite = Lax - -# Default setting for wrap toggle on DAG code and TI log views. -default_wrap = False - -# Allow the UI to be rendered in a frame -x_frame_enabled = True - -# Send anonymous user activity to your analytics tool -# choose from google_analytics, segment, or metarouter -# analytics_tool = - -# Unique ID of your account in the analytics tool -# analytics_id = - -# 'Recent Tasks' stats will show for old DagRuns if set -show_recent_stats_for_completed_runs = True - -# Update FAB permissions and sync security manager roles -# on webserver startup -update_fab_perms = True - -# The UI cookie lifetime in minutes. User will be logged out from UI after -# ``session_lifetime_minutes`` of non-activity -session_lifetime_minutes = 43200 - -# Sets a custom page title for the DAGs overview page and site title for all pages -# instance_name = - -# Whether the custom page title for the DAGs overview page contains any Markup language -instance_name_has_markup = False - -# How frequently, in seconds, the DAG data will auto-refresh in graph or grid view -# when auto-refresh is turned on -auto_refresh_interval = 3 - -# Boolean for displaying warning for publicly viewable deployment -warn_deployment_exposure = True - -# Comma separated string of view events to exclude from dag audit view. -# All other events will be added minus the ones passed here. -# The audit logs in the db will not be affected by this parameter. -audit_view_excluded_events = gantt,landing_times,tries,duration,calendar,graph,grid,tree,tree_data - -# Comma separated string of view events to include in dag audit view. -# If passed, only these events will populate the dag audit view. -# The audit logs in the db will not be affected by this parameter. -# Example: audit_view_included_events = dagrun_cleared,failed -# audit_view_included_events = - -# Boolean for running SwaggerUI in the webserver. -enable_swagger_ui = True - -# Boolean for running Internal API in the webserver. -run_internal_api = False - -# Boolean for enabling rate limiting on authentication endpoints. -auth_rate_limited = True - -# Rate limit for authentication endpoints. -auth_rate_limit = 5 per 40 second - -# The caching algorithm used by the webserver. Must be a valid hashlib function name. -# Example: caching_hash_method = sha256 -caching_hash_method = md5 - -[email] - -# Configuration email backend and whether to -# send email alerts on retry or failure -# Email backend to use -email_backend = airflow.utils.email.send_email_smtp - -# Email connection to use -email_conn_id = smtp_default - -# Whether email alerts should be sent when a task is retried -default_email_on_retry = True - -# Whether email alerts should be sent when a task failed -default_email_on_failure = True - -# File that will be used as the template for Email subject (which will be rendered using Jinja2). -# If not set, Airflow uses a base template. -# Example: subject_template = /path/to/my_subject_template_file -# subject_template = - -# File that will be used as the template for Email content (which will be rendered using Jinja2). -# If not set, Airflow uses a base template. -# Example: html_content_template = /path/to/my_html_content_template_file -# html_content_template = - -# Email address that will be used as sender address. -# It can either be raw email or the complete address in a format ``Sender Name `` -# Example: from_email = Airflow -# from_email = - -[smtp] - -# If you want airflow to send emails on retries, failure, and you want to use -# the airflow.utils.email.send_email_smtp function, you have to configure an -# smtp server here -smtp_host = localhost -smtp_starttls = True -smtp_ssl = False -# Example: smtp_user = airflow -# smtp_user = -# Example: smtp_password = airflow -# smtp_password = -smtp_port = 25 -smtp_mail_from = airflow@example.com -smtp_timeout = 30 -smtp_retry_limit = 5 - -[sentry] - -# Sentry (https://docs.sentry.io) integration. Here you can supply -# additional configuration options based on the Python platform. See: -# https://docs.sentry.io/error-reporting/configuration/?platform=python. -# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, -# ``ignore_errors``, ``before_breadcrumb``, ``transport``. -# Enable error reporting to Sentry -sentry_on = false -sentry_dsn = - -# Dotted path to a before_send function that the sentry SDK should be configured to use. -# before_send = - -[local_kubernetes_executor] - -# This section only applies if you are using the ``LocalKubernetesExecutor`` in -# ``[core]`` section above -# Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``. -# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), -# the task is executed via ``KubernetesExecutor``, -# otherwise via ``LocalExecutor`` -kubernetes_queue = kubernetes - -[celery_kubernetes_executor] - -# This section only applies if you are using the ``CeleryKubernetesExecutor`` in -# ``[core]`` section above -# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. -# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), -# the task is executed via ``KubernetesExecutor``, -# otherwise via ``CeleryExecutor`` -kubernetes_queue = kubernetes - -[celery] - -# This section only applies if you are using the CeleryExecutor in -# ``[core]`` section above -# The app name that will be used by celery -celery_app_name = airflow.providers.celery.executors.celery_executor - -# The concurrency that will be used when starting workers with the -# ``airflow celery worker`` command. This defines the number of task instances that -# a worker will take, so size up your workers based on the resources on -# your worker box and the nature of your tasks -worker_concurrency = 16 - -# The maximum and minimum concurrency that will be used when starting workers with the -# ``airflow celery worker`` command (always keep minimum processes, but grow -# to maximum if necessary). Note the value should be max_concurrency,min_concurrency -# Pick these numbers based on resources on worker box and the nature of the task. -# If autoscale option is available, worker_concurrency will be ignored. -# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale -# Example: worker_autoscale = 16,12 -# worker_autoscale = - -# Used to increase the number of tasks that a worker prefetches which can improve performance. -# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks -# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily -# blocked if there are multiple workers and one worker prefetches tasks that sit behind long -# running tasks while another worker has unutilized processes that are unable to process the already -# claimed blocked tasks. -# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits -worker_prefetch_multiplier = 1 - -# Specify if remote control of the workers is enabled. -# In some cases when the broker does not support remote control, Celery creates lots of -# ``.*reply-celery-pidbox`` queues. You can prevent this by setting this to false. -# However, with this disabled Flower won't work. -# https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html#broker-overview -worker_enable_remote_control = true - -# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally -# a sqlalchemy database. Refer to the Celery documentation for more information. -broker_url = redis://redis:6379/0 - -# The Celery result_backend. When a job finishes, it needs to update the -# metadata of the job. Therefore it will post a message on a message bus, -# or insert it into a database (depending of the backend) -# This status is used by the scheduler to update the state of the task -# The use of a database is highly recommended -# When not specified, sql_alchemy_conn with a db+ scheme prefix will be used -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings -# Example: result_backend = db+postgresql://postgres:airflow@postgres/airflow -# result_backend = - -# Optional configuration dictionary to pass to the Celery result backend SQLAlchemy engine. -# Example: result_backend_sqlalchemy_engine_options = {{"pool_recycle": 1800}} -result_backend_sqlalchemy_engine_options = - -# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start -# it ``airflow celery flower``. This defines the IP that Celery Flower runs on -flower_host = 0.0.0.0 - -# The root URL for Flower -# Example: flower_url_prefix = /flower -flower_url_prefix = - -# This defines the port that Celery Flower runs on -flower_port = 5555 - -# Securing Flower with Basic Authentication -# Accepts user:password pairs separated by a comma -# Example: flower_basic_auth = user1:password1,user2:password2 -flower_basic_auth = - -# How many processes CeleryExecutor uses to sync task state. -# 0 means to use max(1, number of cores - 1) processes. -sync_parallelism = 0 - -# Import path for celery configuration options -celery_config_options = airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG -ssl_active = False - -# Path to the client key. -ssl_key = - -# Path to the client certificate. -ssl_cert = - -# Path to the CA certificate. -ssl_cacert = - -# Celery Pool implementation. -# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. -# See: -# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency -# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html -pool = prefork - -# The number of seconds to wait before timing out ``send_task_to_executor`` or -# ``fetch_celery_task_state`` operations. -operation_timeout = 1.0 - -# Celery task will report its status as 'started' when the task is executed by a worker. -# This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted -# or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. -task_track_started = True - -# The Maximum number of retries for publishing task messages to the broker when failing -# due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. -task_publish_max_retries = 3 - -# Worker initialisation check to validate Metadata Database connection -worker_precheck = False - -[celery_broker_transport_options] - -# This section is for specifying options which can be passed to the -# underlying celery broker transport. See: -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options -# The visibility timeout defines the number of seconds to wait for the worker -# to acknowledge the task before the message is redelivered to another worker. -# Make sure to increase the visibility timeout to match the time of the longest -# ETA you're planning to use. -# visibility_timeout is only supported for Redis and SQS celery brokers. -# See: -# https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#visibility-timeout -# Example: visibility_timeout = 21600 -# visibility_timeout = - -# The sentinel_kwargs parameter allows passing additional options to the Sentinel client. -# In a typical scenario where Redis Sentinel is used as the broker and Redis servers are -# password-protected, the password needs to be passed through this parameter. Although its -# type is string, it is required to pass a string that conforms to the dictionary format. -# See: -# https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#configuration -# Example: sentinel_kwargs = {{"password": "password_for_redis_server"}} -# sentinel_kwargs = - -[dask] - -# This section only applies if you are using the DaskExecutor in -# [core] section above -# The IP address and port of the Dask cluster's scheduler. -cluster_address = 127.0.0.1:8786 - -# Path to a CA certificate file encoded in PEM format to access a secured Dask scheduler. -tls_ca = - -# Path to a certificate file for the client, encoded in PEM format. -tls_cert = - -# Path to a key file for the client, encoded in PEM format. -tls_key = - -[scheduler] -# Task instances listen for external kill signal (when you clear tasks -# from the CLI or the UI), this defines the frequency at which they should -# listen (in seconds). -job_heartbeat_sec = 5 - -# The scheduler constantly tries to trigger new tasks (look at the -# scheduler section in the docs for more information). This defines -# how often the scheduler should run (in seconds). -scheduler_heartbeat_sec = 5 - -# The frequency (in seconds) at which the LocalTaskJob should send heartbeat signals to the -# scheduler to notify it's still alive. If this value is set to 0, the heartbeat interval will default -# to the value of scheduler_zombie_task_threshold. -local_task_job_heartbeat_sec = 0 - -# The number of times to try to schedule each DAG file -# -1 indicates unlimited number -num_runs = -1 - -# Controls how long the scheduler will sleep between loops, but if there was nothing to do -# in the loop. i.e. if it scheduled something then it will start the next loop -# iteration straight away. -scheduler_idle_sleep_time = 1 - -# Number of seconds after which a DAG file is parsed. The DAG file is parsed every -# ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after -# this interval. Keeping this number low will increase CPU usage. -min_file_process_interval = 30 - -# How often (in seconds) to check for stale DAGs (DAGs which are no longer present in -# the expected files) which should be deactivated, as well as datasets that are no longer -# referenced and should be marked as orphaned. -parsing_cleanup_interval = 60 - -# How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale -# DAGs (DAGs which are no longer present in the expected files). The reason why we need -# this threshold is to account for the time between when the file is parsed and when the -# DAG is loaded. The absolute maximum that this could take is `dag_file_processor_timeout`, -# but when you have a long timeout configured, it results in a significant delay in the -# deactivation of stale dags. -stale_dag_threshold = 50 - -# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. -dag_dir_list_interval = 300 - -# How often should stats be printed to the logs. Setting to 0 will disable printing stats -print_stats_interval = 30 - -# How often (in seconds) should pool usage stats be sent to StatsD (if statsd_on is enabled) -pool_metrics_interval = 5.0 - -# If the last scheduler heartbeat happened more than scheduler_health_check_threshold -# ago (in seconds), scheduler is considered unhealthy. -# This is used by the health check in the "/health" endpoint and in `airflow jobs check` CLI -# for SchedulerJob. -scheduler_health_check_threshold = 30 - -# When you start a scheduler, airflow starts a tiny web server -# subprocess to serve a health check if this is set to True -enable_health_check = False - -# When you start a scheduler, airflow starts a tiny web server -# subprocess to serve a health check on this port -scheduler_health_check_server_port = 8974 - -# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs -orphaned_tasks_check_interval = 300.0 -child_process_log_directory = {AIRFLOW_HOME}/logs/scheduler - -# Local task jobs periodically heartbeat to the DB. If the job has -# not heartbeat in this many seconds, the scheduler will mark the -# associated task instance as failed and will re-schedule the task. -scheduler_zombie_task_threshold = 300 - -# How often (in seconds) should the scheduler check for zombie tasks. -zombie_detection_interval = 10.0 - -# Turn off scheduler catchup by setting this to ``False``. -# Default behavior is unchanged and -# Command Line Backfills still work, but the scheduler -# will not do scheduler catchup if this is ``False``, -# however it can be set on a per DAG basis in the -# DAG definition (catchup) -catchup_by_default = True - -# Setting this to True will make first task instance of a task -# ignore depends_on_past setting. A task instance will be considered -# as the first task instance of a task when there is no task instance -# in the DB with an execution_date earlier than it., i.e. no manual marking -# success will be needed for a newly added task to be scheduled. -ignore_first_depends_on_past_by_default = True - -# This changes the batch size of queries in the scheduling main loop. -# If this is too high, SQL query performance may be impacted by -# complexity of query predicate, and/or excessive locking. -# Additionally, you may hit the maximum allowable query length for your db. -# Set this to 0 for no limit (not advised) -max_tis_per_query = 512 - -# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. -# If this is set to False then you should not run more than a single -# scheduler at once -use_row_level_locking = True - -# Max number of DAGs to create DagRuns for per scheduler loop. -max_dagruns_to_create_per_loop = 10 - -# How many DagRuns should a scheduler examine (and lock) when scheduling -# and queuing tasks. -max_dagruns_per_loop_to_schedule = 20 - -# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the -# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other -# dags in some circumstances -schedule_after_task_execution = True - -# The scheduler reads dag files to extract the airflow modules that are going to be used, -# and imports them ahead of time to avoid having to re-do it for each parsing process. -# This flag can be set to False to disable this behavior in case an airflow module needs to be freshly -# imported each time (at the cost of increased DAG parsing time). -parsing_pre_import_modules = True - -# The scheduler can run multiple processes in parallel to parse dags. -# This defines how many processes will run. -parsing_processes = 2 - -# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. -# The scheduler will list and sort the dag files to decide the parsing order. +# airflow config list --defaults > "${AIRFLOW_HOME}/airflow.cfg" # -# * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the -# recently modified DAGs first. -# * ``random_seeded_by_host``: Sort randomly across multiple Schedulers but with same order on the -# same host. This is useful when running with Scheduler in HA mode where each scheduler can -# parse different DAG files. -# * ``alphabetical``: Sort by filename -file_parsing_sort_mode = modified_time - -# Whether the dag processor is running as a standalone process or it is a subprocess of a scheduler -# job. -standalone_dag_processor = False - -# Only applicable if `[scheduler]standalone_dag_processor` is true and callbacks are stored -# in database. Contains maximum number of callbacks that are fetched during a single loop. -max_callbacks_per_loop = 20 - -# Only applicable if `[scheduler]standalone_dag_processor` is true. -# Time in seconds after which dags, which were not updated by Dag Processor are deactivated. -dag_stale_not_seen_duration = 600 - -# Turn off scheduler use of cron intervals by setting this to False. -# DAGs submitted manually in the web UI or with trigger_dag will still run. -use_job_schedule = True - -# Allow externally triggered DagRuns for Execution Dates in the future -# Only has effect if schedule_interval is set to None in DAG -allow_trigger_in_future = False - -# How often to check for expired trigger requests that have not run yet. -trigger_timeout_check_interval = 15 - -# Amount of time a task can be in the queued state before being retried or set to failed. -task_queued_timeout = 600.0 - -# How often to check for tasks that have been in the queued state for -# longer than `[scheduler] task_queued_timeout`. -task_queued_timeout_check_interval = 120.0 - -# The run_id pattern used to verify the validity of user input to the run_id parameter when -# triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id -# for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. -allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$ - -[triggerer] -# How many triggers a single Triggerer will run at once, by default. -default_capacity = 1000 - -# How often to heartbeat the Triggerer job to ensure it hasn't been killed. -job_heartbeat_sec = 5 - -[kerberos] -ccache = /tmp/airflow_krb5_ccache - -# gets augmented with fqdn -principal = airflow -reinit_frequency = 3600 -kinit_path = kinit -keytab = airflow.keytab - -# Allow to disable ticket forwardability. -forwardable = True - -# Allow to remove source IP from token, useful when using token behind NATted Docker host. -include_ip = True - -[elasticsearch] -# Elasticsearch host -host = - -# Format of the log_id, which is used to query for a given tasks logs -log_id_template = {{dag_id}}-{{task_id}}-{{run_id}}-{{map_index}}-{{try_number}} - -# Used to mark the end of a log stream for a task -end_of_log_mark = end_of_log - -# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id -# Code will construct log_id using the log_id template from the argument above. -# NOTE: scheme will default to https if one is not provided -# Example: frontend = http://localhost:5601/app/kibana#/discover?_a=(columns:!(message),query:(language:kuery,query:'log_id: "{{log_id}}"'),sort:!(log.offset,asc)) -frontend = - -# Write the task logs to the stdout of the worker, rather than the default files -write_stdout = False - -# Instead of the default log formatter, write the log lines as JSON -json_format = False - -# Log fields to also attach to the json output, if enabled -json_fields = asctime, filename, lineno, levelname, message - -# The field where host name is stored (normally either `host` or `host.name`) -host_field = host - -# The field where offset is stored (normally either `offset` or `log.offset`) -offset_field = offset - -# Comma separated list of index patterns to use when searching for logs (default: `_all`). -# Example: index_patterns = something-* -index_patterns = _all - -[elasticsearch_configs] -use_ssl = False -verify_certs = True - -[kubernetes_executor] -# Kwargs to override the default urllib3 Retry used in the kubernetes API client -# Example: api_client_retry_configuration = {{ "total": 3, "backoff_factor": 0.5 }} -api_client_retry_configuration = - -# Flag to control the information added to kubernetes executor logs for better traceability -logs_task_metadata = False - -# Path to the YAML pod file that forms the basis for KubernetesExecutor workers. -pod_template_file = - -# The repository of the Kubernetes Image for the Worker to Run -worker_container_repository = - -# The tag of the Kubernetes Image for the Worker to Run -worker_container_tag = - -# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` -namespace = default - -# If True, all worker pods will be deleted upon termination -delete_worker_pods = True - -# If False (and delete_worker_pods is True), -# failed worker pods will not be deleted so users can investigate them. -# This only prevents removal of worker pods where the worker itself failed, -# not when the task it ran failed. -delete_worker_pods_on_failure = False - -# Number of Kubernetes Worker Pod creation calls per scheduler loop. -# Note that the current default of "1" will only launch a single pod -# per-heartbeat. It is HIGHLY recommended that users increase this -# number to match the tolerance of their kubernetes cluster for -# better performance. -worker_pods_creation_batch_size = 1 - -# Allows users to launch pods in multiple namespaces. -# Will require creating a cluster-role for the scheduler, -# or use multi_namespace_mode_namespace_list configuration. -multi_namespace_mode = False - -# If multi_namespace_mode is True while scheduler does not have a cluster-role, -# give the list of namespaces where the scheduler will schedule jobs -# Scheduler needs to have the necessary permissions in these namespaces. -multi_namespace_mode_namespace_list = - -# Use the service account kubernetes gives to pods to connect to kubernetes cluster. -# It's intended for clients that expect to be running inside a pod running on kubernetes. -# It will raise an exception if called from a process not running in a kubernetes environment. -in_cluster = True - -# When running with in_cluster=False change the default cluster_context or config_file -# options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. -# cluster_context = - -# Path to the kubernetes configfile to be used when ``in_cluster`` is set to False -# config_file = - -# Keyword parameters to pass while calling a kubernetes client core_v1_api methods -# from Kubernetes Executor provided as a single line formatted JSON dictionary string. -# List of supported params are similar for all core_v1_apis, hence a single config -# variable for all apis. See: -# https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py -kube_client_request_args = - -# Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client -# ``core_v1_api`` method when using the Kubernetes Executor. -# This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` -# class defined here: -# https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 -# Example: delete_option_kwargs = {{"grace_period_seconds": 10}} -delete_option_kwargs = - -# Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely -# when idle connection is time-outed on services like cloud load balancers or firewalls. -enable_tcp_keepalive = True - -# When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has -# been idle for `tcp_keep_idle` seconds. -tcp_keep_idle = 120 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. -tcp_keep_intvl = 30 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before -# a connection is considered to be broken. -tcp_keep_cnt = 6 - -# Set this to false to skip verifying SSL certificate of Kubernetes python client. -verify_ssl = True - -# How often in seconds to check for task instances stuck in "queued" status without a pod -worker_pods_queued_check_interval = 60 - -# Path to a CA certificate to be used by the Kubernetes client to verify the server's SSL certificate. -ssl_ca_cert = - -[sensors] -# Sensor default timeout, 7 days by default (7 * 24 * 60 * 60). -default_timeout = 604800 diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg deleted file mode 100644 index 523f52cb69a0..000000000000 --- a/airflow/config_templates/default_test.cfg +++ /dev/null @@ -1,68 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# This is the template for Airflow's unit test configuration. When Airflow runs -# unit tests, it looks for a configuration file at $AIRFLOW_HOME/unittests.cfg. -# If it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# unittests.cfg instead. - - -# ----------------------- TEMPLATE BEGINS HERE ----------------------- - -[core] -unit_test_mode = True -dags_folder = {TEST_DAGS_FOLDER} -plugins_folder = {TEST_PLUGINS_FOLDER} -dags_are_paused_at_creation = False -fernet_key = {FERNET_KEY} -killed_task_cleanup_time = 5 -allowed_deserialization_classes = airflow\..* - tests\..* - -[database] -sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/unittests.db - -[logging] -celery_logging_level = WARN - -[api] -auth_backends = airflow.api.auth.backend.default - -[hive] -default_hive_mapred_queue = airflow - -[smtp] -smtp_user = airflow -smtp_password = airflow - -[celery] -broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow -result_backend = db+mysql://airflow:airflow@localhost:3306/airflow - -[scheduler] -job_heartbeat_sec = 1 -schedule_after_task_execution = False -scheduler_health_check_server_port = 8794 -dag_dir_list_interval = 0 - -[elasticsearch] -log_id_template = {{dag_id}}-{{task_id}}-{{run_id}}-{{map_index}}-{{try_number}} diff --git a/airflow/config_templates/unit_tests.cfg b/airflow/config_templates/unit_tests.cfg new file mode 100644 index 000000000000..540c3f3be42b --- /dev/null +++ b/airflow/config_templates/unit_tests.cfg @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE FOR CONTRIBUTORS: +# +# Values specified in this file are automatically used in our unit tests +# run by pytest and override default airflow configuration values provided by config.yml. +# +# These configuration settings should provide consistent environment to run tests - +# no matter if you are in Breeze env or use local venv or even run tests in the CI environment. +# +# If you want to have all unit tests to get some default configuration value, you should set it here. +# +# You can instead use ``conf_vars`` context manager or override the ``AIRFLOW__
    __`` environment +# variable to set the configuration value for a specific test or a group of tests. +# +# The test configuration is loaded via setting AIRFLOW__CORE__UNIT_TEST_MODE=True +# in a pytest fixture in tests/conftest.py. This in turn triggers reloading of the configuration +# from this file after cleaning the respective configuration retrieved during initialization +# of configuration. See ``load_test_config`` function in ``airflow/config.py`` for details. +# + + +[core] +# We want to read dags from the test dags folder +dags_folder = {TEST_DAGS_FOLDER} +# we want to read plugins from example_dags/plugins folder +plugins_folder = {TEST_PLUGINS_FOLDER} +# we want to read fernet key generated dynamically in load_test_config +fernet_key = {FERNET_KEY} +# for tests we use local executor by default +executor = LocalExecutor +# We do not want dags to be paused at creation +dags_are_paused_at_creation = False +# We want to load examples +load_examples = True +# We want pickling to be enabled in tests +donot_pickle = False +# No default impersonation in tests - tests that run impersonation should set this explicitly +default_impersonation = +# We want to use unit test mode (though technically it is not needed as we set it in pytest fixture +# via AIRFLOW__CORE__UNIT_TEST_MODE environment variable) +unit_test_mode = True +# We want to use a shorter timeout for task cleanup +killed_task_cleanup_time = 5 +# We only allow our own classes to be deserialized in tests +allowed_deserialization_classes = airflow\..* tests\..* +[database] +# we want to have default connections loaded in unit tests +load_default_connections = True + +[logging] +# celery tests rely on it being set +celery_logging_level = INFO + +[smtp] +# Used as default values for SMTP unit tests +smtp_user = airflow +smtp_password = airflow +smtp_mail_from = airflow@example.com + +[api] +auth_backends = airflow.api.auth.backend.default + +[hive] +# Hive uses the configuration below to run the tests +default_hive_mapred_queue: airflow + +[celery] +# used to test that we can load celery config from airflow.cfg +broker_url = amqp://guest:guest@rabbitmq:5672/ +# and same for result backend +result_backend = db+mysql://airflow:airflow@localhost:3306/airflow + +[scheduler] +# Those values are set so that during unit tests things run faster than usual +job_heartbeat_sec = 1 +scheduler_heartbeat_sec = 5 +parsing_processes = 2 +dag_dir_list_interval = 0 + +[example_section] +# This section is used to test coercions of configuration values retrieval +string_value = 21600 +only_bool_value = true +only_float_value = 12.0 +only_integer_value = 21600 +only_string_value = this is a test diff --git a/airflow/configuration.py b/airflow/configuration.py index 8cb5971cd5e8..c8e19651d5b0 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -18,6 +18,7 @@ import datetime import functools +import io import json import logging import multiprocessing @@ -30,15 +31,15 @@ import warnings from base64 import b64encode from collections import OrderedDict - -# Ignored Mypy on configparser because it thinks the configparser module has no _UNSET attribute -from configparser import _UNSET, ConfigParser, NoOptionError, NoSectionError # type: ignore -from contextlib import contextmanager, suppress +from configparser import ConfigParser, NoOptionError, NoSectionError +from contextlib import contextmanager from json.decoder import JSONDecodeError +from threading import Lock from typing import IO, Any, Dict, Iterable, Pattern, Set, Tuple, Union from urllib.parse import urlsplit import re2 +from packaging.version import parse as parse_version from typing_extensions import overload from airflow.auth.managers.base_auth_manager import BaseAuthManager @@ -64,6 +65,8 @@ ENV_VAR_PREFIX = "AIRFLOW__" +EMPTY_SET: Set[tuple[str, str]] = set() # noqa: UP006 + def _parse_sqlite_version(s: str) -> tuple[int, ...]: match = _SQLITE3_VERSION_PATTERN.match(s) @@ -131,35 +134,155 @@ def _get_config_value_from_secret_backend(config_key: str) -> str | None: ) +def _is_template(configuration_description: dict[str, dict[str, Any]], section: str, key: str) -> bool: + """ + Check if the config is a template. + + :param configuration_description: description of configuration + :param section: section + :param key: key + :return: True if the config is a template + """ + return configuration_description.get(section, {}).get(key, {}).get("is_template", False) + + def _default_config_file_path(file_name: str) -> str: templates_dir = os.path.join(os.path.dirname(__file__), "config_templates") return os.path.join(templates_dir, file_name) -def default_config_yaml() -> dict[str, Any]: +def retrieve_configuration_description() -> dict[str, dict[str, Any]]: """ - Read Airflow configs from YAML file. + Read Airflow configuration description from YAML file. :return: Python dictionary containing configs & their info """ + base_configuration_description: dict[str, dict[str, Any]] = {} with open(_default_config_file_path("config.yml")) as config_file: - return yaml.safe_load(config_file) + base_configuration_description.update(yaml.safe_load(config_file)) + return base_configuration_description class AirflowConfigParser(ConfigParser): - """Custom Airflow Configparser supporting defaults and deprecated options.""" + """ + Custom Airflow Configparser supporting defaults and deprecated options. + + This is a subclass of ConfigParser that supports defaults and deprecated options. + + The defaults are stored in the ``_default_values ConfigParser. The configuration description keeps + description of all the options available in Airflow (description follow config.yaml.schema). + + :param default_config: default configuration (in the form of ini file). + :param configuration_description: description of configuration to use + """ + + def __init__( + self, + default_config: str | None = None, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.configuration_description = retrieve_configuration_description() + self.upgraded_values = {} + # For those who would like to use a different data structure to keep defaults: + # We have to keep the default values in a ConfigParser rather than in any other + # data structure, because the values we have might contain %% which are ConfigParser + # interpolation placeholders. The _default_values config parser will interpolate them + # properly when we call get() on it. + self._default_values = create_default_config_parser(self.configuration_description) + if default_config is not None: + self._update_defaults_from_string(default_config) + self._update_logging_deprecated_template_to_one_from_defaults() + self.is_validated = False + self._suppress_future_warnings = False + self._interpolation_lock = Lock() + + def _update_logging_deprecated_template_to_one_from_defaults(self): + default = self.get_default_value("logging", "log_filename_template") + if default: + # Tuple does not support item assignment, so we have to create a new tuple and replace it + original_replacement = self.deprecated_values["logging"]["log_filename_template"] + self.deprecated_values["logging"]["log_filename_template"] = ( + original_replacement[0], + default, + original_replacement[2], + ) + + def is_template(self, section: str, key) -> bool: + """ + Return whether the value is templated. + + :param section: section of the config + :param key: key in the section + :return: True if the value is templated + """ + if self.configuration_description is None: + return False + return _is_template(self.configuration_description, section, key) + + def _update_defaults_from_string(self, config_string: str): + """ + The defaults in _default_values are updated based on values in config_string ("ini" format). + Note that those values are not validated and cannot contain variables because we are using + regular config parser to load them. This method is used to test the config parser in unit tests. + + :param config_string: ini-formatted config string + """ + parser = ConfigParser() + parser.read_string(config_string) + for section in parser.sections(): + if section not in self._default_values.sections(): + self._default_values.add_section(section) + errors = False + for key, value in parser.items(section): + if not self.is_template(section, key) and "{" in value: + errors = True + log.error( + f"The {section}.{key} value {value} read from string contains " + "variable. This is not supported" + ) + self._default_values.set(section, key, value) + if errors: + raise Exception( + f"The string config passed as default contains variables. " + f"This is not supported. String config: {config_string}" + ) + + def get_default_value(self, section: str, key: str, fallback: Any = None, raw=False, **kwargs) -> Any: + """ + Retrieves default value from default config parser. + + This will retrieve the default value from the default config parser. Optionally a raw, stored + value can be retrieved by setting skip_interpolation to True. This is useful for example when + we want to write the default value to a file, and we don't want the interpolation to happen + as it is going to be done later when the config is read. + + :param section: section of the config + :param key: key to use + :param fallback: fallback value to use + :param raw: if raw, then interpolation will be reversed + :param kwargs: other args + :return: + """ + value = self._default_values.get(section, key, fallback=fallback, **kwargs) + if raw and value is not None: + return value.replace("%", "%%") + return value # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}_cmd" pattern, the idea behind this # is to not store password on boxes in text files. # These configs can also be fetched from Secrets backend # following the "{section}__{name}__secret" pattern - @functools.cached_property def sensitive_config_values(self) -> Set[tuple[str, str]]: # noqa: UP006 - default_config = default_config_yaml() + if self.configuration_description is None: + return EMPTY_SET.copy() # we can't use set() here because set is defined below # ¯\_(ツ)_/¯ flattened = { - (s, k): item for s, s_c in default_config.items() for k, item in s_c.get("options").items() + (s, k): item + for s, s_c in self.configuration_description.items() + for k, item in s_c.get("options").items() # type: ignore[union-attr] } sensitive = {(section, key) for (section, key), v in flattened.items() if v.get("sensitive") is True} depr_option = {self.deprecated_options[x][:-1] for x in sensitive if x in self.deprecated_options} @@ -285,6 +408,7 @@ def inversed_deprecated_sections(self): "logging": { "log_filename_template": ( re2.compile(re2.escape("{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log")), + # The actual replacement value will be updated after defaults are loaded from config.yml "XX-set-after-default-config-loaded-XX", "3.0", ), @@ -321,38 +445,151 @@ def inversed_deprecated_sections(self): upgraded_values: dict[tuple[str, str], str] """Mapping of (section,option) to the old value that was upgraded""" - # This method transforms option names on every read, get, or set operation. - # This changes from the default behaviour of ConfigParser from lower-casing - # to instead be case-preserving - def optionxform(self, optionstr: str) -> str: - return optionstr + def get_sections_including_defaults(self) -> list[str]: + """ + Retrieves all sections from the configuration parser, including sections defined by built-in defaults. - def __init__(self, default_config: str | None = None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.upgraded_values = {} + :return: list of section names + """ + my_own_sections = self.sections() + all_sections_from_defaults = list(self.configuration_description.keys()) + for section in my_own_sections: + if section not in all_sections_from_defaults: + all_sections_from_defaults.append(section) + return all_sections_from_defaults + + def get_options_including_defaults(self, section: str) -> list[str]: + """ + Retrieves all possible option from the configuration parser for the section given, + including options defined by built-in defaults. - self.airflow_defaults = ConfigParser(*args, **kwargs) - if default_config is not None: - self.airflow_defaults.read_string(default_config) - # Set the upgrade value based on the current loaded default - default = self.airflow_defaults.get("logging", "log_filename_template", fallback=None) - if default: - replacement = self.deprecated_values["logging"]["log_filename_template"] - self.deprecated_values["logging"]["log_filename_template"] = ( - replacement[0], - default, - replacement[2], - ) - else: - # In case of tests it might not exist - with suppress(KeyError): - del self.deprecated_values["logging"]["log_filename_template"] - else: - with suppress(KeyError): - del self.deprecated_values["logging"]["log_filename_template"] + :return: list of option names for the section given + """ + my_own_options = self.options(section) if self.has_section(section) else [] + all_options_from_defaults = list( + self.configuration_description.get(section, {}).get("options", {}).keys() + ) + for option in my_own_options: + if option not in all_options_from_defaults: + all_options_from_defaults.append(option) + return all_options_from_defaults - self.is_validated = False - self._suppress_future_warnings = False + def optionxform(self, optionstr: str) -> str: + """ + This method transforms option names on every read, get, or set operation. + This changes from the default behaviour of ConfigParser from lower-casing + to instead be case-preserving. + + :param optionstr: + :return: + """ + return optionstr + + def write( # type: ignore[override] + self, + file: IO[str], + section: str | None = None, + include_examples: bool = True, + include_descriptions: bool = True, + include_sources: bool = True, + include_env_vars: bool = True, + comment_out_everything: bool = False, + hide_sensitive_values: bool = False, + extra_spacing: bool = True, + only_defaults: bool = False, + **kwargs: Any, + ) -> None: + """ + Writes configuration with comments and examples to a file. + + :param file: file to write to + :param section: section of the config to write, defaults to all sections + :param include_examples: Include examples in the output + :param include_descriptions: Include descriptions in the output + :param include_sources: Include the source of each config option + :param include_env_vars: Include environment variables corresponding to each config option + :param hide_sensitive_values: Include sensitive values in the output + :param extra_spacing: Add extra spacing before examples and after variables + :param only_defaults: Only include default values when writing the config, not the actual values + """ + from airflow import __version__ as airflow_version + + sources_dict = {} + if include_sources: + sources_dict = self.as_dict(display_source=True) + if self._default_values is None: + raise RuntimeError("Cannot write default config, no default config set") + if self.configuration_description is None: + raise RuntimeError("Cannot write default config, no default configuration description set") + for section_to_write in self.get_sections_including_defaults(): + section_config_description = self.configuration_description.get(section_to_write, {}) + if section_to_write != section and section is not None: + continue + if self._default_values.has_section(section_to_write) or self.has_section(section_to_write): + needs_separation = False + file.write(f"[{section_to_write}]\n") + section_description = section_config_description.get("description") + if section_description and include_descriptions: + for line in section_description.splitlines(): + file.write(f"# {line}\n") + file.write("\n") + for key in self.get_options_including_defaults(section_to_write): + option_config_description = ( + section_config_description.get("options", {}).get(key, {}) + if section_config_description + else {} + ) + version_added = option_config_description.get("version_added") + if version_added is not None and parse_version(version_added) > parse_version( + parse_version(airflow_version).base_version + ): + # skip if option is going to be added in the future version + continue + description = option_config_description.get("description") + needs_separation = False + if description and include_descriptions: + for line in description.splitlines(): + file.write(f"# {line}\n") + needs_separation = True + example = option_config_description.get("example") + if example is not None and include_examples: + if extra_spacing: + file.write("#\n") + file.write(f"# Example: {key} = {example}\n") + needs_separation = True + if include_sources and sources_dict: + sources_section = sources_dict.get(section_to_write) + value_with_source = sources_section.get(key) if sources_section else None + if value_with_source is None: + file.write("#\n# Source: not defined\n") + else: + file.write(f"#\n# Source: {value_with_source[1]}\n") + needs_separation = True + if include_env_vars: + file.write(f"#\n# Variable: AIRFLOW__{section_to_write.upper()}__{key.upper()}\n") + if extra_spacing: + file.write("#\n") + needs_separation = True + if self._default_values is None: + default_value = None + else: + default_value = self.get_default_value(section_to_write, key, raw=True) + if only_defaults: + value = default_value + else: + value = self.get(section_to_write, key, fallback=default_value, raw=True) + if value is None: + file.write(f"# {key} = \n") + else: + if comment_out_everything: + file.write(f"# {key} = {value}\n") + else: + file.write(f"{key} = {value}\n") + if needs_separation: + file.write("\n") + if include_descriptions and not needs_separation: + # extra separation between sections in case last option did not need it + file.write("\n") def validate(self): self._validate_sqlite3_version() @@ -437,8 +674,8 @@ def _validate_enums(self): """Validate that enum type config has an accepted value.""" for (section_key, option_key), enum_options in self.enums_options.items(): if self.has_option(section_key, option_key): - value = self.get(section_key, option_key) - if value not in enum_options: + value = self.get(section_key, option_key, fallback=None) + if value and value not in enum_options: raise AirflowConfigException( f"`[{section_key}] {option_key}` should not be " f"{value!r}. Possible values: {', '.join(enum_options)}." @@ -561,17 +798,18 @@ def get_mandatory_value(self, section: str, key: str, **kwargs) -> str: return value @overload # type: ignore[override] - def get(self, section: str, key: str, fallback: str = ..., **kwargs) -> str: # type: ignore[override] + def get(self, section: str, key: str, fallback: str = ..., **kwargs) -> str: ... @overload # type: ignore[override] - def get(self, section: str, key: str, **kwargs) -> str | None: # type: ignore[override] + def get(self, section: str, key: str, **kwargs) -> str | None: ... - def get( # type: ignore[override, misc] + def get( # type: ignore[override,misc] self, section: str, key: str, + suppress_warnings: bool = False, _extra_stacklevel: int = 0, **kwargs, ) -> str | None: @@ -667,10 +905,10 @@ def get( # type: ignore[override, misc] return option # ...then the default config - if self.airflow_defaults.has_option(section, key) or "fallback" in kwargs: - return expand_env_var(self.airflow_defaults.get(section, key, **kwargs)) - - log.warning("section/key [%s/%s] not found in config", section, key) + if self.get_default_value(section, key) is not None or "fallback" in kwargs: + return expand_env_var(self.get_default_value(section, key, **kwargs)) + if not suppress_warnings: + log.warning("section/key [%s/%s] not found in config", section, key) raise AirflowConfigException(f"section/key [{section}/{key}] not found in config") @@ -825,27 +1063,20 @@ def getimport(self, section: str, key: str, **kwargs) -> Any: ) def getjson( - self, section: str, key: str, fallback=_UNSET, **kwargs + self, section: str, key: str, fallback=None, **kwargs ) -> dict | list | str | int | float | None: """ Return a config value parsed from a JSON string. ``fallback`` is *not* JSON parsed but used verbatim when no config value is given. """ - # get always returns the fallback value as a string, so for this if - # someone gives us an object we want to keep that - default = _UNSET - if fallback is not _UNSET: - default = fallback - fallback = _UNSET - try: - data = self.get(section=section, key=key, fallback=fallback, _extra_stacklevel=1, **kwargs) + data = self.get(section=section, key=key, fallback=None, _extra_stacklevel=1, **kwargs) except (NoSectionError, NoOptionError): - return default + data = None - if not data: - return default if default is not _UNSET else None + if data is None or data == "": + return fallback try: return json.loads(data) @@ -896,19 +1127,33 @@ def read( ): super().read(filenames=filenames, encoding=encoding) - # The RawConfigParser defines "Mapping" from abc.collections is not subscriptable - so we have - # to use Dict here. def read_dict( # type: ignore[override] self, dictionary: dict[str, dict[str, Any]], source: str = "" ): + """ + We define a different signature here to add better type hints and checking. + + :param dictionary: dictionary to read from + :param source: source to be used to store the configuration + :return: + """ super().read_dict(dictionary=dictionary, source=source) def has_option(self, section: str, option: str) -> bool: + """ + Check if option is defined. + + Uses self.get() to avoid reimplementing the priority order of config variables + (env, config, cmd, defaults). + + :param section: section to get option from + :param option: option to get + :return: + """ try: - # Using self.get() to avoid reimplementing the priority order - # of config variables (env, config, cmd, defaults) - # UNSET to avoid logging a warning about missing values - self.get(section, option, fallback=_UNSET, _extra_stacklevel=1) + value = self.get(section, option, fallback=None, _extra_stacklevel=1, suppress_warnings=True) + if value is None: + return False return True except (NoOptionError, NoSectionError): return False @@ -923,8 +1168,8 @@ def remove_option(self, section: str, option: str, remove_default: bool = True): if super().has_option(section, option): super().remove_option(section, option) - if self.airflow_defaults.has_option(section, option) and remove_default: - self.airflow_defaults.remove_option(section, option) + if self.get_default_value(section, option) is not None and remove_default: + self._default_values.remove_option(section, option) def getsection(self, section: str) -> ConfigOptionsDictType | None: """ @@ -934,10 +1179,10 @@ def getsection(self, section: str) -> ConfigOptionsDictType | None: :param section: section from the config """ - if not self.has_section(section) and not self.airflow_defaults.has_section(section): + if not self.has_section(section) and not self._default_values.has_section(section): return None - if self.airflow_defaults.has_section(section): - _section: ConfigOptionsDictType = OrderedDict(self.airflow_defaults.items(section)) + if self._default_values.has_section(section): + _section: ConfigOptionsDictType = OrderedDict(self._default_values.items(section)) else: _section = OrderedDict() @@ -971,29 +1216,6 @@ def getsection(self, section: str) -> ConfigOptionsDictType | None: _section[key] = False return _section - def write( # type: ignore[override] - self, fp: IO, space_around_delimiters: bool = True, section: str | None = None - ) -> None: - # This is based on the configparser.RawConfigParser.write method code to add support for - # reading options from environment variables. - # Various type ignores below deal with less-than-perfect RawConfigParser superclass typing - if space_around_delimiters: - delimiter = f" {self._delimiters[0]} " # type: ignore[attr-defined] - else: - delimiter = self._delimiters[0] # type: ignore[attr-defined] - if self._defaults: # type: ignore - self._write_section( # type: ignore[attr-defined] - fp, self.default_section, self._defaults.items(), delimiter # type: ignore[attr-defined] - ) - sections = ( - {section: dict(self.getsection(section))} # type: ignore[arg-type] - if section - else self._sections # type: ignore[attr-defined] - ) - for sect in sections: - item_section: ConfigOptionsDictType = self.getsection(sect) # type: ignore[assignment] - self._write_section(fp, sect, item_section.items(), delimiter) # type: ignore[attr-defined] - def as_dict( self, display_source: bool = False, @@ -1044,14 +1266,15 @@ def as_dict( ) config_sources: ConfigSourcesType = {} - configs = [ - ("default", self.airflow_defaults), + configs: Iterable[tuple[str, ConfigParser]] = [ + ("default", self._default_values), ("airflow.cfg", self), ] self._replace_config_with_display_sources( config_sources, configs, + self.configuration_description if self.configuration_description else {}, display_source, raw, self.deprecated_options, @@ -1212,7 +1435,7 @@ def _filter_by_source( if not getter_opt: continue # Check to see that there is a default value - if not self.airflow_defaults.has_option(section, key): + if self.get_default_value(section, key) is None: continue # Check to see if bare setting is the same as defaults if display_source: @@ -1220,13 +1443,14 @@ def _filter_by_source( opt, source = config_sources[section][key] # type: ignore else: opt = config_sources[section][key] - if opt == self.airflow_defaults.get(section, key): + if opt == self.get_default_value(section, key): del config_sources[section][key] @staticmethod def _replace_config_with_display_sources( config_sources: ConfigSourcesType, configs: Iterable[tuple[str, ConfigParser]], + configuration_description: dict[str, dict[str, Any]], display_source: bool, raw: bool, deprecated_options: dict[tuple[str, str], tuple[str, str, str]], @@ -1235,10 +1459,12 @@ def _replace_config_with_display_sources( include_secret: bool, ): for source_name, config in configs: - for section in config.sections(): + sections = config.sections() + for section in sections: AirflowConfigParser._replace_section_config_with_display_sources( config, config_sources, + configuration_description, display_source, raw, section, @@ -1277,7 +1503,9 @@ def _deprecated_variable_is_set(deprecated_section: str, deprecated_key: str) -> @staticmethod def _deprecated_command_is_set_in_config( - deprecated_section: str, deprecated_key: str, configs: Iterable[tuple[str, ConfigParser]] + deprecated_section: str, + deprecated_key: str, + configs: Iterable[tuple[str, ConfigParser]], ) -> bool: return AirflowConfigParser._deprecated_value_is_set_in_config( deprecated_section=deprecated_section, deprecated_key=deprecated_key + "_cmd", configs=configs @@ -1292,7 +1520,9 @@ def _deprecated_variable_command_is_set(deprecated_section: str, deprecated_key: @staticmethod def _deprecated_secret_is_set_in_config( - deprecated_section: str, deprecated_key: str, configs: Iterable[tuple[str, ConfigParser]] + deprecated_section: str, + deprecated_key: str, + configs: Iterable[tuple[str, ConfigParser]], ) -> bool: return AirflowConfigParser._deprecated_value_is_set_in_config( deprecated_section=deprecated_section, deprecated_key=deprecated_key + "_secret", configs=configs @@ -1316,6 +1546,7 @@ def suppress_future_warnings(self): def _replace_section_config_with_display_sources( config: ConfigParser, config_sources: ConfigSourcesType, + configuration_description: dict[str, dict[str, Any]], display_source: bool, raw: bool, section: str, @@ -1329,7 +1560,7 @@ def _replace_section_config_with_display_sources( sect = config_sources.setdefault(section, OrderedDict()) if isinstance(config, AirflowConfigParser): with config.suppress_future_warnings(): - items = config.items(section=section, raw=raw) + items: Iterable[tuple[str, Any]] = config.items(section=section, raw=raw) else: items = config.items(section=section, raw=raw) for k, val in items: @@ -1372,22 +1603,46 @@ def _replace_section_config_with_display_sources( def load_test_config(self): """ - Load the unit test configuration. + Uses test configuration rather than the configuration coming from airflow defaults. - Note: this is not reversible. + When running tests we use special the unit_test configuration to avoid accidental modifications and + different behaviours when running the tests. Values for those test configuration are stored in + the "unit_tests.cfg" configuration file in the ``airflow/config_templates`` folder + and you need to change values there if you want to make some specific configuration to be used """ - # remove all sections, falling back to defaults + # We need those globals before we run "get_all_expansion_variables" because this is where + # the variables are expanded from in the configuration + global FERNET_KEY, AIRFLOW_HOME + from cryptography.fernet import Fernet + + unit_test_config_file = pathlib.Path(__file__).parent / "config_templates" / "unit_tests.cfg" + unit_test_config = unit_test_config_file.read_text() + self.remove_all_read_configurations() + with io.StringIO(unit_test_config) as test_config_file: + self.read_file(test_config_file) + # set fernet key to a random value + global FERNET_KEY + FERNET_KEY = Fernet.generate_key().decode() + self.expand_all_configuration_values() + log.info("Unit test configuration loaded from 'config_unit_tests.cfg'") + + def expand_all_configuration_values(self): + """Expands all configuration values using global and local variables defined in this module.""" + all_vars = get_all_expansion_variables() for section in self.sections(): - self.remove_section(section) - - # then read test config + for key, value in self.items(section): + if value is not None: + if self.has_option(section, key): + self.remove_option(section, key) + if self.is_template(section, key) or not isinstance(value, str): + self.set(section, key, value) + else: + self.set(section, key, value.format(**all_vars)) - path = _default_config_file_path("default_test.cfg") - log.info("Reading default test configuration from %s", path) - self.read_string(_parameterized_config_from_template("default_test.cfg")) - # then read any "custom" test settings - log.info("Reading test configuration from %s", TEST_CONFIG_FILE) - self.read(TEST_CONFIG_FILE) + def remove_all_read_configurations(self): + """Removes all read configurations, leaving only default values in the config.""" + for section in self.sections(): + self.remove_section(section) @staticmethod def _warn_deprecate( @@ -1414,7 +1669,9 @@ def __getstate__(self): for name in [ "_sections", "is_validated", - "airflow_defaults", + "configuration_description", + "upgraded_values", + "_default_values", ] } @@ -1430,7 +1687,7 @@ def get_airflow_home() -> str: return expand_env_var(os.environ.get("AIRFLOW_HOME", "~/airflow")) -def get_airflow_config(airflow_home) -> str: +def get_airflow_config(airflow_home: str) -> str: """Get Path to airflow.cfg path.""" airflow_config_var = os.environ.get("AIRFLOW_CONFIG") if airflow_config_var is None: @@ -1438,34 +1695,8 @@ def get_airflow_config(airflow_home) -> str: return expand_env_var(airflow_config_var) -def _parameterized_config_from_template(filename) -> str: - TEMPLATE_START = "# ----------------------- TEMPLATE BEGINS HERE -----------------------\n" - - path = _default_config_file_path(filename) - with open(path) as fh: - for line in fh: - if line != TEMPLATE_START: - continue - return parameterized_config(fh.read().strip()) - raise RuntimeError(f"Template marker not found in {path!r}") - - -def parameterized_config(template) -> str: - """ - Generates configuration from provided template & variables defined in current scope. - - :param template: a config content templated with {{variables}} - """ - all_vars = {k: v for d in [globals(), locals()] for k, v in d.items()} - return template.format(**all_vars) - - -def get_airflow_test_config(airflow_home) -> str: - """Get path to unittests.cfg.""" - if "AIRFLOW_TEST_CONFIG" not in os.environ: - return os.path.join(airflow_home, "unittests.cfg") - # It will never return None - return expand_env_var(os.environ["AIRFLOW_TEST_CONFIG"]) # type: ignore[return-value] +def get_all_expansion_variables() -> dict[str, Any]: + return {k: v for d in [globals(), locals()] for k, v in d.items()} def _generate_fernet_key() -> str: @@ -1474,84 +1705,109 @@ def _generate_fernet_key() -> str: return Fernet.generate_key().decode() -def initialize_config() -> AirflowConfigParser: - """ - Load the Airflow config files. - - Called for you automatically as part of the Airflow boot process. +def create_default_config_parser(configuration_description: dict[str, dict[str, Any]]) -> ConfigParser: """ - global FERNET_KEY, AIRFLOW_HOME, WEBSERVER_CONFIG + Creates default config parser based on configuration description. - default_config = _parameterized_config_from_template("default_airflow.cfg") + It creates ConfigParser with all default values retrieved from the configuration description and + expands all the variables from the global and local variables defined in this module. - local_conf = AirflowConfigParser(default_config=default_config) - - if local_conf.getboolean("core", "unit_test_mode"): - # Load test config only - if not os.path.isfile(TEST_CONFIG_FILE): - from cryptography.fernet import Fernet - - log.info("Creating new Airflow config file for unit tests in: %s", TEST_CONFIG_FILE) - pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) - - FERNET_KEY = Fernet.generate_key().decode() + :param configuration_description: configuration description - retrieved from config.yaml files + following the schema defined in "config.yml.schema.json" in the config_templates folder. + :return: Default Config Parser that can be used to read configuration values from. + """ + parser = ConfigParser() + all_vars = get_all_expansion_variables() + for section, section_desc in configuration_description.items(): + parser.add_section(section) + options = section_desc["options"] + for key in options: + default_value = options[key]["default"] + is_template = options[key].get("is_template", False) + if default_value is not None: + if is_template or not isinstance(default_value, str): + parser.set(section, key, default_value) + else: + parser.set(section, key, default_value.format(**all_vars)) + return parser - with open(TEST_CONFIG_FILE, "w") as file: - cfg = _parameterized_config_from_template("default_test.cfg") - file.write(cfg) - make_group_other_inaccessible(TEST_CONFIG_FILE) - local_conf.load_test_config() - else: - # Load normal config - if not os.path.isfile(AIRFLOW_CONFIG): - from cryptography.fernet import Fernet +def load_standard_airflow_configuration(airflow_config_parser: AirflowConfigParser): + """ + Loads standard airflow configuration. - log.info("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) - pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) + In case it finds that the configuration file is missing, it will create it and write the default + configuration values there, based on defaults passed, and will add the comments and examples + from the default configuration. - FERNET_KEY = Fernet.generate_key().decode() + :param airflow_config_parser: parser to which the configuration will be loaded + """ + global FERNET_KEY, AIRFLOW_HOME + if not os.path.isfile(AIRFLOW_CONFIG): + from cryptography.fernet import Fernet + + log.info("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) + pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) + FERNET_KEY = Fernet.generate_key().decode() + if airflow_config_parser._default_values and airflow_config_parser.configuration_description: with open(AIRFLOW_CONFIG, "w") as file: - file.write(default_config) - make_group_other_inaccessible(AIRFLOW_CONFIG) - - log.info("Reading the config from %s", AIRFLOW_CONFIG) - - local_conf.read(AIRFLOW_CONFIG) - - if local_conf.has_option("core", "AIRFLOW_HOME"): - msg = ( - "Specifying both AIRFLOW_HOME environment variable and airflow_home " - "in the config file is deprecated. Please use only the AIRFLOW_HOME " - "environment variable and remove the config file entry." - ) - if "AIRFLOW_HOME" in os.environ: - warnings.warn(msg, category=DeprecationWarning) - elif local_conf.get("core", "airflow_home") == AIRFLOW_HOME: - warnings.warn( - "Specifying airflow_home in the config file is deprecated. As you " - "have left it at the default value you should remove the setting " - "from your airflow.cfg and suffer no change in behaviour.", - category=DeprecationWarning, + airflow_config_parser.write( + file, + include_sources=False, + include_env_vars=True, + extra_spacing=True, + only_defaults=True, ) - else: - # there - AIRFLOW_HOME = local_conf.get("core", "airflow_home") # type: ignore[assignment] - warnings.warn(msg, category=DeprecationWarning) + make_group_other_inaccessible(AIRFLOW_CONFIG) + log.info("Reading the config from %s", AIRFLOW_CONFIG) + airflow_config_parser.read(AIRFLOW_CONFIG) + if airflow_config_parser.has_option("core", "AIRFLOW_HOME"): + msg = ( + "Specifying both AIRFLOW_HOME environment variable and airflow_home " + "in the config file is deprecated. Please use only the AIRFLOW_HOME " + "environment variable and remove the config file entry." + ) + if "AIRFLOW_HOME" in os.environ: + warnings.warn(msg, category=DeprecationWarning) + elif airflow_config_parser.get("core", "airflow_home") == AIRFLOW_HOME: + warnings.warn( + "Specifying airflow_home in the config file is deprecated. As you " + "have left it at the default value you should remove the setting " + "from your airflow.cfg and suffer no change in behaviour.", + category=DeprecationWarning, + ) + else: + # there + AIRFLOW_HOME = airflow_config_parser.get("core", "airflow_home") # type: ignore[assignment] + warnings.warn(msg, category=DeprecationWarning) - # They _might_ have set unit_test_mode in the airflow.cfg, we still - # want to respect that and then load the unittests.cfg - if local_conf.getboolean("core", "unit_test_mode"): - local_conf.load_test_config() - WEBSERVER_CONFIG = local_conf.get("webserver", "config_file") +def initialize_config() -> AirflowConfigParser: + """ + Load the Airflow config files. + + Called for you automatically as part of the Airflow boot process. + """ + global WEBSERVER_CONFIG + airflow_config_parser = AirflowConfigParser() + if airflow_config_parser.getboolean("core", "unit_test_mode"): + airflow_config_parser.load_test_config() + else: + load_standard_airflow_configuration(airflow_config_parser) + # If the user set unit_test_mode in the airflow.cfg, we still + # want to respect that and then load the default unit test configuration + # file on top of it. + if airflow_config_parser.getboolean("core", "unit_test_mode"): + airflow_config_parser.load_test_config() + + WEBSERVER_CONFIG = airflow_config_parser.get("webserver", "config_file") if not os.path.isfile(WEBSERVER_CONFIG): import shutil log.info("Creating new FAB webserver config file in: %s", WEBSERVER_CONFIG) shutil.copy(_default_config_file_path("default_webserver_config.py"), WEBSERVER_CONFIG) - return local_conf + return airflow_config_parser def make_group_other_inaccessible(file_path: str): @@ -1761,39 +2017,6 @@ def initialize_auth_manager() -> BaseAuthManager: return auth_manager_cls() -@functools.lru_cache(maxsize=None) -def _DEFAULT_CONFIG() -> str: - path = _default_config_file_path("default_airflow.cfg") - with open(path) as fh: - return fh.read() - - -@functools.lru_cache(maxsize=None) -def _TEST_CONFIG() -> str: - path = _default_config_file_path("default_test.cfg") - with open(path) as fh: - return fh.read() - - -_deprecated = { - "DEFAULT_CONFIG": _DEFAULT_CONFIG, - "TEST_CONFIG": _TEST_CONFIG, - "TEST_CONFIG_FILE_PATH": functools.partial(_default_config_file_path, "default_test.cfg"), - "DEFAULT_CONFIG_FILE_PATH": functools.partial(_default_config_file_path, "default_airflow.cfg"), -} - - -def __getattr__(name): - if name in _deprecated: - warnings.warn( - f"{__name__}.{name} is deprecated and will be removed in future", - DeprecationWarning, - stacklevel=2, - ) - return _deprecated[name]() - raise AttributeError(f"module {__name__} has no attribute {name}") - - # Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using # "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults. AIRFLOW_HOME = get_airflow_home() @@ -1818,8 +2041,6 @@ def __getattr__(name): else: TEST_PLUGINS_FOLDER = os.path.join(AIRFLOW_HOME, "plugins") -TEST_CONFIG_FILE = get_airflow_test_config(AIRFLOW_HOME) - SECRET_KEY = b64encode(os.urandom(16)).decode("utf-8") FERNET_KEY = "" # Set only if needed when generating a new file WEBSERVER_CONFIG = "" # Set by initialize_config diff --git a/airflow/utils/db.py b/airflow/utils/db.py index f4c2ae8b1bd0..d6c5664a96f4 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -911,8 +911,8 @@ def synchronize_log_template(*, session: Session = NEW_SESSION) -> None: # If we have an empty table, and the default values exist, we will seed the # table with values from pre 2.3.0, so old logs will still be retrievable. if not stored: - is_default_log_id = elasticsearch_id == conf.airflow_defaults.get("elasticsearch", "log_id_template") - is_default_filename = filename == conf.airflow_defaults.get("logging", "log_filename_template") + is_default_log_id = elasticsearch_id == conf.get_default_value("elasticsearch", "log_id_template") + is_default_filename = filename == conf.get_default_value("logging", "log_filename_template") if is_default_log_id and is_default_filename: session.add( LogTemplate( diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index ed7a06d4a466..0ef8ef65b908 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -27,7 +27,6 @@ "black", "blacken-docs", "check-aiobotocore-optional", - "check-airflow-config-yaml-consistent", "check-airflow-provider-compatibility", "check-apache-license-rat", "check-base-operator-partial-arguments", diff --git a/docs/apache-airflow/howto/index.rst b/docs/apache-airflow/howto/index.rst index 617975d2958d..15a5249147f0 100644 --- a/docs/apache-airflow/howto/index.rst +++ b/docs/apache-airflow/howto/index.rst @@ -47,7 +47,6 @@ configuring an Airflow environment. variable run-behind-proxy run-with-systemd - use-test-config define-extra-link email-config dynamic-dag-generation diff --git a/docs/apache-airflow/howto/set-config.rst b/docs/apache-airflow/howto/set-config.rst index b443cff9c81a..14da8198d5d9 100644 --- a/docs/apache-airflow/howto/set-config.rst +++ b/docs/apache-airflow/howto/set-config.rst @@ -21,8 +21,31 @@ Setting Configuration Options ============================= The first time you run Airflow, it will create a file called ``airflow.cfg`` in -your ``$AIRFLOW_HOME`` directory (``~/airflow`` by default). This file contains Airflow's configuration and you -can edit it to change any of the settings. You can also set options with environment variables by using this format: +your ``$AIRFLOW_HOME`` directory (``~/airflow`` by default). This is in order to make it easy to +"play" with airflow configuration. + +However, for production case you are advised to generate the configuration using command line: + +.. code-block:: bash + + airflow config list --defaults + +This command will produce the output that you can copy to your configuration file and edit. + +It will contain all the default configuration options, with examples, nicely commented out +so you need only un-comment and modify those that you want to change. +This way you can easily keep track of all the configuration options that you changed from default +and you can also easily upgrade your installation to new versions of Airflow when they come out and +automatically use the defaults for existing options if they changed there. + +You can redirect it to your configuration file and edit it: + +.. code-block:: bash + + airflow config list --defaults > "${AIRFLOW_HOME}/airflow.cfg" + + +You can also set options with environment variables by using this format: :envvar:`AIRFLOW__{SECTION}__{KEY}` (note the double underscores). For example, the metadata database connection string can either be set in ``airflow.cfg`` like this: diff --git a/docs/apache-airflow/howto/use-test-config.rst b/docs/apache-airflow/howto/use-test-config.rst deleted file mode 100644 index 1c7731efe597..000000000000 --- a/docs/apache-airflow/howto/use-test-config.rst +++ /dev/null @@ -1,36 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - - -Using the Test Mode Configuration -================================= - -Airflow has a fixed set of "test mode" configuration options. You can load these -at any time by calling ``airflow.configuration.load_test_config()``. Please **note** that this -operation is **not reversible**. - -Some options for example, ``DAG_FOLDER``, are loaded before you have a chance to call ``load_test_config()``. -In order to eagerly load the test configuration, set ``test_mode`` in ``airflow.cfg``: - -.. code-block:: ini - - [tests] - unit_test_mode = True - -Due to Airflow's automatic environment variable expansion (see :doc:`set-config`), you can also set the -environment variable ``AIRFLOW__CORE__UNIT_TEST_MODE`` to temporarily overwrite ``airflow.cfg``. diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt index 9d8a5527fbbd..01deeddc8717 100644 --- a/docs/apache-airflow/redirects.txt +++ b/docs/apache-airflow/redirects.txt @@ -141,3 +141,6 @@ python-api-ref.rst public-airflow-interface.rst # Typos howto/define_extra_link.rst howto/define-extra-link.rst + +# Use test config (it's not a howto for users but a howto for developers so we redirect it back to index) +howto/use-test-config.rst index.rst diff --git a/docs/conf.py b/docs/conf.py index 9f1645e42599..a8939833fd6b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,7 +43,7 @@ from packaging.version import parse as parse_version import airflow -from airflow.configuration import AirflowConfigParser, default_config_yaml +from airflow.configuration import AirflowConfigParser, retrieve_configuration_description sys.path.append(str(Path(__file__).parent / "exts")) @@ -419,7 +419,7 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): for deprecated_section, deprecated_key, since_version in deprecated: deprecated_options[deprecated_section][deprecated_key] = section, key, since_version - configs = default_config_yaml() + configs = retrieve_configuration_description() # We want the default/example we show in the docs to reflect the value _after_ # the config has been templated, not before diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index a0bd58e32e2c..a678af7996c6 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,7 +62,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:cdf58a9c27af2877dc5a658ed0a1d99c shell:87e7bdcebe1180395adfec86b2a065f5 start-airflow:f7216a8126ecf14b033e2ea677e1a105 -static-checks:bb1248670000271a9e011c12747079c1 +static-checks:f9ec0d7edaba84180403d95469d94ea0 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index ea72db06740e..11b7038cecc5 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -217,59 +217,59 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index da730e9da702..6220e1db62e0 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - + - + @@ -264,12 +264,9 @@ - - - - Command: static-checks + Command: static-checks @@ -280,80 +277,79 @@ -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... Run static checks. -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run.                                             │ -│(all | black | blacken-docs | check-aiobotocore-optional |                       │ -│check-airflow-config-yaml-consistent | check-airflow-provider-compatibility |    │ -│check-apache-license-rat | check-base-operator-partial-arguments |               │ -│check-base-operator-usage | check-boring-cyborg-configuration |                  │ -│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ -│check-changelog-has-no-duplicates | check-core-deprecation-classes |             │ -│check-daysago-import-from-utils | check-decorated-operator-implements-custom-name│ -│| check-deferrable-default-value | check-docstring-param-types |                 │ -│check-example-dags-urls | check-executables-have-shebangs |                      │ -│check-extra-packages-references | check-extras-order |                           │ -│check-for-inclusive-language | check-hooks-apply |                               │ -│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ -│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ -│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ -│check-no-relative-imports | check-only-new-session-with-provide-session |        │ -│check-persist-credentials-disabled-in-github-workflows |                         │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ -│check-provider-yaml-valid | check-providers-init-file-missing |                  │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ -│check-start-date-not-used-in-defaults | check-system-tests-present |             │ -│check-system-tests-tocs | check-tests-unittest-testcase |                        │ -│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ -│| compile-www-assets | compile-www-assets-dev |                                  │ -│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ -│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ -│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ -│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ -│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ -│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ -│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ -│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ -│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ -│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-local-yml-file |                │ -│update-migration-references | update-providers-dependencies |                    │ -│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ -│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--initialize-environmentInitialize environment before running checks.│ -│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ -│(INTEGER RANGE)                                                       │ -│[default: 3; 1<=x<=10]                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ -│exclusive with --last-commit.                                                                │ -│(TEXT)                                                                                       │ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ -│branch and HEAD of your branch.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ -│--skip-image-checkSkip checking if the CI image is up to date. Useful if you run non-image checks only│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run.                                             │ +│(all | black | blacken-docs | check-aiobotocore-optional |                       │ +│check-airflow-provider-compatibility | check-apache-license-rat |                │ +│check-base-operator-partial-arguments | check-base-operator-usage |              │ +│check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |      │ +│check-builtin-literals | check-changelog-has-no-duplicates |                     │ +│check-core-deprecation-classes | check-daysago-import-from-utils |               │ +│check-decorated-operator-implements-custom-name | check-deferrable-default-value │ +│| check-docstring-param-types | check-example-dags-urls |                        │ +│check-executables-have-shebangs | check-extra-packages-references |              │ +│check-extras-order | check-for-inclusive-language | check-hooks-apply |          │ +│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ +│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ +│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ +│check-no-relative-imports | check-only-new-session-with-provide-session |        │ +│check-persist-credentials-disabled-in-github-workflows |                         │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ +│check-provider-yaml-valid | check-providers-init-file-missing |                  │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ +│check-start-date-not-used-in-defaults | check-system-tests-present |             │ +│check-system-tests-tocs | check-tests-unittest-testcase |                        │ +│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ +│| compile-www-assets | compile-www-assets-dev |                                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ +│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ +│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ +│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ +│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ +│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ +│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ +│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ +│update-installed-providers-to-be-sorted | update-local-yml-file |                │ +│update-migration-references | update-providers-dependencies |                    │ +│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ +│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--initialize-environmentInitialize environment before running checks.│ +│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ +│(INTEGER RANGE)                                                       │ +│[default: 3; 1<=x<=10]                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ +│exclusive with --last-commit.                                                                │ +│(TEXT)                                                                                       │ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ +│branch and HEAD of your branch.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ +│--skip-image-checkSkip checking if the CI image is up to date. Useful if you run non-image checks only│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py deleted file mode 100755 index 45aff2c2a12b..000000000000 --- a/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Module to convert Airflow configs in config.yml to default_airflow.cfg file -""" -from __future__ import annotations - -import os -import re -from pathlib import Path - -import yaml -from packaging.version import parse as parse_version - -FILE_HEADER = """# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This is the template for Airflow's default configuration. When Airflow is -# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If -# it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# airflow.cfg instead. - - -# ----------------------- TEMPLATE BEGINS HERE ----------------------- -""" - - -def read_default_config_yaml(file_path: str) -> dict: - """ - Read Airflow configs from YAML file - - :param file_path: Full path to config.yaml - - :return: Python dictionary containing configs & their info - """ - - with open(file_path) as config_file: - return yaml.safe_load(config_file) - - -def write_config(yaml_config_file_path: str, default_cfg_file_path: str): - """ - Write config to default_airflow.cfg file - - :param yaml_config_file_path: Full path to config.yaml - :param default_cfg_file_path: Full path to default_airflow.cfg - """ - print(f"Converting {yaml_config_file_path} to {default_cfg_file_path}") - with open(default_cfg_file_path, "w") as configfile: - configfile.writelines(FILE_HEADER) - config_yaml = read_default_config_yaml(yaml_config_file_path) - - for section_name, section in config_yaml.items(): - _write_section(configfile, section_name, section) - - -def _write_section(configfile, section_name, section): - configfile.write(f"\n[{section_name}]\n") - section_description = None - if section["description"] is not None: - section_description = list( - filter(lambda x: (x is not None) or x != "", section["description"].splitlines()) - ) - if section_description: - configfile.write("\n") - for single_line_desc in section_description: - if single_line_desc == "": - configfile.write("#\n") - else: - configfile.write(f"# {single_line_desc}\n") - for idx, (option_name, option) in enumerate(section["options"].items()): - _write_option(configfile, idx, option_name, option) - - -def _write_option(configfile, idx, option_name, option): - option_description = None - version_added = option["version_added"] - if version_added is not None and parse_version(version_added) > airflow_version: - # skip if option is going to be added in the future version - return - if option["description"] is not None: - option_description = list(filter(lambda x: x is not None, option["description"].splitlines())) - - if option_description: - if idx != 0: - configfile.write("\n") - for single_line_desc in option_description: - if single_line_desc == "": - configfile.write("#\n") - else: - configfile.write(f"# {single_line_desc}\n") - - if option["example"]: - if not str(option_name).endswith("_template"): - option["example"] = option["example"].replace("{", "{{").replace("}", "}}") - configfile.write(f"# Example: {option_name} = {option['example']}\n") - - if option["default"] is not None: - if not isinstance(option["default"], str): - raise Exception( - f'Key "default" in element with name="{option_name}" has an invalid type. ' - f"Current type: {type(option['default'])}" - ) - # Remove trailing whitespace on empty string - if option["default"]: - value = " " + option["default"] - else: - value = "" - configfile.write(f"{option_name} ={value}\n") - else: - configfile.write(f"# {option_name} =\n") - - -if __name__ == "__main__": - airflow_config_dir = os.path.join( - os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "airflow", "config_templates" - ) - airflow_default_config_path = os.path.join(airflow_config_dir, "default_airflow.cfg") - airflow_config_yaml_file_path = os.path.join(airflow_config_dir, "config.yml") - - airflow_version = parse_version( - re.search( # type: ignore[union-attr,arg-type] - r"__version__ = \"([0-9\.]*)(\.dev[0-9]*)?\"", - (Path(__file__).parents[3] / "airflow" / "__init__.py").read_text(), - ).groups(0)[0] - ) - - write_config( - yaml_config_file_path=airflow_config_yaml_file_path, default_cfg_file_path=airflow_default_config_path - ) - - providers_dir = os.path.join( - os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "airflow", "providers" - ) - for root, dir_names, file_names in os.walk(providers_dir): - for file_name in file_names: - if ( - root.endswith("config_templates") - and file_name == "config.yml" - and os.path.isfile(os.path.join(root, "default_config.cfg")) - ): - write_config( - yaml_config_file_path=os.path.join(root, "config.yml"), - default_cfg_file_path=os.path.join(root, "default_config.cfg"), - ) diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 287bcbca0e09..237c97a300f8 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -262,7 +262,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then unset AIRFLOW__CORE__UNIT_TEST_MODE mkdir -pv "${AIRFLOW_HOME}/logs/" - cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" # Change the default worker_concurrency for tests export AIRFLOW__CELERY__WORKER_CONCURRENCY=8 diff --git a/scripts/in_container/airflow_ci.cfg b/scripts/in_container/airflow_ci.cfg deleted file mode 100644 index 57863bd9c69f..000000000000 --- a/scripts/in_container/airflow_ci.cfg +++ /dev/null @@ -1,59 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[core] -dags_folder = ~/airflow/dags -executor = LocalExecutor -sql_alchemy_conn = # overridden by the startup scripts -#sql_engine_collation_for_ids = overridden by the startup scripts -unit_test_mode = True -load_examples = True -donot_pickle = False -dags_are_paused_at_creation = False -default_impersonation = -fernet_key = af7CN0q6ag5U3g08IsPsw3K45U7Xa0axgVFhoh-3zB8= - -[database] -load_default_connections = True - -[hive] -default_hive_mapred_queue = airflow - -[logging] -base_log_folder = ~/airflow/logs - -[smtp] -smtp_user = airflow -smtp_password = airflow -smtp_mail_from = airflow@example.com - -[celery] -broker_url = amqp://guest:guest@rabbitmq:5672/ -result_backend = # overridden by startup scripts - - -[celery_broker_transport_options] -visibility_timeout = 21600 -_test_only_bool = True -_test_only_float = 12.0 -_test_only_string = this is a test - -[scheduler] -job_heartbeat_sec = 1 -scheduler_heartbeat_sec = 5 -parsing_processes = 2 diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index f5f90bd2f347..f5b427f0cfca 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -37,7 +37,14 @@ def setup_class(cls): def test_cli_show_config_should_write_data(self, mock_conf, mock_stringio): config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) mock_conf.write.assert_called_once_with( - mock_stringio.return_value.__enter__.return_value, section=None + mock_stringio.return_value.__enter__.return_value, + section=None, + include_examples=False, + include_descriptions=False, + include_sources=False, + include_env_vars=False, + comment_out_everything=False, + only_defaults=False, ) @mock.patch("airflow.cli.commands.config_command.io.StringIO") @@ -47,16 +54,141 @@ def test_cli_show_config_should_write_data_specific_section(self, mock_conf, moc self.parser.parse_args(["config", "list", "--section", "core", "--color", "off"]) ) mock_conf.write.assert_called_once_with( - mock_stringio.return_value.__enter__.return_value, section="core" + mock_stringio.return_value.__enter__.return_value, + section="core", + include_examples=False, + include_descriptions=False, + include_sources=False, + include_env_vars=False, + comment_out_everything=False, + only_defaults=False, ) @conf_vars({("core", "testkey"): "test_value"}) def test_cli_show_config_should_display_key(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) - assert "[core]" in temp_stdout.getvalue() + output = temp_stdout.getvalue() + assert "[core]" in output assert "testkey = test_value" in temp_stdout.getvalue() + def test_cli_show_config_should_only_show_comments_when_no_defaults(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert all(not line.startswith("#") or line.endswith("= ") for line in lines if line) + + def test_cli_show_config_shows_descriptions(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--include-descriptions"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + # comes from metrics description + assert all(not line.startswith("# Source: ") for line in lines if line) + assert any(line.startswith("# StatsD") for line in lines if line) + assert all(not line.startswith("# Example:") for line in lines if line) + assert all(not line.startswith("# Variable:") for line in lines if line) + + def test_cli_show_config_shows_examples(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--include-examples"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert all(not line.startswith("# Source: ") for line in lines if line) + assert all(not line.startswith("# StatsD") for line in lines if line) + assert any(line.startswith("# Example:") for line in lines if line) + assert all(not line.startswith("# Variable:") for line in lines if line) + + def test_cli_show_config_shows_variables(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--include-env-vars"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert all(not line.startswith("# Source: ") for line in lines if line) + assert all(not line.startswith("# StatsD") for line in lines if line) + assert all(not line.startswith("# Example:") for line in lines if line) + assert any(line.startswith("# Variable:") for line in lines if line) + + def test_cli_show_config_shows_sources(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--include-sources"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("# Source: ") for line in lines if line) + assert all(not line.startswith("# StatsD") for line in lines if line) + assert all(not line.startswith("# Example:") for line in lines if line) + assert all(not line.startswith("# Variable:") for line in lines if line) + + def test_cli_show_config_defaults(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert all(not line.startswith("# Source: ") for line in lines if line) + assert any(line.startswith("# StatsD") for line in lines if line) + assert any(not line.startswith("# Example:") for line in lines if line) + assert any(not line.startswith("# Example:") for line in lines if line) + assert any(line.startswith("# Variable:") for line in lines if line) + assert any(line.startswith("# task_runner = StandardTaskRunner") for line in lines if line) + + @conf_vars({("core", "task_runner"): "test-runner"}) + def test_cli_show_config_defaults_not_show_conf_changes(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("# task_runner = StandardTaskRunner") for line in lines if line) + + @mock.patch("os.environ", {"AIRFLOW__CORE__TASK_RUNNER": "test-env-runner"}) + def test_cli_show_config_defaults_do_not_show_env_changes(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("# task_runner = StandardTaskRunner") for line in lines if line) + + @conf_vars({("core", "task_runner"): "test-runner"}) + def test_cli_show_changed_defaults_when_overridden_in_conf(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("task_runner = test-runner") for line in lines if line) + + @mock.patch("os.environ", {"AIRFLOW__CORE__TASK_RUNNER": "test-env-runner"}) + def test_cli_show_changed_defaults_when_overridden_in_env(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("task_runner = test-env-runner") for line in lines if line) + + def test_cli_comment_out_everything(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config( + self.parser.parse_args(["config", "list", "--color", "off", "--comment-out-everything"]) + ) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert all( + line.startswith("#") or line.strip() == "" or line.startswith("[") for line in lines if line + ) + class TestCliConfigGetValue: @classmethod diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 378ebd4e715e..7cac8f7ad1d1 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -198,13 +198,26 @@ def test_positive_int(self): cli_config.positive_int(allow_zero=False)("0") cli_config.positive_int(allow_zero=True)("-1") - def test_dag_parser_celery_command_require_celery_executor(self): + @pytest.mark.parametrize( + "command", + [ + ["celery"], + ["celery", "--help"], + ["celery", "worker", "--help"], + ["celery", "worker"], + ["celery", "flower", "--help"], + ["celery", "flower"], + ["celery", "stop_worker", "--help"], + ["celery", "stop_worker"], + ], + ) + def test_dag_parser_require_celery_executor(self, command): with conf_vars({("core", "executor"): "SequentialExecutor"}), contextlib.redirect_stderr( io.StringIO() ) as stderr: parser = cli_parser.get_parser() with pytest.raises(SystemExit): - parser.parse_args(["celery"]) + parser.parse_args(command) stderr = stderr.getvalue() assert ( "airflow command error: argument GROUP_OR_COMMAND: celery subcommand " diff --git a/tests/config_templates/deprecated.cfg b/tests/config_templates/deprecated.cfg index cab0b6e7fa2d..1a7904542481 100644 --- a/tests/config_templates/deprecated.cfg +++ b/tests/config_templates/deprecated.cfg @@ -15,14 +15,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - -# This is the template for Airflow's unit test configuration. When Airflow runs -# unit tests, it looks for a configuration file at $AIRFLOW_HOME/unittests.cfg. -# If it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# unittests.cfg instead. [core] sql_alchemy_conn = mysql:// diff --git a/tests/config_templates/deprecated_cmd.cfg b/tests/config_templates/deprecated_cmd.cfg index ff2a9268dd69..dbe819fbb63f 100644 --- a/tests/config_templates/deprecated_cmd.cfg +++ b/tests/config_templates/deprecated_cmd.cfg @@ -15,15 +15,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - -# This is the template for Airflow's unit test configuration. When Airflow runs -# unit tests, it looks for a configuration file at $AIRFLOW_HOME/unittests.cfg. -# If it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# unittests.cfg instead. - [core] sql_alchemy_conn_cmd = echo -n "postgresql://" diff --git a/tests/config_templates/deprecated_secret.cfg b/tests/config_templates/deprecated_secret.cfg index 6c4cda091af6..ca2f5aa63719 100644 --- a/tests/config_templates/deprecated_secret.cfg +++ b/tests/config_templates/deprecated_secret.cfg @@ -15,15 +15,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - -# This is the template for Airflow's unit test configuration. When Airflow runs -# unit tests, it looks for a configuration file at $AIRFLOW_HOME/unittests.cfg. -# If it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# unittests.cfg instead. - [core] sql_alchemy_conn_secret = secret_path diff --git a/tests/config_templates/empty.cfg b/tests/config_templates/empty.cfg index 36906c07cfde..217e5db96078 100644 --- a/tests/config_templates/empty.cfg +++ b/tests/config_templates/empty.cfg @@ -15,12 +15,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - -# This is the template for Airflow's unit test configuration. When Airflow runs -# unit tests, it looks for a configuration file at $AIRFLOW_HOME/unittests.cfg. -# If it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# unittests.cfg instead. diff --git a/tests/conftest.py b/tests/conftest.py index 9fdda1467cf7..b2ef0b1e6e90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,7 @@ import sys from contextlib import ExitStack, suppress from datetime import datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING import pytest @@ -32,9 +33,10 @@ from itsdangerous import URLSafeSerializer assert "airflow" not in sys.modules, "No airflow module can be imported before these lines" -tests_directory = os.path.dirname(os.path.realpath(__file__)) +AIRFLOW_TESTS_DIR = Path(os.path.dirname(os.path.realpath(__file__))).resolve() +AIRFLOW_SOURCES_ROOT_DIR = AIRFLOW_TESTS_DIR.parent.parent -os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = os.path.join(tests_directory, "dags") +os.environ["AIRFLOW__CORE__DAGS_FOLDER"] = os.fspath(AIRFLOW_TESTS_DIR / "dags") os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "True" os.environ["AWS_DEFAULT_REGION"] = os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" os.environ["CREDENTIALS_DIR"] = os.environ.get("CREDENTIALS_DIR") or "/files/airflow-breeze-config/keys" @@ -882,6 +884,15 @@ def _clear_db(request): """Clear DB before each test module run.""" if not request.config.option.db_cleanup: return + from airflow.configuration import conf + + sql_alchemy_conn = conf.get("database", "sql_alchemy_conn") + if sql_alchemy_conn.startswith("sqlite"): + sql_alchemy_file = sql_alchemy_conn.replace("sqlite:///", "") + if not os.path.exists(sql_alchemy_file): + print(f"The sqlite file `{sql_alchemy_file}` does not exist. Attempt to initialize it.") + initial_db_init() + dist_option = getattr(request.config.option, "dist", "no") if dist_option != "no" or hasattr(request.config, "workerinput"): # Skip if pytest-xdist detected (controller or worker) diff --git a/tests/core/test_config_templates.py b/tests/core/test_config_templates.py deleted file mode 100644 index 574bcf9537ed..000000000000 --- a/tests/core/test_config_templates.py +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import configparser -import os - -import pytest - -from tests.test_utils import AIRFLOW_MAIN_FOLDER - -CONFIG_TEMPLATES_FOLDER = os.path.join(AIRFLOW_MAIN_FOLDER, "airflow", "config_templates") - -DEFAULT_AIRFLOW_SECTIONS = [ - "core", - "database", - "logging", - "metrics", - "secrets", - "cli", - "debug", - "api", - "lineage", - "openlineage", - "atlas", - "operators", - "hive", - "webserver", - "email", - "smtp", - "sentry", - "local_kubernetes_executor", - "celery_kubernetes_executor", - "celery", - "celery_broker_transport_options", - "dask", - "scheduler", - "triggerer", - "kerberos", - "elasticsearch", - "elasticsearch_configs", - "kubernetes_executor", - "sensors", -] - -DEFAULT_TEST_SECTIONS = [ - "core", - "database", - "logging", - "api", - "hive", - "smtp", - "celery", - "scheduler", - "elasticsearch", -] - - -class TestAirflowCfg: - @pytest.mark.parametrize("filename", ["default_airflow.cfg", "default_test.cfg"]) - def test_should_be_ascii_file(self, filename: str): - with open(os.path.join(CONFIG_TEMPLATES_FOLDER, filename), "rb") as f: - content = f.read().decode("ascii") - assert content - - @pytest.mark.parametrize( - "filename, expected_sections", - [ - ( - "default_airflow.cfg", - DEFAULT_AIRFLOW_SECTIONS, - ), - ( - "default_test.cfg", - DEFAULT_TEST_SECTIONS, - ), - ], - ) - def test_should_be_ini_file(self, filename: str, expected_sections): - filepath = os.path.join(CONFIG_TEMPLATES_FOLDER, filename) - config = configparser.ConfigParser() - config.read(filepath) - - assert expected_sections == config.sections() diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 90aa7959ea6a..aa7820cf37b5 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -36,11 +36,10 @@ AirflowConfigException, AirflowConfigParser, conf, - default_config_yaml, expand_env_var, get_airflow_config, get_airflow_home, - parameterized_config, + get_all_expansion_variables, run_command, ) from tests.test_utils.config import conf_vars @@ -61,6 +60,16 @@ def restore_env(): yield +def parameterized_config(template) -> str: + """ + Generates configuration from provided template & variables defined in current scope. + + :param template: a config content templated with {{variables}} + """ + all_vars = get_all_expansion_variables() + return template.format(**all_vars) + + @mock.patch.dict( "os.environ", { @@ -566,12 +575,19 @@ def test_kubernetes_environment_variables_section(self): "kubernetes_environment_variables" ) - def test_broker_transport_options(self): - section_dict = conf.getsection("celery_broker_transport_options") - assert isinstance(section_dict["visibility_timeout"], int) - assert isinstance(section_dict["_test_only_bool"], bool) - assert isinstance(section_dict["_test_only_float"], float) - assert isinstance(section_dict["_test_only_string"], str) + @pytest.mark.parametrize( + "key, type", + [ + ("string_value", int), # Coercion happens here + ("only_bool_value", bool), + ("only_float_value", float), + ("only_integer_value", int), + ("only_string_value", str), + ], + ) + def test_config_value_types(self, key, type): + section_dict = conf.getsection("example_section") + assert isinstance(section_dict[key], type) def test_auth_backends_adds_session(self): test_conf = AirflowConfigParser(default_config="") @@ -608,7 +624,7 @@ def test_command_from_env(self): # sensitive_config_values and therefore should return 'OK' from the environment variable's # echo command, and must not return 'NOT OK' from the configuration assert test_cmdenv_conf.get("testcmdenv", "itsacommand") == "OK" - # AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in sensitive_config_values and therefore + # AIRFLOW__TESTCMDENV__xNOTACOMMAND_CMD maps to no entry in sensitive_config_values and therefore # the option should return 'OK' from the configuration, and must not return 'NOT OK' from # the environment variable's echo command assert test_cmdenv_conf.get("testcmdenv", "notacommand") == "OK" @@ -687,11 +703,20 @@ def test_config_override_original_when_empty_envvar_is_provided(self): @mock.patch.dict("os.environ", {"AIRFLOW__CORE__DAGS_FOLDER": "/tmp/test_folder"}) def test_write_should_respect_env_variable(self): + parser = AirflowConfigParser() with io.StringIO() as string_file: - conf.write(string_file) + parser.write(string_file) content = string_file.getvalue() assert "dags_folder = /tmp/test_folder" in content + @mock.patch.dict("os.environ", {"AIRFLOW__CORE__DAGS_FOLDER": "/tmp/test_folder"}) + def test_write_with_only_defaults_should_not_respect_env_variable(self): + parser = AirflowConfigParser() + with io.StringIO() as string_file: + parser.write(string_file, only_defaults=True) + content = string_file.getvalue() + assert "dags_folder = /tmp/test_folder" not in content + def test_run_command(self): write = r'sys.stdout.buffer.write("\u1000foo".encode("utf8"))' @@ -734,15 +759,15 @@ def test_as_dict_works_without_sensitive_cmds(self): conf_materialize_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=True) conf_maintain_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=False) - assert "sql_alchemy_conn" in conf_materialize_cmds["core"] - assert "sql_alchemy_conn_cmd" not in conf_materialize_cmds["core"] + assert "sql_alchemy_conn" in conf_materialize_cmds["database"] + assert "sql_alchemy_conn_cmd" not in conf_materialize_cmds["database"] - assert "sql_alchemy_conn" in conf_maintain_cmds["core"] - assert "sql_alchemy_conn_cmd" not in conf_maintain_cmds["core"] + assert "sql_alchemy_conn" in conf_maintain_cmds["database"] + assert "sql_alchemy_conn_cmd" not in conf_maintain_cmds["database"] assert ( - conf_materialize_cmds["core"]["sql_alchemy_conn"] - == conf_maintain_cmds["core"]["sql_alchemy_conn"] + conf_materialize_cmds["database"]["sql_alchemy_conn"] + == conf_maintain_cmds["database"]["sql_alchemy_conn"] ) def test_as_dict_respects_sensitive_cmds(self): @@ -763,13 +788,13 @@ def test_as_dict_respects_sensitive_cmds(self): assert "sql_alchemy_conn" in conf_materialize_cmds["database"] assert "sql_alchemy_conn_cmd" not in conf_materialize_cmds["database"] - if conf_conn == test_conf.airflow_defaults["database"]["sql_alchemy_conn"]: + if conf_conn == test_conf._default_values["database"]["sql_alchemy_conn"]: assert conf_materialize_cmds["database"]["sql_alchemy_conn"] == "my-super-secret-conn" assert "sql_alchemy_conn_cmd" in conf_maintain_cmds["database"] assert conf_maintain_cmds["database"]["sql_alchemy_conn_cmd"] == "echo -n my-super-secret-conn" - if conf_conn == test_conf.airflow_defaults["database"]["sql_alchemy_conn"]: + if conf_conn == test_conf._default_values["database"]["sql_alchemy_conn"]: assert "sql_alchemy_conn" not in conf_maintain_cmds["database"] else: assert "sql_alchemy_conn" in conf_maintain_cmds["database"] @@ -1449,6 +1474,93 @@ def test_future_warning_only_for_code_ref(self, key): assert "your `conf.get*` call to use the new name" in str(w.message) assert w.category == FutureWarning + def test_as_dict_raw(self): + test_conf = AirflowConfigParser() + raw_dict = test_conf.as_dict(raw=True) + assert "%%" in raw_dict["logging"]["log_format"] + + def test_as_dict_not_raw(self): + test_conf = AirflowConfigParser() + raw_dict = test_conf.as_dict(raw=False) + assert "%%" not in raw_dict["logging"]["log_format"] + + def test_default_value_raw(self): + test_conf = AirflowConfigParser() + log_format = test_conf.get_default_value("logging", "log_format", raw=True) + assert "%%" in log_format + + def test_default_value_not_raw(self): + test_conf = AirflowConfigParser() + log_format = test_conf.get_default_value("logging", "log_format", raw=False) + assert "%%" not in log_format + + def test_default_value_raw_with_fallback(self): + test_conf = AirflowConfigParser() + log_format = test_conf.get_default_value("logging", "missing", fallback="aa %%", raw=True) + assert "%%" in log_format + + def test_default_value_not_raw_with_fallback(self): + test_conf = AirflowConfigParser() + log_format = test_conf.get_default_value("logging", "missing", fallback="aa %%", raw=False) + # Note that fallback is never interpolated so we expect the value passed as-is + assert "%%" in log_format + + def test_written_defaults_are_raw_for_defaults(self): + test_conf = AirflowConfigParser() + with io.StringIO() as f: + test_conf.write(f, only_defaults=True) + string_written = f.getvalue() + assert "%%(asctime)s" in string_written + + def test_written_defaults_are_raw_for_non_defaults(self): + test_conf = AirflowConfigParser() + with io.StringIO() as f: + test_conf.write(f) + string_written = f.getvalue() + assert "%%(asctime)s" in string_written + + def test_get_sections_including_defaults(self): + airflow_cfg = AirflowConfigParser() + airflow_cfg.remove_all_read_configurations() + default_sections = airflow_cfg.get_sections_including_defaults() + assert "core" in default_sections + assert "test-section" not in default_sections + airflow_cfg.add_section("test-section") + airflow_cfg.set("test-section", "test-key", "test-value") + all_sections_including_defaults = airflow_cfg.get_sections_including_defaults() + assert "core" in all_sections_including_defaults + assert "test-section" in all_sections_including_defaults + airflow_cfg.add_section("core") + airflow_cfg.set("core", "new-test-key", "test-value") + all_sections_including_defaults = airflow_cfg.get_sections_including_defaults() + assert "core" in all_sections_including_defaults + assert "test-section" in all_sections_including_defaults + assert len([section for section in all_sections_including_defaults if section == "core"]) == 1 + + def test_get_options_including_defaults(self): + airflow_cfg = AirflowConfigParser() + airflow_cfg.remove_all_read_configurations() + default_options = airflow_cfg.get_options_including_defaults("core") + assert "task_runner" in default_options + assert "StandardTaskRunner" == airflow_cfg.get("core", "task_runner") + assert "test-key" not in default_options + no_options = airflow_cfg.get_options_including_defaults("test-section") + assert no_options == [] + airflow_cfg.add_section("test-section") + airflow_cfg.set("test-section", "test-key", "test-value") + test_section_options = airflow_cfg.get_options_including_defaults("test-section") + assert "test-key" in test_section_options + assert "StandardTaskRunner" == airflow_cfg.get("core", "task_runner") + airflow_cfg.add_section("core") + airflow_cfg.set("core", "new-test-key", "test-value") + airflow_cfg.set("core", "task_runner", "test-runner") + all_core_options_including_defaults = airflow_cfg.get_options_including_defaults("core") + assert "new-test-key" in all_core_options_including_defaults + assert "dags_folder" in all_core_options_including_defaults + assert "test-value" == airflow_cfg.get("core", "new-test-key") + assert "test-runner" == airflow_cfg.get("core", "task_runner") + assert len([option for option in all_core_options_including_defaults if option == "task_runner"]) == 1 + def test_sensitive_values(): from airflow.settings import conf @@ -1471,8 +1583,7 @@ def test_sensitive_values(): ("database", "sql_alchemy_engine_args"), ("core", "sql_alchemy_conn"), } - default_config = default_config_yaml() - all_keys = {(s, k) for s, v in default_config.items() for k in v.get("options")} + all_keys = {(s, k) for s, v in conf.configuration_description.items() for k in v.get("options")} suspected_sensitive = {(s, k) for (s, k) in all_keys if k.endswith(("password", "kwargs"))} exclude_list = { ("kubernetes_executor", "delete_option_kwargs"), From dff360e5610f6fc4805b710f1119b469818dad24 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 20 Jul 2023 15:12:42 +0800 Subject: [PATCH 235/533] build(pre-commit): add list of supported deferrable operators to doc (#32514) * build(pre-commit): add list of supported deferrable operators to doc * docs(providers): move providers list to apache-airflow-providers core-extension --- .../deferrable-operator-ref.rst | 26 +++ docs/exts/deferrable_operatos_list.rst.jinja2 | 31 ++++ docs/exts/operators_and_hooks_ref.py | 64 ++++++- images/breeze/output_release-management.svg | 171 ------------------ 4 files changed, 120 insertions(+), 172 deletions(-) create mode 100644 docs/apache-airflow-providers/core-extensions/deferrable-operator-ref.rst create mode 100644 docs/exts/deferrable_operatos_list.rst.jinja2 delete mode 100644 images/breeze/output_release-management.svg diff --git a/docs/apache-airflow-providers/core-extensions/deferrable-operator-ref.rst b/docs/apache-airflow-providers/core-extensions/deferrable-operator-ref.rst new file mode 100644 index 000000000000..7b40ed0119a9 --- /dev/null +++ b/docs/apache-airflow-providers/core-extensions/deferrable-operator-ref.rst @@ -0,0 +1,26 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Supported Deferrable Operators +========================================== + +List of operators that supports deferrable mode + +.. airflow-deferrable-operators:: + :tags: None + :header-separator: " diff --git a/docs/exts/deferrable_operatos_list.rst.jinja2 b/docs/exts/deferrable_operatos_list.rst.jinja2 new file mode 100644 index 000000000000..e6eeb66385cf --- /dev/null +++ b/docs/exts/deferrable_operatos_list.rst.jinja2 @@ -0,0 +1,31 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} +{%- for provider in providers %} +**{{ provider["name"] }}** + +.. list-table:: + :header-rows: 1 + + * - Modules + - Operators + {% for module, operator in provider["operators"] %} + * - :mod:`{{ module }}` + - :mod:`{{ operator}}` + {% endfor %} +{% endfor %} diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index cc47cb46fe69..eeaf00726c8a 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -16,12 +16,15 @@ # under the License. from __future__ import annotations +import ast import os from functools import lru_cache -from typing import Iterable +from pathlib import Path +from typing import Any, Iterable, Iterator import jinja2 import rich_click as click +import yaml from docutils import nodes from docutils.nodes import Element @@ -172,6 +175,47 @@ def _render_transfer_content(*, tags: set[str] | None, header_separator: str): ) +def iter_deferrable_operators(module_filename: str) -> Iterator[tuple[str, str]]: + ast_obj = ast.parse(open(module_filename).read()) + cls_nodes = (node for node in ast.iter_child_nodes(ast_obj) if isinstance(node, ast.ClassDef)) + init_method_nodes = ( + (cls_node, node) + for cls_node in cls_nodes + for node in ast.iter_child_nodes(cls_node) + if isinstance(node, ast.FunctionDef) and node.name == "__init__" + ) + for cls_node, node in init_method_nodes: + args = node.args + for argument in [*args.args, *args.kwonlyargs]: + if argument.arg == "deferrable": + module_name = module_filename.replace("/", ".")[:-3] + op_name = cls_node.name + yield (module_name, op_name) + + +def _render_deferrable_operator_content(*, header_separator: str): + providers = [] + for provider_yaml_path in get_provider_yaml_paths(): + provider_parent_path = Path(provider_yaml_path).parent + provider_info: dict[str, Any] = {"name": "", "operators": []} + for root, _, file_names in os.walk(provider_parent_path): + if all([target not in root for target in ["operators", "sensors"]]): + continue + + for file_name in file_names: + if not file_name.endswith(".py") or file_name == "__init__.py": + continue + provider_info["operators"].extend( + iter_deferrable_operators(f"{os.path.relpath(root)}/{file_name}") + ) + + if provider_info["operators"]: + provider_yaml_content = yaml.safe_load(Path(provider_yaml_path).read_text()) + provider_info["name"] = provider_yaml_content["package-name"] + providers.append(provider_info) + return _render_template("deferrable_operatos_list.rst.jinja2", providers=providers) + + class BaseJinjaReferenceDirective(Directive): """The base directive for OperatorsHooksReferenceDirective and TransfersReferenceDirective""" @@ -325,6 +369,15 @@ def render_content( ) +class DeferrableOperatorDirective(BaseJinjaReferenceDirective): + """Generate list of deferrable operators""" + + def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): + return _render_deferrable_operator_content( + header_separator=header_separator, + ) + + def setup(app): """Setup plugin""" app.add_directive("operators-hooks-ref", OperatorsHooksReferenceDirective) @@ -336,6 +389,7 @@ def setup(app): app.add_directive("airflow-extra-links", ExtraLinksDirective) app.add_directive("airflow-notifications", NotificationsDirective) app.add_directive("airflow-executors", ExecutorsDirective) + app.add_directive("airflow-deferrable-operators", DeferrableOperatorDirective) return {"parallel_read_safe": True, "parallel_write_safe": True} @@ -433,5 +487,13 @@ def extra_links(header_separator: str): ) +@cli.command() +@option_tag +@option_header_separator +def deferrable_operators(tag: Iterable[str], header_separator: str): + """Renders Deferrable Operators content""" + print(_render_deferrable_operator_content(header_separator=header_separator)) + + if __name__ == "__main__": cli() diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg deleted file mode 100644 index b9b424c01833..000000000000 --- a/images/breeze/output_release-management.svg +++ /dev/null @@ -1,171 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: release-management - - - - - - - - - - -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... - -Tools that release managers can use to prepare and manage Airflow releases - -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Airflow release commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│prepare-airflow-package      Prepare sdist/whl package of Airflow.                                                │ -│create-minor-branch          Create a new version branch and update the default branches in main                  │ -│start-rc-process             Start RC process                                                                     │ -│start-release                Start Airflow release process                                                        │ -│release-prod-images          Release production images to DockerHub (needs DockerHub permissions).                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Providers release commands ─────────────────────────────────────────────────────────────────────────────────────────╮ -│prepare-provider-documentation      Prepare CHANGELOG, README and COMMITS information for providers.              │ -│prepare-provider-packages           Prepare sdist/whl packages of Airflow Providers.                              │ -│install-provider-packages           Installs provider packages that can be found in dist.                         │ -│verify-provider-packages            Verifies if all provider code is following expectations for providers.        │ -│generate-providers-metadata         Generates metadata for providers.                                             │ -│generate-issue-content-providers    Generates content for issue to test the release.                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│publish-docs            Command to publish generated documentation to airflow-site                                │ -│generate-constraints    Generates pinned constraint files with all extras from setup.py in parallel.              │ -│add-back-references     Command to add back references for documentation to make it backward compatible           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - From eea53a22c92bdcb8ef532e41d3a912f05b5736c0 Mon Sep 17 00:00:00 2001 From: Maham Ali <50532268+mahammi@users.noreply.github.com> Date: Thu, 20 Jul 2023 00:16:53 -0700 Subject: [PATCH 236/533] Add Eventbridge PutEvents operator and hook (#32498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Eventbridge PutEvents operator and hook --------- Co-authored-by: Maham Ali Co-authored-by: Raphaël Vandon --- .../providers/amazon/aws/hooks/eventbridge.py | 27 ++++++ .../amazon/aws/operators/eventbridge.py | 87 ++++++++++++++++++ airflow/providers/amazon/provider.yaml | 12 +++ .../operators/eventbridge.rst | 53 +++++++++++ .../aws/Amazon-EventBridge_64.png | Bin 0 -> 3221 bytes .../amazon/aws/hooks/test_eventbridge.py | 28 ++++++ .../amazon/aws/operators/test_eventbridge.py | 73 +++++++++++++++ .../amazon/aws/example_eventbridge.py | 52 +++++++++++ 8 files changed, 332 insertions(+) create mode 100644 airflow/providers/amazon/aws/hooks/eventbridge.py create mode 100644 airflow/providers/amazon/aws/operators/eventbridge.py create mode 100644 docs/apache-airflow-providers-amazon/operators/eventbridge.rst create mode 100644 docs/integration-logos/aws/Amazon-EventBridge_64.png create mode 100644 tests/providers/amazon/aws/hooks/test_eventbridge.py create mode 100644 tests/providers/amazon/aws/operators/test_eventbridge.py create mode 100644 tests/system/providers/amazon/aws/example_eventbridge.py diff --git a/airflow/providers/amazon/aws/hooks/eventbridge.py b/airflow/providers/amazon/aws/hooks/eventbridge.py new file mode 100644 index 000000000000..d9450ae7e6c6 --- /dev/null +++ b/airflow/providers/amazon/aws/hooks/eventbridge.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook + + +class EventBridgeHook(AwsBaseHook): + """Amazon EventBridge Hook.""" + + def __init__(self, *args, **kwargs): + """Creating object.""" + super().__init__(client_type="events", *args, **kwargs) diff --git a/airflow/providers/amazon/aws/operators/eventbridge.py b/airflow/providers/amazon/aws/operators/eventbridge.py new file mode 100644 index 000000000000..ceeab782a706 --- /dev/null +++ b/airflow/providers/amazon/aws/operators/eventbridge.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from functools import cached_property +from typing import TYPE_CHECKING, Sequence + +from airflow import AirflowException +from airflow.models import BaseOperator +from airflow.providers.amazon.aws.hooks.eventbridge import EventBridgeHook +from airflow.providers.amazon.aws.utils import trim_none_values + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class EventBridgePutEventsOperator(BaseOperator): + """ + Put Events onto Amazon EventBridge. + + :param entries: the list of events to be put onto EventBridge, each event is a dict (required) + :param endpoint_id: the URL subdomain of the endpoint + :param aws_conn_id: the AWS connection to use + :param region_name: the region where events are to be sent + + """ + + template_fields: Sequence[str] = ("entries", "endpoint_id", "aws_conn_id", "region_name") + + def __init__( + self, + *, + entries: list[dict], + endpoint_id: str | None = None, + aws_conn_id: str = "aws_default", + region_name: str | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.entries = entries + self.endpoint_id = endpoint_id + self.aws_conn_id = aws_conn_id + self.region_name = region_name + + @cached_property + def hook(self) -> EventBridgeHook: + """Create and return an EventBridgeHook.""" + return EventBridgeHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) + + def execute(self, context: Context): + + response = self.hook.conn.put_events( + **trim_none_values( + { + "Entries": self.entries, + "EndpointId": self.endpoint_id, + } + ) + ) + + self.log.info("Sent %d events to EventBridge.", len(self.entries)) + + if response.get("FailedEntryCount"): + for event in response["Entries"]: + if "ErrorCode" in event: + self.log.error(event) + + raise AirflowException( + f"{response['FailedEntryCount']} entries in this request have failed to send." + ) + + if self.do_xcom_push: + return [e["EventId"] for e in response["Entries"]] diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index e8ba23afc4d9..ff7ed915255a 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -154,6 +154,12 @@ integrations: - /docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png tags: [aws] + - integration-name: Amazon EventBridge + external-doc-url: https://docs.aws.amazon.com/eventbridge/latest/APIReference/Welcome.html + how-to-guide: + - /docs/apache-airflow-providers-amazon/operators/eventbridge.rst + logo: /integration-logos/aws/Amazon-EventBridge_64.png + tags: [aws] - integration-name: Amazon Glacier external-doc-url: https://aws.amazon.com/glacier/ logo: /integration-logos/aws/Amazon-S3-Glacier_light-bg@4x.png @@ -307,6 +313,9 @@ operators: - integration-name: Amazon EMR on EKS python-modules: - airflow.providers.amazon.aws.operators.emr + - integration-name: Amazon EventBridge + python-modules: + - airflow.providers.amazon.aws.operators.eventbridge - integration-name: Amazon Glacier python-modules: - airflow.providers.amazon.aws.operators.glacier @@ -457,6 +466,9 @@ hooks: - integration-name: Amazon EMR on EKS python-modules: - airflow.providers.amazon.aws.hooks.emr + - integration-name: Amazon EventBridge + python-modules: + - airflow.providers.amazon.aws.hooks.eventbridge - integration-name: Amazon Glacier python-modules: - airflow.providers.amazon.aws.hooks.glacier diff --git a/docs/apache-airflow-providers-amazon/operators/eventbridge.rst b/docs/apache-airflow-providers-amazon/operators/eventbridge.rst new file mode 100644 index 000000000000..ce0c449f056d --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/eventbridge.rst @@ -0,0 +1,53 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +======================================== +Amazon EventBridge +======================================== + +`Amazon Eventbridge `__ is a serverless event bus service that makes it easy +to connect your applications with data from a variety of sources. EventBridge delivers a stream of real-time data from +your own applications, software-as-a-service (SaaS) applications, and AWS services and routes that data to targets such +as AWS Lambda. You can set up routing rules to determine where to send your data to build application architectures +that react in real time to all of your data sources. EventBridge enables you to build event-driven architectures +that are loosely coupled and distributed. + +Prerequisite Tasks +------------------ + +.. include:: ../_partials/prerequisite_tasks.rst + +Operators +--------- + +Send events to Amazon EventBridge +========================================== + +To send custom events to Amazon EventBridge, use +:class:`~airflow.providers.amazon.aws.operators.eventbridge.EventBridgePutEventsOperator`. + +.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_eventbridge.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_eventbridge_put_events] + :end-before: [END howto_operator_eventbridge_put_events] + + +Reference +--------- + +* `AWS boto3 library documentation for EventBridge `__ diff --git a/docs/integration-logos/aws/Amazon-EventBridge_64.png b/docs/integration-logos/aws/Amazon-EventBridge_64.png new file mode 100644 index 0000000000000000000000000000000000000000..51085d2c2648201ad14fafb4e696a0e6410bad3e GIT binary patch literal 3221 zcmV;G3~KXPx#L}ge>W=%~1DgXcg2mk?xX#fNO00031000^Q000001E2u_0{{R30RRC20H6W@ z1ONa40RR91P@n?<1ONa40RR91Pyhe`05RZ9PXGW5|4BqaRCodHTM3jD)fMgPs@}I* z7+^q!k!3&_HemoG%Ndbf0wO9=g99NR!ih!?o8ktD0tNyik;6&kU}D4r5+HyAB5Pz5 zlzow~h%7VCFf+YQFV&S>6zM5`S5;SaH$hdcZ=6rxYokA*%zo?(zlsGZ|tg_(32P@iG3k1I0u$=Bz) zxT-+t>oeYWzlxmvDP8>2#fM5u@?h-NBxAv2`uYyBSX?mTnYP})Xr~x>)7xpVFrDkn zTs)j|nuA|`@0+tFeDpGXcZTIx_uEEyb-p!#JzXCB^5jj|GbTV9_ONs2BhWUm{6NK% zOG10jao%9$Sc&Jgb?ynPF^y}^5Xx+b&3DrqsajZ{X)!&VuZu)IudE@I@qe_}GNRB> zkZbJGB)I-KfkjuJV~+CEN;g%7>AY+mt&beN9IXt*%Mspl4hl0(>ICC) z6jk@gq2!fSvGjTPjaF|fS% z2f8R%15crqs=|>|W$?yyUKZUbi#>NmED2?#jSuDl5;<~N48PXqxYAZLB;Xg0{_@2!X)2J)oEda1wPrYqbS$>M=7hZMi7d!zuE_1A#$QLwCijQTuX~T z*ru$<2*)cAQ+AV~0JV0pq_a1S z;UcVi+-gJnhWh$WuEHPMaVD_xpx{9?F-(ub{FHhMJI&Om1zpd9&CFh`vW@M=1;S+y z%w;cBX_6_P*foEs<%#wLJbdtC)!5}&0t8&Sn?8l(=E+$2xcSjG)vFFwj{Y2U(@%Bvf9L^Ig>;9gm51NKOEGK(--+AkZPKJU8*yQZ_ zN;$HGaL1c{&3#*jzrAo}z#>GcFll)AxxmN!n5K2G+6^u11(to6kV%XnRWBW56yIPk zxxI6?0PcKeAcB*40yPkLe(SS^IE1UlE{j(9V?*G;t6o|eE%zCFG-X=VkHHf*l9VGM zUxC2C@Zap2a;!#caiPi~1R2w!E|xS-d;Ae7&Rre)`YeEkb`1%XLiffh%!OF@tLrem zePS34a^BgSQ8OUm>{&i~!y!)@8N}5D*3jWDKm-~BCD%>T!j38ZkQJB*wU*-> zgF_~!WxZHewxOr&FZP7vB?#CbE8+Yoo6$a@h(K8$YiDSk8?6Y0cAO?q3f*KZZ0gz6 z)H@$r$UEap1=c)@qMUCJ#J{}tW^Pg-cFuUjI5eb;^BUT4Gbge3^c(CCv92J_p3iYhSLbzB7j3-05nM)@7fgROOq2G z!c6Yh*C9_J#3H)3-*-c~E4=^5z-I>tlw{YxcsHt?2q$bKJI52iGgXfGxTMjP(){Tq zji**V$474v1|r21D?qd<%ukSM32i+U+;BpHDNyuX8Md(x>Nw7g(hsLDcSoHF-%+Qt zjVsb;S*7_$Id~~BgaE-Uryz`A0BK?=s0M}MLiwO45*3bCDQ-0|$LoDi?tpD!;TmY7 z7V4hZz?y5C$T}`*ww~8WiSt7o&%hNSL4)3mcgS7htDh_bSXSiQ5Fw=R z!`+CzozoMm;hnV!A_=}-TvkL z#cN(#nN$XAACI%ZiV7Bv0~wNQVBSlQFG5EUPnlT2s#O@y6E2iLhbJcTpB(3zycXOM z4B{MCBZQa+=P;j3vZ~GkJ_*aP2)?5>9UdE7#SaFv1`NLpTPUZzcAhSYupuScs+N=8lI8 z{x({Lv_vV>5C-xgJ&wt%@mV}8If%BJ7NQ8t)17cah7Vn$8)jG#?)T1T8fUR39?pB+ zl^1d;WU$5u@(lOYNBv1|`s?6~KJ%lekqIFIN|^DXJZgWbC!Bd@@h4n$DO zHPQf6uYADoDKxg68WLiB_z6k9+E6Gg`s9{M9ERc@#4T?hX+5gvFI0jMEK#>J^-^k#^ z)8NwxkrsBcP^l4Pi8iFa=5N)cC8N92_hw&>w6x`hn3N$VPSC{$o;?^eflt46k6(>l zD$v8!nM_%6n=VL+kvSmyUqYaUd< z2h!Sq+LpATEng6={?}nbTL1$q4k8-E-4AX_#bV$S_)#iIm?2&z-~$_%Sv*{_jfgTi zZNW|3lExhe#zZel+!etsrZnDyC6Yz@Q`dV(-v{8$0P>xsaoxKLGIkz;n+6C2`L|YAZw66AOy0X32U07aS~1Elz!&^E%}J8 z|03M<<2eJaTB?^HpbRu}FJtFM@R~YSU7=foeVYDf8t!{MZ*K&D#~Xb#-PRH!Axg6h z?*JeYCh!zvJ+n6P_Y%!=P}~#4wYIfIS_<3)OZOwD$D@m!gA;X;1h+7yt!GLGn{ZAz z1>mA3mPsUvyJ+5-wayK0I;kLuGAeBfBZGBMT9dVMRKhBsl0d>W+W(R;f>KD%C`McC zKrl(4!a5z^yp}ef4hKWbA<>n|CD&fMyW*o94)L;^yJ8Xc6&HEHPnx)RapJE zx-}_lEqrU*O}*5$7FPeQZcWNs3*VY{Q!jO`h1GwnTa)s?^ik1&-oSOe00000NkvXX Hu0mjfBl_Mg literal 0 HcmV?d00001 diff --git a/tests/providers/amazon/aws/hooks/test_eventbridge.py b/tests/providers/amazon/aws/hooks/test_eventbridge.py new file mode 100644 index 000000000000..680686854eec --- /dev/null +++ b/tests/providers/amazon/aws/hooks/test_eventbridge.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from moto import mock_events + +from airflow.providers.amazon.aws.hooks.eventbridge import EventBridgeHook + + +@mock_events +class TestEventBridgeHook: + def test_conn_returns_a_boto3_connection(self): + hook = EventBridgeHook(aws_conn_id="aws_default") + assert hook.conn is not None diff --git a/tests/providers/amazon/aws/operators/test_eventbridge.py b/tests/providers/amazon/aws/operators/test_eventbridge.py new file mode 100644 index 000000000000..b09a1817b9a5 --- /dev/null +++ b/tests/providers/amazon/aws/operators/test_eventbridge.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import MagicMock + +import pytest + +from airflow import AirflowException +from airflow.providers.amazon.aws.hooks.eventbridge import EventBridgeHook +from airflow.providers.amazon.aws.operators.eventbridge import EventBridgePutEventsOperator + +TASK_ID = "events_putevents_job" +ENTRIES = [{"Detail": "test-detail", "Source": "test-source", "DetailType": "test-detail-type"}] +FAILED_ENTRIES_RESPONSE = [{"ErrorCode": "test_code"}, {"ErrorCode": "test_code"}] + + +class TestEventBridgePutEventsOperator: + def test_init(self): + operator = EventBridgePutEventsOperator( + task_id=TASK_ID, + entries=ENTRIES, + ) + + assert operator.entries == ENTRIES + + @mock.patch.object(EventBridgeHook, "conn") + def test_execute(self, mock_conn: MagicMock): + hook_response = {"FailedEntryCount": 0, "Entries": [{"EventId": "foobar"}]} + + mock_conn.put_events.return_value = hook_response + + operator = EventBridgePutEventsOperator( + task_id=TASK_ID, + entries=ENTRIES, + ) + + result = operator.execute(None) + + assert result == ["foobar"] + + @mock.patch.object(EventBridgeHook, "conn") + def test_failed_to_send(self, mock_conn: MagicMock): + + hook_response = { + "FailedEntryCount": 1, + "Entries": FAILED_ENTRIES_RESPONSE, + } + + mock_conn.put_events.return_value = hook_response + + operator = EventBridgePutEventsOperator( + task_id=TASK_ID, + entries=ENTRIES, + ) + + with pytest.raises(AirflowException): + operator.execute(None) diff --git a/tests/system/providers/amazon/aws/example_eventbridge.py b/tests/system/providers/amazon/aws/example_eventbridge.py new file mode 100644 index 000000000000..f6208108c967 --- /dev/null +++ b/tests/system/providers/amazon/aws/example_eventbridge.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime + +from airflow import DAG +from airflow.providers.amazon.aws.operators.eventbridge import EventBridgePutEventsOperator + +DAG_ID = "example_eventbridge" +ENTRIES = [ + { + "Detail": '{"event-name": "custom-event"}', + "EventBusName": "custom-bus", + "Source": "example.myapp", + "DetailType": "Sample Custom Event", + } +] + +with DAG( + dag_id=DAG_ID, + schedule="@once", + start_date=datetime(2021, 1, 1), + tags=["example"], + catchup=False, +) as dag: + + # [START howto_operator_eventbridge_put_events] + + put_events = EventBridgePutEventsOperator(task_id="put_events_task", entries=ENTRIES) + + # [END howto_operator_eventbridge_put_events] + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 8b5da2134d287abbe6b9e733e3613339e5f37b1f Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:45:21 +0530 Subject: [PATCH 237/533] Update Dag trigger API and command docs (#32696) trigger dagrun API and CLI command does not execute task if dag is paused. document the same in API docs and CLI command docs Co-authored-by: Wei Lee --- airflow/api_connexion/openapi/v1.yaml | 5 ++++- airflow/cli/cli_config.py | 5 ++++- airflow/www/static/js/types/api-generated.ts | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 8c2b4b191202..d69bbf0b9c8f 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -735,7 +735,10 @@ paths: $ref: '#/components/responses/Unauthenticated' post: - summary: Trigger a new DAG run + summary: Trigger a new DAG run. + description: > + This will initiate a dagrun. + If DAG is paused then dagrun state will remain queued, and the task won't run. x-openapi-router-controller: airflow.api_connexion.endpoints.dag_run_endpoint operationId: post_dag_run tags: [DAGRun] diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 79327a4701ac..b661f9361ab8 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -1236,7 +1236,10 @@ class GroupCommand(NamedTuple): ), ActionCommand( name="trigger", - help="Trigger a DAG run", + help=( + "Trigger a new DAG run. If DAG is paused then dagrun state will remain queued, " + "and the task won't run." + ), func=lazy_load_command("airflow.cli.commands.dag_command.dag_trigger"), args=( ARG_DAG_ID, diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index 9d60b6de8fc0..5387da0cc13b 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -143,6 +143,7 @@ export interface paths { "/dags/{dag_id}/dagRuns": { /** This endpoint allows specifying `~` as the dag_id to retrieve DAG runs for all DAGs. */ get: operations["get_dag_runs"]; + /** This will initiate a dagrun. If DAG is paused then dagrun state will remain queued, and the task won't run. */ post: operations["post_dag_run"]; parameters: { path: { @@ -2931,6 +2932,7 @@ export interface operations { 401: components["responses"]["Unauthenticated"]; }; }; + /** This will initiate a dagrun. If DAG is paused then dagrun state will remain queued, and the task won't run. */ post_dag_run: { parameters: { path: { From 27b5f696a48a088a23294c542acb46bd6e544809 Mon Sep 17 00:00:00 2001 From: Syed Hussain <103602455+syedahsn@users.noreply.github.com> Date: Thu, 20 Jul 2023 07:15:58 -0700 Subject: [PATCH 238/533] Add Deferrable mode for EMR Serverless Start Job Operator (#32534) * Add Deferrable mode for EMR Serverless Start Job Operator --- airflow/providers/amazon/aws/operators/emr.py | 52 +++++++++++-- airflow/providers/amazon/aws/triggers/emr.py | 74 ++++++++++++++++++- .../operators/emr/emr_serverless.rst | 2 + .../aws/operators/test_emr_serverless.py | 41 +++++++++- .../aws/triggers/test_emr_serverless.py | 63 ++++++++++++++++ 5 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 tests/providers/amazon/aws/triggers/test_emr_serverless.py diff --git a/airflow/providers/amazon/aws/operators/emr.py b/airflow/providers/amazon/aws/operators/emr.py index 4cb070da733b..c01dbbbe91d6 100644 --- a/airflow/providers/amazon/aws/operators/emr.py +++ b/airflow/providers/amazon/aws/operators/emr.py @@ -33,6 +33,8 @@ EmrAddStepsTrigger, EmrContainerTrigger, EmrCreateJobFlowTrigger, + EmrServerlessStartApplicationTrigger, + EmrServerlessStartJobTrigger, EmrTerminateJobFlowTrigger, ) from airflow.providers.amazon.aws.utils.waiter import waiter @@ -1107,6 +1109,9 @@ class EmrServerlessStartJobOperator(BaseOperator): :waiter_max_attempts: Number of times the waiter should poll the application to check the state. If not set, the waiter will use its default value. :param waiter_delay: Number of seconds between polling the state of the job run. + :param deferrable: If True, the operator will wait asynchronously for the crawl to complete. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False, but can be overridden in config file by setting default_deferrable to True) """ template_fields: Sequence[str] = ( @@ -1137,6 +1142,7 @@ def __init__( waiter_check_interval_seconds: int | ArgNotSet = NOTSET, waiter_max_attempts: int | ArgNotSet = NOTSET, waiter_delay: int | ArgNotSet = NOTSET, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if waiter_check_interval_seconds is NOTSET: @@ -1171,6 +1177,7 @@ def __init__( self.waiter_max_attempts = int(waiter_max_attempts) # type: ignore[arg-type] self.waiter_delay = int(waiter_delay) # type: ignore[arg-type] self.job_id: str | None = None + self.deferrable = deferrable super().__init__(**kwargs) self.client_request_token = client_request_token or str(uuid4()) @@ -1180,14 +1187,25 @@ def hook(self) -> EmrServerlessHook: """Create and return an EmrServerlessHook.""" return EmrServerlessHook(aws_conn_id=self.aws_conn_id) - def execute(self, context: Context) -> str | None: - self.log.info("Starting job on Application: %s", self.application_id) + def execute(self, context: Context, event: dict[str, Any] | None = None) -> str | None: app_state = self.hook.conn.get_application(applicationId=self.application_id)["application"]["state"] if app_state not in EmrServerlessHook.APPLICATION_SUCCESS_STATES: + self.log.info("Application state is %s", app_state) + self.log.info("Starting application %s", self.application_id) self.hook.conn.start_application(applicationId=self.application_id) waiter = self.hook.get_waiter("serverless_app_started") - + if self.deferrable: + self.defer( + trigger=EmrServerlessStartApplicationTrigger( + application_id=self.application_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + ), + method_name="execute", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) wait( waiter=waiter, waiter_max_attempts=self.waiter_max_attempts, @@ -1197,7 +1215,7 @@ def execute(self, context: Context) -> str | None: status_message="Serverless Application status is", status_args=["application.state", "application.stateDetails"], ) - + self.log.info("Starting job on Application: %s", self.application_id) response = self.hook.conn.start_job_run( clientToken=self.client_request_token, applicationId=self.application_id, @@ -1213,6 +1231,18 @@ def execute(self, context: Context) -> str | None: self.job_id = response["jobRunId"] self.log.info("EMR serverless job started: %s", self.job_id) + if self.deferrable: + self.defer( + trigger=EmrServerlessStartJobTrigger( + application_id=self.application_id, + job_id=self.job_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + ), + method_name="execute_complete", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) if self.wait_for_completion: waiter = self.hook.get_waiter("serverless_job_completed") wait( @@ -1227,8 +1257,20 @@ def execute(self, context: Context) -> str | None: return self.job_id + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "success": + self.log.info("Serverless job completed") + return event["job_id"] + def on_kill(self) -> None: - """Cancel the submitted job run.""" + """ + Cancel the submitted job run. + + Note: this method will not run in deferrable mode. + """ if self.job_id: self.log.info("Stopping job run with jobId - %s", self.job_id) response = self.hook.conn.cancel_job_run(applicationId=self.application_id, jobRunId=self.job_id) diff --git a/airflow/providers/amazon/aws/triggers/emr.py b/airflow/providers/amazon/aws/triggers/emr.py index 7deadc1f3746..471a8a747a08 100644 --- a/airflow/providers/amazon/aws/triggers/emr.py +++ b/airflow/providers/amazon/aws/triggers/emr.py @@ -24,7 +24,7 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook -from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook +from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.triggers.base import BaseTrigger, TriggerEvent @@ -283,3 +283,75 @@ def __init__( def hook(self) -> AwsGenericHook: return EmrHook(self.aws_conn_id) + + +class EmrServerlessStartApplicationTrigger(AwsBaseWaiterTrigger): + """ + Poll an Emr Serverless application and wait for it to be started. + + :param application_id: The ID of the application being polled. + :waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made + :param aws_conn_id: Reference to AWS connection id + """ + + def __init__( + self, + application_id: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, + aws_conn_id: str = "aws_default", + ): + super().__init__( + serialized_fields={"application_id": application_id}, + waiter_name="serverless_app_started", + waiter_args={"applicationId": application_id}, + failure_message="Application failed to start", + status_message="Application status is", + status_queries=["application.state", "application.stateDetails"], + return_key="application_id", + return_value=application_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return EmrServerlessHook(self.aws_conn_id) + + +class EmrServerlessStartJobTrigger(AwsBaseWaiterTrigger): + """ + Poll an Emr Serverless job run and wait for it to be completed. + + :param application_id: The ID of the application the job in being run on. + :param job_id: The ID of the job run. + :waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made + :param aws_conn_id: Reference to AWS connection id + """ + + def __init__( + self, + application_id: str, + job_id: str | None, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, + aws_conn_id: str = "aws_default", + ) -> None: + super().__init__( + serialized_fields={"application_id": application_id, "job_id": job_id}, + waiter_name="serverless_job_completed", + waiter_args={"applicationId": application_id, "jobRunId": job_id}, + failure_message="Serverless Job failed", + status_message="Serverless Job status is", + status_queries=["jobRun.state", "jobRun.stateDetails"], + return_key="job_id", + return_value=job_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return EmrServerlessHook(self.aws_conn_id) diff --git a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst index 4f74690da00c..76638815b8da 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst @@ -54,6 +54,8 @@ Start an EMR Serverless Job You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrServerlessStartJobOperator` to start an EMR Serverless Job. +This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires +the aiobotocore module to be installed. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py :language: python diff --git a/tests/providers/amazon/aws/operators/test_emr_serverless.py b/tests/providers/amazon/aws/operators/test_emr_serverless.py index 8cb4eb1707b5..2d9f37830fd0 100644 --- a/tests/providers/amazon/aws/operators/test_emr_serverless.py +++ b/tests/providers/amazon/aws/operators/test_emr_serverless.py @@ -23,7 +23,7 @@ import pytest from botocore.exceptions import WaiterError -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, TaskDeferred from airflow.providers.amazon.aws.hooks.emr import EmrServerlessHook from airflow.providers.amazon.aws.operators.emr import ( EmrServerlessCreateApplicationOperator, @@ -730,6 +730,45 @@ def test_start_job_waiter_params( assert operator.waiter_delay == expected[0] assert operator.waiter_max_attempts == expected[1] + @mock.patch.object(EmrServerlessHook, "conn") + def test_start_job_deferrable(self, mock_conn): + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + configuration_overrides=configuration_overrides, + deferrable=True, + ) + + with pytest.raises(TaskDeferred): + operator.execute(None) + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + def test_start_job_deferrable_app_not_started(self, mock_conn, mock_get_waiter): + mock_get_waiter.return_value = True + mock_conn.get_application.return_value = {"application": {"state": "CREATING"}} + mock_conn.start_application.return_value = { + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + configuration_overrides=configuration_overrides, + deferrable=True, + ) + + with pytest.raises(TaskDeferred): + operator.execute(None) + class TestEmrServerlessDeleteOperator: @mock.patch.object(EmrServerlessHook, "get_waiter") diff --git a/tests/providers/amazon/aws/triggers/test_emr_serverless.py b/tests/providers/amazon/aws/triggers/test_emr_serverless.py new file mode 100644 index 000000000000..029fc4ccbf73 --- /dev/null +++ b/tests/providers/amazon/aws/triggers/test_emr_serverless.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.amazon.aws.triggers.emr import ( + EmrServerlessStartApplicationTrigger, + EmrServerlessStartJobTrigger, +) + +TEST_APPLICATION_ID = "test-application-id" +TEST_WAITER_DELAY = 10 +TEST_WAITER_MAX_ATTEMPTS = 10 +TEST_AWS_CONN_ID = "test-aws-id" +AWS_CONN_ID = "aws_emr_conn" +TEST_JOB_ID = "test-job-id" + + +class TestEmrTriggers: + @pytest.mark.parametrize( + "trigger", + [ + EmrServerlessStartApplicationTrigger( + application_id=TEST_APPLICATION_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + EmrServerlessStartJobTrigger( + application_id=TEST_APPLICATION_ID, + job_id=TEST_JOB_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 From 848c69a194c03ed3a5badc909e26b5c1bda03050 Mon Sep 17 00:00:00 2001 From: Freddy Demiane Date: Thu, 20 Jul 2023 16:16:32 +0200 Subject: [PATCH 239/533] Refresh GKE OAuth2 tokens (#32673) * Refresh token for sync mode --- .../google/cloud/hooks/kubernetes_engine.py | 4 ++ .../cloud/hooks/test_kubernetes_engine.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/airflow/providers/google/cloud/hooks/kubernetes_engine.py b/airflow/providers/google/cloud/hooks/kubernetes_engine.py index d8b1d92ffae6..00df2b9b28d9 100644 --- a/airflow/providers/google/cloud/hooks/kubernetes_engine.py +++ b/airflow/providers/google/cloud/hooks/kubernetes_engine.py @@ -387,8 +387,12 @@ def get_xcom_sidecar_container_resources(self): def get_conn(self) -> client.ApiClient: configuration = self._get_config() + configuration.refresh_api_key_hook = self._refresh_api_key_hook return client.ApiClient(configuration) + def _refresh_api_key_hook(self, configuration: client.configuration.Configuration): + configuration.api_key = {"authorization": self._get_token(self.get_credentials())} + def _get_config(self) -> client.configuration.Configuration: configuration = client.Configuration( host=self._cluster_url, diff --git a/tests/providers/google/cloud/hooks/test_kubernetes_engine.py b/tests/providers/google/cloud/hooks/test_kubernetes_engine.py index 53963c3a329f..a3f0c59c1219 100644 --- a/tests/providers/google/cloud/hooks/test_kubernetes_engine.py +++ b/tests/providers/google/cloud/hooks/test_kubernetes_engine.py @@ -30,6 +30,7 @@ GKEAsyncHook, GKEHook, GKEPodAsyncHook, + GKEPodHook, ) from airflow.providers.google.common.consts import CLIENT_INFO from tests.providers.google.cloud.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id @@ -397,3 +398,48 @@ async def test_get_operation(self, mock_get_client, async_gke_hook, mock_async_g mock_async_gke_cluster_client.get_operation.assert_called_once_with( name=operation_path, ) + + +class TestGKEPodHook: + def setup_method(self): + with mock.patch( + BASE_STRING.format("GoogleBaseHook.__init__"), new=mock_base_gcp_hook_default_project_id + ): + self.gke_hook = GKEPodHook(gcp_conn_id="test", ssl_ca_cert=None, cluster_url=None) + self.gke_hook._client = mock.Mock() + + def refresh_token(request): + self.credentials.token = "New" + + self.credentials = mock.MagicMock() + self.credentials.token = "Old" + self.credentials.expired = False + self.credentials.refresh = refresh_token + + @mock.patch(GKE_STRING.format("google_requests.Request")) + def test_get_connection_update_hook_with_invalid_token(self, mock_request): + self.gke_hook._get_config = self._get_config + self.gke_hook.get_credentials = self._get_credentials + self.gke_hook.get_credentials().expired = True + the_client: kubernetes.client.ApiClient = self.gke_hook.get_conn() + + the_client.configuration.refresh_api_key_hook(the_client.configuration) + + assert self.gke_hook.get_credentials().token == "New" + + @mock.patch(GKE_STRING.format("google_requests.Request")) + def test_get_connection_update_hook_with_valid_token(self, mock_request): + self.gke_hook._get_config = self._get_config + self.gke_hook.get_credentials = self._get_credentials + self.gke_hook.get_credentials().expired = False + the_client: kubernetes.client.ApiClient = self.gke_hook.get_conn() + + the_client.configuration.refresh_api_key_hook(the_client.configuration) + + assert self.gke_hook.get_credentials().token == "Old" + + def _get_config(self): + return kubernetes.client.configuration.Configuration() + + def _get_credentials(self): + return self.credentials From 8b7ae760261109f1bfa6c60abbbc9803bd93bb74 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:26:40 +0200 Subject: [PATCH 240/533] Fixup docs and optimize system test for DataprocSubmitJobOperator (Hadoop job) (#32722) --- .../google/cloud/operators/dataproc.py | 8 ++++- .../cloud/dataproc/example_dataproc_hadoop.py | 32 ++++++------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index bac0c4eb9bf6..3fa6ae1e771a 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1424,6 +1424,10 @@ def execute(self, context: Context): class DataprocSubmitHadoopJobOperator(DataprocJobBaseOperator): """Start a Hadoop Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + :param main_jar: The HCFS URI of the jar file containing the main class (use this or the main_class, not both together). :param main_class: Name of the job class. (use this or the main_jar, not both @@ -1931,7 +1935,9 @@ class DataprocSubmitJobOperator(GoogleCloudBaseOperator): :param region: Required. The Cloud Dataproc region in which to handle the request. :param job: Required. The job resource. If a dict is provided, it must be of the same form as the protobuf message - :class:`~google.cloud.dataproc_v1.types.Job` + :class:`~google.cloud.dataproc_v1.types.Job`. + For the complete list of supported job types please take a look here + https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs :param request_id: Optional. A unique id used to identify the request. If the server receives two ``SubmitJobRequest`` requests with the same id, then the second request will be ignored and the first ``Job`` created and stored in the backend is returned. diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py index 1eb030717884..7af2654bfed7 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py @@ -28,7 +28,6 @@ DataprocCreateClusterOperator, DataprocDeleteClusterOperator, DataprocSubmitJobOperator, - DataprocUpdateClusterOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.utils.trigger_rule import TriggerRule @@ -53,20 +52,12 @@ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 1024}, }, "worker_config": { - "num_instances": 2, + "num_instances": 3, "machine_type_uri": "n1-standard-4", "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 1024}, }, } -# Update options -CLUSTER_UPDATE = { - "config": {"worker_config": {"num_instances": 3}, "secondary_worker_config": {"num_instances": 3}} -} -UPDATE_MASK = { - "paths": ["config.worker_config.num_instances", "config.secondary_worker_config.num_instances"] -} - TIMEOUT = {"seconds": 1 * 24 * 60 * 60} # Jobs definitions @@ -87,7 +78,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "hadoop"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID @@ -101,16 +92,6 @@ cluster_name=CLUSTER_NAME, ) - scale_cluster = DataprocUpdateClusterOperator( - task_id="scale_cluster", - cluster_name=CLUSTER_NAME, - cluster=CLUSTER_UPDATE, - update_mask=UPDATE_MASK, - graceful_decommission_timeout=TIMEOUT, - project_id=PROJECT_ID, - region=REGION, - ) - hadoop_task = DataprocSubmitJobOperator( task_id="hadoop_task", job=HADOOP_JOB, region=REGION, project_id=PROJECT_ID ) @@ -127,7 +108,14 @@ task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE ) - create_bucket >> create_cluster >> scale_cluster >> hadoop_task >> delete_cluster >> delete_bucket + ( + # TEST SETUP + [create_bucket, create_cluster] + # TEST BODY + >> hadoop_task + # TEST TEARDOWN + >> [delete_cluster, delete_bucket] + ) from tests.system.utils.watcher import watcher From e8287734cbc29d45eeab565936afe8c9eb2e5340 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:27:32 +0200 Subject: [PATCH 241/533] Fixup docstring for deprecated DataprocSubmitHiveJobOperator (#32723) --- airflow/providers/google/cloud/operators/dataproc.py | 4 ++++ .../google/cloud/dataproc/example_dataproc_hive.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 3fa6ae1e771a..e84cbe32a869 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1204,6 +1204,10 @@ def execute(self, context: Context): class DataprocSubmitHiveJobOperator(DataprocJobBaseOperator): """Start a Hive query Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + :param query: The query or reference to the query file (q extension). :param query_uri: The HCFS URI of the script that contains the Hive queries. :param variables: Map of named parameters for the query. diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py index ea98725b2993..8d345f818d52 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py @@ -74,7 +74,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "hive"], ) as dag: # [START how_to_cloud_dataproc_create_cluster_operator] create_cluster = DataprocCreateClusterOperator( @@ -100,7 +100,14 @@ # [END how_to_cloud_dataproc_delete_cluster_operator] delete_cluster.trigger_rule = TriggerRule.ALL_DONE - create_cluster >> hive_task >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> hive_task + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher From dda3dcdcfcc4d687f0b66e3cd993e0f5faf0caee Mon Sep 17 00:00:00 2001 From: Beata Kossakowska <109511937+bkossakowska@users.noreply.github.com> Date: Thu, 20 Jul 2023 19:33:28 +0200 Subject: [PATCH 242/533] Add deferrable mode to ExternalTaskSensor (#29260) --------- Co-authored-by: Beata Kossakowska Co-authored-by: VladaZakharova --- airflow/sensors/external_task.py | 43 ++++++++++ airflow/triggers/external_task.py | 80 ++++++++++++++---- .../howto/operator/external_task_sensor.rst | 9 ++ tests/sensors/test_external_task_sensor.py | 82 ++++++++++++++++++- tests/system/providers/core/__init__.py | 16 ++++ .../example_external_task_child_deferrable.py | 40 +++++++++ ...example_external_task_parent_deferrable.py | 70 ++++++++++++++++ tests/triggers/test_external_task.py | 6 ++ 8 files changed, 328 insertions(+), 18 deletions(-) create mode 100644 tests/system/providers/core/__init__.py create mode 100644 tests/system/providers/core/example_external_task_child_deferrable.py create mode 100644 tests/system/providers/core/example_external_task_parent_deferrable.py diff --git a/airflow/sensors/external_task.py b/airflow/sensors/external_task.py index b5c1c1321f7e..5e42820ffe99 100644 --- a/airflow/sensors/external_task.py +++ b/airflow/sensors/external_task.py @@ -25,6 +25,7 @@ import attr from sqlalchemy import func +from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowSkipException, RemovedInAirflow3Warning from airflow.models.baseoperator import BaseOperatorLink from airflow.models.dag import DagModel @@ -33,11 +34,13 @@ from airflow.models.taskinstance import TaskInstance from airflow.operators.empty import EmptyOperator from airflow.sensors.base import BaseSensorOperator +from airflow.triggers.external_task import TaskStateTrigger from airflow.utils.file import correct_maybe_zipped from airflow.utils.helpers import build_airflow_url_with_query from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.sqlalchemy import tuple_in_condition from airflow.utils.state import State, TaskInstanceState +from airflow.utils.timezone import utcnow if TYPE_CHECKING: from sqlalchemy.orm import Query, Session @@ -126,6 +129,8 @@ class ExternalTaskSensor(BaseSensorOperator): external_task_id is not None) or check if the DAG to wait for exists (when external_task_id is None), and immediately cease waiting if the external task or DAG does not exist (default value: False). + :param poll_interval: polling period in seconds to check for the status + :param deferrable: Run sensor in deferrable mode """ template_fields = ["external_dag_id", "external_task_id", "external_task_ids", "external_task_group_id"] @@ -145,9 +150,12 @@ def __init__( execution_delta: datetime.timedelta | None = None, execution_date_fn: Callable | None = None, check_existence: bool = False, + poll_interval: float = 2.0, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) + self.allowed_states = list(allowed_states) if allowed_states else [TaskInstanceState.SUCCESS.value] self.skipped_states = list(skipped_states) if skipped_states else [] self.failed_states = list(failed_states) if failed_states else [] @@ -211,6 +219,8 @@ def __init__( self.external_task_group_id = external_task_group_id self.check_existence = check_existence self._has_checked_existence = False + self.deferrable = deferrable + self.poll_interval = poll_interval def _get_dttm_filter(self, context): if self.execution_delta: @@ -318,6 +328,39 @@ def poke(self, context: Context, session: Session = NEW_SESSION) -> bool: count_allowed = self.get_count(dttm_filter, session, self.allowed_states) return count_allowed == len(dttm_filter) + def execute(self, context: Context) -> None: + """ + Airflow runs this method on the worker and defers using the triggers + if deferrable is set to True. + """ + if not self.deferrable: + super().execute(context) + else: + self.defer( + trigger=TaskStateTrigger( + dag_id=self.external_dag_id, + task_id=self.external_task_id, + execution_dates=self._get_dttm_filter(context), + states=self.allowed_states, + trigger_start_time=utcnow(), + poll_interval=self.poll_interval, + ), + method_name="execute_complete", + ) + + def execute_complete(self, context, event=None): + """Callback for when the trigger fires - returns immediately.""" + if event["status"] == "success": + self.log.info("External task %s has executed successfully.", self.external_task_id) + return None + elif event["status"] == "timeout": + raise AirflowException("Dag was not started within 1 minute, assuming fail.") + else: + raise AirflowException( + "Error occurred while trying to retrieve task status. Please, check the " + "name of executed task and Dag." + ) + def _check_for_existence(self, session) -> None: dag_to_wait = DagModel.get_current(self.external_dag_id, session) diff --git a/airflow/triggers/external_task.py b/airflow/triggers/external_task.py index e739c7a7cb4e..f179cba259ca 100644 --- a/airflow/triggers/external_task.py +++ b/airflow/triggers/external_task.py @@ -17,8 +17,8 @@ from __future__ import annotations import asyncio -import datetime import typing +from datetime import datetime from asgiref.sync import sync_to_async from sqlalchemy import func @@ -27,7 +27,8 @@ from airflow.models import DagRun, TaskInstance from airflow.triggers.base import BaseTrigger, TriggerEvent from airflow.utils.session import NEW_SESSION, provide_session -from airflow.utils.state import DagRunState +from airflow.utils.state import DagRunState, TaskInstanceState +from airflow.utils.timezone import utcnow class TaskStateTrigger(BaseTrigger): @@ -36,20 +37,26 @@ class TaskStateTrigger(BaseTrigger): :param dag_id: The dag_id that contains the task you want to wait for :param task_id: The task_id that contains the task you want to - wait for. If ``None`` (default value) the sensor waits for the DAG + wait for. :param states: allowed states, default is ``['success']`` - :param execution_dates: + :param execution_dates: task execution time interval :param poll_interval: The time interval in seconds to check the state. The default value is 5 sec. + :param trigger_start_time: time in Datetime format when the trigger was started. Is used + to control the execution of trigger to prevent infinite loop in case if specified name + of the dag does not exist in database. It will wait period of time equals _timeout_sec parameter + from the time, when the trigger was started and if the execution lasts more time than expected, + the trigger will terminate with 'timeout' status. """ def __init__( self, dag_id: str, - task_id: str, - states: list[str], - execution_dates: list[datetime.datetime], - poll_interval: float = 5.0, + execution_dates: list[datetime], + trigger_start_time: datetime, + states: list[str] | None = None, + task_id: str | None = None, + poll_interval: float = 2.0, ): super().__init__() self.dag_id = dag_id @@ -57,6 +64,9 @@ def __init__( self.states = states self.execution_dates = execution_dates self.poll_interval = poll_interval + self.trigger_start_time = trigger_start_time + self.states = states if states else [TaskInstanceState.SUCCESS.value] + self._timeout_sec = 60 def serialize(self) -> tuple[str, dict[str, typing.Any]]: """Serializes TaskStateTrigger arguments and classpath.""" @@ -68,17 +78,52 @@ def serialize(self) -> tuple[str, dict[str, typing.Any]]: "states": self.states, "execution_dates": self.execution_dates, "poll_interval": self.poll_interval, + "trigger_start_time": self.trigger_start_time, }, ) async def run(self) -> typing.AsyncIterator[TriggerEvent]: - """Checks periodically in the database to see if the task exists and has hit one of the states.""" + """ + Checks periodically in the database to see if the dag exists and is in the running state. If found, + wait until the task specified will reach one of the expected states. If dag with specified name was + not in the running state after _timeout_sec seconds after starting execution process of the trigger, + terminate with status 'timeout'. + """ while True: - # mypy confuses typing here - num_tasks = await self.count_tasks() # type: ignore[call-arg] - if num_tasks == len(self.execution_dates): - yield TriggerEvent(True) - await asyncio.sleep(self.poll_interval) + try: + delta = utcnow() - self.trigger_start_time + if delta.total_seconds() < self._timeout_sec: + # mypy confuses typing here + if await self.count_running_dags() == 0: # type: ignore[call-arg] + self.log.info("Waiting for DAG to start execution...") + await asyncio.sleep(self.poll_interval) + else: + yield TriggerEvent({"status": "timeout"}) + return + # mypy confuses typing here + if await self.count_tasks() == len(self.execution_dates): # type: ignore[call-arg] + yield TriggerEvent({"status": "success"}) + return + self.log.info("Task is still running, sleeping for %s seconds...", self.poll_interval) + await asyncio.sleep(self.poll_interval) + except Exception: + yield TriggerEvent({"status": "failed"}) + return + + @sync_to_async + @provide_session + def count_running_dags(self, session: Session): + """Count how many dag instances in running state in the database.""" + dags = ( + session.query(func.count("*")) + .filter( + TaskInstance.dag_id == self.dag_id, + TaskInstance.execution_date.in_(self.execution_dates), + TaskInstance.state.in_(["running", "success"]), + ) + .scalar() + ) + return dags @sync_to_async @provide_session @@ -112,7 +157,7 @@ def __init__( self, dag_id: str, states: list[DagRunState], - execution_dates: list[datetime.datetime], + execution_dates: list[datetime], poll_interval: float = 5.0, ): super().__init__() @@ -134,7 +179,10 @@ def serialize(self) -> tuple[str, dict[str, typing.Any]]: ) async def run(self) -> typing.AsyncIterator[TriggerEvent]: - """Checks periodically in the database to see if the dag run exists and has hit one of the states.""" + """ + Checks periodically in the database to see if the dag run exists, and has + hit one of the states yet, or not. + """ while True: # mypy confuses typing here num_dags = await self.count_dags() # type: ignore[call-arg] diff --git a/docs/apache-airflow/howto/operator/external_task_sensor.rst b/docs/apache-airflow/howto/operator/external_task_sensor.rst index 923f8ec3d116..f6f53f87e55f 100644 --- a/docs/apache-airflow/howto/operator/external_task_sensor.rst +++ b/docs/apache-airflow/howto/operator/external_task_sensor.rst @@ -53,6 +53,15 @@ via ``allowed_states`` and ``failed_states`` parameters. :start-after: [START howto_operator_external_task_sensor] :end-before: [END howto_operator_external_task_sensor] +Also for this action you can use sensor in the deferrable mode: + +.. exampleinclude:: /../../tests/system/providers/core/example_external_task_parent_deferrable.py + :language: python + :dedent: 4 + :start-after: [START howto_external_task_async_sensor] + :end-before: [END howto_external_task_async_sensor] + + ExternalTaskSensor with task_group dependency --------------------------------------------- In Addition, we can also use the :class:`~airflow.sensors.external_task.ExternalTaskSensor` to make tasks on a DAG diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index a5259084b13f..e84b3f69f48e 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -22,12 +22,13 @@ import tempfile import zipfile from datetime import time, timedelta +from unittest import mock import pytest from airflow import exceptions, settings from airflow.decorators import task as task_deco -from airflow.exceptions import AirflowException, AirflowSensorTimeout +from airflow.exceptions import AirflowException, AirflowSensorTimeout, TaskDeferred from airflow.models import DagBag, DagRun, TaskInstance from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel @@ -35,9 +36,14 @@ from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor, ExternalTaskSensorLink +from airflow.sensors.external_task import ( + ExternalTaskMarker, + ExternalTaskSensor, + ExternalTaskSensorLink, +) from airflow.sensors.time_sensor import TimeSensor from airflow.serialization.serialized_objects import SerializedBaseOperator +from airflow.triggers.external_task import TaskStateTrigger from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import create_session, provide_session from airflow.utils.state import DagRunState, State, TaskInstanceState @@ -54,6 +60,9 @@ TEST_TASK_ID_ALTERNATE = "time_sensor_check_alternate" TEST_TASK_GROUP_ID = "time_sensor_group_id" DEV_NULL = "/dev/null" +TASK_ID = "external_task_sensor_check" +EXTERNAL_DAG_ID = "child_dag" # DAG the external task sensor is waiting on +EXTERNAL_TASK_ID = "child_task" # Task the external task sensor is waiting on @pytest.fixture(autouse=True) @@ -829,6 +838,75 @@ def test_external_task_group_when_there_is_no_TIs(self): ) +class TestExternalTaskAsyncSensor: + TASK_ID = "external_task_sensor_check" + EXTERNAL_DAG_ID = "child_dag" # DAG the external task sensor is waiting on + EXTERNAL_TASK_ID = "child_task" # Task the external task sensor is waiting on + + def test_defer_and_fire_task_state_trigger(self): + """ + Asserts that a task is deferred and TaskStateTrigger will be fired + when the ExternalTaskAsyncSensor is provided with all required arguments + (i.e. including the external_task_id). + """ + sensor = ExternalTaskSensor( + task_id=TASK_ID, + external_task_id=EXTERNAL_TASK_ID, + external_dag_id=EXTERNAL_DAG_ID, + deferrable=True, + ) + + with pytest.raises(TaskDeferred) as exc: + sensor.execute(context=mock.MagicMock()) + + assert isinstance(exc.value.trigger, TaskStateTrigger), "Trigger is not a TaskStateTrigger" + + def test_defer_and_fire_failed_state_trigger(self): + """Tests that an AirflowException is raised in case of error event""" + sensor = ExternalTaskSensor( + task_id=TASK_ID, + external_task_id=EXTERNAL_TASK_ID, + external_dag_id=EXTERNAL_DAG_ID, + deferrable=True, + ) + + with pytest.raises(AirflowException): + sensor.execute_complete( + context=mock.MagicMock(), event={"status": "error", "message": "test failure message"} + ) + + def test_defer_and_fire_timeout_state_trigger(self): + """Tests that an AirflowException is raised in case of timeout event""" + sensor = ExternalTaskSensor( + task_id=TASK_ID, + external_task_id=EXTERNAL_TASK_ID, + external_dag_id=EXTERNAL_DAG_ID, + deferrable=True, + ) + + with pytest.raises(AirflowException): + sensor.execute_complete( + context=mock.MagicMock(), + event={"status": "timeout", "message": "Dag was not started within 1 minute, assuming fail."}, + ) + + def test_defer_execute_check_correct_logging(self): + """Asserts that logging occurs as expected""" + sensor = ExternalTaskSensor( + task_id=TASK_ID, + external_task_id=EXTERNAL_TASK_ID, + external_dag_id=EXTERNAL_DAG_ID, + deferrable=True, + ) + + with mock.patch.object(sensor.log, "info") as mock_log_info: + sensor.execute_complete( + context=mock.MagicMock(), + event={"status": "success"}, + ) + mock_log_info.assert_called_with("External task %s has executed successfully.", EXTERNAL_TASK_ID) + + def test_external_task_sensor_check_zipped_dag_existence(dag_zip_maker): with dag_zip_maker("test_external_task_sensor_check_existense.py") as dagbag: with create_session() as session: diff --git a/tests/system/providers/core/__init__.py b/tests/system/providers/core/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/system/providers/core/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/core/example_external_task_child_deferrable.py b/tests/system/providers/core/example_external_task_child_deferrable.py new file mode 100644 index 000000000000..f75eb4f23479 --- /dev/null +++ b/tests/system/providers/core/example_external_task_child_deferrable.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime + +from airflow import DAG +from airflow.operators.bash import BashOperator + +with DAG( + dag_id="child_dag", + start_date=datetime(2022, 1, 1), + schedule="@once", + catchup=False, + tags=["example", "async", "core"], +) as dag: + dummy_task = BashOperator( + task_id="child_task", + bash_command="echo 1; sleep 1; echo 2; sleep 2; echo 3; sleep 3", + ) + + +from tests.system.utils import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/tests/system/providers/core/example_external_task_parent_deferrable.py b/tests/system/providers/core/example_external_task_parent_deferrable.py new file mode 100644 index 000000000000..7cec2ce13815 --- /dev/null +++ b/tests/system/providers/core/example_external_task_parent_deferrable.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow import DAG +from airflow.operators.dummy import DummyOperator +from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.timezone import datetime + +with DAG( + dag_id="example_external_task", + start_date=datetime(2022, 1, 1), + schedule="@once", + catchup=False, + tags=["example", "async", "core"], +) as dag: + start = DummyOperator(task_id="start") + + # [START howto_external_task_async_sensor] + external_task_sensor = ExternalTaskSensor( + task_id="parent_task_sensor", + external_task_id="child_task", + external_dag_id="child_dag", + deferrable=True, + ) + # [END howto_external_task_async_sensor] + + trigger_child_task = TriggerDagRunOperator( + task_id="trigger_child_task", + trigger_dag_id="child_dag", + allowed_states=[ + "success", + "failed", + ], + execution_date="{{execution_date}}", + poke_interval=5, + reset_dag_run=True, + wait_for_completion=True, + ) + + end = DummyOperator(task_id="end") + + start >> [trigger_child_task, external_task_sensor] >> end + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "teardown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + + +from tests.system.utils import get_test_run + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/tests/triggers/test_external_task.py b/tests/triggers/test_external_task.py index e8a4a67ba5d5..a8569a9c558d 100644 --- a/tests/triggers/test_external_task.py +++ b/tests/triggers/test_external_task.py @@ -26,6 +26,7 @@ from airflow.triggers.external_task import DagStateTrigger, TaskStateTrigger from airflow.utils import timezone from airflow.utils.state import DagRunState, TaskInstanceState +from airflow.utils.timezone import utcnow class TestTaskStateTrigger: @@ -40,6 +41,7 @@ async def test_task_state_trigger(self, session): Asserts that the TaskStateTrigger only goes off on or after a TaskInstance reaches an allowed state (i.e. SUCCESS). """ + trigger_start_time = utcnow() dag = DAG(self.DAG_ID, start_date=timezone.datetime(2022, 1, 1)) dag_run = DagRun( dag_id=dag.dag_id, @@ -61,6 +63,7 @@ async def test_task_state_trigger(self, session): states=self.STATES, execution_dates=[timezone.datetime(2022, 1, 1)], poll_interval=0.2, + trigger_start_time=trigger_start_time, ) task = asyncio.create_task(trigger.run().__anext__()) @@ -83,12 +86,14 @@ def test_serialization(self): Asserts that the TaskStateTrigger correctly serializes its arguments and classpath. """ + trigger_start_time = utcnow() trigger = TaskStateTrigger( dag_id=self.DAG_ID, task_id=self.TASK_ID, states=self.STATES, execution_dates=[timezone.datetime(2022, 1, 1)], poll_interval=5, + trigger_start_time=trigger_start_time, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.triggers.external_task.TaskStateTrigger" @@ -98,6 +103,7 @@ def test_serialization(self): "states": self.STATES, "execution_dates": [timezone.datetime(2022, 1, 1)], "poll_interval": 5, + "trigger_start_time": trigger_start_time, } From ac524826f3fdd384e15b9a302e47a0c4a8e9f280 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 20 Jul 2023 19:11:54 +0100 Subject: [PATCH 243/533] Refactor setup/teardown ctx mgr to operate freely with other task definitions (#32687) * Refactor setup/teardown ctx mgr to operate freely with other task definitions This refactor is aimed at both simplifying the code for easy understanding and reducing assumption of how setups/teardowns should link. Only link when necessary and don't force a link. This is particularly important as we now have .as_teardown(setups=[]) * Apply suggestions from code review Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> * Fix setup not being connected to work task if roots doesnt exist --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- airflow/utils/setup_teardown.py | 246 ++++++++++++++++++------ tests/decorators/test_setup_teardown.py | 44 +++++ 2 files changed, 230 insertions(+), 60 deletions(-) diff --git a/airflow/utils/setup_teardown.py b/airflow/utils/setup_teardown.py index 8ca7fa385d4c..cedd5b7a7a5c 100644 --- a/airflow/utils/setup_teardown.py +++ b/airflow/utils/setup_teardown.py @@ -39,16 +39,22 @@ class BaseSetupTeardownContext: _previous_context_managed_setup_task: list[AbstractOperator | list[AbstractOperator]] = [] _context_managed_teardown_task: AbstractOperator | list[AbstractOperator] = [] _previous_context_managed_teardown_task: list[AbstractOperator | list[AbstractOperator]] = [] + _teardown_downstream_of_setup: AbstractOperator | list[AbstractOperator] = [] + _previous_teardown_downstream_of_setup: list[AbstractOperator | list[AbstractOperator]] = [] + _setup_upstream_of_teardown: AbstractOperator | list[AbstractOperator] = [] + _previous_setup_upstream_of_teardown: list[AbstractOperator | list[AbstractOperator]] = [] @classmethod def push_context_managed_setup_task(cls, task: AbstractOperator | list[AbstractOperator]): - if cls._context_managed_setup_task: + setup_task = cls._context_managed_setup_task + if setup_task and setup_task != task: cls._previous_context_managed_setup_task.append(cls._context_managed_setup_task) cls._context_managed_setup_task = task @classmethod def push_context_managed_teardown_task(cls, task: AbstractOperator | list[AbstractOperator]): - if cls._context_managed_teardown_task: + teardown_task = cls._context_managed_teardown_task + if teardown_task and teardown_task != task: cls._previous_context_managed_teardown_task.append(cls._context_managed_teardown_task) cls._context_managed_teardown_task = task @@ -59,15 +65,67 @@ def pop_context_managed_setup_task(cls) -> AbstractOperator | list[AbstractOpera cls._context_managed_setup_task = cls._previous_context_managed_setup_task.pop() setup_task = cls._context_managed_setup_task if setup_task and old_setup_task: - if isinstance(setup_task, list): - for task in setup_task: - task.set_downstream(old_setup_task) - else: - setup_task.set_downstream(old_setup_task) + cls.set_dependency(old_setup_task, setup_task, upstream=False) else: cls._context_managed_setup_task = [] return old_setup_task + @classmethod + def pop_context_managed_teardown_task(cls) -> AbstractOperator | list[AbstractOperator]: + old_teardown_task = cls._context_managed_teardown_task + if cls._previous_context_managed_teardown_task: + cls._context_managed_teardown_task = cls._previous_context_managed_teardown_task.pop() + teardown_task = cls._context_managed_teardown_task + if teardown_task and old_teardown_task: + cls.set_dependency(old_teardown_task, teardown_task) + else: + cls._context_managed_teardown_task = [] + return old_teardown_task + + @classmethod + def pop_teardown_downstream_of_setup(cls) -> AbstractOperator | list[AbstractOperator]: + old_teardown_task = cls._teardown_downstream_of_setup + if cls._previous_teardown_downstream_of_setup: + cls._teardown_downstream_of_setup = cls._previous_teardown_downstream_of_setup.pop() + teardown_task = cls._teardown_downstream_of_setup + if teardown_task and old_teardown_task: + cls.set_dependency(old_teardown_task, teardown_task) + else: + cls._teardown_downstream_of_setup = [] + return old_teardown_task + + @classmethod + def pop_setup_upstream_of_teardown(cls) -> AbstractOperator | list[AbstractOperator]: + old_setup_task = cls._setup_upstream_of_teardown + if cls._previous_setup_upstream_of_teardown: + cls._setup_upstream_of_teardown = cls._previous_setup_upstream_of_teardown.pop() + setup_task = cls._setup_upstream_of_teardown + if setup_task and old_setup_task: + cls.set_dependency(old_setup_task, setup_task, upstream=False) + else: + cls._setup_upstream_of_teardown = [] + return old_setup_task + + @classmethod + def set_dependency( + cls, + receiving_task: AbstractOperator | list[AbstractOperator], + new_task: AbstractOperator | list[AbstractOperator], + upstream=True, + ): + if isinstance(new_task, (list, tuple)): + for task in new_task: + cls._set_dependency(task, receiving_task, upstream) + else: + cls._set_dependency(new_task, receiving_task, upstream) + + @staticmethod + def _set_dependency(task, receiving_task, upstream): + if upstream: + task.set_upstream(receiving_task) + else: + task.set_downstream(receiving_task) + @classmethod def update_context_map(cls, task: DependencyMixin): from airflow.models.abstractoperator import AbstractOperator @@ -77,46 +135,22 @@ def update_context_map(cls, task: DependencyMixin): return ctx = cls.context_map - def _get_or_set_item(item): + def _append_or_set_item(item): if ctx.get(item) is None: ctx[item] = [task_] else: ctx[item].append(task_) - if setup_task := cls.get_context_managed_setup_task(): + if setup_task := cls._context_managed_setup_task: if isinstance(setup_task, list): - _get_or_set_item(tuple(setup_task)) + _append_or_set_item(tuple(setup_task)) else: - _get_or_set_item(setup_task) - if teardown_task := cls.get_context_managed_teardown_task(): + _append_or_set_item(setup_task) + if teardown_task := cls._context_managed_teardown_task: if isinstance(teardown_task, list): - _get_or_set_item(tuple(teardown_task)) + _append_or_set_item(tuple(teardown_task)) else: - _get_or_set_item(teardown_task) - - @classmethod - def pop_context_managed_teardown_task(cls) -> AbstractOperator | list[AbstractOperator]: - old_teardown_task = cls._context_managed_teardown_task - if cls._previous_context_managed_teardown_task: - cls._context_managed_teardown_task = cls._previous_context_managed_teardown_task.pop() - teardown_task = cls._context_managed_teardown_task - if teardown_task and old_teardown_task: - if isinstance(teardown_task, list): - for task in teardown_task: - task.set_upstream(old_teardown_task) - else: - teardown_task.set_upstream(old_teardown_task) - else: - cls._context_managed_teardown_task = [] - return old_teardown_task - - @classmethod - def get_context_managed_setup_task(cls) -> AbstractOperator | list[AbstractOperator]: - return cls._context_managed_setup_task - - @classmethod - def get_context_managed_teardown_task(cls) -> AbstractOperator | list[AbstractOperator]: - return cls._context_managed_teardown_task + _append_or_set_item(teardown_task) @classmethod def push_setup_teardown_task(cls, operator: AbstractOperator | list[AbstractOperator]): @@ -134,57 +168,149 @@ def push_setup_teardown_task(cls, operator: AbstractOperator | list[AbstractOper @classmethod def _push_tasks(cls, operator: AbstractOperator | list[AbstractOperator], setup: bool = False): if isinstance(operator, list): - upstream_tasks = operator[0].upstream_list - downstream_list = operator[0].downstream_list if not all(task.is_setup == operator[0].is_setup for task in operator): cls.error("All tasks in the list must be either setup or teardown tasks") - else: - upstream_tasks = operator.upstream_list - downstream_list = operator.downstream_list if setup: cls.push_context_managed_setup_task(operator) - if downstream_list: - cls.push_context_managed_teardown_task(list(downstream_list)) + # workout the teardown + cls._update_teardown_downstream(operator) else: cls.push_context_managed_teardown_task(operator) - if upstream_tasks: - cls.push_context_managed_setup_task(list(upstream_tasks)) + # workout the setups + cls._update_setup_upstream(operator) + + @classmethod + def _update_teardown_downstream(cls, operator: AbstractOperator | list[AbstractOperator]): + """This recursively goes through the tasks downstream of the setup in the context manager, + if found, updates the _teardown_downstream_of_setup accordingly. + """ + operator = operator[0] if isinstance(operator, list) else operator + + def _get_teardowns(tasks): + teardowns = [i for i in tasks if i.is_teardown] + if not teardowns: + all_lists = [task.downstream_list + task.upstream_list for task in tasks] + new_list = [ + x + for sublist in all_lists + for x in sublist + if (isinstance(operator, list) and x in operator) or x != operator + ] + if not new_list: + return [] + return _get_teardowns(new_list) + return teardowns + + teardowns = _get_teardowns(operator.downstream_list) + teardown_task = cls._teardown_downstream_of_setup + if teardown_task and teardown_task != teardowns: + cls._previous_teardown_downstream_of_setup.append(cls._teardown_downstream_of_setup) + cls._teardown_downstream_of_setup = teardowns + + @classmethod + def _update_setup_upstream(cls, operator: AbstractOperator | list[AbstractOperator]): + """This recursively goes through the tasks upstream of the teardown task in the context manager, + if found, updates the _setup_upstream_of_teardown accordingly. + """ + operator = operator[0] if isinstance(operator, list) else operator + + def _get_setups(tasks): + setups = [i for i in tasks if i.is_setup] + if not setups: + all_lists = [task.downstream_list + task.upstream_list for task in tasks] + new_list = [ + x + for sublist in all_lists + for x in sublist + if (isinstance(operator, list) and x in operator) or x != operator + ] + if not new_list: + return [] + return _get_setups(new_list) + return setups + + setups = _get_setups(operator.upstream_list) + setup_task = cls._setup_upstream_of_teardown + if setup_task and setup_task != setups: + cls._previous_setup_upstream_of_teardown.append(cls._setup_upstream_of_teardown) + cls._setup_upstream_of_teardown = setups + + @classmethod + def set_teardown_task_as_leaves(cls, leaves): + teardown_task = cls._teardown_downstream_of_setup + if cls._context_managed_teardown_task: + cls.set_dependency(cls._context_managed_teardown_task, teardown_task) + else: + cls.set_dependency(leaves, teardown_task) + + @classmethod + def set_setup_task_as_roots(cls, roots): + setup_task = cls._setup_upstream_of_teardown + if cls._context_managed_setup_task: + cls.set_dependency(cls._context_managed_setup_task, setup_task, upstream=False) + else: + cls.set_dependency(roots, setup_task, upstream=False) @classmethod def set_work_task_roots_and_leaves(cls): - if setup_task := cls.get_context_managed_setup_task(): + """Sets the work task roots and leaves.""" + if setup_task := cls._context_managed_setup_task: if isinstance(setup_task, list): setup_task = tuple(setup_task) - tasks_in_context = cls.context_map.get(setup_task, []) + tasks_in_context = [ + x for x in cls.context_map.get(setup_task, []) if not x.is_teardown and not x.is_setup + ] if tasks_in_context: roots = [task for task in tasks_in_context if not task.upstream_list] if not roots: setup_task >> tasks_in_context[0] - elif isinstance(setup_task, tuple): - for task in setup_task: - task >> roots else: - setup_task >> roots - if teardown_task := cls.get_context_managed_teardown_task(): + cls.set_dependency(roots, setup_task, upstream=False) + leaves = [task for task in tasks_in_context if not task.downstream_list] + if not leaves: + leaves = tasks_in_context[-1] + cls.set_teardown_task_as_leaves(leaves) + + if teardown_task := cls._context_managed_teardown_task: if isinstance(teardown_task, list): teardown_task = tuple(teardown_task) - tasks_in_context = cls.context_map.get(teardown_task, []) + tasks_in_context = [ + x for x in cls.context_map.get(teardown_task, []) if not x.is_teardown and not x.is_setup + ] if tasks_in_context: leaves = [task for task in tasks_in_context if not task.downstream_list] if not leaves: teardown_task << tasks_in_context[-1] - elif isinstance(teardown_task, tuple): - for task in teardown_task: - task << leaves else: - teardown_task << leaves + cls.set_dependency(leaves, teardown_task) + roots = [task for task in tasks_in_context if not task.upstream_list] + if not roots: + roots = tasks_in_context[0] + cls.set_setup_task_as_roots(roots) + cls.set_setup_teardown_relationships() + cls.active = False + + @classmethod + def set_setup_teardown_relationships(cls): + """ + Here we set relationship between setup to setup and + teardown to teardown. + + code:: python + with setuptask >> teardowntask: + with setuptask2 >> teardowntask2: + ... + + We set setuptask >> setuptask2, teardowntask >> teardowntask2 + """ setup_task = cls.pop_context_managed_setup_task() teardown_task = cls.pop_context_managed_teardown_task() if isinstance(setup_task, list): setup_task = tuple(setup_task) if isinstance(teardown_task, list): teardown_task = tuple(teardown_task) - cls.active = False + cls.pop_teardown_downstream_of_setup() + cls.pop_setup_upstream_of_teardown() cls.context_map.pop(setup_task, None) cls.context_map.pop(teardown_task, None) diff --git a/tests/decorators/test_setup_teardown.py b/tests/decorators/test_setup_teardown.py index 4fe195ae3e7b..bcfb81b0ba30 100644 --- a/tests/decorators/test_setup_teardown.py +++ b/tests/decorators/test_setup_teardown.py @@ -1313,3 +1313,47 @@ def mytask(): assert dag.task_group.children["mytask2"].upstream_task_ids == {"setuptask2"} assert dag.task_group.children["mytask2"].downstream_task_ids == {"teardowntask2"} assert dag.task_group.children["teardowntask2"].upstream_task_ids == {"mytask2", "setuptask2"} + + def test_check_for_circular_dependency(self, dag_maker): + + with dag_maker() as dag: + s1 = make_task("s1", type_="classic", setup_=True) + s2 = make_task("s2", type_="classic", setup_=True) + t1 = make_task("t1", type_="classic", teardown_=True) + t2 = make_task("t2", type_="classic", teardown_=True) + + s1 >> s2 + s1 >> t1 + s2 >> t1 + s1 >> t2 + s2 >> t2 + with t1, t2: + make_task("work_task", type_="classic") + + dag.validate() + + assert dag.task_group.children.keys() == {"s1", "s2", "t1", "t2", "work_task"} + assert dag.task_group.children["s1"].downstream_task_ids == {"s2", "work_task", "t1", "t2"} + assert dag.task_group.children["s2"].downstream_task_ids == {"work_task", "t1", "t2"} + assert dag.task_group.children["t2"].downstream_task_ids == {"t1"} + + def test_mixing_construct_with_add_task(self, dag_maker): + + with dag_maker() as dag: + s1 = make_task("s1", type_="classic") + s2 = make_task("s2", type_="classic") + t1 = make_task("t1", type_="classic") + t2 = make_task("t2", type_="classic") + t1.as_teardown(setups=s1) + t2.as_teardown(setups=s2) + with t1: + work = make_task("work", type_="classic") + with t2 as scope: + scope.add_task(work) + + assert dag.task_group.children.keys() == {"s1", "s2", "t1", "t2", "work"} + assert dag.task_group.children["s1"].downstream_task_ids == {"work", "t1"} + assert dag.task_group.children["s2"].downstream_task_ids == {"work", "t2"} + assert not dag.task_group.children["t1"].downstream_task_ids + assert not dag.task_group.children["t2"].downstream_task_ids + assert dag.task_group.children["work"].downstream_task_ids == {"t1", "t2"} From 3c14753b03872b259ce2248eda92f7fb6f4d751b Mon Sep 17 00:00:00 2001 From: Avinash Holla Pandeshwar Date: Thu, 20 Jul 2023 23:52:45 +0530 Subject: [PATCH 244/533] Fix BigQueryGetDataOperator where project_id is not being respected in deferrable mode (#32488) * fixing BigQueryGetDataOperator to respect project_id as compute project. A new parameter table_project_id will be used for specifying table storage project. --- .../google/cloud/operators/bigquery.py | 34 +++++++++++++++---- .../google/cloud/operators/test_bigquery.py | 13 ++++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 70ab30d61a23..f5e5a9634f72 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -810,7 +810,11 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator): :param dataset_id: The dataset ID of the requested table. (templated) :param table_id: The table ID of the requested table. (templated) - :param project_id: (Optional) The name of the project where the data + :param table_project_id: (Optional) The project ID of the requested table. + If None, it will be derived from the hook's project ID. (templated) + :param job_project_id: (Optional) Google Cloud Project where the job is running. + If None, it will be derived from the hook's project ID. (templated) + :param project_id: (Deprecated) (Optional) The name of the project where the data will be returned from. If None, it will be derived from the hook's project ID. (templated) :param max_results: The maximum number of records (rows) to be fetched from the table. (templated) @@ -837,6 +841,8 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator): template_fields: Sequence[str] = ( "dataset_id", "table_id", + "table_project_id", + "job_project_id", "project_id", "max_results", "selected_fields", @@ -849,6 +855,8 @@ def __init__( *, dataset_id: str, table_id: str, + table_project_id: str | None = None, + job_project_id: str | None = None, project_id: str | None = None, max_results: int = 100, selected_fields: str | None = None, @@ -863,8 +871,10 @@ def __init__( ) -> None: super().__init__(**kwargs) + self.table_project_id = table_project_id self.dataset_id = dataset_id self.table_id = table_id + self.job_project_id = job_project_id self.max_results = int(max_results) self.selected_fields = selected_fields self.gcp_conn_id = gcp_conn_id @@ -887,7 +897,7 @@ def _submit_job( return hook.insert_job( configuration=configuration, location=self.location, - project_id=hook.project_id, + project_id=self.job_project_id or hook.project_id, job_id=job_id, nowait=True, ) @@ -900,12 +910,22 @@ def generate_query(self, hook: BigQueryHook) -> str: else: query += "*" query += ( - f" from `{self.project_id or hook.project_id}.{self.dataset_id}" + f" from `{self.table_project_id or hook.project_id}.{self.dataset_id}" f".{self.table_id}` limit {self.max_results}" ) return query def execute(self, context: Context): + if self.project_id: + self.log.warning( + "The project_id parameter is deprecated, and will be removed in a future release." + " Please use table_project_id instead.", + ) + if not self.table_project_id: + self.table_project_id = self.project_id + else: + self.log.info("Ignoring project_id parameter, as table_project_id is found.") + hook = BigQueryHook( gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain, @@ -915,7 +935,7 @@ def execute(self, context: Context): if not self.deferrable: self.log.info( "Fetching Data from %s.%s.%s max results: %s", - self.project_id or hook.project_id, + self.table_project_id or hook.project_id, self.dataset_id, self.table_id, self.max_results, @@ -924,7 +944,7 @@ def execute(self, context: Context): schema: dict[str, list] = hook.get_schema( dataset_id=self.dataset_id, table_id=self.table_id, - project_id=self.project_id, + project_id=self.table_project_id or hook.project_id, ) if "fields" in schema: self.selected_fields = ",".join([field["name"] for field in schema["fields"]]) @@ -935,7 +955,7 @@ def execute(self, context: Context): max_results=self.max_results, selected_fields=self.selected_fields, location=self.location, - project_id=self.project_id, + project_id=self.table_project_id or hook.project_id, ) if isinstance(rows, RowIterator): @@ -961,7 +981,7 @@ def execute(self, context: Context): job_id=job.job_id, dataset_id=self.dataset_id, table_id=self.table_id, - project_id=hook.project_id, + project_id=self.job_project_id or hook.project_id, poll_interval=self.poll_interval, as_dict=self.as_dict, ), diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 226d1e709595..4026b4ba4507 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -64,6 +64,7 @@ TEST_DATASET = "test-dataset" TEST_DATASET_LOCATION = "EU" TEST_GCP_PROJECT_ID = "test-project" +TEST_JOB_PROJECT_ID = "test-job-project" TEST_DELETE_CONTENTS = True TEST_TABLE_ID = "test-table-id" TEST_GCS_BUCKET = "test-bucket" @@ -804,7 +805,7 @@ def test_execute(self, mock_hook, as_dict): task_id=TASK_ID, dataset_id=TEST_DATASET, table_id=TEST_TABLE_ID, - project_id=TEST_GCP_PROJECT_ID, + table_project_id=TEST_GCP_PROJECT_ID, max_results=max_results, selected_fields=selected_fields, location=TEST_DATASET_LOCATION, @@ -823,13 +824,13 @@ def test_execute(self, mock_hook, as_dict): ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") - def test_generate_query__with_project_id(self, mock_hook): + def test_generate_query__with_table_project_id(self, mock_hook): operator = BigQueryGetDataOperator( gcp_conn_id=GCP_CONN_ID, task_id=TASK_ID, dataset_id=TEST_DATASET, table_id=TEST_TABLE_ID, - project_id=TEST_GCP_PROJECT_ID, + table_project_id=TEST_GCP_PROJECT_ID, max_results=100, use_legacy_sql=False, ) @@ -839,7 +840,7 @@ def test_generate_query__with_project_id(self, mock_hook): ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") - def test_generate_query__without_project_id(self, mock_hook): + def test_generate_query__without_table_project_id(self, mock_hook): hook_project_id = mock_hook.project_id operator = BigQueryGetDataOperator( gcp_conn_id=GCP_CONN_ID, @@ -868,6 +869,7 @@ def test_bigquery_get_data_operator_async_with_selected_fields( task_id="get_data_from_bq", dataset_id=TEST_DATASET, table_id=TEST_TABLE_ID, + job_project_id=TEST_JOB_PROJECT_ID, max_results=100, selected_fields="value,name", deferrable=True, @@ -896,6 +898,7 @@ def test_bigquery_get_data_operator_async_without_selected_fields( task_id="get_data_from_bq", dataset_id=TEST_DATASET, table_id=TEST_TABLE_ID, + job_project_id=TEST_JOB_PROJECT_ID, max_results=100, deferrable=True, as_dict=as_dict, @@ -917,6 +920,7 @@ def test_bigquery_get_data_operator_execute_failure(self, as_dict): task_id="get_data_from_bq", dataset_id=TEST_DATASET, table_id="any", + job_project_id=TEST_JOB_PROJECT_ID, max_results=100, deferrable=True, as_dict=as_dict, @@ -936,6 +940,7 @@ def test_bigquery_get_data_op_execute_complete_with_records(self, as_dict): task_id="get_data_from_bq", dataset_id=TEST_DATASET, table_id="any", + job_project_id=TEST_JOB_PROJECT_ID, max_results=100, deferrable=True, as_dict=as_dict, From 99b8a90346b8826756ac165b73464a701e2c33aa Mon Sep 17 00:00:00 2001 From: Kristopher Kane Date: Thu, 20 Jul 2023 14:24:22 -0400 Subject: [PATCH 245/533] Filtering and ordering results of DataprocListBatchesOperator (#32500) --- airflow/providers/google/cloud/hooks/dataproc.py | 12 ++++++++++++ airflow/providers/google/cloud/operators/dataproc.py | 8 ++++++++ tests/providers/google/cloud/hooks/test_dataproc.py | 4 ++++ .../google/cloud/operators/test_dataproc.py | 6 ++++++ 4 files changed, 30 insertions(+) diff --git a/airflow/providers/google/cloud/hooks/dataproc.py b/airflow/providers/google/cloud/hooks/dataproc.py index 19e711c26388..e7eb48bef105 100644 --- a/airflow/providers/google/cloud/hooks/dataproc.py +++ b/airflow/providers/google/cloud/hooks/dataproc.py @@ -950,6 +950,8 @@ def list_batches( retry: Retry | _MethodDefault = DEFAULT, timeout: float | None = None, metadata: Sequence[tuple[str, str]] = (), + filter: str | None = None, + order_by: str | None = None, ): """List batch workloads. @@ -966,6 +968,8 @@ def list_batches( to complete. If *retry* is specified, the timeout applies to each individual attempt. :param metadata: Additional metadata that is provided to the method. + :param filter: Result filters as specified in ListBatchesRequest + :param order_by: How to order results as specified in ListBatchesRequest """ client = self.get_batch_client(region) parent = f"projects/{project_id}/regions/{region}" @@ -975,6 +979,8 @@ def list_batches( "parent": parent, "page_size": page_size, "page_token": page_token, + "filter": filter, + "order_by": order_by, }, retry=retry, timeout=timeout, @@ -1768,6 +1774,8 @@ async def list_batches( retry: Retry | _MethodDefault = DEFAULT, timeout: float | None = None, metadata: Sequence[tuple[str, str]] = (), + filter: str | None = None, + order_by: str | None = None, ): """List batch workloads. @@ -1784,6 +1792,8 @@ async def list_batches( to complete. If *retry* is specified, the timeout applies to each individual attempt. :param metadata: Additional metadata that is provided to the method. + :param filter: Result filters as specified in ListBatchesRequest + :param order_by: How to order results as specified in ListBatchesRequest """ client = self.get_batch_client(region) parent = f"projects/{project_id}/regions/{region}" @@ -1793,6 +1803,8 @@ async def list_batches( "parent": parent, "page_size": page_size, "page_token": page_token, + "filter": filter, + "order_by": order_by, }, retry=retry, timeout=timeout, diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index e84cbe32a869..5ac1b1cb3df0 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -2582,6 +2582,8 @@ class DataprocListBatchesOperator(GoogleCloudBaseOperator): If set as a sequence, the identities from the list must grant Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). + :param filter: Result filters as specified in ListBatchesRequest + :param order_by: How to order results as specified in ListBatchesRequest """ template_fields: Sequence[str] = ("region", "project_id", "impersonation_chain") @@ -2599,6 +2601,8 @@ def __init__( metadata: Sequence[tuple[str, str]] = (), gcp_conn_id: str = "google_cloud_default", impersonation_chain: str | Sequence[str] | None = None, + filter: str | None = None, + order_by: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -2611,6 +2615,8 @@ def __init__( self.metadata = metadata self.gcp_conn_id = gcp_conn_id self.impersonation_chain = impersonation_chain + self.filter = filter + self.order_by = order_by def execute(self, context: Context): hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain) @@ -2622,6 +2628,8 @@ def execute(self, context: Context): retry=self.retry, timeout=self.timeout, metadata=self.metadata, + filter=self.filter, + order_by=self.order_by, ) DataprocListLink.persist(context=context, task_instance=self, url=DATAPROC_BATCHES_LINK) return [Batch.to_dict(result) for result in results] diff --git a/tests/providers/google/cloud/hooks/test_dataproc.py b/tests/providers/google/cloud/hooks/test_dataproc.py index 1957e28d7cd5..c340d5b02d74 100644 --- a/tests/providers/google/cloud/hooks/test_dataproc.py +++ b/tests/providers/google/cloud/hooks/test_dataproc.py @@ -491,6 +491,8 @@ def test_list_batches(self, mock_client): parent=PARENT.format(GCP_PROJECT, GCP_LOCATION), page_size=None, page_token=None, + filter=None, + order_by=None, ), metadata=(), retry=DEFAULT, @@ -939,6 +941,8 @@ async def test_list_batches(self, mock_client): parent=PARENT.format(GCP_PROJECT, GCP_LOCATION), page_size=None, page_token=None, + filter=None, + order_by=None, ), metadata=(), retry=DEFAULT, diff --git a/tests/providers/google/cloud/operators/test_dataproc.py b/tests/providers/google/cloud/operators/test_dataproc.py index 5494a88ba933..b37c6ebc765c 100644 --- a/tests/providers/google/cloud/operators/test_dataproc.py +++ b/tests/providers/google/cloud/operators/test_dataproc.py @@ -2220,6 +2220,8 @@ class TestDataprocListBatchesOperator: def test_execute(self, mock_hook): page_token = "page_token" page_size = 42 + filter = 'batch_id=~"a-batch-id*" AND create_time>="2023-07-05T14:25:04.643818Z"' + order_by = "create_time desc" op = DataprocListBatchesOperator( task_id=TASK_ID, @@ -2232,6 +2234,8 @@ def test_execute(self, mock_hook): retry=RETRY, timeout=TIMEOUT, metadata=METADATA, + filter=filter, + order_by=order_by, ) op.execute(context=MagicMock()) mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN) @@ -2243,6 +2247,8 @@ def test_execute(self, mock_hook): retry=RETRY, timeout=TIMEOUT, metadata=METADATA, + filter=filter, + order_by=order_by, ) @mock.patch(DATAPROC_PATH.format("DataprocHook")) From 8e6754666096996738e738ec6bf7708a89696482 Mon Sep 17 00:00:00 2001 From: Beata Kossakowska <109511937+bkossakowska@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:45:05 +0200 Subject: [PATCH 246/533] Fix DataformCreateWorkflowInvocationOperator system test (#32599) Co-authored-by: Beata Kossakowska --- .../google/cloud/dataform/example_dataform.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/system/providers/google/cloud/dataform/example_dataform.py b/tests/system/providers/google/cloud/dataform/example_dataform.py index 18a738917a3b..5ea66b6fa8ce 100644 --- a/tests/system/providers/google/cloud/dataform/example_dataform.py +++ b/tests/system/providers/google/cloud/dataform/example_dataform.py @@ -26,6 +26,7 @@ from google.cloud.dataform_v1beta1 import WorkflowInvocation from airflow import models +from airflow.providers.google.cloud.operators.bigquery import BigQueryDeleteDatasetOperator from airflow.providers.google.cloud.operators.dataform import ( DataformCancelWorkflowInvocationOperator, DataformCreateCompilationResultOperator, @@ -54,6 +55,7 @@ REPOSITORY_ID = f"example_dataform_repository_{ENV_ID}" REGION = "us-central1" WORKSPACE_ID = f"example_dataform_workspace_{ENV_ID}" +DEFAULT_DATASET = "dataform" # This DAG is not self-run we need to do some extra configuration to execute it in automation process with models.DAG( @@ -82,6 +84,13 @@ ) # [END howto_operator_create_workspace] + # Delete the default dataset if it exists in the bigquery + delete_dataset = BigQueryDeleteDatasetOperator( + task_id="delete_dataset", + dataset_id=DEFAULT_DATASET, + delete_contents=True, + ) + # [START howto_initialize_workspace] first_initialization_step, last_initialization_step = make_initialization_workspace_flow( project_id=PROJECT_ID, @@ -250,6 +259,13 @@ ) # [END howto_operator_remove_directory] + delete_dataset_2 = BigQueryDeleteDatasetOperator( + task_id="delete_dataset_2", + dataset_id=DEFAULT_DATASET, + delete_contents=True, + trigger_rule=TriggerRule.ALL_DONE, + ) + # [START howto_operator_delete_workspace] delete_workspace = DataformDeleteWorkspaceOperator( task_id="delete-workspace", @@ -273,7 +289,7 @@ delete_repository.trigger_rule = TriggerRule.ALL_DONE - (make_repository >> make_workspace >> first_initialization_step) + (make_repository >> make_workspace >> delete_dataset >> first_initialization_step) ( last_initialization_step >> install_npm_packages @@ -289,6 +305,7 @@ >> write_test_file >> remove_test_file >> remove_test_directory + >> delete_dataset_2 >> delete_workspace >> delete_repository ) From 75ed3bc3f80ac03c301535f38544a4923ee6d43f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Thu, 20 Jul 2023 12:11:24 -0700 Subject: [PATCH 247/533] [bugfix] fix AWS triggers where deserialization would crash if region was not specified (#32729) --- airflow/providers/amazon/aws/triggers/batch.py | 2 +- airflow/providers/amazon/aws/triggers/ecs.py | 4 ++-- airflow/providers/amazon/aws/triggers/eks.py | 6 +++--- airflow/providers/amazon/aws/triggers/rds.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/airflow/providers/amazon/aws/triggers/batch.py b/airflow/providers/amazon/aws/triggers/batch.py index 87be13933293..8966540a3849 100644 --- a/airflow/providers/amazon/aws/triggers/batch.py +++ b/airflow/providers/amazon/aws/triggers/batch.py @@ -209,7 +209,7 @@ class BatchJobTrigger(AwsBaseWaiterTrigger): def __init__( self, job_id: str | None, - region_name: str | None, + region_name: str | None = None, aws_conn_id: str | None = "aws_default", waiter_delay: int = 5, waiter_max_attempts: int = 720, diff --git a/airflow/providers/amazon/aws/triggers/ecs.py b/airflow/providers/amazon/aws/triggers/ecs.py index af6f72e7714e..d0fdfb63b6b6 100644 --- a/airflow/providers/amazon/aws/triggers/ecs.py +++ b/airflow/providers/amazon/aws/triggers/ecs.py @@ -49,7 +49,7 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str | None, - region_name: str | None, + region_name: str | None = None, ): super().__init__( serialized_fields={"cluster_arn": cluster_arn}, @@ -88,7 +88,7 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str | None, - region_name: str | None, + region_name: str | None = None, ): super().__init__( serialized_fields={"cluster_arn": cluster_arn}, diff --git a/airflow/providers/amazon/aws/triggers/eks.py b/airflow/providers/amazon/aws/triggers/eks.py index ff99b512001e..b5419b76e778 100644 --- a/airflow/providers/amazon/aws/triggers/eks.py +++ b/airflow/providers/amazon/aws/triggers/eks.py @@ -47,7 +47,7 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, + region_name: str | None = None, ): super().__init__( serialized_fields={"cluster_name": cluster_name, "region_name": region_name}, @@ -309,7 +309,7 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, + region_name: str | None = None, ): super().__init__( serialized_fields={ @@ -357,7 +357,7 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, + region_name: str | None = None, ): super().__init__( serialized_fields={"cluster_name": cluster_name, "nodegroup_name": nodegroup_name}, diff --git a/airflow/providers/amazon/aws/triggers/rds.py b/airflow/providers/amazon/aws/triggers/rds.py index b8f20b9c9bb7..82354c187411 100644 --- a/airflow/providers/amazon/aws/triggers/rds.py +++ b/airflow/providers/amazon/aws/triggers/rds.py @@ -126,9 +126,9 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, response: dict[str, Any], db_type: RdsDbType, + region_name: str | None = None, ) -> None: super().__init__( serialized_fields={ @@ -172,9 +172,9 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, response: dict[str, Any], db_type: RdsDbType, + region_name: str | None = None, ) -> None: super().__init__( serialized_fields={ @@ -218,9 +218,9 @@ def __init__( waiter_delay: int, waiter_max_attempts: int, aws_conn_id: str, - region_name: str | None, response: dict[str, Any], db_type: RdsDbType, + region_name: str | None = None, ) -> None: super().__init__( serialized_fields={ From d70fecfaf68dbc34c818290ad35f48bf01044dd9 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 20 Jul 2023 13:12:18 -0700 Subject: [PATCH 248/533] Add initial docs for setup / teardown (#32169) Co-authored-by: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Co-authored-by: Tzu-ping Chung --- .../example_setup_teardown_taskflow.py | 85 ++++---- docs/apache-airflow/core-concepts/dags.rst | 18 +- docs/apache-airflow/howto/index.rst | 1 + .../howto/setup-and-teardown.rst | 184 ++++++++++++++++++ docs/spelling_wordlist.txt | 3 + 5 files changed, 247 insertions(+), 44 deletions(-) create mode 100644 docs/apache-airflow/howto/setup-and-teardown.rst diff --git a/airflow/example_dags/example_setup_teardown_taskflow.py b/airflow/example_dags/example_setup_teardown_taskflow.py index 128534f1d23c..4dcdbf253f6c 100644 --- a/airflow/example_dags/example_setup_teardown_taskflow.py +++ b/airflow/example_dags/example_setup_teardown_taskflow.py @@ -31,68 +31,75 @@ ) as dag: @task - def task_1(): + def my_first_task(): print("Hello 1") @task - def task_2(): + def my_second_task(): print("Hello 2") @task - def task_3(): + def my_third_task(): print("Hello 3") # you can set setup / teardown relationships with the `as_teardown` method. - t1 = task_1() - t2 = task_2() - t3 = task_3() - t1 >> t2 >> t3.as_teardown(setups=t1) + task_1 = my_first_task() + task_2 = my_second_task() + task_3 = my_third_task() + task_1 >> task_2 >> task_3.as_teardown(setups=task_1) - # the method `as_teadrown` will mark t3 as teardown, t1 as setup, and arrow t1 >> t3 - # now if you clear t2 (downstream), then t1 will be cleared in addition to t3 + # The method `as_teardown` will mark task_3 as teardown, task_1 as setup, and + # arrow task_1 >> task_3. + # Now if you clear task_2, then it's setup task, task_1, will be cleared in + # addition to its teardown task, task_3 # it's also possible to use a decorator to mark a task as setup or # teardown when you define it. see below. @setup - def dag_setup(): - print("I am dag_setup") + def outer_setup(): + print("I am outer_setup") + return "some cluster id" @teardown - def dag_teardown(): - print("I am dag_teardown") + def outer_teardown(cluster_id): + print("I am outer_teardown") + print(f"Tearing down cluster: {cluster_id}") @task - def dag_normal_task(): + def outer_work(): print("I am just a normal task") - s = dag_setup() - t = dag_teardown() - - # by using the decorators, dag_setup and dag_teardown are already marked as setup / teardown - # now we just need to make sure they are linked directly - # what we need to do is this:: - # s >> t - # s >> dag_normal_task() >> t - # but we can use a context manager to make it cleaner - with s >> t: - dag_normal_task() - @task_group def section_1(): - @task - def my_setup(): + @setup + def inner_setup(): print("I set up") + return "some_cluster_id" @task - def my_teardown(): - print("I tear down") - - @task - def hello(): - print("I say hello") - - (s := my_setup()) >> hello() >> my_teardown().as_teardown(setups=s) - - # and let's put section 1 inside the "dag setup" and "dag teardown" - s >> section_1() >> t + def inner_work(cluster_id): + print(f"doing some work with {cluster_id=}") + + @teardown + def inner_teardown(cluster_id): + print(f"tearing down {cluster_id=}") + + # this passes the return value of `inner_setup` to both `inner_work` and `inner_teardown` + inner_setup_task = inner_setup() + inner_work(inner_setup_task) >> inner_teardown(inner_setup_task) + + # by using the decorators, outer_setup and outer_teardown are already marked as setup / teardown + # now we just need to make sure they are linked directly. At a low level, what we need + # to do so is the following:: + # s = outer_setup() + # t = outer_teardown() + # s >> t + # s >> outer_work() >> t + # Thus, s and t are linked directly, and outer_work runs in between. We can take advantage of + # the fact that we are in taskflow, along with the context manager on teardowns, as follows: + with outer_teardown(outer_setup()): + outer_work() + + # and let's put section 1 inside the outer setup and teardown tasks + section_1() diff --git a/docs/apache-airflow/core-concepts/dags.rst b/docs/apache-airflow/core-concepts/dags.rst index ab3e3695e841..aa95699ab8ce 100644 --- a/docs/apache-airflow/core-concepts/dags.rst +++ b/docs/apache-airflow/core-concepts/dags.rst @@ -299,11 +299,11 @@ Control Flow By default, a DAG will only run a Task when all the Tasks it depends on are successful. There are several ways of modifying this, however: -* :ref:`concepts:branching`, where you can select which Task to move onto based on a condition -* :ref:`concepts:latest-only`, a special form of branching that only runs on DAGs running against the present -* :ref:`concepts:depends-on-past`, where tasks can depend on themselves *from a previous run* -* :ref:`concepts:trigger-rules`, which let you set the conditions under which a DAG will run a task. - +* :ref:`concepts:branching` - select which Task to move onto based on a condition +* :ref:`concepts:trigger-rules` - set the conditions under which a DAG will run a task +* :doc:`/howto/setup-and-teardown` - define setup and teardown relationships +* :ref:`concepts:latest-only` - a special form of branching that only runs on DAGs running against the present +* :ref:`concepts:depends-on-past` - tasks can depend on themselves *from a previous run* .. _concepts:branching: @@ -484,6 +484,14 @@ You can also combine this with the :ref:`concepts:depends-on-past` functionality .. image:: /img/branch_with_trigger.png +Setup and teardown +------------------ + +In data workflows it's common to create a resource (such as a compute resource), use it to do some work, and then tear it down. Airflow provides setup and teardown tasks to support this need. + +Please see main article :doc:`/howto/setup-and-teardown` for details on how to use this feature. + + Dynamic DAGs ------------ diff --git a/docs/apache-airflow/howto/index.rst b/docs/apache-airflow/howto/index.rst index 15a5249147f0..03c34820c4cc 100644 --- a/docs/apache-airflow/howto/index.rst +++ b/docs/apache-airflow/howto/index.rst @@ -45,6 +45,7 @@ configuring an Airflow environment. export-more-env-vars connection variable + setup-and-teardown run-behind-proxy run-with-systemd define-extra-link diff --git a/docs/apache-airflow/howto/setup-and-teardown.rst b/docs/apache-airflow/howto/setup-and-teardown.rst new file mode 100644 index 000000000000..c897031a6c6c --- /dev/null +++ b/docs/apache-airflow/howto/setup-and-teardown.rst @@ -0,0 +1,184 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Setup and Teardown +~~~~~~~~~~~~~~~~~~ + +In data workflows it's common to create a resource (such as a compute resource), use it to do some work, and then tear it down. Airflow provides setup and teardown tasks to support this need. + +Key features of setup and teardown tasks: + + * If you clear a task, its setups and teardowns will be cleared. + * By default, teardown tasks are ignored for the purpose of evaluating dag run state. + * A teardown task will run if its setup was successful, even if its work tasks failed. + * Teardown tasks are ignored when setting dependencies against task groups. + +How setup and teardown works +"""""""""""""""""""""""""""" + +Basic usage +""""""""""" + +Suppose you have a dag that creates a cluster, runs a query, and deletes the cluster. Without using setup and teardown tasks you might set these relationships: + +.. code-block:: python + + create_cluster >> run_query >> delete_cluster + +To enable create_cluster and delete_cluster as setup and teardown tasks, we mark them as such methods ``as_setup`` and ``as_teardown`` and add an upstream / downstream relationship between them: + +.. code-block:: python + + create_cluster.as_setup() >> run_query >> delete_cluster.as_teardown() + create_cluster >> delete_cluster + +For convenience we can do this in one line by passing ``create_cluster`` to the ``as_teardown`` method: + +.. code-block:: python + + create_cluster >> run_query >> delete_cluster.as_teardown(setups=create_cluster) + +Observations: + + * If you clear ``run_query`` to run it again, then both ``create_cluster`` and ``delete_cluster`` will be cleared. + * If ``run_query`` fails, then ``delete_cluster`` will still run. + * The success of the dag run will depend *only* on the success of ``run_query``. + +Additionally, if we have multiple tasks to wrap, we can use the teardown as a context manager: + +.. code-block:: python + + with delete_cluster.as_teardown(setups=create_cluster): + [RunQueryOne(), RunQueryTwo()] >> DoSomeOtherStuff() + WorkOne() >> [do_this_stuff(), do_other_stuff()] + +This will set create_cluster to run before the tasks in the context, and delete_cluster after them. + +Note that if you are attempting to add an already-instantiated task to a setup context you need to do it explicitly: + +.. code-block:: python + + with my_teardown_task as scope: + scope.add_task(work_task) # work_task was already instantiated elsewhere + +Setup "scope" +""""""""""""" + +Tasks between a setup and its teardown are in the "scope" of the setup / teardown pair. + +Let's look at an example: + +.. code-block:: python + + s1 >> w1 >> w2 >> t1.as_teardown(setups=s1) >> w3 + w2 >> w4 + +In the above example, ``w1`` and ``w2`` are "between" ``s1`` and ``t1`` and therefore are assumed to require ``s1``. Thus if ``w1`` or ``w2`` is cleared, so too will be ``s1`` and ``t1``. But if ``w3`` or ``w4`` is cleared, neither ``s1`` nor ``t1`` will be cleared. + +You can have multiple setup tasks wired to a single teardown. The teardown will run if at least one of the setups completed successfully. + +You can have a setup without a teardown: + +.. code-block:: python + + create_cluster >> run_query >> other_task + +In this case, everything downstream of create_cluster is assumed to require it. So if you clear query_two, it will also clear create_cluster. Suppose that we add a teardown for create_cluster after run_query: + +.. code-block:: python + + create_cluster >> run_query >> other_task + run_query >> delete_cluster.as_teardown(setups=create_cluster) + +Now, Airflow can infer that other_task does not require create_cluster, so if we clear other_task, create_cluster will not also be cleared. + +In that example, we (in our pretend docs land) actually wanted to delete the cluster. But supposing we didn't, and we just wanted to say "other_task does not require create_cluster", then we could use an EmptyOperator to limit the setup's scope: + +.. code-block:: python + + create_cluster >> run_query >> other_task + run_query >> EmptyOperator(task_id="cluster_teardown").as_teardown(setups=create_cluster) + + +Controlling dag run state +""""""""""""""""""""""""" + +Another feature of setup / teardown tasks is you can choose whether or not the teardown task should have an impact on dag run state. Perhaps you don't care if the "cleanup" work performed by your teardown task fails, and you only consider the dag run a failure if the "work" tasks fail. By default, teardown tasks are not considered for dag run state. + +Continuing with the example above, if you want the run's success to depend on ``delete_cluster``, then set``on_failure_fail_dagrun=True`` when setting ``delete_cluster`` as teardown. For example: + +.. code-block:: python + + create_cluster >> run_query >> delete_cluster.as_teardown(setups=create_cluster, on_failure_fail_dagrun=True) + +Authoring with task groups +"""""""""""""""""""""""""" + +When arrowing from task group to task group, or from task group to task, we ignore teardowns. This allows teardowns to run in parallel, and allows dag execution to proceed even if teardown tasks fail. + +Consider this example: + +.. code-block:: python + + with TaskGroup("my_group") as tg: + s1 = my_setup() + w1 = my_work() + t1 = my_teardown() + s1 >> w1 >> t1.as_teardown(setups=s1) + w2 = other_work() + tg >> w2 + +If ``t1`` were not a teardown task, then this dag would effectively be ``s1 >> w1 >> t1 >> w2``. But since we have marked ``t1`` as a teardown, it's ignored in ``tg >> w2``. So the dag is equivalent to the following: + +.. code-block:: python + + s1 >> w1 >> [t1.as_teardown(setups=s1), w2] + +Now let's consider an example with nesting: + +.. code-block:: python + + with TaskGroup("my_group") as tg: + s1 = my_setup() + w1 = my_work() + t1 = my_teardown() + s1 >> w1 >> t1.as_teardown(setups=s1) + w2 = other_work() + tg >> w2 + dag_s1 = dag_setup1() + dag_t1 = dag_teardown1() + dag_s1 >> [tg, w2] >> dag_t1.as_teardown(dag_s1) + +In this example ``s1`` is downstream of ``dag_s1``, so it must wait for ``dag_s1`` to complete successfully. But ``t1`` and ``dag_t1`` can run concurrently, because ``t1`` is ignored in the expression ``tg >> dag_t1``. If you clear ``w2``, it will clear ``dag_s1`` and ``dag_t1``, but not anything in the task group. + +Running setups and teardowns in parallel +"""""""""""""""""""""""""""""""""""""""" + +You can run setup tasks in parallel: + +.. code-block:: python + + ( + [create_cluster, create_bucket] + >> run_query + >> [delete_cluster.as_teardown(setups=create_cluster), delete_bucket.as_teardown(setups=create_bucket)] + ) + +Trigger rule behavior for teardowns +""""""""""""""""""""""""""""""""""" + +Teardowns use a (non-configurable) trigger rule called ALL_DONE_SETUP_SUCCESS. With this rule, as long as all upstreams are done and at least one directly connected setup is successful, the teardown will run. If all of a teardown's setups were skipped or failed, those states will propagate to the teardown. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index ad3b7cd8f8fe..8effe2cc92ca 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -481,6 +481,7 @@ Dont DOS'ing DownloadReportV downscaling +downstreams Drillbit Drivy dropdown @@ -1493,6 +1494,7 @@ tblproperties TCP tcp teardown +teardowns templatable templateable Templated @@ -1597,6 +1599,7 @@ Upsert upsert upserts Upsight +upstreams Uptime uri uris From 15d42b4320d535cf54743929f134e36f59c615bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Thu, 20 Jul 2023 13:29:20 -0700 Subject: [PATCH 249/533] Reduce default for max TIs per query, enforce <= parallelism (#32572) Co-authored-by: Tzu-ping Chung Co-authored-by: Simon Liu Co-authored-by: lyso Co-authored-by: root --- airflow/config_templates/config.yml | 5 ++-- airflow/configuration.py | 20 ++++++++++++++ .../scheduler.rst | 7 ++--- newsfragments/32572.significant.rst | 10 +++++++ tests/core/test_configuration.py | 27 +++++++++++++++++++ 5 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 newsfragments/32572.significant.rst diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 0629cb5d1195..fbaac698b0e3 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2463,14 +2463,15 @@ scheduler: max_tis_per_query: description: | This changes the batch size of queries in the scheduling main loop. + This should not be greater than ``core.parallelism``. If this is too high, SQL query performance may be impacted by complexity of query predicate, and/or excessive locking. Additionally, you may hit the maximum allowable query length for your db. - Set this to 0 for no limit (not advised) + Set this to 0 to use the value of ``core.parallelism`` version_added: ~ type: integer example: ~ - default: "512" + default: "16" use_row_level_locking: description: | Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. diff --git a/airflow/configuration.py b/airflow/configuration.py index c8e19651d5b0..10f4cf5a704a 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -594,6 +594,7 @@ def write( # type: ignore[override] def validate(self): self._validate_sqlite3_version() self._validate_enums() + self._validate_max_tis_per_query() for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): @@ -615,6 +616,25 @@ def validate(self): self._upgrade_postgres_metastore_conn() self.is_validated = True + def _validate_max_tis_per_query(self) -> None: + """ + Check if config ``scheduler.max_tis_per_query`` is not greater than ``core.parallelism``. + If not met, a warning message is printed to guide the user to correct it. + + More info: https://github.com/apache/airflow/pull/32572 + """ + max_tis_per_query = self.getint("scheduler", "max_tis_per_query") + parallelism = self.getint("core", "parallelism") + + if max_tis_per_query > parallelism: + warnings.warn( + f"Config scheduler.max_tis_per_query (value: {max_tis_per_query}) " + f"should NOT be greater than core.parallelism (value: {parallelism}). " + "Will now use core.parallelism as the max task instances per query " + "instead of specified value.", + UserWarning, + ) + def _upgrade_auth_backends(self): """ Ensure a custom auth_backends setting contains session. diff --git a/docs/apache-airflow/administration-and-deployment/scheduler.rst b/docs/apache-airflow/administration-and-deployment/scheduler.rst index 1a2a41b136ec..06769d159502 100644 --- a/docs/apache-airflow/administration-and-deployment/scheduler.rst +++ b/docs/apache-airflow/administration-and-deployment/scheduler.rst @@ -368,11 +368,12 @@ However, you can also look at other non-performance-related scheduler configurat The scheduler will list and sort the DAG files to decide the parsing order. - :ref:`config:scheduler__max_tis_per_query` - The batch size of queries in the scheduling main loop. If this is too high, SQL query - performance may be impacted by complexity of query predicate, and/or excessive locking. + The batch size of queries in the scheduling main loop. This should not be greater than + ``core.parallelism``. If this is too high then SQL query performance may be impacted by + complexity of query predicate, and/or excessive locking. Additionally, you may hit the maximum allowable query length for your db. - Set this to 0 for no limit (not advised). + Set this to 0 to use the value of ``core.parallelism``. - :ref:`config:scheduler__min_file_process_interval` Number of seconds after which a DAG file is re-parsed. The DAG file is parsed every diff --git a/newsfragments/32572.significant.rst b/newsfragments/32572.significant.rst new file mode 100644 index 000000000000..3105e9831598 --- /dev/null +++ b/newsfragments/32572.significant.rst @@ -0,0 +1,10 @@ +The default value for ``scheduler.max_tis_per_query`` is changed from 512 to 16. + +This change is expected to make the Scheduler more responsive. + +``scheduler.max_tis_per_query`` needs to be lower than ``core.parallelism``. +If both were left to their default value previously, the effective default value of ``scheduler.max_tis_per_query`` was 32 +(because it was capped at ``core.parallelism``). + +To keep the behavior as close as possible to the old config, one can set ``scheduler.max_tis_per_query = 0``, +in which case it'll always use the value of ``core.parallelism``. diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index aa7820cf37b5..ecb01f5beeec 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -78,6 +78,9 @@ def parameterized_config(template) -> str: "AIRFLOW__TESTSECTION__TESTPERCENT": "with%percent", "AIRFLOW__TESTCMDENV__ITSACOMMAND_CMD": 'echo -n "OK"', "AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD": 'echo -n "NOT OK"', + # also set minimum conf values required to pass validation + "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "16", + "AIRFLOW__CORE__PARALLELISM": "32", }, ) class TestConf: @@ -755,6 +758,22 @@ def test_enum_logging_levels(self): ) assert message == exception + @mock.patch.dict( + "os.environ", + { + "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "200", + "AIRFLOW__CORE__PARALLELISM": "100", + }, + ) + def test_max_tis_per_query_too_high(self): + test_conf = AirflowConfigParser() + + with pytest.warns(UserWarning) as ctx: + test_conf._validate_max_tis_per_query() + + captured_warnings_msg = str(ctx.pop().message) + assert "max_tis_per_query" in captured_warnings_msg and "core.parallelism" in captured_warnings_msg + def test_as_dict_works_without_sensitive_cmds(self): conf_materialize_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=True) conf_maintain_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=False) @@ -879,6 +898,14 @@ def test_gettimedelta(self): assert test_conf.gettimedelta("default", "key7") is None +@mock.patch.dict( + "os.environ", + { + # set minimum conf values required to pass validation + "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "16", + "AIRFLOW__CORE__PARALLELISM": "32", + }, +) class TestDeprecatedConf: @conf_vars( { From c7c0deecb4fd7c080c6d7fda0cfd5b0be00dc1d1 Mon Sep 17 00:00:00 2001 From: Phani Kumar <94376113+phanikumv@users.noreply.github.com> Date: Fri, 21 Jul 2023 08:12:52 +0530 Subject: [PATCH 250/533] Refactor Sqlalchemy queries to 2.0 style (Part 6) (#32645) --- airflow/models/serialized_dag.py | 50 +++++++++++---------- airflow/models/trigger.py | 75 ++++++++++++++++++-------------- airflow/utils/cli.py | 3 +- airflow/utils/db_cleanup.py | 6 +-- 4 files changed, 73 insertions(+), 61 deletions(-) diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index 90480d8a3343..82d4c6e31c3c 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -24,7 +24,7 @@ from typing import Collection import sqlalchemy_jsonfield -from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, and_, or_ +from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, and_, or_, select from sqlalchemy.orm import Session, backref, foreign, relationship from sqlalchemy.sql.expression import func, literal @@ -144,23 +144,19 @@ def write_dag( # If Yes, does nothing # If No or the DAG does not exists, updates / writes Serialized DAG to DB if min_update_interval is not None: - if ( - session.query(literal(True)) - .filter( - and_( - cls.dag_id == dag.dag_id, - (timezone.utcnow() - timedelta(seconds=min_update_interval)) < cls.last_updated, - ) + if session.scalar( + select(literal(True)).where( + cls.dag_id == dag.dag_id, + (timezone.utcnow() - timedelta(seconds=min_update_interval)) < cls.last_updated, ) - .scalar() ): return False log.debug("Checking if DAG (%s) changed", dag.dag_id) new_serialized_dag = cls(dag, processor_subdir) - serialized_dag_db = ( - session.query(cls.dag_hash, cls.processor_subdir).filter(cls.dag_id == dag.dag_id).first() - ) + serialized_dag_db = session.execute( + select(cls.dag_hash, cls.processor_subdir).where(cls.dag_id == dag.dag_id) + ).first() if ( serialized_dag_db is not None @@ -183,7 +179,7 @@ def read_all_dags(cls, session: Session = NEW_SESSION) -> dict[str, SerializedDA :param session: ORM Session :returns: a dict of DAGs read from database """ - serialized_dags = session.query(cls) + serialized_dags = session.scalars(select(cls)) dags = {} for row in serialized_dags: @@ -275,7 +271,7 @@ def has_dag(cls, dag_id: str, session: Session = NEW_SESSION) -> bool: :param dag_id: the DAG to check :param session: ORM Session """ - return session.query(literal(True)).filter(cls.dag_id == dag_id).first() is not None + return session.scalar(select(literal(True)).where(cls.dag_id == dag_id).limit(1)) is not None @classmethod @provide_session @@ -296,15 +292,15 @@ def get(cls, dag_id: str, session: Session = NEW_SESSION) -> SerializedDagModel :param dag_id: the DAG to fetch :param session: ORM Session """ - row = session.query(cls).filter(cls.dag_id == dag_id).one_or_none() + row = session.scalar(select(cls).where(cls.dag_id == dag_id)) if row: return row # If we didn't find a matching DAG id then ask the DAG table to find # out the root dag - root_dag_id = session.query(DagModel.root_dag_id).filter(DagModel.dag_id == dag_id).scalar() + root_dag_id = session.scalar(select(DagModel.root_dag_id).where(DagModel.dag_id == dag_id)) - return session.query(cls).filter(cls.dag_id == root_dag_id).one_or_none() + return session.scalar(select(cls).where(cls.dag_id == root_dag_id)) @staticmethod @provide_session @@ -340,7 +336,7 @@ def get_last_updated_datetime(cls, dag_id: str, session: Session = NEW_SESSION) :param dag_id: DAG ID :param session: ORM Session """ - return session.query(cls.last_updated).filter(cls.dag_id == dag_id).scalar() + return session.scalar(select(cls.last_updated).where(cls.dag_id == dag_id)) @classmethod @provide_session @@ -350,7 +346,7 @@ def get_max_last_updated_datetime(cls, session: Session = NEW_SESSION) -> dateti :param session: ORM Session """ - return session.query(func.max(cls.last_updated)).scalar() + return session.scalar(select(func.max(cls.last_updated))) @classmethod @provide_session @@ -362,7 +358,7 @@ def get_latest_version_hash(cls, dag_id: str, session: Session = NEW_SESSION) -> :param session: ORM Session :return: DAG Hash, or None if the DAG is not found """ - return session.query(cls.dag_hash).filter(cls.dag_id == dag_id).scalar() + return session.scalar(select(cls.dag_hash).where(cls.dag_id == dag_id)) @classmethod def get_latest_version_hash_and_updated_datetime( @@ -379,7 +375,9 @@ def get_latest_version_hash_and_updated_datetime( :param session: ORM Session :return: A tuple of DAG Hash and last updated datetime, or None if the DAG is not found """ - return session.query(cls.dag_hash, cls.last_updated).filter(cls.dag_id == dag_id).one_or_none() + return session.execute( + select(cls.dag_hash, cls.last_updated).where(cls.dag_id == dag_id) + ).one_or_none() @classmethod @provide_session @@ -390,11 +388,15 @@ def get_dag_dependencies(cls, session: Session = NEW_SESSION) -> dict[str, list[ :param session: ORM Session """ if session.bind.dialect.name in ["sqlite", "mysql"]: - query = session.query(cls.dag_id, func.json_extract(cls._data, "$.dag.dag_dependencies")) + query = session.execute( + select(cls.dag_id, func.json_extract(cls._data, "$.dag.dag_dependencies")) + ) iterator = ((dag_id, json.loads(deps_data) if deps_data else []) for dag_id, deps_data in query) elif session.bind.dialect.name == "mssql": - query = session.query(cls.dag_id, func.json_query(cls._data, "$.dag.dag_dependencies")) + query = session.execute(select(cls.dag_id, func.json_query(cls._data, "$.dag.dag_dependencies"))) iterator = ((dag_id, json.loads(deps_data) if deps_data else []) for dag_id, deps_data in query) else: - iterator = session.query(cls.dag_id, func.json_extract_path(cls._data, "dag", "dag_dependencies")) + iterator = session.execute( + select(cls.dag_id, func.json_extract_path(cls._data, "dag", "dag_dependencies")) + ) return {dag_id: [DagDependency(**d) for d in (deps_data or [])] for dag_id, deps_data in iterator} diff --git a/airflow/models/trigger.py b/airflow/models/trigger.py index 4f50918a09d4..bba60c94493f 100644 --- a/airflow/models/trigger.py +++ b/airflow/models/trigger.py @@ -20,7 +20,7 @@ from traceback import format_exception from typing import Any, Iterable -from sqlalchemy import Column, Integer, String, delete, func, or_ +from sqlalchemy import Column, Integer, String, delete, func, or_, select, update from sqlalchemy.orm import Session, joinedload, relationship from airflow.api_internal.internal_api_call import internal_api_call @@ -93,9 +93,9 @@ def from_object(cls, trigger: BaseTrigger) -> Trigger: @provide_session def bulk_fetch(cls, ids: Iterable[int], session: Session = NEW_SESSION) -> dict[int, Trigger]: """Fetches all the Triggers by ID and returns a dict mapping ID -> Trigger instance.""" - query = ( - session.query(cls) - .filter(cls.id.in_(ids)) + query = session.scalars( + select(cls) + .where(cls.id.in_(ids)) .options( joinedload("task_instance"), joinedload("task_instance.trigger"), @@ -117,19 +117,21 @@ def clean_unused(cls, session: Session = NEW_SESSION) -> None: # Update all task instances with trigger IDs that are not DEFERRED to remove them for attempt in run_with_db_retries(): with attempt: - session.query(TaskInstance).filter( - TaskInstance.state != TaskInstanceState.DEFERRED, TaskInstance.trigger_id.isnot(None) - ).update({TaskInstance.trigger_id: None}) + session.execute( + update(TaskInstance) + .where( + TaskInstance.state != TaskInstanceState.DEFERRED, TaskInstance.trigger_id.is_not(None) + ) + .values(trigger_id=None) + ) + # Get all triggers that have no task instances depending on them... - ids = [ - trigger_id - for (trigger_id,) in ( - session.query(cls.id) - .join(TaskInstance, cls.id == TaskInstance.trigger_id, isouter=True) - .group_by(cls.id) - .having(func.count(TaskInstance.trigger_id) == 0) - ) - ] + ids = session.scalars( + select(cls.id) + .join(TaskInstance, cls.id == TaskInstance.trigger_id, isouter=True) + .group_by(cls.id) + .having(func.count(TaskInstance.trigger_id) == 0) + ).all() # ...and delete them (we can't do this in one query due to MySQL) session.execute( delete(Trigger).where(Trigger.id.in_(ids)).execution_options(synchronize_session=False) @@ -140,8 +142,10 @@ def clean_unused(cls, session: Session = NEW_SESSION) -> None: @provide_session def submit_event(cls, trigger_id, event, session: Session = NEW_SESSION) -> None: """Takes an event from an instance of itself, and triggers all dependent tasks to resume.""" - for task_instance in session.query(TaskInstance).filter( - TaskInstance.trigger_id == trigger_id, TaskInstance.state == TaskInstanceState.DEFERRED + for task_instance in session.scalars( + select(TaskInstance).where( + TaskInstance.trigger_id == trigger_id, TaskInstance.state == TaskInstanceState.DEFERRED + ) ): # Add the event's payload into the kwargs for the task next_kwargs = task_instance.next_kwargs or {} @@ -171,8 +175,10 @@ def submit_failure(cls, trigger_id, exc=None, session: Session = NEW_SESSION) -> workers as first-class concepts, we can run the failure code here in-process, but we can't do that right now. """ - for task_instance in session.query(TaskInstance).filter( - TaskInstance.trigger_id == trigger_id, TaskInstance.state == TaskInstanceState.DEFERRED + for task_instance in session.scalars( + select(TaskInstance).where( + TaskInstance.trigger_id == trigger_id, TaskInstance.state == TaskInstanceState.DEFERRED + ) ): # Add the error and set the next_method to the fail state traceback = format_exception(type(exc), exc, exc.__traceback__) if exc else None @@ -188,7 +194,7 @@ def submit_failure(cls, trigger_id, exc=None, session: Session = NEW_SESSION) -> @provide_session def ids_for_triggerer(cls, triggerer_id, session: Session = NEW_SESSION) -> list[int]: """Retrieves a list of triggerer_ids.""" - return [row[0] for row in session.query(cls.id).filter(cls.triggerer_id == triggerer_id)] + return session.scalars(select(cls.id).where(cls.triggerer_id == triggerer_id)).all() @classmethod @internal_api_call @@ -203,7 +209,7 @@ def assign_unassigned(cls, triggerer_id, capacity, heartrate, session: Session = """ from airflow.jobs.job import Job # To avoid circular import - count = session.query(func.count(cls.id)).filter(cls.triggerer_id == triggerer_id).scalar() + count = session.scalar(select(func.count(cls.id)).filter(cls.triggerer_id == triggerer_id)) capacity -= count if capacity <= 0: @@ -211,14 +217,13 @@ def assign_unassigned(cls, triggerer_id, capacity, heartrate, session: Session = # we multiply heartrate by a grace_multiplier to give the triggerer # a chance to heartbeat before we consider it dead health_check_threshold = heartrate * 2.1 - alive_triggerer_ids = [ - row[0] - for row in session.query(Job.id).filter( + alive_triggerer_ids = session.scalars( + select(Job.id).where( Job.end_date.is_(None), Job.latest_heartbeat > timezone.utcnow() - datetime.timedelta(seconds=health_check_threshold), Job.job_type == "TriggererJob", ) - ] + ).all() # Find triggers who do NOT have an alive triggerer_id, and then assign # up to `capacity` of those to us. @@ -226,19 +231,23 @@ def assign_unassigned(cls, triggerer_id, capacity, heartrate, session: Session = capacity=capacity, alive_triggerer_ids=alive_triggerer_ids, session=session ) if trigger_ids_query: - session.query(cls).filter(cls.id.in_([i.id for i in trigger_ids_query])).update( - {cls.triggerer_id: triggerer_id}, - synchronize_session=False, + session.execute( + update(cls) + .where(cls.id.in_([i.id for i in trigger_ids_query])) + .values(triggerer_id=triggerer_id) + .execution_options(synchronize_session=False) ) + session.commit() @classmethod def get_sorted_triggers(cls, capacity, alive_triggerer_ids, session): - return with_row_locks( - session.query(cls.id) - .filter(or_(cls.triggerer_id.is_(None), cls.triggerer_id.notin_(alive_triggerer_ids))) + query = with_row_locks( + select(cls.id) + .where(or_(cls.triggerer_id.is_(None), cls.triggerer_id.not_in(alive_triggerer_ids))) .order_by(cls.created_date) .limit(capacity), session, skip_locked=True, - ).all() + ) + return session.execute(query).all() diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index 354c2b236c07..b601c4c07e2c 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Callable, TypeVar, cast import re2 +from sqlalchemy import select from sqlalchemy.orm import Session from airflow import settings @@ -267,7 +268,7 @@ def get_dag_by_pickle(pickle_id: int, session: Session = NEW_SESSION) -> DAG: """Fetch DAG from the database using pickling.""" from airflow.models import DagPickle - dag_pickle = session.query(DagPickle).filter(DagPickle.id == pickle_id).first() + dag_pickle = session.scalar(select(DagPickle).where(DagPickle.id == pickle_id)).first() if not dag_pickle: raise AirflowException(f"pickle_id could not be found in DagPickle.id list: {pickle_id}") pickle_dag = dag_pickle.pickle diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 36bced947348..f9dae372429d 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -30,7 +30,7 @@ from typing import Any from pendulum import DateTime -from sqlalchemy import and_, column, false, func, inspect, table, text +from sqlalchemy import and_, column, false, func, inspect, select, table, text from sqlalchemy.exc import OperationalError, ProgrammingError from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import Query, Session, aliased @@ -179,7 +179,7 @@ def _do_delete(*, query, orm_model, skip_archive, session): pk_cols = source_table.primary_key.columns delete = source_table.delete().where( tuple_(*pk_cols).in_( - session.query(*[target_table.c[x.name] for x in source_table.primary_key.columns]).subquery() + select(*[target_table.c[x.name] for x in source_table.primary_key.columns]).subquery() ) ) else: @@ -196,7 +196,7 @@ def _do_delete(*, query, orm_model, skip_archive, session): def _subquery_keep_last(*, recency_column, keep_last_filters, group_by_columns, max_date_colname, session): - subquery = session.query(*group_by_columns, func.max(recency_column).label(max_date_colname)) + subquery = select(*group_by_columns, func.max(recency_column).label(max_date_colname)) if keep_last_filters is not None: for entry in keep_last_filters: From b09e1f97d57ed9c57591b02e3d578427f3890da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20P=C3=A1ez?= Date: Fri, 21 Jul 2023 10:35:16 +0200 Subject: [PATCH 251/533] Fix dagProcessor not including webserver-config volume (#32644) --- .../dag-processor-deployment.yaml | 5 +++ helm_tests/airflow_core/test_dag_processor.py | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index 0e0e52578a9f..e748e9dff401 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -220,6 +220,11 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }} + - name: webserver-config + configMap: + name: {{ template "airflow_webserver_config_configmap_name" . }} + {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags persistentVolumeClaim: diff --git a/helm_tests/airflow_core/test_dag_processor.py b/helm_tests/airflow_core/test_dag_processor.py index c25fea08ab0c..c1c13634740f 100644 --- a/helm_tests/airflow_core/test_dag_processor.py +++ b/helm_tests/airflow_core/test_dag_processor.py @@ -579,6 +579,45 @@ def test_should_add_component_specific_annotations(self): assert "annotations" in jmespath.search("metadata", docs[0]) assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value" + @pytest.mark.parametrize( + "webserver_config, should_add_volume", + [ + ("CSRF_ENABLED = True", True), + (None, False), + ], + ) + def test_should_add_webserver_config_volume_and_volume_mount_when_exists( + self, webserver_config, should_add_volume + ): + expected_volume = { + "name": "webserver-config", + "configMap": {"name": "release-name-webserver-config"}, + } + expected_volume_mount = { + "name": "webserver-config", + "mountPath": "/opt/airflow/webserver_config.py", + "subPath": "webserver_config.py", + "readOnly": True, + } + + docs = render_chart( + values={ + "dagProcessor": {"enabled": True}, + "webserver": {"webserverConfig": webserver_config}, + }, + show_only=["templates/dag-processor/dag-processor-deployment.yaml"], + ) + + created_volumes = jmespath.search("spec.template.spec.volumes", docs[0]) + created_volume_mounts = jmespath.search("spec.template.spec.containers[1].volumeMounts", docs[0]) + + if should_add_volume: + assert expected_volume in created_volumes + assert expected_volume_mount in created_volume_mounts + else: + assert expected_volume not in created_volumes + assert expected_volume_mount not in created_volume_mounts + class TestDagProcessorLogGroomer(LogGroomerTestBase): """DAG processor log groomer.""" From d5bf74c7b059cd51d72bc4ac0b5ca893e8ac2263 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 21 Jul 2023 15:14:29 +0530 Subject: [PATCH 252/533] Removing hardcoded flag value and using constant (#32713) --- dev/breeze/src/airflow_breeze/utils/docs_errors.py | 4 ++-- dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py | 3 +-- dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py | 2 ++ dev/breeze/src/airflow_breeze/utils/spelling_checks.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/docs_errors.py b/dev/breeze/src/airflow_breeze/utils/docs_errors.py index 441c4c13d12c..57006a8a6ccb 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_errors.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_errors.py @@ -23,13 +23,13 @@ from rich.console import Console -from airflow_breeze.utils.publish_docs_helpers import prepare_code_snippet +from airflow_breeze.utils.publish_docs_helpers import CONSOLE_WIDTH, prepare_code_snippet CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") -console = Console(force_terminal=True, color_system="standard", width=180) +console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH) @total_ordering diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py index 5a64cbd9976b..cb0fba3d8157 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_builder.py @@ -29,11 +29,10 @@ from airflow_breeze.global_constants import get_airflow_version from airflow_breeze.utils.docs_errors import DocBuildError, parse_sphinx_warnings from airflow_breeze.utils.helm_chart_utils import chart_version -from airflow_breeze.utils.publish_docs_helpers import load_package_data, pretty_format_path +from airflow_breeze.utils.publish_docs_helpers import CONSOLE_WIDTH, load_package_data, pretty_format_path from airflow_breeze.utils.spelling_checks import SpellingError, parse_spelling_warnings PROCESS_TIMEOUT = 15 * 60 -CONSOLE_WIDTH = 180 ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py index d570b1992bf9..1d7887f96f00 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py @@ -26,6 +26,8 @@ import yaml +CONSOLE_WIDTH = 180 + ROOT_DIR = Path(__file__).parents[5].resolve() PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR / "airflow" / "provider.yaml.schema.json" diff --git a/dev/breeze/src/airflow_breeze/utils/spelling_checks.py b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py index 42d9bf772375..f0d8a1e6c1a4 100644 --- a/dev/breeze/src/airflow_breeze/utils/spelling_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/spelling_checks.py @@ -24,13 +24,13 @@ from rich.console import Console -from airflow_breeze.utils.publish_docs_helpers import prepare_code_snippet +from airflow_breeze.utils.publish_docs_helpers import CONSOLE_WIDTH, prepare_code_snippet CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) ROOT_PROJECT_DIR = Path(__file__).parents[5].resolve() DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs") -console = Console(force_terminal=True, color_system="standard", width=180) +console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH) @total_ordering From 978adb309aee755df02aadab72fdafb61bec5c80 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 21 Jul 2023 11:57:04 +0200 Subject: [PATCH 253/533] Install sqlalchemy-spanner package into Google provider (#31925) --- .../providers/google/cloud/hooks/spanner.py | 49 +++++++++++++++++-- airflow/providers/google/provider.yaml | 1 + generated/provider_dependencies.json | 3 +- .../google/cloud/hooks/test_spanner.py | 35 ++++++++++++- 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/spanner.py b/airflow/providers/google/cloud/hooks/spanner.py index 00fe9a880456..a3e652dcbd88 100644 --- a/airflow/providers/google/cloud/hooks/spanner.py +++ b/airflow/providers/google/cloud/hooks/spanner.py @@ -18,7 +18,7 @@ """This module contains a Google Cloud Spanner Hook.""" from __future__ import annotations -from typing import Callable, Sequence +from typing import Callable, NamedTuple, Sequence from google.api_core.exceptions import AlreadyExists, GoogleAPICallError from google.cloud.spanner_v1.client import Client @@ -26,13 +26,23 @@ from google.cloud.spanner_v1.instance import Instance from google.cloud.spanner_v1.transaction import Transaction from google.longrunning.operations_grpc_pb2 import Operation +from sqlalchemy import create_engine from airflow.exceptions import AirflowException +from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.providers.google.common.consts import CLIENT_INFO -from airflow.providers.google.common.hooks.base_google import GoogleBaseHook +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook, get_field -class SpannerHook(GoogleBaseHook): +class SpannerConnectionParams(NamedTuple): + """Information about Google Spanner connection parameters.""" + + project_id: str | None + instance_id: str | None + database_id: str | None + + +class SpannerHook(GoogleBaseHook, DbApiHook): """ Hook for Google Cloud Spanner APIs. @@ -40,6 +50,11 @@ class SpannerHook(GoogleBaseHook): keyword arguments rather than positional. """ + conn_name_attr = "gcp_conn_id" + default_conn_name = "google_cloud_spanner_default" + conn_type = "gcpspanner" + hook_name = "Google Cloud Spanner" + def __init__( self, gcp_conn_id: str = "google_cloud_default", @@ -70,6 +85,34 @@ def _get_client(self, project_id: str) -> Client: ) return self._client + def _get_conn_params(self) -> SpannerConnectionParams: + """Extract spanner database connection parameters.""" + extras = self.get_connection(self.gcp_conn_id).extra_dejson + project_id = get_field(extras, "project_id") or self.project_id + instance_id = get_field(extras, "instance_id") + database_id = get_field(extras, "database_id") + return SpannerConnectionParams(project_id, instance_id, database_id) + + def get_uri(self) -> str: + """Override DbApiHook get_uri method for get_sqlalchemy_engine().""" + project_id, instance_id, database_id = self._get_conn_params() + if not all([instance_id, database_id]): + raise AirflowException("The instance_id or database_id were not specified") + return f"spanner+spanner:///projects/{project_id}/instances/{instance_id}/databases/{database_id}" + + def get_sqlalchemy_engine(self, engine_kwargs=None): + """ + Get an sqlalchemy_engine object. + + :param engine_kwargs: Kwargs used in :func:`~sqlalchemy.create_engine`. + :return: the created engine. + """ + if engine_kwargs is None: + engine_kwargs = {} + project_id, _, _ = self._get_conn_params() + spanner_client = self._get_client(project_id=project_id) + return create_engine(self.get_uri(), connect_args={"client": spanner_client}, **engine_kwargs) + @GoogleBaseHook.fallback_to_default_project_id def get_instance( self, diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 5a7b272f420e..3a8471de26d1 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -126,6 +126,7 @@ dependencies: - proto-plus>=1.19.6 - PyOpenSSL - sqlalchemy-bigquery>=1.2.1 + - sqlalchemy-spanner>=1.6.2 integrations: - integration-name: Google Analytics360 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 09c2211b3aaf..abf1088cd45b 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -440,7 +440,8 @@ "pandas-gbq", "pandas>=0.17.1", "proto-plus>=1.19.6", - "sqlalchemy-bigquery>=1.2.1" + "sqlalchemy-bigquery>=1.2.1", + "sqlalchemy-spanner>=1.6.2" ], "cross-providers-deps": [ "amazon", diff --git a/tests/providers/google/cloud/hooks/test_spanner.py b/tests/providers/google/cloud/hooks/test_spanner.py index abb346f0016f..1633d558166d 100644 --- a/tests/providers/google/cloud/hooks/test_spanner.py +++ b/tests/providers/google/cloud/hooks/test_spanner.py @@ -18,9 +18,10 @@ from __future__ import annotations from unittest import mock -from unittest.mock import PropertyMock +from unittest.mock import MagicMock, PropertyMock import pytest +import sqlalchemy from airflow.providers.google.cloud.hooks.spanner import SpannerHook from airflow.providers.google.common.consts import CLIENT_INFO @@ -33,6 +34,8 @@ SPANNER_INSTANCE = "instance" SPANNER_CONFIGURATION = "configuration" SPANNER_DATABASE = "database-name" +SPANNER_PROJECT_ID = "test_project_id" +SPANNER_CONN_PARAMS = (SPANNER_PROJECT_ID, SPANNER_INSTANCE, SPANNER_DATABASE) class TestGcpSpannerHookDefaultProjectId: @@ -431,6 +434,21 @@ def test_execute_dml_overridden_project_id(self, get_client): run_in_transaction_method.assert_called_once_with(mock.ANY) assert res is None + def test_get_uri(self): + self.spanner_hook_default_project_id._get_conn_params = MagicMock(return_value=SPANNER_CONN_PARAMS) + uri = self.spanner_hook_default_project_id.get_uri() + assert ( + uri + == f"spanner+spanner:///projects/{SPANNER_PROJECT_ID}/instances/{SPANNER_INSTANCE}/databases/{SPANNER_DATABASE}" + ) + + @mock.patch("airflow.providers.google.cloud.hooks.spanner.SpannerHook._get_client") + def test_get_sqlalchemy_engine(self, get_client): + self.spanner_hook_default_project_id._get_conn_params = MagicMock(return_value=SPANNER_CONN_PARAMS) + engine = self.spanner_hook_default_project_id.get_sqlalchemy_engine() + assert isinstance(engine, sqlalchemy.engine.Engine) + assert engine.name == "spanner+spanner" + class TestGcpSpannerHookNoDefaultProjectID: def setup_method(self): @@ -675,3 +693,18 @@ def test_execute_dml_overridden_project_id(self, get_client): database_method.assert_called_once_with(database_id="database-name") run_in_transaction_method.assert_called_once_with(mock.ANY) assert res is None + + def test_get_uri(self): + self.spanner_hook_no_default_project_id._get_conn_params = MagicMock(return_value=SPANNER_CONN_PARAMS) + uri = self.spanner_hook_no_default_project_id.get_uri() + assert ( + uri + == f"spanner+spanner:///projects/{SPANNER_PROJECT_ID}/instances/{SPANNER_INSTANCE}/databases/{SPANNER_DATABASE}" + ) + + @mock.patch("airflow.providers.google.cloud.hooks.spanner.SpannerHook._get_client") + def test_get_sqlalchemy_engine(self, get_client): + self.spanner_hook_no_default_project_id._get_conn_params = MagicMock(return_value=SPANNER_CONN_PARAMS) + engine = self.spanner_hook_no_default_project_id.get_sqlalchemy_engine() + assert isinstance(engine, sqlalchemy.engine.Engine) + assert engine.name == "spanner+spanner" From 531eb41bff032e10ffd1f8941113e2a872ef78fd Mon Sep 17 00:00:00 2001 From: Deepchand Swami Date: Fri, 21 Jul 2023 15:37:21 +0530 Subject: [PATCH 254/533] Metrics - Enabled encrypted OTel Endpoint (#32524) * Configured https on otel collector endpoint * added otel ssl flag to config.yml * better coding style used for otel ssl * doc updated for enabling ssl at opentelemetry collector end * Update the documentation to provide instructions on how to establish an HTTPS connection to the OpenTelemetry collector * renamed code block to yaml from yml as doc tests are failing --------- Co-authored-by: Deep Swami --- airflow/config_templates/config.yml | 10 ++++++++++ airflow/metrics/otel_logger.py | 6 ++++-- .../logging-monitoring/metrics.rst | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index fbaac698b0e3..9432b03d1f69 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1059,6 +1059,16 @@ metrics: type: string example: ~ default: "False" + otel_ssl_active: + description: | + If True, SSL will be enabled. Defaults to False. + To establish an HTTPS connection to the OpenTelemetry collector, + you need to configure the SSL certificate and key within the OpenTelemetry collector's + config.yml file. + version_added: 2.7.0 + type: string + example: ~ + default: "False" secrets: description: ~ options: diff --git a/airflow/metrics/otel_logger.py b/airflow/metrics/otel_logger.py index 0cb3289cb50a..3168e9ae65d1 100644 --- a/airflow/metrics/otel_logger.py +++ b/airflow/metrics/otel_logger.py @@ -386,6 +386,7 @@ def get_otel_logger(cls) -> SafeOtelLogger: host = conf.get("metrics", "otel_host") # ex: "breeze-otel-collector" port = conf.getint("metrics", "otel_port") # ex: 4318 prefix = conf.get("metrics", "otel_prefix") # ex: "airflow" + ssl_active = conf.getboolean("metrics", "otel_ssl_active") # PeriodicExportingMetricReader will default to an interval of 60000 millis. interval = conf.getint("metrics", "otel_interval_milliseconds", fallback=None) # ex: 30000 debug = conf.getboolean("metrics", "otel_debugging_on") @@ -394,8 +395,9 @@ def get_otel_logger(cls) -> SafeOtelLogger: allow_list_validator = AllowListValidator(allow_list) resource = Resource(attributes={SERVICE_NAME: "Airflow"}) - # TODO: figure out https instead of http ?? - endpoint = f"http://{host}:{port}/v1/metrics" + + protocol = "https" if ssl_active else "http" + endpoint = f"{protocol}://{host}:{port}/v1/metrics" logging.info("[Metric Exporter] Connecting to OpenTelemetry Collector at %s", endpoint) readers = [ diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst index 02279fbb0366..e4bb1da68a26 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst @@ -73,7 +73,24 @@ Add the following lines to your configuration file e.g. ``airflow.cfg`` otel_port = 8889 otel_prefix = airflow otel_interval_milliseconds = 30000 # The interval between exports, defaults to 60000 + otel_ssl_active = False +Enable Https +----------------- + +To establish an HTTPS connection to the OpenTelemetry collector +You need to configure the SSL certificate and key within the OpenTelemetry collector's ``config.yml`` file. + +.. code-block:: yaml + + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + tls: + cert_file: "/path/to/cert/cert.crt" + key_file: "/path/to/key/key.pem" Allow/Block Lists ----------------- From 49921763eb15f68f91da826a86690ba4c4155c35 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Fri, 21 Jul 2023 12:20:41 +0200 Subject: [PATCH 255/533] Migrate system test for PostgresToGCSOperator to new design AIP-47 (#32641) Co-authored-by: Niko Oliveira --- .../example_dags/example_postgres_to_gcs.py | 51 ---- .../google/cloud/transfers/postgres_to_gcs.py | 4 + airflow/providers/google/provider.yaml | 1 + .../operators/transfer/postgres_to_gcs.rst | 52 ++++ .../transfers/test_postgres_to_gcs_system.py | 2 +- .../transfers/example_postgres_to_gcs.py | 227 ++++++++++++++++++ 6 files changed, 285 insertions(+), 52 deletions(-) delete mode 100644 airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py create mode 100644 docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst create mode 100644 tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py diff --git a/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py deleted file mode 100644 index eca37996eee0..000000000000 --- a/airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Example DAG using PostgresToGoogleCloudStorageOperator. -""" -from __future__ import annotations - -import os -from datetime import datetime - -from airflow import models -from airflow.providers.google.cloud.transfers.postgres_to_gcs import PostgresToGCSOperator - -PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET_NAME", "INVALID BUCKET NAME") -FILENAME = "test_file" -SQL_QUERY = "select * from test_table;" - -with models.DAG( - dag_id="example_postgres_to_gcs", - start_date=datetime(2021, 1, 1), - catchup=False, - tags=["example"], -) as dag: - upload_data = PostgresToGCSOperator( - task_id="get_data", sql=SQL_QUERY, bucket=GCS_BUCKET, filename=FILENAME, gzip=False - ) - - upload_data_server_side_cursor = PostgresToGCSOperator( - task_id="get_data_with_server_side_cursor", - sql=SQL_QUERY, - bucket=GCS_BUCKET, - filename=FILENAME, - gzip=False, - use_server_side_cursor=True, - ) diff --git a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py index f83b47ea325c..9120f7c95afd 100644 --- a/airflow/providers/google/cloud/transfers/postgres_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/postgres_to_gcs.py @@ -69,6 +69,10 @@ class PostgresToGCSOperator(BaseSQLToGCSOperator): """ Copy data from Postgres to Google Cloud Storage in JSON, CSV or Parquet format. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:PostgresToGCSOperator` + :param postgres_conn_id: Reference to a specific Postgres hook. :param use_server_side_cursor: If server-side cursor should be used for querying postgres. For detailed info, check https://www.psycopg.org/docs/usage.html#server-side-cursors diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 3a8471de26d1..14dc8c78900a 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -927,6 +927,7 @@ transfers: python-module: airflow.providers.google.cloud.transfers.gcs_to_sftp - source-integration-name: PostgreSQL target-integration-name: Google Cloud Storage (GCS) + how-to-guide: /docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst python-module: airflow.providers.google.cloud.transfers.postgres_to_gcs - source-integration-name: Google BigQuery target-integration-name: Common SQL diff --git a/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst new file mode 100644 index 000000000000..b6b01ce864a8 --- /dev/null +++ b/docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Postgres To Google Cloud Storage Operator +========================================= +The `Google Cloud Storage `__ (GCS) service is +used to store large data from various applications. This page shows how to copy +data from Postgres to GCS. + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: /operators/_partials/prerequisite_tasks.rst + +.. _howto/operator:PostgresToGCSOperator: + +PostgresToGCSOperator +~~~~~~~~~~~~~~~~~~~~~ + +:class:`~airflow.providers.google.cloud.transfers.postgres_to_gcs.PostgresToGCSOperator` allows you to upload +data from Postgres database to GCS. + +When you use this operator, you can optionally compress the data being uploaded to gzip format. + +Below is an example of using this operator to upload data to GCS. + +.. exampleinclude:: /../../tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py + :language: python + :dedent: 0 + :start-after: [START howto_operator_postgres_to_gcs] + :end-before: [END howto_operator_postgres_to_gcs] + + +Reference +--------- + +For further information, look at: +* `Google Cloud Storage Documentation `__ diff --git a/tests/providers/google/cloud/transfers/test_postgres_to_gcs_system.py b/tests/providers/google/cloud/transfers/test_postgres_to_gcs_system.py index 8b88f3f2d996..262b41ca244e 100644 --- a/tests/providers/google/cloud/transfers/test_postgres_to_gcs_system.py +++ b/tests/providers/google/cloud/transfers/test_postgres_to_gcs_system.py @@ -20,9 +20,9 @@ import pytest from psycopg2 import ProgrammingError -from airflow.providers.google.cloud.example_dags.example_postgres_to_gcs import GCS_BUCKET from airflow.providers.postgres.hooks.postgres import PostgresHook from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY +from tests.system.providers.google.cloud.transfers.example_postgres_to_gcs import BUCKET_NAME as GCS_BUCKET from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context CREATE_QUERY = """ diff --git a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py new file mode 100644 index 000000000000..64e1e1574d5a --- /dev/null +++ b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py @@ -0,0 +1,227 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG using PostgresToGoogleCloudStorageOperator. +""" +from __future__ import annotations + +import logging +import os +from datetime import datetime + +from googleapiclient import discovery + +from airflow import models +from airflow.decorators import task +from airflow.models import Connection +from airflow.operators.bash import BashOperator +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.google.cloud.operators.cloud_sql import ( + CloudSQLCreateInstanceDatabaseOperator, + CloudSQLCreateInstanceOperator, + CloudSQLDeleteInstanceOperator, +) +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, +) +from airflow.providers.google.cloud.transfers.postgres_to_gcs import PostgresToGCSOperator +from airflow.settings import Session +from airflow.utils.trigger_rule import TriggerRule + +DAG_ID = "example_postgres_to_gcs" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "example-project") + +CLOUD_SQL_INSTANCE = f"cloud-sql-{DAG_ID}-{ENV_ID}".replace("_", "-") +CLOUD_SQL_INSTANCE_CREATION_BODY = { + "name": CLOUD_SQL_INSTANCE, + "settings": { + "tier": "db-custom-1-3840", + "dataDiskSizeGb": 30, + "ipConfiguration": { + "ipv4Enabled": True, + "requireSsl": False, + # Consider specifying your network mask + # for allowing requests only from the trusted sources, not from anywhere + "authorizedNetworks": [ + {"value": "0.0.0.0/0"}, + ], + }, + "pricingPlan": "PER_USE", + }, + "databaseVersion": "POSTGRES_15", + "region": "us-central1", +} +DB_NAME = f"{DAG_ID}-{ENV_ID}-db".replace("-", "_") +DB_PORT = 5432 +DB_CREATE_BODY = {"instance": CLOUD_SQL_INSTANCE, "name": DB_NAME, "project": PROJECT_ID} +DB_USER_NAME = "demo_user" +DB_USER_PASSWORD = "demo_password" +CONNECTION_ID = f"postgres_{DAG_ID}_{ENV_ID}".replace("-", "_") + +BUCKET_NAME = f"{DAG_ID}_{ENV_ID}_bucket" +FILE_NAME = "result.json" + +SQL_TABLE = "test_table" +SQL_CREATE = f"CREATE TABLE IF NOT EXISTS {SQL_TABLE} (col_1 INT, col_2 VARCHAR(8))" +SQL_INSERT = f"INSERT INTO {SQL_TABLE} (col_1, col_2) VALUES (1, 'one'), (2, 'two')" +SQL_SELECT = f"SELECT * FROM {SQL_TABLE}" + +log = logging.getLogger(__name__) + + +with models.DAG( + dag_id=DAG_ID, + schedule="@once", + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "postgres", "gcs"], +) as dag: + create_cloud_sql_instance = CloudSQLCreateInstanceOperator( + task_id="create_cloud_sql_instance", + project_id=PROJECT_ID, + instance=CLOUD_SQL_INSTANCE, + body=CLOUD_SQL_INSTANCE_CREATION_BODY, + ) + + create_database = CloudSQLCreateInstanceDatabaseOperator( + task_id="create_database", body=DB_CREATE_BODY, instance=CLOUD_SQL_INSTANCE + ) + + @task + def create_user() -> None: + with discovery.build("sqladmin", "v1beta4") as service: + request = service.users().insert( + project=PROJECT_ID, + instance=CLOUD_SQL_INSTANCE, + body={ + "name": DB_USER_NAME, + "password": DB_USER_PASSWORD, + }, + ) + request.execute() + + create_user_task = create_user() + + @task + def get_public_ip() -> str | None: + with discovery.build("sqladmin", "v1beta4") as service: + request = service.connect().get( + project=PROJECT_ID, instance=CLOUD_SQL_INSTANCE, fields="ipAddresses" + ) + response = request.execute() + for ip_item in response.get("ipAddresses", []): + if ip_item["type"] == "PRIMARY": + return ip_item["ipAddress"] + + get_public_ip_task = get_public_ip() + + @task + def setup_postgres_connection(**kwargs) -> None: + public_ip = kwargs["ti"].xcom_pull(task_ids="get_public_ip") + connection = Connection( + conn_id=CONNECTION_ID, + description="Example PostgreSQL connection", + conn_type="postgres", + host=public_ip, + login=DB_USER_NAME, + password=DB_USER_PASSWORD, + schema=DB_NAME, + port=DB_PORT, + ) + session: Session = Session() + if session.query(Connection).filter(Connection.conn_id == CONNECTION_ID).first(): + log.warning("Connection %s already exists", CONNECTION_ID) + return None + + session.add(connection) + session.commit() + + setup_postgres_connection_task = setup_postgres_connection() + + create_bucket = GCSCreateBucketOperator( + task_id="create_bucket", + bucket_name=BUCKET_NAME, + ) + + create_sql_table = SQLExecuteQueryOperator( + task_id="create_sql_table", + conn_id=CONNECTION_ID, + sql=SQL_CREATE, + ) + + insert_data = SQLExecuteQueryOperator( + task_id="insert_data", + conn_id=CONNECTION_ID, + sql=SQL_INSERT, + ) + + # [START howto_operator_postgres_to_gcs] + get_data = PostgresToGCSOperator( + task_id="get_data", + postgres_conn_id=CONNECTION_ID, + sql=SQL_SELECT, + bucket=BUCKET_NAME, + filename=FILE_NAME, + gzip=False, + ) + # [END howto_operator_postgres_to_gcs] + + delete_cloud_sql_instance = CloudSQLDeleteInstanceOperator( + task_id="delete_cloud_sql_instance", + project_id=PROJECT_ID, + instance=CLOUD_SQL_INSTANCE, + trigger_rule=TriggerRule.ALL_DONE, + ) + + delete_postgres_connection = BashOperator( + task_id="delete_postgres_connection", + bash_command=f"airflow connections delete {CONNECTION_ID}", + trigger_rule=TriggerRule.ALL_DONE, + ) + + delete_bucket = GCSDeleteBucketOperator( + task_id="delete_bucket", + bucket_name=BUCKET_NAME, + trigger_rule=TriggerRule.ALL_DONE, + ) + + # TEST SETUP + create_cloud_sql_instance >> [create_database, create_user_task, get_public_ip_task] + [create_user_task, get_public_ip_task] >> setup_postgres_connection_task + create_database >> setup_postgres_connection_task >> create_sql_table >> insert_data + ( + [insert_data, create_bucket] + # TEST BODY + >> get_data + # TEST TEARDOWN + >> [delete_cloud_sql_instance, delete_postgres_connection, delete_bucket] + ) + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 440c9eb2b4c834d040584a492e23ff4f1212f7f0 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:35:14 +0200 Subject: [PATCH 256/533] Fixup system test for DataprocSubmitJobOperator (PySpark job) (#32740) --- .../google/cloud/operators/dataproc.py | 6 +- .../dataproc/example_dataproc_pyspark.py | 65 +++++++++++++++---- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 5ac1b1cb3df0..bf420e76c010 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1504,6 +1504,10 @@ def execute(self, context: Context): class DataprocSubmitPySparkJobOperator(DataprocJobBaseOperator): """Start a PySpark Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + :param main: [Required] The Hadoop Compatible Filesystem (HCFS) URI of the main Python file to use as the driver. Must be a .py file. (templated) :param arguments: Arguments for the job. (templated) @@ -1940,7 +1944,7 @@ class DataprocSubmitJobOperator(GoogleCloudBaseOperator): :param job: Required. The job resource. If a dict is provided, it must be of the same form as the protobuf message :class:`~google.cloud.dataproc_v1.types.Job`. - For the complete list of supported job types please take a look here + For the complete list of supported job types and their configurations please take a look here https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs :param request_id: Optional. A unique id used to identify the request. If the server receives two ``SubmitJobRequest`` requests with the same id, then the second request will be ignored and the first diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py index b23482f5ac8a..ff3e619d7d8e 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py @@ -22,15 +22,18 @@ import os from datetime import datetime -from pathlib import Path from airflow import models +from airflow.decorators import task from airflow.providers.google.cloud.operators.dataproc import ( DataprocCreateClusterOperator, DataprocDeleteClusterOperator, DataprocSubmitJobOperator, ) -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.google.cloud.operators.gcs import ( + GCSCreateBucketOperator, + GCSDeleteBucketOperator, +) from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator from airflow.utils.trigger_rule import TriggerRule @@ -43,11 +46,7 @@ REGION = "europe-west1" ZONE = "europe-west1-b" -PYSPARK_SRC = str(Path(__file__).parent / "resources" / "hello_world.py") -PYSPARK_FILE = "hello_world.py" - # Cluster definition - CLUSTER_CONFIG = { "master_config": { "num_instances": 1, @@ -61,14 +60,35 @@ }, } -TIMEOUT = {"seconds": 1 * 24 * 60 * 60} +JOB_FILE_NAME = "dataproc-pyspark-job.py" +JOB_FILE_LOCAL_PATH = f"/tmp/{JOB_FILE_NAME}" +JOB_FILE_CONTENT = """from operator import add +from random import random + +from pyspark.sql import SparkSession + + +def f(_: int) -> float: + x = random() * 2 - 1 + y = random() * 2 - 1 + return 1 if x**2 + y**2 <= 1 else 0 + + +spark = SparkSession.builder.appName("PythonPi").getOrCreate() +partitions = 2 +n = 100000 * partitions +count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add) +print(f"Pi is roughly {4.0 * count / n:f}") + +spark.stop() +""" # Jobs definitions # [START how_to_cloud_dataproc_pyspark_config] PYSPARK_JOB = { "reference": {"project_id": PROJECT_ID}, "placement": {"cluster_name": CLUSTER_NAME}, - "pyspark_job": {"main_python_file_uri": f"gs://{BUCKET_NAME}/{PYSPARK_FILE}"}, + "pyspark_job": {"main_python_file_uri": f"gs://{BUCKET_NAME}/{JOB_FILE_NAME}"}, } # [END how_to_cloud_dataproc_pyspark_config] @@ -78,15 +98,23 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "pyspark"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID ) + + @task + def create_job_file(): + with open(JOB_FILE_LOCAL_PATH, "w") as job_file: + job_file.write(JOB_FILE_CONTENT) + + create_job_file_task = create_job_file() + upload_file = LocalFilesystemToGCSOperator( task_id="upload_file", - src=PYSPARK_SRC, - dst=PYSPARK_FILE, + src=JOB_FILE_LOCAL_PATH, + dst=JOB_FILE_NAME, bucket=BUCKET_NAME, ) @@ -116,14 +144,23 @@ task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE ) + @task(trigger_rule=TriggerRule.ALL_DONE) + def delete_job_file(): + try: + os.remove(JOB_FILE_LOCAL_PATH) + except FileNotFoundError: + pass + return 0 + + delete_job_file_task = delete_job_file() + ( # TEST SETUP - create_bucket - >> [upload_file, create_cluster] + [[create_job_file_task, create_bucket] >> upload_file, create_cluster] # TEST BODY >> pyspark_task # TEST TEARDOWN - >> [delete_cluster, delete_bucket] + >> [delete_cluster, delete_bucket, delete_job_file_task] ) from tests.system.utils.watcher import watcher From fbeddc30178eec7bddbafc1d560ff1eb812ae37a Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:36:13 +0200 Subject: [PATCH 257/533] Fixup docstring for deprecated DataprocSubmitPigJobOperator (#32739) --- .../providers/google/cloud/operators/dataproc.py | 4 ++++ .../google/cloud/dataproc/example_dataproc_pig.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index bf420e76c010..b4123f08e994 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1096,6 +1096,10 @@ def on_kill(self) -> None: class DataprocSubmitPigJobOperator(DataprocJobBaseOperator): """Start a Pig query Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + The parameters of the operation will be passed to the cluster. It's a good practice to define dataproc_* parameters in the default_args of the dag diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py index 4510ab09b500..a449437cb31a 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py @@ -31,8 +31,8 @@ ) from airflow.utils.trigger_rule import TriggerRule -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "dataproc_pig" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") CLUSTER_NAME = f"cluster-dataproc-pig-{ENV_ID}" @@ -41,7 +41,6 @@ # Cluster definition - CLUSTER_CONFIG = { "master_config": { "num_instances": 1, @@ -72,7 +71,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "pig"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", @@ -94,7 +93,14 @@ trigger_rule=TriggerRule.ALL_DONE, ) - create_cluster >> pig_task >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> pig_task + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher From 0fbb05a459f62a14896aa5d863685b449ab3e442 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Fri, 21 Jul 2023 18:24:09 +0200 Subject: [PATCH 258/533] Fixup system test for DataprocSubmitJobOperator (SparkSQL job) (#32745) --- airflow/providers/google/cloud/operators/dataproc.py | 4 ++++ .../cloud/dataproc/example_dataproc_spark_sql.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index b4123f08e994..0d9cf592a1cb 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1286,6 +1286,10 @@ def execute(self, context: Context): class DataprocSubmitSparkSqlJobOperator(DataprocJobBaseOperator): """Start a Spark SQL query Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + :param query: The query or reference to the query file (q extension). (templated) :param query_uri: The HCFS URI of the script that contains the SQL queries. :param variables: Map of named parameters for the query. (templated) diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py index ff742aef33be..3db2f881bcd5 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py @@ -37,7 +37,6 @@ CLUSTER_NAME = f"dataproc-spark-sql-{ENV_ID}" REGION = "europe-west1" -ZONE = "europe-west1-b" # Cluster definition CLUSTER_CONFIG = { @@ -53,8 +52,6 @@ }, } -TIMEOUT = {"seconds": 1 * 24 * 60 * 60} - # Jobs definitions # [START how_to_cloud_dataproc_sparksql_config] SPARK_SQL_JOB = { @@ -92,7 +89,14 @@ trigger_rule=TriggerRule.ALL_DONE, ) - create_cluster >> spark_sql_task >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> spark_sql_task + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher From 98a999034cab576b001340c9274ed293dcfce2cd Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Fri, 21 Jul 2023 18:35:22 +0200 Subject: [PATCH 259/533] Add OpenLineage support for MySQL. (#31609) Signed-off-by: Jakub Dardzinski --- airflow/providers/mysql/hooks/mysql.py | 25 ++++++ generated/provider_dependencies.json | 1 + tests/providers/mysql/operators/test_mysql.py | 77 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/airflow/providers/mysql/hooks/mysql.py b/airflow/providers/mysql/hooks/mysql.py index e105d8f96a13..e5023aac1af3 100644 --- a/airflow/providers/mysql/hooks/mysql.py +++ b/airflow/providers/mysql/hooks/mysql.py @@ -298,3 +298,28 @@ def bulk_load_custom( cursor.close() conn.commit() conn.close() # type: ignore[misc] + + def get_openlineage_database_info(self, connection): + """Returns MySQL specific information for OpenLineage.""" + from airflow.providers.openlineage.sqlparser import DatabaseInfo + + return DatabaseInfo( + scheme=self.get_openlineage_database_dialect(connection), + authority=DbApiHook.get_openlineage_authority_part(connection), + information_schema_columns=[ + "table_schema", + "table_name", + "column_name", + "ordinal_position", + "column_type", + ], + normalize_name_method=lambda name: name.upper(), + ) + + def get_openlineage_database_dialect(self, _): + """Returns database dialect.""" + return "mysql" + + def get_openlineage_default_schema(self): + """MySQL has no concept of schema.""" + return None diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index abf1088cd45b..7a160c96dcf0 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -602,6 +602,7 @@ "cross-providers-deps": [ "amazon", "common.sql", + "openlineage", "presto", "trino", "vertica" diff --git a/tests/providers/mysql/operators/test_mysql.py b/tests/providers/mysql/operators/test_mysql.py index 379459985aa6..9202767f9d93 100644 --- a/tests/providers/mysql/operators/test_mysql.py +++ b/tests/providers/mysql/operators/test_mysql.py @@ -20,9 +20,13 @@ import os from contextlib import closing from tempfile import NamedTemporaryFile +from unittest.mock import MagicMock import pytest +from openlineage.client.facet import SchemaDatasetFacet, SchemaField, SqlJobFacet +from openlineage.client.run import Dataset +from airflow.models.connection import Connection from airflow.models.dag import DAG from airflow.providers.mysql.hooks.mysql import MySqlHook from airflow.providers.mysql.operators.mysql import MySqlOperator @@ -111,3 +115,76 @@ def test_mysql_operator_resolve_parameters_template_json_file(self): assert isinstance(task.parameters, dict) assert task.parameters["foo"] == "{{ ds }}" + + @pytest.mark.parametrize("client", ["mysqlclient", "mysql-connector-python"]) + def test_mysql_operator_openlineage(self, client): + with MySqlContext(client): + sql = """ + CREATE TABLE IF NOT EXISTS test_airflow ( + dummy VARCHAR(50) + ); + """ + op = MySqlOperator(task_id="basic_mysql", sql=sql, dag=self.dag) + + lineage = op.get_openlineage_facets_on_start() + assert len(lineage.inputs) == 0 + assert len(lineage.outputs) == 0 + op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + + # OpenLineage provider runs same method on complete by default + lineage_on_complete = op.get_openlineage_facets_on_start() + assert len(lineage_on_complete.inputs) == 0 + assert len(lineage_on_complete.outputs) == 1 + + +def test_execute_openlineage_events(): + class MySqlHookForTests(MySqlHook): + conn_name_attr = "sql_default" + get_conn = MagicMock(name="conn") + get_connection = MagicMock() + + dbapi_hook = MySqlHookForTests() + + class SQLExecuteQueryOperatorForTest(MySqlOperator): + def get_db_hook(self): + return dbapi_hook + + sql = """CREATE TABLE IF NOT EXISTS popular_orders_day_of_week ( + order_day_of_week VARCHAR(64) NOT NULL, + order_placed_on TIMESTAMP NOT NULL, + orders_placed INTEGER NOT NULL + ); +FORGOT TO COMMENT""" + op = SQLExecuteQueryOperatorForTest(task_id="mysql-operator", sql=sql) + DB_SCHEMA_NAME = "PUBLIC" + rows = [ + (DB_SCHEMA_NAME, "popular_orders_day_of_week", "order_day_of_week", 1, "varchar"), + (DB_SCHEMA_NAME, "popular_orders_day_of_week", "order_placed_on", 2, "timestamp"), + (DB_SCHEMA_NAME, "popular_orders_day_of_week", "orders_placed", 3, "int4"), + ] + dbapi_hook.get_connection.return_value = Connection( + conn_id="mysql_default", conn_type="mysql", host="host", port=1234 + ) + dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [rows, []] + + lineage = op.get_openlineage_facets_on_start() + assert len(lineage.inputs) == 0 + assert lineage.outputs == [ + Dataset( + namespace="mysql://host:1234", + name="PUBLIC.popular_orders_day_of_week", + facets={ + "schema": SchemaDatasetFacet( + fields=[ + SchemaField(name="order_day_of_week", type="varchar"), + SchemaField(name="order_placed_on", type="timestamp"), + SchemaField(name="orders_placed", type="int4"), + ] + ) + }, + ) + ] + + assert lineage.job_facets == {"sql": SqlJobFacet(query=sql)} + + assert lineage.run_facets["extractionError"].failedTasks == 1 From 5b082c38a66b1a0b6b496e0d3b15a6684339e1d1 Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Fri, 21 Jul 2023 18:44:03 +0200 Subject: [PATCH 260/533] openlineage, snowflake: add OpenLineage support for Snowflake (#31696) * Add OpenLineage support for SnowflakeOperator. Signed-off-by: Jakub Dardzinski * Change how default schema is retrieved. Signed-off-by: Jakub Dardzinski --------- Signed-off-by: Jakub Dardzinski --- .../providers/snowflake/hooks/snowflake.py | 71 ++++++++++++++++++- generated/provider_dependencies.json | 1 + .../snowflake/hooks/test_snowflake.py | 30 ++++++++ .../snowflake/operators/test_snowflake_sql.py | 68 ++++++++++++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index 76b39be6acc3..3cb0c2a704d6 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -22,7 +22,8 @@ from functools import wraps from io import StringIO from pathlib import Path -from typing import Any, Callable, Iterable, Mapping, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, TypeVar, overload +from urllib.parse import urlparse from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -36,6 +37,9 @@ from airflow.utils.strings import to_boolean T = TypeVar("T") +if TYPE_CHECKING: + from airflow.providers.openlineage.extractors import OperatorLineage + from airflow.providers.openlineage.sqlparser import DatabaseInfo def _try_to_boolean(value: Any): @@ -448,3 +452,68 @@ def _get_cursor(self, conn: Any, return_dictionaries: bool): finally: if cursor is not None: cursor.close() + + def get_openlineage_database_info(self, connection) -> DatabaseInfo: + from airflow.providers.openlineage.sqlparser import DatabaseInfo + + database = self.database or self._get_field(connection.extra_dejson, "database") + + return DatabaseInfo( + scheme=self.get_openlineage_database_dialect(connection), + authority=self._get_openlineage_authority(connection), + information_schema_columns=[ + "table_schema", + "table_name", + "column_name", + "ordinal_position", + "data_type", + ], + database=database, + is_information_schema_cross_db=True, + is_uppercase_names=True, + ) + + def get_openlineage_database_dialect(self, _) -> str: + return "snowflake" + + def get_openlineage_default_schema(self) -> str | None: + """ + Attempts to get current schema. + + Usually ``SELECT CURRENT_SCHEMA();`` should work. + However, apparently you may set ``database`` without ``schema`` + and get results from ``SELECT CURRENT_SCHEMAS();`` but not + from ``SELECT CURRENT_SCHEMA();``. + It still may return nothing if no database is set in connection. + """ + schema = self._get_conn_params()["schema"] + if not schema: + current_schemas = self.get_first("SELECT PARSE_JSON(CURRENT_SCHEMAS())[0]::string;")[0] + if current_schemas: + _, schema = current_schemas.split(".") + return schema + + def _get_openlineage_authority(self, _) -> str: + from openlineage.common.provider.snowflake import fix_snowflake_sqlalchemy_uri + + uri = fix_snowflake_sqlalchemy_uri(self.get_uri()) + return urlparse(uri).hostname + + def get_openlineage_database_specific_lineage(self, _) -> OperatorLineage | None: + from openlineage.client.facet import ExternalQueryRunFacet + + from airflow.providers.openlineage.extractors import OperatorLineage + from airflow.providers.openlineage.sqlparser import SQLParser + + connection = self.get_connection(getattr(self, self.conn_name_attr)) + namespace = SQLParser.create_namespace(self.get_database_info(connection)) + + if self.query_ids: + return OperatorLineage( + run_facets={ + "externalQuery": ExternalQueryRunFacet( + externalQueryId=self.query_ids[0], source=namespace + ) + } + ) + return None diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 7a160c96dcf0..03d821eff15e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -814,6 +814,7 @@ ], "cross-providers-deps": [ "common.sql", + "openlineage", "slack" ], "excluded-python-versions": [] diff --git a/tests/providers/snowflake/hooks/test_snowflake.py b/tests/providers/snowflake/hooks/test_snowflake.py index 6a738952d90f..fef57bc8fce1 100644 --- a/tests/providers/snowflake/hooks/test_snowflake.py +++ b/tests/providers/snowflake/hooks/test_snowflake.py @@ -621,3 +621,33 @@ def test___ensure_prefixes(self): "extra__snowflake__private_key_content", "extra__snowflake__insecure_mode", ] + + @pytest.mark.parametrize( + "returned_schema,expected_schema", + [([None], ""), (["DATABASE.SCHEMA"], "SCHEMA")], + ) + @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook.get_first") + def test_get_openlineage_default_schema_with_no_schema_set( + self, mock_get_first, returned_schema, expected_schema + ): + connection_kwargs = { + **BASE_CONNECTION_KWARGS, + "schema": None, + } + with mock.patch.dict("os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()): + hook = SnowflakeHook(snowflake_conn_id="test_conn") + mock_get_first.return_value = returned_schema + assert hook.get_openlineage_default_schema() == expected_schema + + @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook.get_first") + def test_get_openlineage_default_schema_with_schema_set(self, mock_get_first): + with mock.patch.dict( + "os.environ", AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri() + ): + hook = SnowflakeHook(snowflake_conn_id="test_conn") + assert hook.get_openlineage_default_schema() == BASE_CONNECTION_KWARGS["schema"] + mock_get_first.assert_not_called() + + hook_with_schema_param = SnowflakeHook(snowflake_conn_id="test_conn", schema="my_schema") + assert hook_with_schema_param.get_openlineage_default_schema() == "my_schema" + mock_get_first.assert_not_called() diff --git a/tests/providers/snowflake/operators/test_snowflake_sql.py b/tests/providers/snowflake/operators/test_snowflake_sql.py index 3794bfc550bf..75edeba40545 100644 --- a/tests/providers/snowflake/operators/test_snowflake_sql.py +++ b/tests/providers/snowflake/operators/test_snowflake_sql.py @@ -21,8 +21,12 @@ import pytest from databricks.sql.types import Row +from openlineage.client.facet import SchemaDatasetFacet, SchemaField, SqlJobFacet +from openlineage.client.run import Dataset +from airflow.models.connection import Connection from airflow.providers.common.sql.hooks.sql import fetch_all_handler +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator DATE = "2017-04-20" @@ -138,3 +142,67 @@ def test_exec_success(sql, return_last, split_statement, hook_results, hook_desc return_last=return_last, split_statements=split_statement, ) + + +def test_execute_openlineage_events(): + DB_NAME = "DATABASE" + DB_SCHEMA_NAME = "PUBLIC" + + class SnowflakeHookForTests(SnowflakeHook): + get_conn = MagicMock(name="conn") + get_connection = MagicMock() + + def get_first(self, *_): + return [f"{DB_NAME}.{DB_SCHEMA_NAME}"] + + dbapi_hook = SnowflakeHookForTests() + + class SnowflakeOperatorForTest(SnowflakeOperator): + def get_db_hook(self): + return dbapi_hook + + sql = """CREATE TABLE IF NOT EXISTS popular_orders_day_of_week ( + order_day_of_week VARCHAR(64) NOT NULL, + order_placed_on TIMESTAMP NOT NULL, + orders_placed INTEGER NOT NULL + ); +FORGOT TO COMMENT""" + op = SnowflakeOperatorForTest(task_id="snowflake-operator", sql=sql) + rows = [ + (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDER_DAY_OF_WEEK", 1, "TEXT"), + (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDER_PLACED_ON", 2, "TIMESTAMP_NTZ"), + (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDERS_PLACED", 3, "NUMBER"), + ] + dbapi_hook.get_connection.return_value = Connection( + conn_id="snowflake_default", + conn_type="snowflake", + extra={ + "account": "test_account", + "region": "us-east", + "warehouse": "snow-warehouse", + "database": DB_NAME, + }, + ) + dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [rows, []] + + lineage = op.get_openlineage_facets_on_start() + assert len(lineage.inputs) == 0 + assert lineage.outputs == [ + Dataset( + namespace="snowflake://test_account.us-east.aws", + name=f"{DB_NAME}.{DB_SCHEMA_NAME}.POPULAR_ORDERS_DAY_OF_WEEK", + facets={ + "schema": SchemaDatasetFacet( + fields=[ + SchemaField(name="ORDER_DAY_OF_WEEK", type="TEXT"), + SchemaField(name="ORDER_PLACED_ON", type="TIMESTAMP_NTZ"), + SchemaField(name="ORDERS_PLACED", type="NUMBER"), + ] + ) + }, + ) + ] + + assert lineage.job_facets == {"sql": SqlJobFacet(query=sql)} + + assert lineage.run_facets["extractionError"].failedTasks == 1 From 00aa6ea72c4d72df6c62ac1dda9bb145bece2e1c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 21 Jul 2023 19:04:30 +0200 Subject: [PATCH 261/533] Fix Pandas2 compatibility for Hive (#32752) Pandas2 removed deprecated iteritems() in a few classes with items() being replacement https://github.com/pandas-dev/pandas/pull/37545 The Hive Hook used iteritems when converting to dataframes. Pandas2 upgrade has been enabled now by databricks-sql upgrade so it is now possible to use Pandas2 and update our constraints. --- airflow/providers/apache/hive/hooks/hive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index c67901445a4d..8e81592f7911 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -376,7 +376,7 @@ def _infer_field_types_from_df(df: pandas.DataFrame) -> dict[Any, Any]: } order_type = OrderedDict() - for col, dtype in df.dtypes.iteritems(): + for col, dtype in df.dtypes.items(): order_type[col] = dtype_kind_hive_type[dtype.kind] return order_type From 8e054bb57dd0b29f302872b4500b799c369bde71 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 21 Jul 2023 19:27:51 +0200 Subject: [PATCH 262/533] Fix ARM image building after Cython 3.0.0 release (#32748) Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using setuptools, there is no easy way to fix version of Cython used to compile packages. This triggers a problem with newer `pip` versions that have build isolation enabled by default because There is no (easy) way to pin build dependencies for dependent packages. If a package does not have limit on build dependencies, it will use the latest version of them to build that particular package. The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing the package. Which is precisely what we are doing here. Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip launches a subprocess `pip` to build the package. This is a temporary solution until the issue is resolved in pymssql or Cython. Issues/discussions that track it: * https://github.com/cython/cython/issues/5541 * https://github.com/pymssql/pymssql/pull/827 * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 Since we have to change Dockerfile around installing `pip`, also version of `pip` has been upgraded to latest - 23.2 --- Dockerfile | 59 +++++++++++++++++-- Dockerfile.ci | 50 +++++++++++++++- IMAGES.rst | 2 +- .../src/airflow_breeze/global_constants.py | 2 +- docs/docker-stack/build-arg-ref.rst | 2 +- scripts/docker/common.sh | 2 +- scripts/docker/install_mssql.sh | 41 +++++++++++++ 7 files changed, 147 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f0807425937..14f02ec90377 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.6.3" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bullseye" -ARG AIRFLOW_PIP_VERSION=23.1.2 +ARG AIRFLOW_PIP_VERSION=23.2 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -266,6 +266,10 @@ EOF COPY <<"EOF" /install_mssql.sh set -euo pipefail +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +: "${AIRFLOW_PIP_VERSION:?Should be set}" + : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" COLOR_BLUE=$'\e[34m' @@ -298,6 +302,40 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* + + # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of + # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using + # setuptools, there is no easy way to fix version of Cython used to compile packages. + # + # This triggers a problem with newer `pip` versions that have build isolation enabled by default because + # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have + # limit on build dependencies, it will use the latest version of them to build that particular package. + # + # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to + # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing + # the package. Which is precisely what we are doing here. + # + # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to + # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation + # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip + # launches a subprocess `pip` to build the package. + # + # This is a temporary solution until the issue is resolved in pymssql or Cython + # Issues/discussions that track it: + # + # * https://github.com/cython/cython/issues/5541 + # * https://github.com/pymssql/pymssql/pull/827 + # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 + # + # TODO: Remove this workaround when the issue is resolved. + # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib + # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) + if [[ "${1}" == "dev" ]]; then + common::install_pip_version + echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt + PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql + rm /tmp/mssql-constraints.txt + fi } install_mssql_client "${@}" @@ -433,7 +471,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.1.2" + export AIRFLOW_PIP_VERSION="23.2" fi fi } @@ -1157,8 +1195,15 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ # scripts which are needed much later will not invalidate the docker layer here COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ +# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED +ARG AIRFLOW_PIP_VERSION=23.2 +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +COPY --from=scripts common.sh /scripts/docker/ + + RUN bash /scripts/docker/install_mysql.sh dev && \ - bash /scripts/docker/install_mssql.sh && \ + bash /scripts/docker/install_mssql.sh dev && \ bash /scripts/docker/install_postgres.sh dev ENV PATH=${PATH}:/opt/mssql-tools/bin @@ -1340,6 +1385,12 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} +# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED +ARG AIRFLOW_PIP_VERSION=23.2 +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +COPY --from=scripts common.sh /scripts/docker/ + # Only copy mysql/mssql installation scripts for now - so that changing the other # scripts which are needed much later will not invalidate the docker layer here. COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ @@ -1348,7 +1399,7 @@ COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scrip # had different umask set and group x bit was not set. In Azure the bit might be not set at all. # That also protects against AUFS Docker backend problem where changing the executable bit required sync RUN bash /scripts/docker/install_mysql.sh prod \ - && bash /scripts/docker/install_mssql.sh \ + && bash /scripts/docker/install_mssql.sh prod \ && bash /scripts/docker/install_postgres.sh prod \ && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 23a24d26dd46..bbea500d8f6a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -226,6 +226,10 @@ EOF COPY <<"EOF" /install_mssql.sh set -euo pipefail +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +: "${AIRFLOW_PIP_VERSION:?Should be set}" + : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" COLOR_BLUE=$'\e[34m' @@ -258,6 +262,40 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* + + # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of + # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using + # setuptools, there is no easy way to fix version of Cython used to compile packages. + # + # This triggers a problem with newer `pip` versions that have build isolation enabled by default because + # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have + # limit on build dependencies, it will use the latest version of them to build that particular package. + # + # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to + # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing + # the package. Which is precisely what we are doing here. + # + # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to + # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation + # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip + # launches a subprocess `pip` to build the package. + # + # This is a temporary solution until the issue is resolved in pymssql or Cython + # Issues/discussions that track it: + # + # * https://github.com/cython/cython/issues/5541 + # * https://github.com/pymssql/pymssql/pull/827 + # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 + # + # TODO: Remove this workaround when the issue is resolved. + # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib + # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) + if [[ "${1}" == "dev" ]]; then + common::install_pip_version + echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt + PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql + rm /tmp/mssql-constraints.txt + fi } install_mssql_client "${@}" @@ -393,7 +431,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.1.2" + export AIRFLOW_PIP_VERSION="23.2" fi fi } @@ -1222,6 +1260,12 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh dev +# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED +ARG AIRFLOW_PIP_VERSION=23.2 +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +COPY --from=scripts common.sh /scripts/docker/ + # Only copy mysql/mssql installation scripts for now - so that changing the other # scripts which are needed much later will not invalidate the docker layer here. COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ @@ -1240,7 +1284,7 @@ ENV HOME=${HOME} \ # That also protects against AUFS Docker backen dproblem where changing the executable bit required sync RUN bash /scripts/docker/install_mysql.sh prod \ && bash /scripts/docker/install_mysql.sh dev \ - && bash /scripts/docker/install_mssql.sh \ + && bash /scripts/docker/install_mssql.sh dev \ && bash /scripts/docker/install_postgres.sh dev # Install Helm @@ -1275,7 +1319,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="4"0 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" -ARG AIRFLOW_PIP_VERSION=23.1.2 +ARG AIRFLOW_PIP_VERSION=23.2 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" diff --git a/IMAGES.rst b/IMAGES.rst index cb049bf00571..2fa95b4a9c0f 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -462,7 +462,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_DEV_APT_ENV`` | | Additional env variables defined | | | | when installing dev deps | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``23.1.2`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``23.2`` | PIP version used. | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 7b79af5ad4b8..18ff5d7828f6 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -87,7 +87,7 @@ ALLOWED_MYSQL_VERSIONS = ["5.7", "8"] ALLOWED_MSSQL_VERSIONS = ["2017-latest", "2019-latest"] -PIP_VERSION = "23.1.2" +PIP_VERSION = "23.2" @lru_cache(maxsize=None) diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 6da9cdf145e2..8544beff6dad 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. | +------------------------------------------+------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``23.1.2`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``23.2`` | PIP version used. | +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``ADDITIONAL_PIP_INSTALL_FLAGS`` | | additional ``pip`` flags passed to the | | | | installation commands (except when | diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index ba04841b7afc..f6066c3806af 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -42,7 +42,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.1.2" + export AIRFLOW_PIP_VERSION="23.2" fi fi } diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh index fa0cfc0033cb..883819a2255e 100644 --- a/scripts/docker/install_mssql.sh +++ b/scripts/docker/install_mssql.sh @@ -17,6 +17,13 @@ # shellcheck shell=bash set -euo pipefail +# REMOVE THOSE 3 LINES BELOW TOGETHER WITH PYMSSQL INSTALLATION BELOW AFTER PYMSSQL +# INSTALLATION WITH CYTHON 3.0.0 IS FIXED SEE BELOW FOR MORE DETAILS +# shellcheck source=scripts/docker/common.sh +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +: "${AIRFLOW_PIP_VERSION:?Should be set}" + : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" COLOR_BLUE=$'\e[34m' @@ -49,6 +56,40 @@ function install_mssql_client() { rm -rf /var/lib/apt/lists/* apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* + + # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of + # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using + # setuptools, there is no easy way to fix version of Cython used to compile packages. + # + # This triggers a problem with newer `pip` versions that have build isolation enabled by default because + # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have + # limit on build dependencies, it will use the latest version of them to build that particular package. + # + # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to + # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing + # the package. Which is precisely what we are doing here. + # + # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to + # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation + # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip + # launches a subprocess `pip` to build the package. + # + # This is a temporary solution until the issue is resolved in pymssql or Cython + # Issues/discussions that track it: + # + # * https://github.com/cython/cython/issues/5541 + # * https://github.com/pymssql/pymssql/pull/827 + # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833 + # + # TODO: Remove this workaround when the issue is resolved. + # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib + # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove) + if [[ "${1}" == "dev" ]]; then + common::install_pip_version + echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt + PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql + rm /tmp/mssql-constraints.txt + fi } install_mssql_client "${@}" From 067365f561ab8a3ab7b5bbf1227e6139607e0abf Mon Sep 17 00:00:00 2001 From: Arjun Anandkumar <102953522+arjunanan6@users.noreply.github.com> Date: Fri, 21 Jul 2023 19:35:46 +0200 Subject: [PATCH 263/533] Arjunanan6/add company (#32742) --- INTHEWILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INTHEWILD.md b/INTHEWILD.md index 9a1e17fbffcc..4a872b80e642 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -514,4 +514,5 @@ Currently, **officially** using Airflow: 1. [Zerodha](https://zerodha.com/) [[@johnnybravo-xyz](https://github.com/johnnybravo-xyz)] 1. [Zymergen](https://www.zymergen.com/) 1. [Zynga](https://www.zynga.com) +1. [Ørsted](https://orsted.com/en) [[@arjunanan6](https://github.com/arjunanan6)] 1. [好大夫在线](https://www.haodf.com/) [[@leiguorui](https://github.com/leiguorui)] From 815655101b7457d60be08648e6cd02af30e0d695 Mon Sep 17 00:00:00 2001 From: Syed Hussain <103602455+syedahsn@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:53:17 -0700 Subject: [PATCH 264/533] Add Deferrable mode to StepFunctionStartExecutionOperator (#32563) * Add stepfunction trigger to provider.yaml * Update docstring about default value of deferrable * Change file name from test_stepfunction.py to test_step_function.py to match other tests --- .../amazon/aws/operators/step_function.py | 36 ++++++++++- .../amazon/aws/triggers/step_function.py | 59 +++++++++++++++++++ .../amazon/aws/waiters/stepfunctions.json | 36 +++++++++++ airflow/providers/amazon/provider.yaml | 3 + .../operators/step_functions.rst | 1 + .../aws/operators/test_step_function.py | 17 ++++++ .../amazon/aws/triggers/test_step_function.py | 53 +++++++++++++++++ 7 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 airflow/providers/amazon/aws/triggers/step_function.py create mode 100644 airflow/providers/amazon/aws/waiters/stepfunctions.json create mode 100644 tests/providers/amazon/aws/triggers/test_step_function.py diff --git a/airflow/providers/amazon/aws/operators/step_function.py b/airflow/providers/amazon/aws/operators/step_function.py index 2aa8bdd8e257..68324df731de 100644 --- a/airflow/providers/amazon/aws/operators/step_function.py +++ b/airflow/providers/amazon/aws/operators/step_function.py @@ -17,11 +17,14 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Sequence +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Sequence +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook +from airflow.providers.amazon.aws.triggers.step_function import StepFunctionsExecutionCompleteTrigger if TYPE_CHECKING: from airflow.utils.context import Context @@ -42,6 +45,11 @@ class StepFunctionStartExecutionOperator(BaseOperator): :param state_machine_input: JSON data input to pass to the State Machine :param aws_conn_id: aws connection to uses :param do_xcom_push: if True, execution_arn is pushed to XCom with key execution_arn. + :param waiter_max_attempts: Maximum number of attempts to poll the execution. + :param waiter_delay: Number of seconds between polling the state of the execution. + :param deferrable: If True, the operator will wait asynchronously for the job to complete. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False, but can be overridden in config file by setting default_deferrable to True) """ template_fields: Sequence[str] = ("state_machine_arn", "name", "input") @@ -56,6 +64,9 @@ def __init__( state_machine_input: dict | str | None = None, aws_conn_id: str = "aws_default", region_name: str | None = None, + waiter_max_attempts: int = 30, + waiter_delay: int = 60, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(**kwargs) @@ -64,6 +75,9 @@ def __init__( self.input = state_machine_input self.aws_conn_id = aws_conn_id self.region_name = region_name + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts + self.deferrable = deferrable def execute(self, context: Context): hook = StepFunctionHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) @@ -74,9 +88,27 @@ def execute(self, context: Context): raise AirflowException(f"Failed to start State Machine execution for: {self.state_machine_arn}") self.log.info("Started State Machine execution for %s: %s", self.state_machine_arn, execution_arn) - + if self.deferrable: + self.defer( + trigger=StepFunctionsExecutionCompleteTrigger( + execution_arn=execution_arn, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + region_name=self.region_name, + ), + method_name="execute_complete", + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + ) return execution_arn + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None or event["status"] != "success": + raise AirflowException(f"Trigger error: event is {event}") + + self.log.info("State Machine execution completed successfully") + return event["execution_arn"] + class StepFunctionGetExecutionOutputOperator(BaseOperator): """ diff --git a/airflow/providers/amazon/aws/triggers/step_function.py b/airflow/providers/amazon/aws/triggers/step_function.py new file mode 100644 index 000000000000..c4875f078f19 --- /dev/null +++ b/airflow/providers/amazon/aws/triggers/step_function.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook +from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook +from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger + + +class StepFunctionsExecutionCompleteTrigger(AwsBaseWaiterTrigger): + """ + Trigger to poll for the completion of a Step Functions execution. + + :param execution_arn: ARN of the state machine to poll + :param waiter_delay: The amount of time in seconds to wait between attempts. + :param waiter_max_attempts: The maximum number of attempts to be made. + :param aws_conn_id: The Airflow connection used for AWS credentials. + """ + + def __init__( + self, + *, + execution_arn: str, + waiter_delay: int = 60, + waiter_max_attempts: int = 30, + aws_conn_id: str | None = None, + region_name: str | None = None, + ) -> None: + + super().__init__( + serialized_fields={"execution_arn": execution_arn, "region_name": region_name}, + waiter_name="step_function_succeeded", + waiter_args={"executionArn": execution_arn}, + failure_message="Step function failed", + status_message="Status of step function execution is", + status_queries=["status", "error", "cause"], + return_key="execution_arn", + return_value=execution_arn, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return StepFunctionHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name) diff --git a/airflow/providers/amazon/aws/waiters/stepfunctions.json b/airflow/providers/amazon/aws/waiters/stepfunctions.json new file mode 100644 index 000000000000..7a7af36786dc --- /dev/null +++ b/airflow/providers/amazon/aws/waiters/stepfunctions.json @@ -0,0 +1,36 @@ +{ + "version": 2, + "waiters": { + "step_function_succeeded": { + "operation": "DescribeExecution", + "delay": 30, + "maxAttempts": 60, + "acceptors": [ + { + "matcher": "path", + "argument": "status", + "expected": "SUCCEEDED", + "state": "success" + }, + { + "matcher": "error", + "argument": "status", + "expected": "RUNNING", + "state": "retry" + }, + { + "matcher": "path", + "argument": "status", + "expected": "FAILED", + "state": "failure" + }, + { + "matcher": "path", + "argument": "status", + "expected": "ABORTED", + "state": "failure" + } + ] + } + } +} diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index ff7ed915255a..66d331b5cdb0 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -566,6 +566,9 @@ triggers: - integration-name: Amazon RDS python-modules: - airflow.providers.amazon.aws.triggers.rds + - integration-name: AWS Step Functions + python-modules: + - airflow.providers.amazon.aws.triggers.step_function transfers: - source-integration-name: Amazon DynamoDB diff --git a/docs/apache-airflow-providers-amazon/operators/step_functions.rst b/docs/apache-airflow-providers-amazon/operators/step_functions.rst index 1f207b4576f9..7736fa9b1674 100644 --- a/docs/apache-airflow-providers-amazon/operators/step_functions.rst +++ b/docs/apache-airflow-providers-amazon/operators/step_functions.rst @@ -38,6 +38,7 @@ Start an AWS Step Functions state machine execution To start a new AWS Step Functions state machine execution you can use :class:`~airflow.providers.amazon.aws.operators.step_function.StepFunctionStartExecutionOperator`. +You can also run this operator in deferrable mode by setting ``deferrable`` param to ``True``. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_step_functions.py :language: python diff --git a/tests/providers/amazon/aws/operators/test_step_function.py b/tests/providers/amazon/aws/operators/test_step_function.py index 566e134a86eb..91ccebf7c6e2 100644 --- a/tests/providers/amazon/aws/operators/test_step_function.py +++ b/tests/providers/amazon/aws/operators/test_step_function.py @@ -22,6 +22,8 @@ import pytest +from airflow.exceptions import TaskDeferred +from airflow.providers.amazon.aws.hooks.step_function import StepFunctionHook from airflow.providers.amazon.aws.operators.step_function import ( StepFunctionGetExecutionOutputOperator, StepFunctionStartExecutionOperator, @@ -132,3 +134,18 @@ def test_execute(self, mock_hook): # Then assert hook_response == result + + @mock.patch.object(StepFunctionHook, "start_execution") + def test_step_function_start_execution_deferrable(self, mock_start_execution): + mock_start_execution.return_value = "test-execution-arn" + operator = StepFunctionStartExecutionOperator( + task_id=self.TASK_ID, + state_machine_arn=STATE_MACHINE_ARN, + name=NAME, + state_machine_input=INPUT, + aws_conn_id=AWS_CONN_ID, + region_name=REGION_NAME, + deferrable=True, + ) + with pytest.raises(TaskDeferred): + operator.execute(None) diff --git a/tests/providers/amazon/aws/triggers/test_step_function.py b/tests/providers/amazon/aws/triggers/test_step_function.py new file mode 100644 index 000000000000..d0c25e096f58 --- /dev/null +++ b/tests/providers/amazon/aws/triggers/test_step_function.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.amazon.aws.triggers.step_function import StepFunctionsExecutionCompleteTrigger + +TEST_EXECUTION_ARN = "test-execution-arn" +TEST_WAITER_DELAY = 10 +TEST_WAITER_MAX_ATTEMPTS = 10 +TEST_AWS_CONN_ID = "test-conn-id" +TEST_REGION_NAME = "test-region-name" + + +class TestStepFunctionsTriggers: + @pytest.mark.parametrize( + "trigger", + [ + StepFunctionsExecutionCompleteTrigger( + execution_arn=TEST_EXECUTION_ARN, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + region_name=TEST_REGION_NAME, + ) + ], + ) + def test_serialize_recreate(self, trigger): + class_path, args = trigger.serialize() + + class_name = class_path.split(".")[-1] + clazz = globals()[class_name] + instance = clazz(**args) + + class_path2, args2 = instance.serialize() + + assert class_path == class_path2 + assert args == args2 From 73b90c48b1933b49086d34176527947bd727ec85 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 21 Jul 2023 20:13:17 +0200 Subject: [PATCH 265/533] Allow configuration to be contributed by providers (#32604) * Allow configuration to be contributed by providers The changes implemented: * provider.yaml files for providers can optionally contribute extra configuration, the configuration is exposed via "get_provider_info" entrypoint, thus allowing Airflow to discover the configuration from both - sources (in Breeze and local development) and from installed packages * Provider configuraitions are lazily loaded - only for commands that actually need them * Documentation for configuration contributed by providers is generated as part of Provider documentation. It is also discoverable by having a "core-extension" page displaying all community providers that contribute their own configuration. * Celery configuration (and in the future Kubernetes configuration) is linked directly from the airflow documentation - the providers are preinstalled, which means that celery (and Kubernetes in the future) configuration is considered as important to be directly mentioned and linked from the core. Similarly Celery and Kubernetes executor documentation remains in the core documentation (still configuration options are detailed only in the provider documentation and only linked from the core. * configuration writing happens in "main" not in the configuration initialization and we will always execute provider configuration initialization. This will make sure that the generated configuration will contain configuration for the providers as well. * Related documentation about custom and community providers have been updated and somewhat refactored - I realized that some of it was quite out-of-date and some of it was really "developer" not user docs. The docs are restructured a bit, cleaned, missing information is added and old/irrelevant parts removed. Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> * Update airflow/configuration.py Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- TESTING.rst | 3 +- airflow/__main__.py | 7 + airflow/cli/cli_config.py | 34 +- airflow/cli/commands/config_command.py | 9 + airflow/cli/commands/provider_command.py | 12 + airflow/config_templates/config.yml | 251 ------------ airflow/configuration.py | 371 +++++++++++++----- .../CREATING_COMMUNITY_PROVIDERS.rst | 99 ----- .../executors/celery_kubernetes_executor.py | 22 +- .../celery/executors/default_celery.py | 2 + airflow/providers/celery/provider.yaml | 253 ++++++++++++ airflow/providers_manager.py | 39 +- dev/README_RELEASE_AIRFLOW.md | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../configurations-ref.rst | 18 + .../apache-airflow-providers-celery/index.rst | 1 + .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../configurations-ref.rst | 71 ---- .../apache-airflow-providers-google/index.rst | 1 - .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../installing-providers-from-sources.rst | 2 +- .../core-extensions/configurations.rst | 31 ++ .../howto/create-custom-providers.rst | 268 +++++++++++++ docs/apache-airflow-providers/index.rst | 241 ++---------- docs/apache-airflow-providers/redirects.txt | 19 + docs/apache-airflow/configurations-ref.rst | 81 +--- .../core-concepts/executor/celery.rst | 2 + .../executor/celery_kubernetes.rst | 2 + .../howto/create-custom-decorator.rst | 3 +- .../howto/setup-and-teardown.rst | 2 +- docs/apache-airflow/tutorial/taskflow.rst | 46 ++- docs/conf.py | 35 +- .../installing-providers-from-sources.rst | 0 .../includes/providers-configurations-ref.rst | 29 ++ docs/exts/includes/sections-and-options.rst | 90 +++++ docs/exts/operators_and_hooks_ref.py | 22 +- .../{ => templates}/auth_backend.rst.jinja2 | 0 docs/exts/templates/configuration.rst.jinja2 | 21 + .../{ => templates}/connections.rst.jinja2 | 0 .../deferrable_operators_list.rst.jinja2} | 2 +- .../exts/{ => templates}/executors.rst.jinja2 | 0 .../{ => templates}/extra_links.rst.jinja2 | 0 docs/exts/{ => templates}/logging.rst.jinja2 | 0 .../{ => templates}/notifications.rst.jinja2 | 0 ...erators_and_hooks_ref-transfers.rst.jinja2 | 0 .../operators_and_hooks_ref.rst.jinja2 | 0 .../{ => templates}/secret_backend.rst.jinja2 | 0 tests/cli/commands/test_celery_command.py | 2 +- tests/cli/commands/test_config_command.py | 9 + tests/cli/test_cli_parser.py | 81 +++- tests/conftest.py | 7 + tests/core/test_configuration.py | 58 ++- .../test_celery_kubernetes_executor.py | 2 +- tests/providers/hashicorp/hooks/test_vault.py | 89 ++++- 132 files changed, 1556 insertions(+), 945 deletions(-) rename docs/apache-airflow-providers/howto/create-update-providers.rst => airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst (71%) create mode 100644 docs/apache-airflow-providers-celery/configurations-ref.rst delete mode 100644 docs/apache-airflow-providers-google/configurations-ref.rst create mode 100644 docs/apache-airflow-providers/core-extensions/configurations.rst create mode 100644 docs/apache-airflow-providers/howto/create-custom-providers.rst create mode 100644 docs/apache-airflow-providers/redirects.txt rename docs/{ => exts/includes}/installing-providers-from-sources.rst (100%) create mode 100644 docs/exts/includes/providers-configurations-ref.rst create mode 100644 docs/exts/includes/sections-and-options.rst rename docs/exts/{ => templates}/auth_backend.rst.jinja2 (100%) create mode 100644 docs/exts/templates/configuration.rst.jinja2 rename docs/exts/{ => templates}/connections.rst.jinja2 (100%) rename docs/exts/{deferrable_operatos_list.rst.jinja2 => templates/deferrable_operators_list.rst.jinja2} (98%) rename docs/exts/{ => templates}/executors.rst.jinja2 (100%) rename docs/exts/{ => templates}/extra_links.rst.jinja2 (100%) rename docs/exts/{ => templates}/logging.rst.jinja2 (100%) rename docs/exts/{ => templates}/notifications.rst.jinja2 (100%) rename docs/exts/{ => templates}/operators_and_hooks_ref-transfers.rst.jinja2 (100%) rename docs/exts/{ => templates}/operators_and_hooks_ref.rst.jinja2 (100%) rename docs/exts/{ => templates}/secret_backend.rst.jinja2 (100%) diff --git a/TESTING.rst b/TESTING.rst index 09b7e9d31491..95bdc5cdeb6d 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -75,7 +75,7 @@ as default for all tests you should add the value to this file. You can also of course override the values in individual test by patching environment variables following the usual ``AIRFLOW__SECTION__KEY`` pattern or ``conf_vars`` context manager. -.. note:: +.. note:: Previous way of setting the test configuration The test configuration for Airflow before July 2023 was automatically generated in a file named ``AIRFLOW_HOME/unittest.cfg``. The template for it was stored in "config_templates" next to the yaml file. @@ -87,6 +87,7 @@ the usual ``AIRFLOW__SECTION__KEY`` pattern or ``conf_vars`` context manager. The unittest.cfg file generated in {AIRFLOW_HOME} will no longer be used and can be removed. + Airflow test types ------------------ diff --git a/airflow/__main__.py b/airflow/__main__.py index 5eca0b84ed6d..893c6bde099d 100644 --- a/airflow/__main__.py +++ b/airflow/__main__.py @@ -45,6 +45,13 @@ def main(): parser = cli_parser.get_parser() argcomplete.autocomplete(parser) args = parser.parse_args() + + # Here we ensure that the default configuration is written if needed before running any command + # that might need it. This used to be done during configuration initialization but having it + # in main ensures that it is not done during tests and other ways airflow imports are used + from airflow.configuration import write_default_airflow_configuration_if_needed + + write_default_airflow_configuration_if_needed() args.func(args) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index b661f9361ab8..a57a8dae1dec 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -330,6 +330,11 @@ def string_lower_type(val): help="Comment out all configuration options. Useful as starting point for new installation", action="store_true", ) +ARG_EXCLUDE_PROVIDERS = Arg( + ("-p", "--exclude-providers"), + help="Exclude provider configuration (they are included by default)", + action="store_true", +) ARG_DEFAULTS = Arg( ("-a", "--defaults"), help="Show only defaults - do not include local configuration, sources," @@ -818,6 +823,16 @@ def string_lower_type(val): action="store_true", ) +# IMPORTANT NOTE! ONLY FOR CELERY ARGUMENTS +# +# Celery configs below have explicit fallback values because celery provider defaults are not yet loaded +# via provider at the time we parse the command line, so in case it is not set, we need to have manual +# fallback. After ProvidersManager.initialize_providers_configuration() is called, the fallbacks are +# not needed anymore and everywhere where you access configuration in provider-specific code and when +# you are sure that providers configuration has been initialized, you can use conf.get() without fallbacks. +# +# DO NOT REMOVE THE FALLBACKS in args parsing even if you are tempted to. +# TODO: possibly move the commands to providers but that could be big performance hit on the CLI # worker ARG_QUEUES = Arg( ("-q", "--queues"), @@ -828,7 +843,7 @@ def string_lower_type(val): ("-c", "--concurrency"), type=int, help="The number of worker processes", - default=conf.get("celery", "worker_concurrency"), + default=conf.getint("celery", "worker_concurrency", fallback=16), ) ARG_CELERY_HOSTNAME = Arg( ("-H", "--celery-hostname"), @@ -855,22 +870,24 @@ def string_lower_type(val): ARG_BROKER_API = Arg(("-a", "--broker-api"), help="Broker API") ARG_FLOWER_HOSTNAME = Arg( ("-H", "--hostname"), - default=conf.get("celery", "FLOWER_HOST"), + default=conf.get("celery", "FLOWER_HOST", fallback="0.0.0.0"), help="Set the hostname on which to run the server", ) ARG_FLOWER_PORT = Arg( ("-p", "--port"), - default=conf.get("celery", "FLOWER_PORT"), + default=conf.getint("celery", "FLOWER_PORT", fallback=5555), type=int, help="The port on which to run the server", ) ARG_FLOWER_CONF = Arg(("-c", "--flower-conf"), help="Configuration file for flower") ARG_FLOWER_URL_PREFIX = Arg( - ("-u", "--url-prefix"), default=conf.get("celery", "FLOWER_URL_PREFIX"), help="URL prefix for Flower" + ("-u", "--url-prefix"), + default=conf.get("celery", "FLOWER_URL_PREFIX", fallback=""), + help="URL prefix for Flower", ) ARG_FLOWER_BASIC_AUTH = Arg( ("-A", "--basic-auth"), - default=conf.get("celery", "FLOWER_BASIC_AUTH"), + default=conf.get("celery", "FLOWER_BASIC_AUTH", fallback=""), help=( "Securing Flower with Basic Authentication. " "Accepts user:password pairs separated by a comma. " @@ -1848,6 +1865,12 @@ class GroupCommand(NamedTuple): func=lazy_load_command("airflow.cli.commands.provider_command.executors_list"), args=(ARG_OUTPUT, ARG_VERBOSE), ), + ActionCommand( + name="configs", + help="Get information about provider configuration", + func=lazy_load_command("airflow.cli.commands.provider_command.config_list"), + args=(ARG_OUTPUT, ARG_VERBOSE), + ), ActionCommand( name="status", help="Get information about provider initialization status", @@ -2038,6 +2061,7 @@ class GroupCommand(NamedTuple): ARG_INCLUDE_SOURCES, ARG_INCLUDE_ENV_VARS, ARG_COMMENT_OUT_EVERYTHING, + ARG_EXCLUDE_PROVIDERS, ARG_DEFAULTS, ARG_VERBOSE, ), diff --git a/airflow/cli/commands/config_command.py b/airflow/cli/commands/config_command.py index 8d664417237d..86130dab102c 100644 --- a/airflow/cli/commands/config_command.py +++ b/airflow/cli/commands/config_command.py @@ -37,6 +37,7 @@ def show_config(args): include_descriptions=args.include_descriptions or args.defaults, include_sources=args.include_sources and not args.defaults, include_env_vars=args.include_env_vars or args.defaults, + include_providers=not args.exclude_providers, comment_out_everything=args.comment_out_everything or args.defaults, only_defaults=args.defaults, ) @@ -48,6 +49,14 @@ def show_config(args): def get_value(args): """Get one value from configuration.""" + # while this will make get_value quite a bit slower we must initialize configuration + # for providers because we do not know what sections and options will be available after + # providers are initialized. Theoretically Providers might add new sections and options + # but also override defaults for existing options, so without loading all providers we + # cannot be sure what is the final value of the option. + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() if not conf.has_option(args.section, args.option): raise SystemExit(f"The option [{args.section}/{args.option}] is not found in config.") diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py index 0256a74089c0..adcb1d18fa14 100644 --- a/airflow/cli/commands/provider_command.py +++ b/airflow/cli/commands/provider_command.py @@ -183,6 +183,18 @@ def executors_list(args): ) +@suppress_logs_and_warning +def config_list(args): + """Lists all configurations at the command line.""" + AirflowConsole().print_as( + data=list(ProvidersManager().provider_configs), + output=args.output, + mapper=lambda x: { + "provider_config": x, + }, + ) + + @suppress_logs_and_warning def status(args): """Informs if providers manager has been initialized. diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 9432b03d1f69..4cef07dee1de 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2010,257 +2010,6 @@ local_kubernetes_executor: type: string example: ~ default: "kubernetes" -celery_kubernetes_executor: - description: | - This section only applies if you are using the ``CeleryKubernetesExecutor`` in - ``[core]`` section above - options: - kubernetes_queue: - description: | - Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. - When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), - the task is executed via ``KubernetesExecutor``, - otherwise via ``CeleryExecutor`` - version_added: 2.0.0 - type: string - example: ~ - default: "kubernetes" -celery: - description: | - This section only applies if you are using the CeleryExecutor in - ``[core]`` section above - options: - celery_app_name: - description: | - The app name that will be used by celery - version_added: ~ - type: string - example: ~ - default: "airflow.providers.celery.executors.celery_executor" - worker_concurrency: - description: | - The concurrency that will be used when starting workers with the - ``airflow celery worker`` command. This defines the number of task instances that - a worker will take, so size up your workers based on the resources on - your worker box and the nature of your tasks - version_added: ~ - type: string - example: ~ - default: "16" - worker_autoscale: - description: | - The maximum and minimum concurrency that will be used when starting workers with the - ``airflow celery worker`` command (always keep minimum processes, but grow - to maximum if necessary). Note the value should be max_concurrency,min_concurrency - Pick these numbers based on resources on worker box and the nature of the task. - If autoscale option is available, worker_concurrency will be ignored. - http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale - version_added: ~ - type: string - example: 16,12 - default: ~ - worker_prefetch_multiplier: - description: | - Used to increase the number of tasks that a worker prefetches which can improve performance. - The number of processes multiplied by worker_prefetch_multiplier is the number of tasks - that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily - blocked if there are multiple workers and one worker prefetches tasks that sit behind long - running tasks while another worker has unutilized processes that are unable to process the already - claimed blocked tasks. - https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits - version_added: 2.0.0 - type: integer - example: ~ - default: "1" - worker_enable_remote_control: - description: | - Specify if remote control of the workers is enabled. - In some cases when the broker does not support remote control, Celery creates lots of - ``.*reply-celery-pidbox`` queues. You can prevent this by setting this to false. - However, with this disabled Flower won't work. - https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html#broker-overview - version_added: 2.3.0 - type: boolean - example: ~ - default: "true" - broker_url: - description: | - The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally - a sqlalchemy database. Refer to the Celery documentation for more information. - version_added: ~ - type: string - sensitive: true - example: ~ - default: "redis://redis:6379/0" - result_backend: - description: | - The Celery result_backend. When a job finishes, it needs to update the - metadata of the job. Therefore it will post a message on a message bus, - or insert it into a database (depending of the backend) - This status is used by the scheduler to update the state of the task - The use of a database is highly recommended - When not specified, sql_alchemy_conn with a db+ scheme prefix will be used - http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings - version_added: ~ - type: string - sensitive: true - example: "db+postgresql://postgres:airflow@postgres/airflow" - default: ~ - result_backend_sqlalchemy_engine_options: - description: | - Optional configuration dictionary to pass to the Celery result backend SQLAlchemy engine. - version_added: 2.6.0 - type: string - example: '{"pool_recycle": 1800}' - default: "" - flower_host: - description: | - Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start - it ``airflow celery flower``. This defines the IP that Celery Flower runs on - version_added: ~ - type: string - example: ~ - default: "0.0.0.0" - flower_url_prefix: - description: | - The root URL for Flower - version_added: ~ - type: string - example: "/flower" - default: "" - flower_port: - description: | - This defines the port that Celery Flower runs on - version_added: ~ - type: string - example: ~ - default: "5555" - flower_basic_auth: - description: | - Securing Flower with Basic Authentication - Accepts user:password pairs separated by a comma - version_added: 1.10.2 - type: string - sensitive: true - example: "user1:password1,user2:password2" - default: "" - sync_parallelism: - description: | - How many processes CeleryExecutor uses to sync task state. - 0 means to use max(1, number of cores - 1) processes. - version_added: 1.10.3 - type: string - example: ~ - default: "0" - celery_config_options: - description: | - Import path for celery configuration options - version_added: ~ - type: string - example: ~ - default: "airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG" - ssl_active: - description: ~ - version_added: ~ - type: string - example: ~ - default: "False" - ssl_key: - description: | - Path to the client key. - version_added: ~ - type: string - example: ~ - default: "" - ssl_cert: - description: | - Path to the client certificate. - version_added: ~ - type: string - example: ~ - default: "" - ssl_cacert: - description: | - Path to the CA certificate. - version_added: ~ - type: string - example: ~ - default: "" - pool: - description: | - Celery Pool implementation. - Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. - See: - https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency - https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html - version_added: 1.10.4 - type: string - example: ~ - default: "prefork" - operation_timeout: - description: | - The number of seconds to wait before timing out ``send_task_to_executor`` or - ``fetch_celery_task_state`` operations. - version_added: 1.10.8 - type: float - example: ~ - default: "1.0" - task_track_started: - description: | - Celery task will report its status as 'started' when the task is executed by a worker. - This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted - or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. - version_added: 2.0.0 - type: boolean - example: ~ - default: "True" - task_publish_max_retries: - description: | - The Maximum number of retries for publishing task messages to the broker when failing - due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. - version_added: 2.0.0 - type: integer - example: ~ - default: "3" - worker_precheck: - description: | - Worker initialisation check to validate Metadata Database connection - version_added: 2.0.0 - type: string - example: ~ - default: "False" -celery_broker_transport_options: - description: | - This section is for specifying options which can be passed to the - underlying celery broker transport. See: - http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options - options: - visibility_timeout: - description: | - The visibility timeout defines the number of seconds to wait for the worker - to acknowledge the task before the message is redelivered to another worker. - Make sure to increase the visibility timeout to match the time of the longest - ETA you're planning to use. - visibility_timeout is only supported for Redis and SQS celery brokers. - See: - https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#visibility-timeout - version_added: ~ - type: string - example: "21600" - default: ~ - sentinel_kwargs: - description: | - The sentinel_kwargs parameter allows passing additional options to the Sentinel client. - In a typical scenario where Redis Sentinel is used as the broker and Redis servers are - password-protected, the password needs to be passed through this parameter. Although its - type is string, it is required to pass a string that conforms to the dictionary format. - See: - https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#configuration - version_added: 2.7.0 - type: string - sensitive: true - example: '{"password": "password_for_redis_server"}' - default: ~ dask: description: | This section only applies if you are using the DaskExecutor in diff --git a/airflow/configuration.py b/airflow/configuration.py index 10f4cf5a704a..86d3ca52528e 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -33,9 +33,9 @@ from collections import OrderedDict from configparser import ConfigParser, NoOptionError, NoSectionError from contextlib import contextmanager +from copy import deepcopy from json.decoder import JSONDecodeError -from threading import Lock -from typing import IO, Any, Dict, Iterable, Pattern, Set, Tuple, Union +from typing import IO, Any, Dict, Generator, Iterable, Pattern, Set, Tuple, Union from urllib.parse import urlsplit import re2 @@ -151,15 +151,31 @@ def _default_config_file_path(file_name: str) -> str: return os.path.join(templates_dir, file_name) -def retrieve_configuration_description() -> dict[str, dict[str, Any]]: +def retrieve_configuration_description( + include_airflow: bool = True, + include_providers: bool = True, + selected_provider: str | None = None, +) -> dict[str, dict[str, Any]]: """ Read Airflow configuration description from YAML file. + :param include_airflow: Include Airflow configs + :param include_providers: Include provider configs + :param selected_provider: If specified, include selected provider only + :param config_file_name: name of the file in "config_templates" directory to read default config from :return: Python dictionary containing configs & their info """ base_configuration_description: dict[str, dict[str, Any]] = {} - with open(_default_config_file_path("config.yml")) as config_file: - base_configuration_description.update(yaml.safe_load(config_file)) + if include_airflow: + with open(_default_config_file_path("config.yml")) as config_file: + base_configuration_description.update(yaml.safe_load(config_file)) + if include_providers: + from airflow.providers_manager import ProvidersManager + + for provider, config in ProvidersManager().provider_configs: + if selected_provider and provider != selected_provider: + continue + base_configuration_description.update(config) return base_configuration_description @@ -183,7 +199,7 @@ def __init__( **kwargs, ): super().__init__(*args, **kwargs) - self.configuration_description = retrieve_configuration_description() + self.configuration_description = retrieve_configuration_description(include_providers=False) self.upgraded_values = {} # For those who would like to use a different data structure to keep defaults: # We have to keep the default values in a ConfigParser rather than in any other @@ -196,7 +212,7 @@ def __init__( self._update_logging_deprecated_template_to_one_from_defaults() self.is_validated = False self._suppress_future_warnings = False - self._interpolation_lock = Lock() + self._providers_configuration_loaded = False def _update_logging_deprecated_template_to_one_from_defaults(self): default = self.get_default_value("logging", "log_filename_template") @@ -452,6 +468,7 @@ def get_sections_including_defaults(self) -> list[str]: :return: list of section names """ my_own_sections = self.sections() + all_sections_from_defaults = list(self.configuration_description.keys()) for section in my_own_sections: if section not in all_sections_from_defaults: @@ -485,6 +502,129 @@ def optionxform(self, optionstr: str) -> str: """ return optionstr + @contextmanager + def make_sure_configuration_loaded(self, with_providers: bool) -> Generator[None, None, None]: + """ + Make sure configuration is loaded with or without providers, regardless if the provider configuration + has been loaded before or not. Restores configuration to the state before entering the context. + + :param with_providers: whether providers should be loaded + """ + reload_providers_when_leaving = False + if with_providers and not self._providers_configuration_loaded: + # make sure providers are initialized + from airflow.providers_manager import ProvidersManager + + # run internal method to initialize providers configuration in ordered to not trigger the + # initialize_providers_configuration cache (because we will be unloading it now + ProvidersManager()._initialize_providers_configuration() + elif not with_providers and self._providers_configuration_loaded: + reload_providers_when_leaving = True + self.restore_core_default_configuration() + yield + if reload_providers_when_leaving: + self.load_providers_configuration() + + @staticmethod + def _write_section_header( + file: IO[str], + include_descriptions: bool, + section_config_description: dict[str, str], + section_to_write: str, + ) -> None: + """Writes header for configuration section.""" + file.write(f"[{section_to_write}]\n") + section_description = section_config_description.get("description") + if section_description and include_descriptions: + for line in section_description.splitlines(): + file.write(f"# {line}\n") + file.write("\n") + + def _write_option_header( + self, + file: IO[str], + option: str, + extra_spacing: bool, + include_descriptions: bool, + include_env_vars: bool, + include_examples: bool, + include_sources: bool, + section_config_description: dict[str, dict[str, Any]], + section_to_write: str, + sources_dict: ConfigSourcesType, + ) -> tuple[bool, bool]: + """Writes header for configuration option. + + Returns tuple of (should_continue, needs_separation) where needs_separation should be + set if the option needs additional separation to visually separate it from the next option. + """ + from airflow import __version__ as airflow_version + + option_config_description = ( + section_config_description.get("options", {}).get(option, {}) + if section_config_description + else {} + ) + version_added = option_config_description.get("version_added") + if version_added is not None and parse_version(version_added) > parse_version( + parse_version(airflow_version).base_version + ): + # skip if option is going to be added in the future version + return False, False + description = option_config_description.get("description") + needs_separation = False + if description and include_descriptions: + for line in description.splitlines(): + file.write(f"# {line}\n") + needs_separation = True + example = option_config_description.get("example") + if example is not None and include_examples: + if extra_spacing: + file.write("#\n") + file.write(f"# Example: {option} = {example}\n") + needs_separation = True + if include_sources and sources_dict: + sources_section = sources_dict.get(section_to_write) + value_with_source = sources_section.get(option) if sources_section else None + if value_with_source is None: + file.write("#\n# Source: not defined\n") + else: + file.write(f"#\n# Source: {value_with_source[1]}\n") + needs_separation = True + if include_env_vars: + file.write(f"#\n# Variable: AIRFLOW__{section_to_write.upper()}__{option.upper()}\n") + if extra_spacing: + file.write("#\n") + needs_separation = True + return True, needs_separation + + def _write_value( + self, + file: IO[str], + option: str, + comment_out_everything: bool, + needs_separation: bool, + only_defaults: bool, + section_to_write: str, + ): + if self._default_values is None: + default_value = None + else: + default_value = self.get_default_value(section_to_write, option, raw=True) + if only_defaults: + value = default_value + else: + value = self.get(section_to_write, option, fallback=default_value, raw=True) + if value is None: + file.write(f"# {option} = \n") + else: + if comment_out_everything: + file.write(f"# {option} = {value}\n") + else: + file.write(f"{option} = {value}\n") + if needs_separation: + file.write("\n") + def write( # type: ignore[override] self, file: IO[str], @@ -493,6 +633,7 @@ def write( # type: ignore[override] include_descriptions: bool = True, include_sources: bool = True, include_env_vars: bool = True, + include_providers: bool = True, comment_out_everything: bool = False, hide_sensitive_values: bool = False, extra_spacing: bool = True, @@ -508,12 +649,12 @@ def write( # type: ignore[override] :param include_descriptions: Include descriptions in the output :param include_sources: Include the source of each config option :param include_env_vars: Include environment variables corresponding to each config option + :param include_providers: Include providers configuration + :param comment_out_everything: Comment out all values :param hide_sensitive_values: Include sensitive values in the output :param extra_spacing: Add extra spacing before examples and after variables :param only_defaults: Only include default values when writing the config, not the actual values """ - from airflow import __version__ as airflow_version - sources_dict = {} if include_sources: sources_dict = self.as_dict(display_source=True) @@ -521,75 +662,49 @@ def write( # type: ignore[override] raise RuntimeError("Cannot write default config, no default config set") if self.configuration_description is None: raise RuntimeError("Cannot write default config, no default configuration description set") - for section_to_write in self.get_sections_including_defaults(): - section_config_description = self.configuration_description.get(section_to_write, {}) - if section_to_write != section and section is not None: - continue - if self._default_values.has_section(section_to_write) or self.has_section(section_to_write): - needs_separation = False - file.write(f"[{section_to_write}]\n") - section_description = section_config_description.get("description") - if section_description and include_descriptions: - for line in section_description.splitlines(): - file.write(f"# {line}\n") - file.write("\n") - for key in self.get_options_including_defaults(section_to_write): - option_config_description = ( - section_config_description.get("options", {}).get(key, {}) - if section_config_description - else {} + with self.make_sure_configuration_loaded(with_providers=include_providers): + for section_to_write in self.get_sections_including_defaults(): + section_config_description = self.configuration_description.get(section_to_write, {}) + if section_to_write != section and section is not None: + continue + if self._default_values.has_section(section_to_write) or self.has_section(section_to_write): + self._write_section_header( + file, include_descriptions, section_config_description, section_to_write ) - version_added = option_config_description.get("version_added") - if version_added is not None and parse_version(version_added) > parse_version( - parse_version(airflow_version).base_version - ): - # skip if option is going to be added in the future version - continue - description = option_config_description.get("description") - needs_separation = False - if description and include_descriptions: - for line in description.splitlines(): - file.write(f"# {line}\n") - needs_separation = True - example = option_config_description.get("example") - if example is not None and include_examples: - if extra_spacing: - file.write("#\n") - file.write(f"# Example: {key} = {example}\n") - needs_separation = True - if include_sources and sources_dict: - sources_section = sources_dict.get(section_to_write) - value_with_source = sources_section.get(key) if sources_section else None - if value_with_source is None: - file.write("#\n# Source: not defined\n") - else: - file.write(f"#\n# Source: {value_with_source[1]}\n") - needs_separation = True - if include_env_vars: - file.write(f"#\n# Variable: AIRFLOW__{section_to_write.upper()}__{key.upper()}\n") - if extra_spacing: - file.write("#\n") - needs_separation = True - if self._default_values is None: - default_value = None - else: - default_value = self.get_default_value(section_to_write, key, raw=True) - if only_defaults: - value = default_value - else: - value = self.get(section_to_write, key, fallback=default_value, raw=True) - if value is None: - file.write(f"# {key} = \n") - else: - if comment_out_everything: - file.write(f"# {key} = {value}\n") - else: - file.write(f"{key} = {value}\n") - if needs_separation: + for option in self.get_options_including_defaults(section_to_write): + should_continue, needs_separation = self._write_option_header( + file=file, + option=option, + extra_spacing=extra_spacing, + include_descriptions=include_descriptions, + include_env_vars=include_env_vars, + include_examples=include_examples, + include_sources=include_sources, + section_config_description=section_config_description, + section_to_write=section_to_write, + sources_dict=sources_dict, + ) + self._write_value( + file=file, + option=option, + comment_out_everything=comment_out_everything, + needs_separation=needs_separation, + only_defaults=only_defaults, + section_to_write=section_to_write, + ) + if include_descriptions and not needs_separation: + # extra separation between sections in case last option did not need it file.write("\n") - if include_descriptions and not needs_separation: - # extra separation between sections in case last option did not need it - file.write("\n") + + def restore_core_default_configuration(self) -> None: + """Restore default configuration for core Airflow. + + It does not restore configuration for providers. If you want to restore configuration for + providers, you need to call ``load_providers_configuration`` method. + """ + self.configuration_description = retrieve_configuration_description(include_providers=False) + self._default_values = create_default_config_parser(self.configuration_description) + self._providers_configuration_loaded = False def validate(self): self._validate_sqlite3_version() @@ -1617,7 +1732,13 @@ def _replace_section_config_with_display_sources( ): continue if display_source: - sect[k] = (val, source_name) + updated_source_name = source_name + if source_name == "default": + source_description_section = configuration_description.get(section, {}) + source_description_key = source_description_section.get("options", {}).get(k, {}) + if source_description_key is not None: + updated_source_name = source_description_key.get("source", source_name) + sect[k] = (val, updated_source_name) else: sect[k] = val @@ -1664,6 +1785,61 @@ def remove_all_read_configurations(self): for section in self.sections(): self.remove_section(section) + @property + def providers_configuration_loaded(self) -> bool: + """Checks if providers have been loaded.""" + return self._providers_configuration_loaded + + def load_providers_configuration(self): + """ + Loads configuration for providers. + + This should be done after initial configuration have been performed. Initializing and discovering + providers is an expensive operation and cannot be performed when we load configuration for the first + time when airflow starts, because we initialize configuration very early, during importing of the + `airflow` package and the module is not yet ready to be used when it happens and until configuration + and settings are loaded. Therefore, in order to reload provider configuration we need to additionally + load provider - specific configuration. + """ + log.debug("Loading providers configuration") + from airflow.providers_manager import ProvidersManager + + self.restore_core_default_configuration() + for provider, config in ProvidersManager().already_initialized_provider_configs: + for provider_section, provider_section_content in config.items(): + provider_options = provider_section_content["options"] + section_in_current_config = self.configuration_description.get(provider_section) + if not section_in_current_config: + self.configuration_description[provider_section] = deepcopy(provider_section_content) + section_in_current_config = self.configuration_description.get(provider_section) + section_in_current_config["source"] = f"default-{provider}" + for option in provider_options: + section_in_current_config["options"][option]["source"] = f"default-{provider}" + else: + section_source = section_in_current_config.get("source", "Airflow's core package").split( + "default-" + )[-1] + raise AirflowConfigException( + f"The provider {provider} is attempting to contribute " + f"configuration section {provider_section} that " + f"has already been added before. The source of it: {section_source}." + "This is forbidden. A provider can only add new sections. It" + "cannot contribute options to existing sections or override other " + "provider's configuration.", + UserWarning, + ) + self._default_values = create_default_config_parser(self.configuration_description) + # sensitive_config_values needs to be refreshed here. This is a cached_property, so we can delete + # the cached values, and it will be refreshed on next access. This has been an implementation + # detail in Python 3.8 but as of Python 3.9 it is documented behaviour. + # See https://docs.python.org/3/library/functools.html#functools.cached_property + try: + del self.sensitive_config_values + except AttributeError: + # no problem if cache is not set yet + pass + self._providers_configuration_loaded = True + @staticmethod def _warn_deprecate( section: str, key: str, deprecated_section: str, deprecated_name: str, extra_stacklevel: int @@ -1752,6 +1928,29 @@ def create_default_config_parser(configuration_description: dict[str, dict[str, return parser +def write_default_airflow_configuration_if_needed(): + if not os.path.isfile(AIRFLOW_CONFIG): + log.debug("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) + pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) + if conf.get("core", "fernet_key", fallback=None) is None: + # We know that FERNET_KEY is not set, so we can generate it, set as global key + # and also write it to the config file so that same key will be used next time + global FERNET_KEY + FERNET_KEY = _generate_fernet_key() + conf.remove_option("core", "fernet_key") + conf.set("core", "fernet_key", FERNET_KEY) + with open(AIRFLOW_CONFIG, "w") as file: + conf.write( + file, + include_sources=False, + include_env_vars=True, + include_providers=True, + extra_spacing=True, + only_defaults=True, + ) + make_group_other_inaccessible(AIRFLOW_CONFIG) + + def load_standard_airflow_configuration(airflow_config_parser: AirflowConfigParser): """ Loads standard airflow configuration. @@ -1763,23 +1962,7 @@ def load_standard_airflow_configuration(airflow_config_parser: AirflowConfigPars :param airflow_config_parser: parser to which the configuration will be loaded """ - global FERNET_KEY, AIRFLOW_HOME - if not os.path.isfile(AIRFLOW_CONFIG): - from cryptography.fernet import Fernet - - log.info("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) - pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) - FERNET_KEY = Fernet.generate_key().decode() - if airflow_config_parser._default_values and airflow_config_parser.configuration_description: - with open(AIRFLOW_CONFIG, "w") as file: - airflow_config_parser.write( - file, - include_sources=False, - include_env_vars=True, - extra_spacing=True, - only_defaults=True, - ) - make_group_other_inaccessible(AIRFLOW_CONFIG) + global AIRFLOW_HOME log.info("Reading the config from %s", AIRFLOW_CONFIG) airflow_config_parser.read(AIRFLOW_CONFIG) if airflow_config_parser.has_option("core", "AIRFLOW_HOME"): diff --git a/docs/apache-airflow-providers/howto/create-update-providers.rst b/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst similarity index 71% rename from docs/apache-airflow-providers/howto/create-update-providers.rst rename to airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst index 723e70bec3e7..be249b53bb04 100644 --- a/docs/apache-airflow-providers/howto/create-update-providers.rst +++ b/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst @@ -263,102 +263,3 @@ main Airflow documentation that involves some steps with the providers is also w breeze build-docs --package-filter apache-airflow-providers- breeze build-docs --package-filter apache-airflow - -Optional provider features --------------------------- - - .. versionadded:: 2.3.0 - - This feature is available in Airflow 2.3+. - -Some providers might provide optional features, which are only available when some packages or libraries -are installed. Such features will typically result in ``ImportErrors``; however, those import errors -should be silently ignored rather than pollute the logs of Airflow with false warnings. False warnings -are a very bad pattern, as they tend to turn into blind spots, so avoiding false warnings is encouraged. -However, until Airflow 2.3, Airflow had no mechanism to selectively ignore "known" ImportErrors. So -Airflow 2.1 and 2.2 silently ignored all ImportErrors coming from providers with actually lead to -ignoring even important import errors - without giving the clue to Airflow users that there is something -missing in provider dependencies. - -Using Providers with dynamic task mapping ------------------------------------------ - -Airflow 2.3 added `Dynamic Task Mapping `_ -and it added the possibility of assigning a unique key to each task. Which means that when such dynamically -mapped task wants to retrieve a value from XCom (for example in case an extra link should calculated) -it should always check if the ti_key value passed is not None an only then retrieve the XCom value using -XCom.get_value. This allows to keep backwards compatibility with earlier versions of Airflow. - -Typical code to access XCom Value in providers that want to keep backwards compatibility should look similar to -this (note the ``if ti_key is not None:`` condition). - - .. code-block:: python - - def get_link( - self, - operator: BaseOperator, - dttm: datetime | None = None, - ti_key: "TaskInstanceKey" | None = None, - ): - if ti_key is not None: - job_ids = XCom.get_value(key="job_id", ti_key=ti_key) - else: - assert dttm is not None - job_ids = XCom.get_one( - key="job_id", - dag_id=operator.dag.dag_id, - task_id=operator.task_id, - execution_date=dttm, - ) - if not job_ids: - return None - if len(job_ids) < self.index: - return None - job_id = job_ids[self.index] - return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) - - -Having sensors return XCOM values ---------------------------------- -In Airflow 2.3, sensor operators will be able to return XCOM values. This is achieved by returning an instance of the ``PokeReturnValue`` object at the end of the ``poke()`` method: - - .. code-block:: python - - from airflow.sensors.base import PokeReturnValue - - - class SensorWithXcomValue(BaseSensorOperator): - def poke(self, context: Context) -> Union[bool, PokeReturnValue]: - # ... - is_done = ... # set to true if the sensor should stop poking. - xcom_value = ... # return value of the sensor operator to be pushed to XCOM. - return PokeReturnValue(is_done, xcom_value) - - -To implement a sensor operator that pushes a XCOM value and supports both version 2.3 and pre-2.3, you need to explicitly push the XCOM value if the version is pre-2.3. - - .. code-block:: python - - try: - from airflow.sensors.base import PokeReturnValue - except ImportError: - PokeReturnValue = None - - - class SensorWithXcomValue(BaseSensorOperator): - def poke(self, context: Context) -> bool: - # ... - is_done = ... # set to true if the sensor should stop poking. - xcom_value = ... # return value of the sensor operator to be pushed to XCOM. - if PokeReturnValue is not None: - return PokeReturnValue(is_done, xcom_value) - else: - if is_done: - context["ti"].xcom_push(key="xcom_key", value=xcom_value) - return is_done - - -How-to Update a community provider ----------------------------------- - -See `Provider packages versioning `_ diff --git a/airflow/providers/celery/executors/celery_kubernetes_executor.py b/airflow/providers/celery/executors/celery_kubernetes_executor.py index 0522b12f41aa..72c037b684ea 100644 --- a/airflow/providers/celery/executors/celery_kubernetes_executor.py +++ b/airflow/providers/celery/executors/celery_kubernetes_executor.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +from functools import cached_property from typing import TYPE_CHECKING, Sequence from airflow.callbacks.base_callback_sink import BaseCallbackSink @@ -55,14 +56,21 @@ class CeleryKubernetesExecutor(LoggingMixin): callback_sink: BaseCallbackSink | None = None - KUBERNETES_QUEUE = conf.get("celery_kubernetes_executor", "kubernetes_queue") + @cached_property + def kubernetes_queue(self) -> str: + # lazily retrieve the value of kubernetes_queue from the configuration + # because it might need providers + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() + return conf.get("celery_kubernetes_executor", "kubernetes_queue") def __init__(self, celery_executor: CeleryExecutor, kubernetes_executor: KubernetesExecutor): super().__init__() self._job_id: int | None = None self.celery_executor = celery_executor self.kubernetes_executor = kubernetes_executor - self.kubernetes_executor.kubernetes_queue = self.KUBERNETES_QUEUE + self.kubernetes_executor.kubernetes_queue = self.kubernetes_queue @property def queued_tasks(self) -> dict[TaskInstanceKey, QueuedTaskInstanceType]: @@ -194,16 +202,16 @@ def try_adopt_task_instances(self, tis: Sequence[TaskInstance]) -> Sequence[Task :return: any TaskInstances that were unable to be adopted """ - celery_tis = [ti for ti in tis if ti.queue != self.KUBERNETES_QUEUE] - kubernetes_tis = [ti for ti in tis if ti.queue == self.KUBERNETES_QUEUE] + celery_tis = [ti for ti in tis if ti.queue != self.kubernetes_queue] + kubernetes_tis = [ti for ti in tis if ti.queue == self.kubernetes_queue] return [ *self.celery_executor.try_adopt_task_instances(celery_tis), *self.kubernetes_executor.try_adopt_task_instances(kubernetes_tis), ] def cleanup_stuck_queued_tasks(self, tis: list[TaskInstance]) -> list[str]: - celery_tis = [ti for ti in tis if ti.queue != self.KUBERNETES_QUEUE] - kubernetes_tis = [ti for ti in tis if ti.queue == self.KUBERNETES_QUEUE] + celery_tis = [ti for ti in tis if ti.queue != self.kubernetes_queue] + kubernetes_tis = [ti for ti in tis if ti.queue == self.kubernetes_queue] return [ *self.celery_executor.cleanup_stuck_queued_tasks(celery_tis), *self.kubernetes_executor.cleanup_stuck_queued_tasks(kubernetes_tis), @@ -226,7 +234,7 @@ def _router(self, simple_task_instance: SimpleTaskInstance) -> CeleryExecutor | :param simple_task_instance: SimpleTaskInstance :return: celery_executor or kubernetes_executor """ - if simple_task_instance.queue == self.KUBERNETES_QUEUE: + if simple_task_instance.queue == self.kubernetes_queue: return self.kubernetes_executor return self.celery_executor diff --git a/airflow/providers/celery/executors/default_celery.py b/airflow/providers/celery/executors/default_celery.py index 8d546952f038..f5c23cbb81a3 100644 --- a/airflow/providers/celery/executors/default_celery.py +++ b/airflow/providers/celery/executors/default_celery.py @@ -25,6 +25,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException +from airflow.providers_manager import ProvidersManager def _broker_supports_visibility_timeout(url): @@ -33,6 +34,7 @@ def _broker_supports_visibility_timeout(url): log = logging.getLogger(__name__) +ProvidersManager().initialize_providers_configuration() broker_url = conf.get("celery", "BROKER_URL") broker_transport_options = conf.getsection("celery_broker_transport_options") or {} diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 32682e51e5b1..26787d4694ff 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -60,3 +60,256 @@ sensors: executors: - airflow.providers.celery.executors.celery_executor.CeleryExecutor - airflow.providers.celery.executors.celery_kubernetes_executor.CeleryKubernetesExecutor + +config: + celery_kubernetes_executor: + description: | + This section only applies if you are using the ``CeleryKubernetesExecutor`` in + ``[core]`` section above + options: + kubernetes_queue: + description: | + Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. + When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), + the task is executed via ``KubernetesExecutor``, + otherwise via ``CeleryExecutor`` + version_added: ~ + type: string + example: ~ + default: "kubernetes" + celery: + description: | + This section only applies if you are using the CeleryExecutor in + ``[core]`` section above + options: + celery_app_name: + description: | + The app name that will be used by celery + version_added: ~ + type: string + example: ~ + default: "airflow.providers.celery.executors.celery_executor" + worker_concurrency: + description: | + The concurrency that will be used when starting workers with the + ``airflow celery worker`` command. This defines the number of task instances that + a worker will take, so size up your workers based on the resources on + your worker box and the nature of your tasks + version_added: ~ + type: string + example: ~ + default: "16" + worker_autoscale: + description: | + The maximum and minimum concurrency that will be used when starting workers with the + ``airflow celery worker`` command (always keep minimum processes, but grow + to maximum if necessary). Note the value should be max_concurrency,min_concurrency + Pick these numbers based on resources on worker box and the nature of the task. + If autoscale option is available, worker_concurrency will be ignored. + http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale + version_added: ~ + type: string + example: 16,12 + default: ~ + worker_prefetch_multiplier: + description: | + Used to increase the number of tasks that a worker prefetches which can improve performance. + The number of processes multiplied by worker_prefetch_multiplier is the number of tasks + that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily + blocked if there are multiple workers and one worker prefetches tasks that sit behind long + running tasks while another worker has unutilized processes that are unable to process the already + claimed blocked tasks. + https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits + version_added: ~ + type: integer + example: ~ + default: "1" + worker_enable_remote_control: + description: | + Specify if remote control of the workers is enabled. + In some cases when the broker does not support remote control, Celery creates lots of + ``.*reply-celery-pidbox`` queues. You can prevent this by setting this to false. + However, with this disabled Flower won't work. + https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html#broker-overview + version_added: ~ + type: boolean + example: ~ + default: "true" + broker_url: + description: | + The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally + a sqlalchemy database. Refer to the Celery documentation for more information. + version_added: ~ + type: string + sensitive: true + example: ~ + default: "redis://redis:6379/0" + result_backend: + description: | + The Celery result_backend. When a job finishes, it needs to update the + metadata of the job. Therefore it will post a message on a message bus, + or insert it into a database (depending of the backend) + This status is used by the scheduler to update the state of the task + The use of a database is highly recommended + When not specified, sql_alchemy_conn with a db+ scheme prefix will be used + http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings + version_added: ~ + type: string + sensitive: true + example: "db+postgresql://postgres:airflow@postgres/airflow" + default: ~ + result_backend_sqlalchemy_engine_options: + description: | + Optional configuration dictionary to pass to the Celery result backend SQLAlchemy engine. + version_added: ~ + type: string + example: '{"pool_recycle": 1800}' + default: "" + flower_host: + description: | + Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start + it ``airflow celery flower``. This defines the IP that Celery Flower runs on + version_added: ~ + type: string + example: ~ + default: "0.0.0.0" + flower_url_prefix: + description: | + The root URL for Flower + version_added: ~ + type: string + example: "/flower" + default: "" + flower_port: + description: | + This defines the port that Celery Flower runs on + version_added: ~ + type: string + example: ~ + default: "5555" + flower_basic_auth: + description: | + Securing Flower with Basic Authentication + Accepts user:password pairs separated by a comma + version_added: ~ + type: string + sensitive: true + example: "user1:password1,user2:password2" + default: "" + sync_parallelism: + description: | + How many processes CeleryExecutor uses to sync task state. + 0 means to use max(1, number of cores - 1) processes. + version_added: ~ + type: string + example: ~ + default: "0" + celery_config_options: + description: | + Import path for celery configuration options + version_added: ~ + type: string + example: ~ + default: "airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG" + ssl_active: + description: ~ + version_added: ~ + type: string + example: ~ + default: "False" + ssl_key: + description: | + Path to the client key. + version_added: ~ + type: string + example: ~ + default: "" + ssl_cert: + description: | + Path to the client certificate. + version_added: ~ + type: string + example: ~ + default: "" + ssl_cacert: + description: | + Path to the CA certificate. + version_added: ~ + type: string + example: ~ + default: "" + pool: + description: | + Celery Pool implementation. + Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. + See: + https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency + https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html + version_added: ~ + type: string + example: ~ + default: "prefork" + operation_timeout: + description: | + The number of seconds to wait before timing out ``send_task_to_executor`` or + ``fetch_celery_task_state`` operations. + version_added: ~ + type: float + example: ~ + default: "1.0" + task_track_started: + description: | + Celery task will report its status as 'started' when the task is executed by a worker. + This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted + or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. + version_added: ~ + type: boolean + example: ~ + default: "True" + task_publish_max_retries: + description: | + The Maximum number of retries for publishing task messages to the broker when failing + due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. + version_added: ~ + type: integer + example: ~ + default: "3" + worker_precheck: + description: | + Worker initialisation check to validate Metadata Database connection + version_added: ~ + type: string + example: ~ + default: "False" + celery_broker_transport_options: + description: | + This section is for specifying options which can be passed to the + underlying celery broker transport. See: + http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options + options: + visibility_timeout: + description: | + The visibility timeout defines the number of seconds to wait for the worker + to acknowledge the task before the message is redelivered to another worker. + Make sure to increase the visibility timeout to match the time of the longest + ETA you're planning to use. + visibility_timeout is only supported for Redis and SQS celery brokers. + See: + https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#visibility-timeout + version_added: ~ + type: string + example: "21600" + default: ~ + sentinel_kwargs: + description: | + The sentinel_kwargs parameter allows passing additional options to the Sentinel client. + In a typical scenario where Redis Sentinel is used as the broker and Redis servers are + password-protected, the password needs to be passed through this parameter. Although its + type is string, it is required to pass a string that conforms to the dictionary format. + See: + https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#configuration + version_added: 3.3.0 + type: string + sensitive: true + example: '{"password": "password_for_redis_server"}' + default: ~ diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 108712d484ad..bda6df3441e8 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -136,7 +136,7 @@ def _read_schema_from_resources_or_local_file(filename: str) -> dict: try: with resource_files("airflow").joinpath(filename).open("rb") as f: schema = json.load(f) - except FileNotFoundError: + except (TypeError, FileNotFoundError): import pathlib with (pathlib.Path(__file__).parent / filename).open("rb") as f: @@ -490,12 +490,25 @@ def initialize_providers_executors(self): @provider_info_cache("config") def initialize_providers_configuration(self): """Lazy initialization of providers configuration information.""" + self._initialize_providers_configuration() + + def _initialize_providers_configuration(self): + """ + Internal method to initialize providers configuration information. + + Should be used if we do not want to trigger caching for ``initialize_providers_configuration`` method. + In some cases we might want to make sure that the configuration is initialized, but we do not want + to cache the initialization method - for example when we just want to write configuration with + providers, but it is used in the context where no providers are loaded yet we will eventually + restore the original configuration and we want the subsequent ``initialize_providers_configuration`` + method to be run in order to load the configuration for providers again. + """ self.initialize_providers_list() self._discover_config() # Now update conf with the new provider configuration from providers from airflow.configuration import conf - conf.load_provider_configuration() + conf.load_providers_configuration() @provider_info_cache("auth_backends") def initialize_providers_auth_backends(self): @@ -551,9 +564,10 @@ def _discover_all_airflow_builtin_providers_from_local_sources(self) -> None: except ImportError: log.info("You have no providers installed.") return - try: - seen = set() - for path in airflow.providers.__path__: # type: ignore[attr-defined] + + seen = set() + for path in airflow.providers.__path__: # type: ignore[attr-defined] + try: # The same path can appear in the __path__ twice, under non-normalized paths (ie. # /path/to/repo/airflow/providers and /path/to/repo/./airflow/providers) path = os.path.realpath(path) @@ -561,8 +575,8 @@ def _discover_all_airflow_builtin_providers_from_local_sources(self) -> None: continue seen.add(path) self._add_provider_info_from_local_source_files_on_path(path) - except Exception as e: - log.warning("Error when loading 'provider.yaml' files from airflow sources: %s", e) + except Exception as e: + log.warning(f"Error when loading 'provider.yaml' files from {path} airflow sources: {e}") def _add_provider_info_from_local_source_files_on_path(self, path) -> None: """ @@ -573,9 +587,14 @@ def _add_provider_info_from_local_source_files_on_path(self, path) -> None: root_path = path for folder, subdirs, files in os.walk(path, topdown=True): for filename in fnmatch.filter(files, "provider.yaml"): - package_name = "apache-airflow-providers" + folder[len(root_path) :].replace(os.sep, "-") - self._add_provider_info_from_local_source_file(os.path.join(folder, filename), package_name) - subdirs[:] = [] + try: + package_name = "apache-airflow-providers" + folder[len(root_path) :].replace(os.sep, "-") + self._add_provider_info_from_local_source_file( + os.path.join(folder, filename), package_name + ) + subdirs[:] = [] + except Exception as e: + log.warning("Error when loading 'provider.yaml' file from %s %e", folder, e) def _add_provider_info_from_local_source_file(self, path, package_name) -> None: """ diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 06d98041fd14..0147f418a0ad 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -949,7 +949,7 @@ EOF This includes: - Modify `./scripts/ci/pre_commit/pre_commit_supported_versions.py` and let pre-commit do the job. -- For major/minor release, update version in `airflow/__init__.py`, `docs/docker-stack/` and `airflow/api_connexion/openapi/v1.yaml` to the next likely minor version release. +- For major/minor release, update version in `airflow/__main__.py`, `docs/docker-stack/` and `airflow/api_connexion/openapi/v1.yaml` to the next likely minor version release. - Update the `REVISION_HEADS_MAP` at airflow/utils/db.py to include the revision head of the release even if there are no migrations. - Sync `RELEASE_NOTES.rst` (including deleting relevant `newsfragments`) and `README.md` changes. - Updating `airflow_bug_report.yml` issue template in `.github/ISSUE_TEMPLATE/` with the new version. diff --git a/docs/apache-airflow-providers-airbyte/installing-providers-from-sources.rst b/docs/apache-airflow-providers-airbyte/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-airbyte/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-airbyte/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst b/docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-alibaba/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-amazon/installing-providers-from-sources.rst b/docs/apache-airflow-providers-amazon/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-amazon/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-amazon/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-beam/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-beam/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-beam/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-beam/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-cassandra/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-cassandra/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-cassandra/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-cassandra/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-drill/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-drill/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-drill/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-drill/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-druid/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-druid/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-druid/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-druid/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-flink/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-flink/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-flink/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-flink/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-hdfs/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-hdfs/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-hdfs/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-hdfs/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-hive/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-hive/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-hive/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-hive/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-impala/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-impala/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-impala/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-impala/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-kafka/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-kafka/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-kafka/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-kafka/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-kylin/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-kylin/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-kylin/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-kylin/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-livy/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-livy/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-livy/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-livy/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-pig/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-pig/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-pig/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-pig/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-pinot/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-pinot/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-pinot/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-pinot/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-spark/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apache-sqoop/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apache-sqoop/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apache-sqoop/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apache-sqoop/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-apprise/installing-providers-from-sources.rst b/docs/apache-airflow-providers-apprise/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-apprise/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-apprise/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-arangodb/installing-providers-from-sources.rst b/docs/apache-airflow-providers-arangodb/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-arangodb/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-arangodb/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-asana/installing-providers-from-sources.rst b/docs/apache-airflow-providers-asana/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-asana/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-asana/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-atlassian-jira/installing-providers-from-sources.rst b/docs/apache-airflow-providers-atlassian-jira/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-atlassian-jira/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-atlassian-jira/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-celery/configurations-ref.rst b/docs/apache-airflow-providers-celery/configurations-ref.rst new file mode 100644 index 000000000000..5885c9d91b6e --- /dev/null +++ b/docs/apache-airflow-providers-celery/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-celery/index.rst b/docs/apache-airflow-providers-celery/index.rst index fbf57bead943..464f12cd92e4 100644 --- a/docs/apache-airflow-providers-celery/index.rst +++ b/docs/apache-airflow-providers-celery/index.rst @@ -34,6 +34,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/celery/index> PyPI Repository Installing from sources diff --git a/docs/apache-airflow-providers-celery/installing-providers-from-sources.rst b/docs/apache-airflow-providers-celery/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-celery/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-celery/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-cloudant/installing-providers-from-sources.rst b/docs/apache-airflow-providers-cloudant/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-cloudant/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-cloudant/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst b/docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst b/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-common-sql/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-databricks/installing-providers-from-sources.rst b/docs/apache-airflow-providers-databricks/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-databricks/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-databricks/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-datadog/installing-providers-from-sources.rst b/docs/apache-airflow-providers-datadog/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-datadog/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-datadog/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-dbt-cloud/installing-providers-from-sources.rst b/docs/apache-airflow-providers-dbt-cloud/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-dbt-cloud/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-dbt-cloud/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-dingding/installing-providers-from-sources.rst b/docs/apache-airflow-providers-dingding/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-dingding/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-dingding/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-discord/installing-providers-from-sources.rst b/docs/apache-airflow-providers-discord/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-discord/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-discord/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-docker/installing-providers-from-sources.rst b/docs/apache-airflow-providers-docker/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-docker/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-docker/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-elasticsearch/installing-providers-from-sources.rst b/docs/apache-airflow-providers-elasticsearch/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-elasticsearch/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-elasticsearch/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-exasol/installing-providers-from-sources.rst b/docs/apache-airflow-providers-exasol/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-exasol/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-exasol/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-facebook/installing-providers-from-sources.rst b/docs/apache-airflow-providers-facebook/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-facebook/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-facebook/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-ftp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-ftp/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-ftp/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-ftp/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-github/installing-providers-from-sources.rst b/docs/apache-airflow-providers-github/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-github/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-github/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-google/configurations-ref.rst b/docs/apache-airflow-providers-google/configurations-ref.rst deleted file mode 100644 index 9f243dc763ac..000000000000 --- a/docs/apache-airflow-providers-google/configurations-ref.rst +++ /dev/null @@ -1,71 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Configuration Reference -======================= - -This page contains the list of all the provider configurations that you -can set in ``airflow.cfg`` file or using environment variables. - -.. note:: - For more information on setting the configuration, see :doc:`apache-airflow:howto/set-config` - -.. contents:: Sections: - :local: - :depth: 1 - -.. jinja:: config_ctx - - {% for section in configs %} - - [{{ section["name"] }}] - {{ "=" * (section["name"]|length + 2) }} - - {% if section["description"] %} - {{ section["description"] }} - {% endif %} - - {% for option in section["options"] %} - - .. _config:{{ section["name"] }}__{{ option["name"] }}: - - {{ option["name"] }} - {{ "-" * option["name"]|length }} - - {% if option["version_added"] %} - .. versionadded:: {{ option["version_added"] }} - {% endif %} - - {% if option["description"] %} - {{ option["description"] }} - {% endif %} - - {% if option.get("see_also") %} - .. seealso:: {{ option["see_also"] }} - {% endif %} - - :Type: {{ option["type"] }} - :Default: ``{{ "''" if option["default"] == "" else option["default"] }}`` - :Environment Variable: ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}`` - {% if option["example"] %} - :Example: - ``{{ option["example"] }}`` - {% endif %} - - {% endfor %} - {% endfor %} diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 0c2fca3e2cdc..40969f38dd91 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -46,7 +46,6 @@ :caption: References Python API <_api/airflow/providers/google/index> - Configuration .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-google/installing-providers-from-sources.rst b/docs/apache-airflow-providers-google/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-google/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-google/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-grpc/installing-providers-from-sources.rst b/docs/apache-airflow-providers-grpc/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-grpc/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-grpc/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-hashicorp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-hashicorp/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-hashicorp/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-hashicorp/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-http/installing-providers-from-sources.rst b/docs/apache-airflow-providers-http/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-http/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-http/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-imap/installing-providers-from-sources.rst b/docs/apache-airflow-providers-imap/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-imap/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-imap/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-influxdb/installing-providers-from-sources.rst b/docs/apache-airflow-providers-influxdb/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-influxdb/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-influxdb/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-jdbc/installing-providers-from-sources.rst b/docs/apache-airflow-providers-jdbc/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-jdbc/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-jdbc/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-jenkins/installing-providers-from-sources.rst b/docs/apache-airflow-providers-jenkins/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-jenkins/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-jenkins/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-microsoft-azure/installing-providers-from-sources.rst b/docs/apache-airflow-providers-microsoft-azure/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-microsoft-azure/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-microsoft-azure/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-microsoft-mssql/installing-providers-from-sources.rst b/docs/apache-airflow-providers-microsoft-mssql/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-microsoft-mssql/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-microsoft-mssql/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-microsoft-psrp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-microsoft-psrp/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-microsoft-psrp/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-microsoft-psrp/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-microsoft-winrm/installing-providers-from-sources.rst b/docs/apache-airflow-providers-microsoft-winrm/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-microsoft-winrm/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-microsoft-winrm/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-mongo/installing-providers-from-sources.rst b/docs/apache-airflow-providers-mongo/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-mongo/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-mongo/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-mysql/installing-providers-from-sources.rst b/docs/apache-airflow-providers-mysql/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-mysql/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-mysql/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-neo4j/installing-providers-from-sources.rst b/docs/apache-airflow-providers-neo4j/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-neo4j/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-neo4j/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-odbc/installing-providers-from-sources.rst b/docs/apache-airflow-providers-odbc/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-odbc/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-odbc/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-openfaas/installing-providers-from-sources.rst b/docs/apache-airflow-providers-openfaas/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-openfaas/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-openfaas/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-openlineage/installing-providers-from-sources.rst b/docs/apache-airflow-providers-openlineage/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-openlineage/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-openlineage/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-opsgenie/installing-providers-from-sources.rst b/docs/apache-airflow-providers-opsgenie/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-opsgenie/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-opsgenie/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-oracle/installing-providers-from-sources.rst b/docs/apache-airflow-providers-oracle/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-oracle/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-oracle/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-pagerduty/installing-providers-from-sources.rst b/docs/apache-airflow-providers-pagerduty/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-pagerduty/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-pagerduty/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-papermill/installing-providers-from-sources.rst b/docs/apache-airflow-providers-papermill/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-papermill/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-papermill/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-plexus/installing-providers-from-sources.rst b/docs/apache-airflow-providers-plexus/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-plexus/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-plexus/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-postgres/installing-providers-from-sources.rst b/docs/apache-airflow-providers-postgres/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-postgres/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-postgres/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-presto/installing-providers-from-sources.rst b/docs/apache-airflow-providers-presto/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-presto/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-presto/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst b/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-redis/installing-providers-from-sources.rst b/docs/apache-airflow-providers-redis/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-redis/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-redis/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-salesforce/installing-providers-from-sources.rst b/docs/apache-airflow-providers-salesforce/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-salesforce/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-salesforce/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-samba/installing-providers-from-sources.rst b/docs/apache-airflow-providers-samba/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-samba/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-samba/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-segment/installing-providers-from-sources.rst b/docs/apache-airflow-providers-segment/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-segment/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-segment/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-sendgrid/installing-providers-from-sources.rst b/docs/apache-airflow-providers-sendgrid/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-sendgrid/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-sendgrid/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-sftp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-sftp/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-sftp/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-sftp/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-singularity/installing-providers-from-sources.rst b/docs/apache-airflow-providers-singularity/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-singularity/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-singularity/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-slack/installing-providers-from-sources.rst b/docs/apache-airflow-providers-slack/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-slack/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-slack/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst b/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-smtp/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-snowflake/installing-providers-from-sources.rst b/docs/apache-airflow-providers-snowflake/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-snowflake/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-snowflake/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-sqlite/installing-providers-from-sources.rst b/docs/apache-airflow-providers-sqlite/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-sqlite/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-sqlite/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-ssh/installing-providers-from-sources.rst b/docs/apache-airflow-providers-ssh/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-ssh/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-ssh/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-tableau/installing-providers-from-sources.rst b/docs/apache-airflow-providers-tableau/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-tableau/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-tableau/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-tabular/installing-providers-from-sources.rst b/docs/apache-airflow-providers-tabular/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-tabular/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-tabular/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-telegram/installing-providers-from-sources.rst b/docs/apache-airflow-providers-telegram/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-telegram/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-telegram/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-trino/installing-providers-from-sources.rst b/docs/apache-airflow-providers-trino/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-trino/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-trino/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-vertica/installing-providers-from-sources.rst b/docs/apache-airflow-providers-vertica/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-vertica/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-vertica/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst b/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-zendesk/installing-providers-from-sources.rst b/docs/apache-airflow-providers-zendesk/installing-providers-from-sources.rst index 1c90205d15b3..b4e730f4ff21 100644 --- a/docs/apache-airflow-providers-zendesk/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-zendesk/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers/core-extensions/configurations.rst b/docs/apache-airflow-providers/core-extensions/configurations.rst new file mode 100644 index 000000000000..c90f28a730c8 --- /dev/null +++ b/docs/apache-airflow-providers/core-extensions/configurations.rst @@ -0,0 +1,31 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Configurations +-------------- + +This is a summary of all Apache Airflow Community provider custom configurations. + + +You can take a look at Configuration available in the core Airflow and how to set the +configuration options in :doc:`apache-airflow:configurations-ref`. + +Those provided by the community-managed providers: + +.. airflow-configurations:: + :tags: None + :header-separator: " diff --git a/docs/apache-airflow-providers/howto/create-custom-providers.rst b/docs/apache-airflow-providers/howto/create-custom-providers.rst new file mode 100644 index 000000000000..6fe27674f226 --- /dev/null +++ b/docs/apache-airflow-providers/howto/create-custom-providers.rst @@ -0,0 +1,268 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +How to create your own provider +------------------------------- + +Custom provider packages +'''''''''''''''''''''''' + +You can develop and release your own providers. Your custom operators, hooks, sensors, transfer operators +can be packaged together in a standard airflow package and installed using the same mechanisms. +Moreover they can also use the same mechanisms to extend the Airflow Core with auth backends, +custom connections, logging, secret backends and extra operator links as described in the previous chapter. + +As mentioned in the `Providers `_ +documentation, custom providers can extend Airflow core - they can add extra links to operators as well +as custom connections. You can use build your own providers and install them as packages if you would like +to use the mechanism for your own, custom providers. + +How to create a provider +'''''''''''''''''''''''' + +Adding a provider to Airflow is just a matter of building a Python package and adding the right meta-data to +the package. We are using standard mechanism of python to define +`entry points `_ . Your package +needs to define appropriate entry-point ``apache_airflow_provider`` which has to point to a callable +implemented by your package and return a dictionary containing the list of discoverable capabilities +of your package. The dictionary has to follow the +`json-schema specification `_. + +Most of the schema provides extension point for the documentation (which you might want to also use for +your own purpose) but the important fields from the extensibility point of view are those: + +Displaying package information in CLI/API: + +* ``package-name`` - Name of the package for the provider. + +* ``name`` - Human-friendly name of the provider. + +* ``description`` - Additional description of the provider. + +* ``version`` - List of versions of the package (in reverse-chronological order). The first version in the + list is the current package version. It is taken from the version of package installed, not from the + provider_info information. + +Exposing customized functionality to the Airflow's core: + +* ``extra-links`` - this field should contain the list of all the operator class names that are adding extra links + capability. See :doc:`apache-airflow:howto/define-extra-link` for description of how to add extra link + capability to the operators of yours. + +* ``connection-types`` - this field should contain the list of all the connection types together with hook + class names implementing those custom connection types (providing custom extra fields and + custom field behaviour). This field is available as of Airflow 2.2.0 and it replaces deprecated + ``hook-class-names``. See :doc:`apache-airflow:howto/connection` for more details. + +* ``secret-backends`` - this field should contain the list of all the secret backends class names that the + provider provides. See :doc:`apache-airflow:security/secrets/secrets-backend/index` for description of how + to add. + +* ``task-decorators`` - this field should contain the list of dictionaries of name/path where the decorators + are available. See :doc:`apache-airflow:howto/create-custom-decorator` for description of how to add + custom decorators. + +* ``logging`` - this field should contain the list of all the logging handler class names that the + provider provides. See :doc:`apache-airflow:administration-and-deployment/logging-monitoring/logging-tasks` + for description of the logging handlers. + +* ``auth-backends`` - this field should contain the authentication backend module names for API/UI. + See :doc:`apache-airflow:security/api` for description of the auth backends. + +* ``notifications`` - this field should contain the notification classes. + See :doc:`apache-airflow:howto/notifications` for description of the notifications. + +* ``executors`` - this field should contain the executor class class names. + See :doc:`apache-airflow:core-concepts/executor/index` for description of the executors. + +* ``config`` - this field should contain dictionary that should conform to the + ``airflow/config_templates/config.yml.schema.json`` with configuration contributed by the providers + See :doc:`apache-airflow:howto/set-config` for details about setting configuration. + +.. note:: Deprecated values + + * ``hook-class-names`` (deprecated) - this field should contain the list of all hook class names that provide + custom connection types with custom extra fields and field behaviour. The ``hook-class-names`` array + is deprecated as of Airflow 2.2.0 (for optimization reasons) and will be removed in Airflow 3. If your + providers are targeting Airflow 2.2.0+ you do not have to include the ``hook-class-names`` array, if + you want to also target earlier versions of Airflow 2, you should include both ``hook-class-names`` and + ``connection-types`` arrays. See :doc:`apache-airflow:howto/connection` for more details. + + +When your providers are installed you can query the installed providers and their capabilities with the +``airflow providers`` command. This way you can verify if your providers are properly recognized and whether +they define the extensions properly. See :doc:`apache-airflow:cli-and-env-variables-ref` for details of available CLI +sub-commands. + +When you write your own provider, consider following the +`Naming conventions for provider packages `_ + +Special considerations +'''''''''''''''''''''' + +Optional provider features +-------------------------- + + .. versionadded:: 2.3.0 + + This feature is available in Airflow 2.3+. + +Some providers might provide optional features, which are only available when some packages or libraries +are installed. Such features will typically result in ``ImportErrors``; however, those import errors +should be silently ignored rather than pollute the logs of Airflow with false warnings. False warnings +are a very bad pattern, as they tend to turn into blind spots, so avoiding false warnings is encouraged. +However, until Airflow 2.3, Airflow had no mechanism to selectively ignore "known" ImportErrors. So +Airflow 2.1 and 2.2 silently ignored all ImportErrors coming from providers with actually lead to +ignoring even important import errors - without giving the clue to Airflow users that there is something +missing in provider dependencies. + +Using Providers with dynamic task mapping +----------------------------------------- + +Airflow 2.3 added `Dynamic Task Mapping `_ +and it added the possibility of assigning a unique key to each task. Which means that when such dynamically +mapped task wants to retrieve a value from XCom (for example in case an extra link should calculated) +it should always check if the ti_key value passed is not None an only then retrieve the XCom value using +XCom.get_value. This allows to keep backwards compatibility with earlier versions of Airflow. + +Typical code to access XCom Value in providers that want to keep backwards compatibility should look similar to +this (note the ``if ti_key is not None:`` condition). + + .. code-block:: python + + def get_link( + self, + operator: BaseOperator, + dttm: datetime | None = None, + ti_key: "TaskInstanceKey" | None = None, + ): + if ti_key is not None: + job_ids = XCom.get_value(key="job_id", ti_key=ti_key) + else: + assert dttm is not None + job_ids = XCom.get_one( + key="job_id", + dag_id=operator.dag.dag_id, + task_id=operator.task_id, + execution_date=dttm, + ) + if not job_ids: + return None + if len(job_ids) < self.index: + return None + job_id = job_ids[self.index] + return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) + + + + +FAQ for custom providers +'''''''''''''''''''''''' + +**When I write my own provider, do I need to do anything special to make it available to others?** + +You do not need to do anything special besides creating the ``apache_airflow_provider`` entry point +returning properly formatted meta-data - dictionary with ``extra-links`` and ``connection-types`` fields +(and deprecated ``hook-class-names`` field if you are also targeting versions of Airflow before 2.2.0). + +Anyone who runs airflow in an environment that has your Python package installed will be able to use the +package as a provider package. + + +**Should I name my provider specifically or should it be created in ``airflow.providers`` package?** + +We have quite a number (>80) of providers managed by the community and we are going to maintain them +together with Apache Airflow. All those providers have well-defined structured and follow the +naming conventions we defined and they are all in ``airflow.providers`` package. If your intention is +to contribute your provider, then you should follow those conventions and make a PR to Apache Airflow +to contribute to it. But you are free to use any package name as long as there are no conflicts with other +names, so preferably choose package that is in your "domain". + +**What do I need to do to turn a package into a provider?** + +You need to do the following to turn an existing Python package into a provider (see below for examples): + +* Add the ``apache_airflow_provider`` entry point in the ``setup.cfg`` - this tells airflow where to get + the required provider metadata +* Create the function that you refer to in the first step as part of your package: this functions returns a + dictionary that contains all meta-data about your provider package +* If you want Airflow to link to documentation of your Provider in the providers page, make sure + to add "project-url/documentation" `metadata `_ to your package. + This will also add link to your documentation in PyPI. +* note that the dictionary should be compliant with ``airflow/provider_info.schema.json`` JSON-schema + specification. The community-managed providers have more fields there that are used to build + documentation, but the requirement for runtime information only contains several fields which are defined + in the schema: + +.. exampleinclude:: /../../airflow/provider_info.schema.json + :language: json + +Example ``setup.cfg``: + +.. code-block:: cfg + + [options.entry_points] + # the function get_provider_info is defined in myproviderpackage.somemodule + apache_airflow_provider= + provider_info=myproviderpackage.somemodule:get_provider_info + +Example ``myproviderpackage/somemodule.py``: + +.. code-block:: Python + + def get_provider_info(): + return { + "package-name": "my-package-name", + "name": "name", + "description": "a description", + "hook-class-names": [ + "myproviderpackage.hooks.source.SourceHook", + ], + } + + + +**Is there a convention for a connection id and type?** + +Very good question. Glad that you asked. We usually follow the convention ``_default`` for connection +id and just ```` for connection type. Few examples: + +* ``google_cloud_default`` id and ``google_cloud_platform`` type +* ``aws_default`` id and ``aws`` type + +You should follow this convention. It is important, to use unique names for connection type, +so it should be unique for your provider. If two providers try to add connection with the same type +only one of them will succeed. + +**Can I contribute my own provider to Apache Airflow?** + +The answer depends on the provider. We have a policy for that in the +`PROVIDERS.rst `_ developer documentation. + +**Can I advertise my own provider to Apache Airflow users and share it with others as package in PyPI?** + +Absolutely! We have an `Ecosystem `_ area on our website where +we share non-community managed extensions and work for Airflow. Feel free to make a PR to the page and +add we will evaluate and merge it when we see that such provider can be useful for the community of +Airflow users. + +**Can I charge for the use of my provider?** + +This is something that is outside of our control and domain. As an Apache project, we are +commercial-friendly and there are many businesses built around Apache Airflow and many other +Apache projects. As a community, we provide all the software for free and this will never +change. What 3rd-party developers are doing is not under control of Apache Airflow community. diff --git a/docs/apache-airflow-providers/index.rst b/docs/apache-airflow-providers/index.rst index 0c960a8eeef4..8b8390e5e6a8 100644 --- a/docs/apache-airflow-providers/index.rst +++ b/docs/apache-airflow-providers/index.rst @@ -32,10 +32,15 @@ for ``Apache Airflow`` are designed in the way that you can write your own provi develop your own providers - the providers you build have exactly the same capability as the providers written by the community, so you can release and share those providers with others. -The full list of community managed providers is available at + +If you want to learn how to build your own custom provider, you can find all the information about it at +:doc:`/howto/create-custom-providers`. + + +The full list of all the community managed providers is available at `Providers Index `_. -You can also see index of all community provider's operators and hooks in +You can also see index of all the community provider's operators and hooks in :doc:`/operators-and-hooks-ref/index` Extending Airflow core functionality @@ -48,9 +53,17 @@ describe all the custom capabilities. Airflow automatically discovers which providers add those additional capabilities and, once you install provider package and re-start Airflow, those become automatically available to Airflow Users. -The summary of the core functionalities that can be extended are available in +The summary of all the core functionalities that can be extended are available in :doc:`/core-extensions/index`. +Configuration +''''''''''''' + +Providers can have their own configuration options which allow you to configure how they work: + +You can see all community-managed providers with their own configuration in +:doc:`/core-extensions/configurations` + Auth backends ''''''''''''' @@ -66,7 +79,7 @@ Custom connections The providers can add custom connection types, extending connection form and handling custom form field behaviour for the connections defined by the provider. -You can see all custom connections available via community-managed providers in +You can see all the custom connections available via community-managed providers in :doc:`/core-extensions/connections`. Extra links @@ -97,7 +110,7 @@ Secret backends Airflow has the capability of reading connections, variables and configuration from Secret Backends rather than from its own Database. -You can see all secret backends available via community-managed providers in +You can see all the secret backends available via community-managed providers in :doc:`/core-extensions/secrets-backends`. Notifications @@ -132,6 +145,8 @@ Providers have the same capacity - no matter if they are provided by the communi third-party providers. This chapter explains how community managed providers are versioned and released and how you can create your own providers. +.. _providers:community-maintained-providers: + Community maintained providers '''''''''''''''''''''''''''''' @@ -167,224 +182,18 @@ provider packages are automatically documented in the release notes of every pro Airflow 1.10. The last release of backport providers was done on March 17, 2021 and the backport providers will no longer be released, since Airflow 1.10 has reached End-Of-Life as of June 17, 2021. -If you want to contribute to ``Apache Airflow``, you can see how to build and extend community -managed providers in :doc:`howto/create-update-providers`. - -.. _providers:community-maintained-providers: - -Custom provider packages -'''''''''''''''''''''''' - -You can develop and release your own providers. Your custom operators, hooks, sensors, transfer operators -can be packaged together in a standard airflow package and installed using the same mechanisms. -Moreover they can also use the same mechanisms to extend the Airflow Core with auth backends, -custom connections, logging, secret backends and extra operator links as described in the previous chapter. - -How to create your own provider -------------------------------- - -As mentioned in the `Providers `_ -documentation, custom providers can extend Airflow core - they can add extra links to operators as well -as custom connections. You can use build your own providers and install them as packages if you would like -to use the mechanism for your own, custom providers. - -Adding a provider to Airflow is just a matter of building a Python package and adding the right meta-data to -the package. We are using standard mechanism of python to define -`entry points `_ . Your package -needs to define appropriate entry-point ``apache_airflow_provider`` which has to point to a callable -implemented by your package and return a dictionary containing the list of discoverable capabilities -of your package. The dictionary has to follow the -`json-schema specification `_. - -Most of the schema provides extension point for the documentation (which you might want to also use for -your own purpose) but the important fields from the extensibility point of view are those: - -Displaying package information in CLI/API: - -* ``package-name`` - Name of the package for the provider. - -* ``name`` - Human-friendly name of the provider. - -* ``description`` - Additional description of the provider. - -* ``version`` - List of versions of the package (in reverse-chronological order). The first version in the - list is the current package version. It is taken from the version of package installed, not from the - provider_info information. - -Exposing customized functionality to the Airflow's core: - -* ``extra-links`` - this field should contain the list of all operator class names that are adding extra links - capability. See :doc:`apache-airflow:howto/define-extra-link` for description of how to add extra link - capability to the operators of yours. - -* ``connection-types`` - this field should contain the list of all connection types together with hook - class names implementing those custom connection types (providing custom extra fields and - custom field behaviour). This field is available as of Airflow 2.2.0 and it replaces deprecated - ``hook-class-names``. See :doc:`apache-airflow:howto/connection` for more details - -* ``hook-class-names`` (deprecated) - this field should contain the list of all hook class names that provide - custom connection types with custom extra fields and field behaviour. The ``hook-class-names`` array - is deprecated as of Airflow 2.2.0 (for optimization reasons) and will be removed in Airflow 3. If your - providers are targeting Airflow 2.2.0+ you do not have to include the ``hook-class-names`` array, if - you want to also target earlier versions of Airflow 2, you should include both ``hook-class-names`` and - ``connection-types`` arrays. See :doc:`apache-airflow:howto/connection` for more details. - - -When your providers are installed you can query the installed providers and their capabilities with the -``airflow providers`` command. This way you can verify if your providers are properly recognized and whether -they define the extensions properly. See :doc:`apache-airflow:cli-and-env-variables-ref` for details of available CLI -sub-commands. - -When you write your own provider, consider following the -`Naming conventions for provider packages `_ - - -FAQ for Airflow and Providers ------------------------------ - -Upgrading Airflow 2.0 and Providers -''''''''''''''''''''''''''''''''''' - -**When upgrading to a new Airflow version such as 2.0, but possibly 2.0.1 and beyond, is the best practice -to also upgrade provider packages at the same time?** - -It depends on your use case. If you have automated or semi-automated verification of your installation, -that you can run a new version of Airflow including all provider packages, then definitely go for it. -If you rely more on manual testing, it is advised that you upgrade in stages. Depending on your choice -you can either upgrade all used provider packages first, and then upgrade Airflow Core or the other way -round. The first approach - when you first upgrade all providers is probably safer, as you can do it -incrementally, step-by-step replacing provider by provider in your environment. - -Creating your own providers -''''''''''''''''''''''''''' - -**When I write my own provider, do I need to do anything special to make it available to others?** - -You do not need to do anything special besides creating the ``apache_airflow_provider`` entry point -returning properly formatted meta-data - dictionary with ``extra-links`` and ``connection-types`` fields -(and deprecated ``hook-class-names`` field if you are also targeting versions of Airflow before 2.2.0). - -Anyone who runs airflow in an environment that has your Python package installed will be able to use the -package as a provider package. - -**What do I need to do to turn a package into a provider?** - -You need to do the following to turn an existing Python package into a provider (see below for examples): - -* Add the ``apache_airflow_provider`` entry point in the ``setup.cfg`` - this tells airflow where to get - the required provider metadata -* Create the function that you refer to in the first step as part of your package: this functions returns a - dictionary that contains all meta-data about your provider package -* If you want Airflow to link to documentation of your Provider in the providers page, make sure - to add "project-url/documentation" `metadata `_ to your package. - This will also add link to your documentation in PyPI. -* note that the dictionary should be compliant with ``airflow/provider_info.schema.json`` JSON-schema - specification. The community-managed providers have more fields there that are used to build - documentation, but the requirement for runtime information only contains several fields which are defined - in the schema: - -.. exampleinclude:: /../../airflow/provider_info.schema.json - :language: json - -Example ``setup.cfg``: - -.. code-block:: cfg - - [options.entry_points] - # the function get_provider_info is defined in myproviderpackage.somemodule - apache_airflow_provider= - provider_info=myproviderpackage.somemodule:get_provider_info - -Example ``myproviderpackage/somemodule.py``: - -.. code-block:: Python - - def get_provider_info(): - return { - "package-name": "my-package-name", - "name": "name", - "description": "a description", - "hook-class-names": [ - "myproviderpackage.hooks.source.SourceHook", - ], - } - -**How do provider packages work under the hood?** - -When running Airflow with your provider package, there will be (at least) three components to your airflow installation: - -* The installation itself (for example, a ``venv`` where you installed airflow with ``pip install apache-airflow``) - together with the related files (e.g. ``dags`` folder) -* The ``apache-airflow`` package -* Your own ``myproviderpackage`` package that is independent of ``apache-airflow`` or your airflow installation, which - can be a local Python package (that you install via ``pip install -e /path/to/my-package``), a normal pip package - (``pip install myproviderpackage``), or any other type of Python package - -In the ``myproviderpackage`` package you need to add the entry point and provide the appropriate metadata as described above. -If you have done that, airflow does the following at runtime: - -* Loop through ALL packages installed in your environment / ``venv`` -* For each package, if the package's ``setup.cfg`` has a section ``[options.entry_points]``, and if that section has a value - for ``apache_airflow_provider``, then get the value for ``provider_info``, e.g. ``myproviderpackage.somemodule:get_provider_info`` -* That value works like an import statement: ``myproviderpackage.somemodule:get_provider_info`` translates to something like - ``from myproviderpackage.somemodule import get_provider_info``, and the ``get_provider_info`` that is being imported should be a - callable, i.e. a function -* This function should return a dictionary with metadata -* If you have custom connection types as part of your package, that metadata will including a field called ``hook-class-names`` - which should be a list of strings of your custom hooks - those strings should also be in an import-like format, e.g. - ``myproviderpackage.hooks.source.SourceHook`` means that there is a class ``SourceHook`` in ``myproviderpackage/hooks/source.py`` - - airflow then imports these hooks and looks for the functions ``get_ui_field_behaviour`` and ``get_connection_form_widgets`` - (both optional) as well as the attributes ``conn_type`` and ``hook_name`` to create the custom connection type in the airflow UI - -**Should I name my provider specifically or should it be created in ``airflow.providers`` package?** - -We have quite a number (>80) of providers managed by the community and we are going to maintain them -together with Apache Airflow. All those providers have well-defined structured and follow the -naming conventions we defined and they are all in ``airflow.providers`` package. If your intention is -to contribute your provider, then you should follow those conventions and make a PR to Apache Airflow -to contribute to it. But you are free to use any package name as long as there are no conflicts with other -names, so preferably choose package that is in your "domain". - -**Is there a convention for a connection id and type?** - -Very good question. Glad that you asked. We usually follow the convention ``_default`` for connection -id and just ```` for connection type. Few examples: - -* ``google_cloud_default`` id and ``google_cloud_platform`` type -* ``aws_default`` id and ``aws`` type - -You should follow this convention. It is important, to use unique names for connection type, -so it should be unique for your provider. If two providers try to add connection with the same type -only one of them will succeed. - -**Can I contribute my own provider to Apache Airflow?** - -The answer depends on the provider. We have a policy for that in the -`PROVIDERS.rst `_ developer documentation. - -**Can I advertise my own provider to Apache Airflow users and share it with others as package in PyPI?** - -Absolutely! We have an `Ecosystem `_ area on our website where -we share non-community managed extensions and work for Airflow. Feel free to make a PR to the page and -add we will evaluate and merge it when we see that such provider can be useful for the community of -Airflow users. - -**Can I charge for the use of my provider?** - -This is something that is outside of our control and domain. As an Apache project, we are -commercial-friendly and there are many businesses built around Apache Airflow and many other -Apache projects. As a community, we provide all the software for free and this will never -change. What 3rd-party developers are doing is not under control of Apache Airflow community. +If you want to contribute to ``Apache Airflow``, you can see how to build and extend community +managed providers in ``https://github.com/apache/airflow/blob/main/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst``. .. toctree:: :hidden: :maxdepth: 2 Providers + Installing from PyPI + Installing from sources + Create custom providers Packages Operators and hooks Core Extensions - Update community providers - Installing from sources - Installing from PyPI diff --git a/docs/apache-airflow-providers/redirects.txt b/docs/apache-airflow-providers/redirects.txt new file mode 100644 index 000000000000..a6f42e867073 --- /dev/null +++ b/docs/apache-airflow-providers/redirects.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Administration and deployment security -> security +howto/create-update-providers.rst index.rst diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index 09f7fd427047..eb34b10c3592 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -17,7 +17,7 @@ Configuration Reference -======================= +....................... This page contains the list of all the available Airflow configurations that you can set in ``airflow.cfg`` file or using environment variables. @@ -32,79 +32,12 @@ generated using the secret key has a short expiry time though - make sure that t that you run airflow components on is synchronized (for example using ntpd) otherwise you might get "forbidden" errors when the logs are accessed. -.. note:: - For more information on setting the configuration, see :doc:`howto/set-config` - -.. contents:: Sections: - :local: - :depth: 1 - -.. jinja:: config_ctx - - {% for section_name, section in configs.items() %} - - .. _config:{{ section_name }}: - - [{{ section_name }}] - {{ "=" * (section_name|length + 2) }} - - {% if 'renamed' in section %} - *Renamed in version {{ section['renamed']['version'] }}, previous name was {{ section['renamed']['previous_name'] }}* - {% endif %} - - {% if section["description"] %} - {{ section["description"] }} - {% endif %} - - {% for option_name, option in section["options"].items() %} - - .. _config:{{ section_name }}__{{ option_name }}: - - {{ option_name }} - {{ "-" * option_name|length }} +Some of the providers have their own configuration options, you will find details of their configuration +in the provider's documentation. The pre-installed providers that you may want to configure are: - {% if option["version_added"] %} - .. versionadded:: {{ option["version_added"] }} - {% endif %} +* :doc:`Configuration Reference for Celery Provider ` - {% if option["description"] %} - {{ option["description"] }} - {% endif %} - - {% if option.get("see_also") %} - .. seealso:: {{ option["see_also"] }} - {% endif %} - - :Type: {{ option["type"] }} - :Default: ``{{ "''" if option["default"] == "" else option["default"] }}`` - {% if option.get("sensitive") %} - :Environment Variables: - ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` - - ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD`` - - ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET`` - {% else %} - :Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` - {% endif %} - {% if option["example"] %} - :Example: - ``{{ option["example"] }}`` - {% endif %} - - {% endfor %} - - {% if section_name in deprecated_options %} - - {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section_name].items() %} - .. _config:{{ section_name }}__{{ deprecated_option_name }}: - - {{ deprecated_option_name }} (Deprecated) - {{ "-" * (deprecated_option_name + " (Deprecated)")|length }} - - .. deprecated:: {{ since_version }} - The option has been moved to :ref:`{{ new_section_name }}.{{ new_option_name }} ` - {% endfor %} - {% endif %} +.. note:: + For more information see :doc:`/howto/set-config`. - {% endfor %} +.. include:: ../exts/includes/sections-and-options.rst diff --git a/docs/apache-airflow/core-concepts/executor/celery.rst b/docs/apache-airflow/core-concepts/executor/celery.rst index ca137fa8e3c3..f69da1570fc3 100644 --- a/docs/apache-airflow/core-concepts/executor/celery.rst +++ b/docs/apache-airflow/core-concepts/executor/celery.rst @@ -29,6 +29,8 @@ change your ``airflow.cfg`` to point the executor parameter to For more information about setting up a Celery broker, refer to the exhaustive `Celery documentation on the topic `_. +The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`. + Here are a few imperative requirements for your workers: - ``airflow`` needs to be installed, and the CLI needs to be in the path diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst index 720fe7e743dd..1eaf9bda5843 100644 --- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst +++ b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst @@ -28,6 +28,8 @@ An executor is chosen to run a task based on the task's queue. ``CeleryKubernetesExecutor`` inherits the scalability of the ``CeleryExecutor`` to handle the high load at the peak time and runtime isolation of the ``KubernetesExecutor``. +The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`. + When to use CeleryKubernetesExecutor #################################### diff --git a/docs/apache-airflow/howto/create-custom-decorator.rst b/docs/apache-airflow/howto/create-custom-decorator.rst index dc7efd21c44c..81312b2335f7 100644 --- a/docs/apache-airflow/howto/create-custom-decorator.rst +++ b/docs/apache-airflow/howto/create-custom-decorator.rst @@ -63,7 +63,8 @@ tasks. The steps to create and register ``@task.foo`` are: 3. Register your new decorator in get_provider_info of your provider - Finally, add a key-value ``task-decorators`` to the dict returned from the provider entrypoint. This should be + Finally, add a key-value ``task-decorators`` to the dict returned from the provider entrypoint as described + in :doc:`apache-airflow-providers:howto/create-custom-providers`. This should be a list with each item containing ``name`` and ``class-name`` keys. When Airflow starts, the ``ProviderManager`` class will automatically import this value and ``task.foo`` will work as a new decorator! diff --git a/docs/apache-airflow/howto/setup-and-teardown.rst b/docs/apache-airflow/howto/setup-and-teardown.rst index c897031a6c6c..08eb4157eff3 100644 --- a/docs/apache-airflow/howto/setup-and-teardown.rst +++ b/docs/apache-airflow/howto/setup-and-teardown.rst @@ -106,7 +106,7 @@ In this case, everything downstream of create_cluster is assumed to require it. Now, Airflow can infer that other_task does not require create_cluster, so if we clear other_task, create_cluster will not also be cleared. -In that example, we (in our pretend docs land) actually wanted to delete the cluster. But supposing we didn't, and we just wanted to say "other_task does not require create_cluster", then we could use an EmptyOperator to limit the setup's scope: +In that example, we (in our pretend docs land) actually wanted to delete the cluster. But supposing we did not, and we just wanted to say "other_task does not require create_cluster", then we could use an EmptyOperator to limit the setup's scope: .. code-block:: python diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index f39eb78fac9a..e007f263f919 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -360,11 +360,51 @@ Notes on using the operator: Using the TaskFlow API with Sensor operators -------------------------------------------- + You can apply the ``@task.sensor`` decorator to convert a regular Python function to an instance of the BaseSensorOperator class. The Python function implements the poke logic and returns an instance of -the ``PokeReturnValue`` class as the ``poke()`` method in the BaseSensorOperator does. The ``PokeReturnValue`` is -a new feature in Airflow 2.3 that allows a sensor operator to push an XCom value as described in -section "Having sensors return XCOM values" of :doc:`apache-airflow-providers:howto/create-update-providers`. +the ``PokeReturnValue`` class as the ``poke()`` method in the BaseSensorOperator does. +In Airflow 2.3, sensor operators will be able to return XCOM values. This is achieved by returning +an instance of the ``PokeReturnValue`` object at the end of the ``poke()`` method: + + .. code-block:: python + + from airflow.sensors.base import PokeReturnValue + + + class SensorWithXcomValue(BaseSensorOperator): + def poke(self, context: Context) -> Union[bool, PokeReturnValue]: + # ... + is_done = ... # set to true if the sensor should stop poking. + xcom_value = ... # return value of the sensor operator to be pushed to XCOM. + return PokeReturnValue(is_done, xcom_value) + + +To implement a sensor operator that pushes a XCOM value and supports both version 2.3 and +pre-2.3, you need to explicitly push the XCOM value if the version is pre-2.3. + + .. code-block:: python + + try: + from airflow.sensors.base import PokeReturnValue + except ImportError: + PokeReturnValue = None + + + class SensorWithXcomValue(BaseSensorOperator): + def poke(self, context: Context) -> bool: + # ... + is_done = ... # set to true if the sensor should stop poking. + xcom_value = ... # return value of the sensor operator to be pushed to XCOM. + if PokeReturnValue is not None: + return PokeReturnValue(is_done, xcom_value) + else: + if is_done: + context["ti"].xcom_push(key="xcom_key", value=xcom_value) + return is_done + + + Alternatively in cases where the sensor doesn't need to push XCOM values: both ``poke()`` and the wrapped function can return a boolean-like value where ``True`` designates the sensor's operation as complete and diff --git a/docs/conf.py b/docs/conf.py index a8939833fd6b..0618a4401d33 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -407,8 +407,10 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): ).groups(0)[0] ) -# Jinja context -if PACKAGE_NAME == "apache-airflow": + +def get_configs_and_deprecations( + package_name: str, +) -> tuple[dict[str, dict[str, tuple[str, str, str]]], dict[str, dict[str, tuple[str, str, str]]]]: deprecated_options: dict[str, dict[str, tuple[str, str, str]]] = defaultdict(dict) for (section, key), ( (deprecated_section, deprecated_key, since_version) @@ -419,7 +421,12 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): for deprecated_section, deprecated_key, since_version in deprecated: deprecated_options[deprecated_section][deprecated_key] = section, key, since_version - configs = retrieve_configuration_description() + if package_name == "apache-airflow": + configs = retrieve_configuration_description(include_providers=False) + else: + configs = retrieve_configuration_description( + include_airflow=False, include_providers=True, selected_provider=package_name + ) # We want the default/example we show in the docs to reflect the value _after_ # the config has been templated, not before @@ -440,7 +447,12 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): configs = {k: v for k, v in sorted(configs.items())} for section in deprecated_options: deprecated_options[section] = {k: v for k, v in sorted(deprecated_options[section].items())} + return configs, deprecated_options + +# Jinja context +if PACKAGE_NAME == "apache-airflow": + configs, deprecated_options = get_configs_and_deprecations(PACKAGE_NAME) jinja_contexts = { "config_ctx": {"configs": configs, "deprecated_options": deprecated_options}, "quick_start_ctx": { @@ -453,18 +465,13 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path): }, } elif PACKAGE_NAME.startswith("apache-airflow-providers-"): - - def _load_config(): - file_path = PACKAGE_DIR / "config_templates" / "config.yml" - if not file_path.exists(): - return {} - - with file_path.open() as f: - return yaml.safe_load(f) - - config = _load_config() + configs, deprecated_options = get_configs_and_deprecations(PACKAGE_NAME) jinja_contexts = { - "config_ctx": {"configs": config}, + "config_ctx": { + "configs": configs, + "deprecated_options": deprecated_options, + "package_name": PACKAGE_NAME, + }, "official_download_page": { "base_url": "https://downloads.apache.org/airflow/providers", "closer_lua_url": "https://www.apache.org/dyn/closer.lua/airflow/providers", diff --git a/docs/installing-providers-from-sources.rst b/docs/exts/includes/installing-providers-from-sources.rst similarity index 100% rename from docs/installing-providers-from-sources.rst rename to docs/exts/includes/installing-providers-from-sources.rst diff --git a/docs/exts/includes/providers-configurations-ref.rst b/docs/exts/includes/providers-configurations-ref.rst new file mode 100644 index 000000000000..d696e126bbc5 --- /dev/null +++ b/docs/exts/includes/providers-configurations-ref.rst @@ -0,0 +1,29 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Configuration Reference +....................... + +.. jinja:: config_ctx + + This page contains the list of all available Airflow configurations for the + ``{{ package_name }}`` provider that can be set in the ``airflow.cfg`` file or using environment variables. + + .. note:: + For more information see :doc:`apache-airflow:howto/set-config`. + +.. include:: ../exts/includes/sections-and-options.rst diff --git a/docs/exts/includes/sections-and-options.rst b/docs/exts/includes/sections-and-options.rst new file mode 100644 index 000000000000..497223fcee1e --- /dev/null +++ b/docs/exts/includes/sections-and-options.rst @@ -0,0 +1,90 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. contents:: Sections: + :local: + :depth: 1 + +.. jinja:: config_ctx + + {% for section_name, section in configs.items() %} + + .. _config:{{ section_name }}: + + [{{ section_name }}] + {{ "=" * (section_name|length + 2) }} + + {% if 'renamed' in section %} + *Renamed in version {{ section['renamed']['version'] }}, previous name was {{ section['renamed']['previous_name'] }}* + {% endif %} + + {% if section["description"] %} + {{ section["description"] }} + {% endif %} + + {% for option_name, option in section["options"].items() %} + + .. _config:{{ section_name }}__{{ option_name }}: + + {{ option_name }} + {{ "-" * option_name|length }} + + {% if option["version_added"] %} + .. versionadded:: {{ option["version_added"] }} + {% endif %} + + {% if option["description"] %} + {{ option["description"] }} + {% endif %} + + {% if option.get("see_also") %} + .. seealso:: {{ option["see_also"] }} + {% endif %} + + :Type: {{ option["type"] }} + :Default: ``{{ "''" if option["default"] == "" else option["default"] }}`` + {% if option.get("sensitive") %} + :Environment Variables: + ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` + + ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD`` + + ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET`` + {% else %} + :Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}`` + {% endif %} + {% if option["example"] %} + :Example: + ``{{ option["example"] }}`` + {% endif %} + + {% endfor %} + + {% if section_name in deprecated_options %} + + {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section_name].items() %} + .. _config:{{ section_name }}__{{ deprecated_option_name }}: + + {{ deprecated_option_name }} (Deprecated) + {{ "-" * (deprecated_option_name + " (Deprecated)")|length }} + + .. deprecated:: {{ since_version }} + The option has been moved to :ref:`{{ new_section_name }}.{{ new_option_name }} ` + {% endfor %} + {% endif %} + + {% endfor %} diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index eeaf00726c8a..06c37cf0b427 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -48,14 +48,15 @@ """ DEFAULT_HEADER_SEPARATOR = "=" -CURRENT_DIR = os.path.dirname(__file__) +CURRENT_DIR = Path(os.path.dirname(__file__)) +TEMPLATE_DIR = CURRENT_DIR / "templates" ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir)) DOCS_DIR = os.path.join(ROOT_DIR, "docs") @lru_cache(maxsize=None) def _get_jinja_env(): - loader = jinja2.FileSystemLoader(CURRENT_DIR, followlinks=True) + loader = jinja2.FileSystemLoader(TEMPLATE_DIR, followlinks=True) env = jinja2.Environment(loader=loader, undefined=jinja2.StrictUndefined) return env @@ -213,7 +214,7 @@ def _render_deferrable_operator_content(*, header_separator: str): provider_yaml_content = yaml.safe_load(Path(provider_yaml_path).read_text()) provider_info["name"] = provider_yaml_content["package-name"] providers.append(provider_info) - return _render_template("deferrable_operatos_list.rst.jinja2", providers=providers) + return _render_template("deferrable_operators_list.rst.jinja2", providers=providers) class BaseJinjaReferenceDirective(Directive): @@ -308,6 +309,20 @@ def render_content( ) +class AuthConfigurations(BaseJinjaReferenceDirective): + """Generate list of configurations""" + + def render_content( + self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR + ) -> str: + tabular_data = [ + provider["package-name"] for provider in load_package_data() if provider.get("config") is not None + ] + return _render_template( + "configuration.rst.jinja2", items=tabular_data, header_separator=header_separator + ) + + class SecretsBackendDirective(BaseJinjaReferenceDirective): """Generate list of secret backend handlers""" @@ -384,6 +399,7 @@ def setup(app): app.add_directive("transfers-ref", TransfersReferenceDirective) app.add_directive("airflow-logging", LoggingDirective) app.add_directive("airflow-auth-backends", AuthBackendDirective) + app.add_directive("airflow-configurations", AuthConfigurations) app.add_directive("airflow-secrets-backends", SecretsBackendDirective) app.add_directive("airflow-connections", ConnectionsDirective) app.add_directive("airflow-extra-links", ExtraLinksDirective) diff --git a/docs/exts/auth_backend.rst.jinja2 b/docs/exts/templates/auth_backend.rst.jinja2 similarity index 100% rename from docs/exts/auth_backend.rst.jinja2 rename to docs/exts/templates/auth_backend.rst.jinja2 diff --git a/docs/exts/templates/configuration.rst.jinja2 b/docs/exts/templates/configuration.rst.jinja2 new file mode 100644 index 000000000000..be43b4706ca7 --- /dev/null +++ b/docs/exts/templates/configuration.rst.jinja2 @@ -0,0 +1,21 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} +{%for provider_package_name in items %} +* :doc:`Configuration for {{ provider_package_name }} <{{ provider_package_name }}:configurations-ref>` +{% endfor %} diff --git a/docs/exts/connections.rst.jinja2 b/docs/exts/templates/connections.rst.jinja2 similarity index 100% rename from docs/exts/connections.rst.jinja2 rename to docs/exts/templates/connections.rst.jinja2 diff --git a/docs/exts/deferrable_operatos_list.rst.jinja2 b/docs/exts/templates/deferrable_operators_list.rst.jinja2 similarity index 98% rename from docs/exts/deferrable_operatos_list.rst.jinja2 rename to docs/exts/templates/deferrable_operators_list.rst.jinja2 index e6eeb66385cf..58f29d295a7c 100644 --- a/docs/exts/deferrable_operatos_list.rst.jinja2 +++ b/docs/exts/templates/deferrable_operators_list.rst.jinja2 @@ -19,7 +19,7 @@ {%- for provider in providers %} **{{ provider["name"] }}** -.. list-table:: +.. list-table::F :header-rows: 1 * - Modules diff --git a/docs/exts/executors.rst.jinja2 b/docs/exts/templates/executors.rst.jinja2 similarity index 100% rename from docs/exts/executors.rst.jinja2 rename to docs/exts/templates/executors.rst.jinja2 diff --git a/docs/exts/extra_links.rst.jinja2 b/docs/exts/templates/extra_links.rst.jinja2 similarity index 100% rename from docs/exts/extra_links.rst.jinja2 rename to docs/exts/templates/extra_links.rst.jinja2 diff --git a/docs/exts/logging.rst.jinja2 b/docs/exts/templates/logging.rst.jinja2 similarity index 100% rename from docs/exts/logging.rst.jinja2 rename to docs/exts/templates/logging.rst.jinja2 diff --git a/docs/exts/notifications.rst.jinja2 b/docs/exts/templates/notifications.rst.jinja2 similarity index 100% rename from docs/exts/notifications.rst.jinja2 rename to docs/exts/templates/notifications.rst.jinja2 diff --git a/docs/exts/operators_and_hooks_ref-transfers.rst.jinja2 b/docs/exts/templates/operators_and_hooks_ref-transfers.rst.jinja2 similarity index 100% rename from docs/exts/operators_and_hooks_ref-transfers.rst.jinja2 rename to docs/exts/templates/operators_and_hooks_ref-transfers.rst.jinja2 diff --git a/docs/exts/operators_and_hooks_ref.rst.jinja2 b/docs/exts/templates/operators_and_hooks_ref.rst.jinja2 similarity index 100% rename from docs/exts/operators_and_hooks_ref.rst.jinja2 rename to docs/exts/templates/operators_and_hooks_ref.rst.jinja2 diff --git a/docs/exts/secret_backend.rst.jinja2 b/docs/exts/templates/secret_backend.rst.jinja2 similarity index 100% rename from docs/exts/secret_backend.rst.jinja2 rename to docs/exts/templates/secret_backend.rst.jinja2 diff --git a/tests/cli/commands/test_celery_command.py b/tests/cli/commands/test_celery_command.py index 0e8dee24570b..f3e3d391180e 100644 --- a/tests/cli/commands/test_celery_command.py +++ b/tests/cli/commands/test_celery_command.py @@ -39,7 +39,7 @@ def test_error(self, mock_validate_session): by mocking validate_session method """ mock_validate_session.return_value = False - with pytest.raises(SystemExit) as ctx: + with pytest.raises(SystemExit) as ctx, conf_vars({("core", "executor"): "CeleryExecutor"}): celery_command.worker(Namespace(queues=1, concurrency=1)) assert str(ctx.value) == "Worker exiting, database connection precheck failed." diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index f5b427f0cfca..59d04de860b6 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -43,6 +43,7 @@ def test_cli_show_config_should_write_data(self, mock_conf, mock_stringio): include_descriptions=False, include_sources=False, include_env_vars=False, + include_providers=True, comment_out_everything=False, only_defaults=False, ) @@ -60,6 +61,7 @@ def test_cli_show_config_should_write_data_specific_section(self, mock_conf, moc include_descriptions=False, include_sources=False, include_env_vars=False, + include_providers=True, comment_out_everything=False, only_defaults=False, ) @@ -178,6 +180,13 @@ def test_cli_show_changed_defaults_when_overridden_in_env(self): lines = output.split("\n") assert any(line.startswith("task_runner = test-env-runner") for line in lines if line) + def test_cli_has_providers(self): + with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: + config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) + output = temp_stdout.getvalue() + lines = output.split("\n") + assert any(line.startswith("celery_config_options") for line in lines if line) + def test_cli_comment_out_everything(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config( diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 7cac8f7ad1d1..e770e19416eb 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,20 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + from __future__ import annotations import argparse import contextlib import io +import os import re import subprocess +import sys import timeit from collections import Counter +from pathlib import Path from unittest.mock import patch import pytest from airflow.cli import cli_config, cli_parser +from airflow.configuration import AIRFLOW_HOME from tests.test_utils.config import conf_vars # Can not be `--snake_case` or contain uppercase letter @@ -284,9 +288,25 @@ def test_invalid_choice_raises_for_export_format_in_db_export_archived_command( "(choose from 'csv'), see help above.\n" ) + +# We need to run it from sources with PYTHONPATH, not command line tool, +# because we need to make sure that we have providers configured from source provider.yaml files + +CONFIG_FILE = Path(AIRFLOW_HOME) / "airflow.cfg" + + +class TestCliSubprocess: + """ + We need to run it from sources using "__main__" and setting the PYTHONPATH, not command line tool, + because we need to make sure that we have providers loaded from source provider.yaml files rather + than from provider packages which might not be installed in the test environment. + """ + def test_cli_run_time(self): setup_code = "import subprocess" - timing_code = 'subprocess.run(["airflow", "--help"])' + command = [sys.executable, "-m", "airflow", "--help"] + env = {"PYTHONPATH": os.pathsep.join(sys.path)} + timing_code = f"subprocess.run({command},env={env})" # Limit the number of samples otherwise the test will take a very long time num_samples = 3 threshold = 3.5 @@ -301,4 +321,59 @@ def test_cli_parsing_does_not_initialize_providers_manager(self): separate subprocess, to make sure we do not have providers manager initialized in the main process from other tests. """ - subprocess.check_call(["airflow", "providers", "status"]) + CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) + CONFIG_FILE.touch(exist_ok=True) + result = subprocess.run( + [sys.executable, "-m", "airflow", "providers", "status"], + env={"PYTHONPATH": os.pathsep.join(sys.path)}, + check=False, + text=True, + ) + assert result.returncode == 0 + + def test_airflow_config_contains_providers(self): + """Test that airflow config has providers included by default. + + This test is run as a separate subprocess, to make sure we do not have providers manager + initialized in the main process from other tests. + """ + CONFIG_FILE.unlink(missing_ok=True) + result = subprocess.run( + [sys.executable, "-m", "airflow", "version"], + env={"PYTHONPATH": os.pathsep.join(sys.path)}, + check=False, + text=True, + ) + assert result.returncode == 0 + assert CONFIG_FILE.exists() + assert "celery_config_options" in CONFIG_FILE.read_text() + + def test_airflow_config_output_contains_providers_by_default(self): + """Test that airflow config has providers excluded in config list when asked for it.""" + CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) + CONFIG_FILE.touch(exist_ok=True) + + result = subprocess.run( + [sys.executable, "-m", "airflow", "config", "list"], + env={"PYTHONPATH": os.pathsep.join(sys.path)}, + check=False, + text=True, + capture_output=True, + ) + assert result.returncode == 0 + assert "celery_config_options" in result.stdout + + def test_airflow_config_output_does_not_contain_providers_when_excluded(self): + """Test that airflow config has providers excluded in config list when asked for it.""" + CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) + CONFIG_FILE.unlink(missing_ok=True) + CONFIG_FILE.touch(exist_ok=True) + result = subprocess.run( + [sys.executable, "-m", "airflow", "config", "list", "--exclude-providers"], + env={"PYTHONPATH": os.pathsep.join(sys.path)}, + check=False, + text=True, + capture_output=True, + ) + assert result.returncode == 0 + assert "celery_config_options" not in result.stdout diff --git a/tests/conftest.py b/tests/conftest.py index b2ef0b1e6e90..49733a7da4e2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -935,3 +935,10 @@ def refuse_to_run_test_from_wrongly_named_files(request): f"contains {request.function} that looks like a test case. Please rename the file to " f"follow the test_* pattern if you want to run the tests in it." ) + + +@pytest.fixture(autouse=True) +def initialize_providers_manager(): + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index ecb01f5beeec..6eb50d919f10 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -27,6 +27,7 @@ import warnings from collections import OrderedDict from unittest import mock +from unittest.mock import patch import pytest from pytest import param @@ -627,7 +628,7 @@ def test_command_from_env(self): # sensitive_config_values and therefore should return 'OK' from the environment variable's # echo command, and must not return 'NOT OK' from the configuration assert test_cmdenv_conf.get("testcmdenv", "itsacommand") == "OK" - # AIRFLOW__TESTCMDENV__xNOTACOMMAND_CMD maps to no entry in sensitive_config_values and therefore + # AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD maps to no entry in sensitive_config_values and therefore # the option should return 'OK' from the configuration, and must not return 'NOT OK' from # the environment variable's echo command assert test_cmdenv_conf.get("testcmdenv", "notacommand") == "OK" @@ -1599,9 +1600,6 @@ def test_sensitive_values(): sensitive_values = { ("database", "sql_alchemy_conn"), ("core", "fernet_key"), - ("celery", "broker_url"), - ("celery", "flower_basic_auth"), - ("celery", "result_backend"), ("atlas", "password"), ("smtp", "smtp_password"), ("webserver", "secret_key"), @@ -1609,6 +1607,10 @@ def test_sensitive_values(): ("sentry", "sentry_dsn"), ("database", "sql_alchemy_engine_args"), ("core", "sql_alchemy_conn"), + ("celery_broker_transport_options", "sentinel_kwargs"), + ("celery", "broker_url"), + ("celery", "flower_basic_auth"), + ("celery", "result_backend"), } all_keys = {(s, k) for s, v in conf.configuration_description.items() for k in v.get("options")} suspected_sensitive = {(s, k) for (s, k) in all_keys if k.endswith(("password", "kwargs"))} @@ -1618,3 +1620,51 @@ def test_sensitive_values(): suspected_sensitive -= exclude_list sensitive_values.update(suspected_sensitive) assert sensitive_values == conf.sensitive_config_values + + +def test_restore_and_reload_provider_configuration(): + from airflow.settings import conf + + assert conf.providers_configuration_loaded is True + assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" + conf.restore_core_default_configuration() + assert conf.providers_configuration_loaded is False + with pytest.raises(AirflowConfigException, match="not found"): + conf.get("celery", "celery_app_name") + conf.load_providers_configuration() + assert conf.providers_configuration_loaded is True + assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" + + +def test_error_when_contributing_to_existing_section(): + from airflow.settings import conf + + try: + assert conf.providers_configuration_loaded is True + assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" + conf.restore_core_default_configuration() + assert conf.providers_configuration_loaded is False + conf.configuration_description["celery"] = { + "description": "Celery Executor configuration", + "options": { + "celery_app_name": { + "default": "test", + } + }, + } + conf._default_values.add_section("celery") + conf._default_values.set("celery", "celery_app_name", "test") + assert conf.get("celery", "celery_app_name") == "test" + # patching restoring_core_default_configuration to avoid reloading the defaults + with patch.object(conf, "restore_core_default_configuration"): + with pytest.raises( + AirflowConfigException, + match="The provider apache-airflow-providers-celery is attempting to contribute " + "configuration section celery that has already been added before. " + "The source of it: Airflow's core package", + ): + conf.load_providers_configuration() + assert conf.get("celery", "celery_app_name") == "test" + finally: + conf.restore_core_default_configuration() + conf.load_providers_configuration() diff --git a/tests/providers/celery/executors/test_celery_kubernetes_executor.py b/tests/providers/celery/executors/test_celery_kubernetes_executor.py index 6302efea21da..1ef45ab22d9b 100644 --- a/tests/providers/celery/executors/test_celery_kubernetes_executor.py +++ b/tests/providers/celery/executors/test_celery_kubernetes_executor.py @@ -27,7 +27,7 @@ from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.providers.celery.executors.celery_kubernetes_executor import CeleryKubernetesExecutor -KUBERNETES_QUEUE = CeleryKubernetesExecutor.KUBERNETES_QUEUE +KUBERNETES_QUEUE = "kubernetes" class TestCeleryKubernetesExecutor: diff --git a/tests/providers/hashicorp/hooks/test_vault.py b/tests/providers/hashicorp/hooks/test_vault.py index 4bd3e90e5614..29bace064241 100644 --- a/tests/providers/hashicorp/hooks/test_vault.py +++ b/tests/providers/hashicorp/hooks/test_vault.py @@ -16,13 +16,17 @@ # under the License. from __future__ import annotations +import re from unittest import mock from unittest.mock import PropertyMock, mock_open, patch import pytest from hvac.exceptions import VaultError +from airflow.configuration import AirflowConfigParser +from airflow.exceptions import AirflowConfigException from airflow.providers.hashicorp.hooks.vault import VaultHook +from tests.test_utils.config import conf_vars class TestVaultHook: @@ -859,7 +863,6 @@ def test_radius_dejson_wrong_port(self, mock_hvac, mock_get_connection): kwargs = { "vault_conn_id": "vault_conn_id", } - with pytest.raises(VaultError, match="Radius port was wrong: wrong"): VaultHook(**kwargs) @@ -1252,3 +1255,87 @@ def test_create_or_update_secret_v1(self, mock_hvac, mock_get_connection, method mock_client.secrets.kv.v1.create_or_update_secret.assert_called_once_with( mount_point="secret", secret_path="path", secret={"key": "value"}, method=expected_method ) + + +class TestConfigurationFromSecrets: + @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") + @conf_vars( + { + ("secrets", "backend"): "airflow.providers.hashicorp.secrets.vault.VaultBackend", + ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200", "token": "token"}', + } + ) + def test_config_from_secret_backend(self, mock_hvac): + """Get Config Value from a Secret Backend""" + mock_client = mock.MagicMock() + mock_hvac.Client.return_value = mock_client + mock_client.secrets.kv.v2.read_secret_version.return_value = { + "request_id": "2d48a2ad-6bcb-e5b6-429d-da35fdf31f56", + "lease_id": "", + "renewable": False, + "lease_duration": 0, + "data": { + "data": {"value": "sqlite:////Users/airflow/airflow/airflow.db"}, + "metadata": { + "created_time": "2020-03-28T02:10:54.301784Z", + "deletion_time": "", + "destroyed": False, + "version": 1, + }, + }, + "wrap_info": None, + "warnings": None, + "auth": None, + } + + test_config = """[test] + sql_alchemy_conn_secret = sql_alchemy_conn + """ + test_config_default = """[test] + sql_alchemy_conn = airflow + """ + + test_conf = AirflowConfigParser(default_config=test_config_default) + test_conf.read_string(test_config) + test_conf.sensitive_config_values = test_conf.sensitive_config_values | { + ("test", "sql_alchemy_conn"), + } + + assert "sqlite:////Users/airflow/airflow/airflow.db" == test_conf.get("test", "sql_alchemy_conn") + + @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") + @conf_vars( + { + ("secrets", "backend"): "airflow.providers.hashicorp.secrets.vault.VaultBackend", + ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200", "token": "token"}', + } + ) + def test_config_raise_exception_from_secret_backend_connection_error(self, mock_hvac): + """Get Config Value from a Secret Backend""" + + mock_client = mock.MagicMock() + # mock_client.side_effect = AirflowConfigException + mock_hvac.Client.return_value = mock_client + mock_client.secrets.kv.v2.read_secret_version.return_value = Exception + + test_config = """[test] +sql_alchemy_conn_secret = sql_alchemy_conn +""" + test_config_default = """[test] +sql_alchemy_conn = airflow +""" + test_conf = AirflowConfigParser(default_config=test_config_default) + test_conf.read_string(test_config) + test_conf.sensitive_config_values = test_conf.sensitive_config_values | { + ("test", "sql_alchemy_conn"), + } + + with pytest.raises( + AirflowConfigException, + match=re.escape( + "Cannot retrieve config from alternative secrets backend. " + "Make sure it is configured properly and that the Backend " + "is accessible." + ), + ): + test_conf.get("test", "sql_alchemy_conn") From 62fe683f33d9b64c1c876c16ca29b9405175fbd0 Mon Sep 17 00:00:00 2001 From: Akash Sharma <35839624+Adaverse@users.noreply.github.com> Date: Sat, 22 Jul 2023 01:50:19 +0530 Subject: [PATCH 266/533] provider_info schema correction (#32754) Co-authored-by: adaverse --- airflow/provider_info.schema.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/provider_info.schema.json b/airflow/provider_info.schema.json index 9a5dcaefbe20..3a37810f7a04 100644 --- a/airflow/provider_info.schema.json +++ b/airflow/provider_info.schema.json @@ -39,12 +39,12 @@ "description": "Hook class name that implements the connection type", "type": "string" } - } - }, - "required": [ - "connection-type", - "hook-class-name" - ] + }, + "required": [ + "connection-type", + "hook-class-name" + ] + } }, "extra-links": { "type": "array", From d7899ecfafb20cc58f8fb43e287d1c6778b8fa9f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 21 Jul 2023 23:22:21 +0200 Subject: [PATCH 267/533] Bring back automated airflow home dir creation (#32755) The #32604 moved initialization of airflow config to after config initialization but webserver config is still in initialization part. Previously when the AIRFLOW_HOME folder was missing, it was created during config writing but it needs to be created now before webserver config is written. --- airflow/configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/configuration.py b/airflow/configuration.py index 86d3ca52528e..317c10c01310 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -2008,6 +2008,7 @@ def initialize_config() -> AirflowConfigParser: if not os.path.isfile(WEBSERVER_CONFIG): import shutil + pathlib.Path(WEBSERVER_CONFIG).parent.mkdir(parents=True, exist_ok=True) log.info("Creating new FAB webserver config file in: %s", WEBSERVER_CONFIG) shutil.copy(_default_config_file_path("default_webserver_config.py"), WEBSERVER_CONFIG) return airflow_config_parser From 3d89e75eb98d16989d91f5c22dd153b6e51cb42e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 22 Jul 2023 09:55:16 +0200 Subject: [PATCH 268/533] Fix failing Celery Executor tests after config migration (#32763) The Celery Executor tests in Helm started to fail after the configuration migration has been merged (#32604). The PR did not have "full tests needed" label and it skipped K8S tests because there was no change related to kubernetes (but some fundamental changes in how configuration were retrieved caused the Celery Executor failed on missing default configuration value. The change adds ProvidersManager configuration initialization when executors are started in order to fix the problem temporarily, however there is an ongoing effort to optimise the path of retrieving provider configuration without having to initialize all provider's configuration and those lines will be removed when it happens. --- airflow/executors/executor_loader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 58cd9ae9168b..816e172140ae 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -98,6 +98,9 @@ def load_executor(cls, executor_name: str) -> BaseExecutor: :return: an instance of executor class via executor_name """ + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() if executor_name == CELERY_KUBERNETES_EXECUTOR: return cls.__load_celery_kubernetes_executor() elif executor_name == LOCAL_KUBERNETES_EXECUTOR: From 53c6305bd0a914738074821d5f5f233e3ed5bee5 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 22 Jul 2023 11:25:16 +0200 Subject: [PATCH 269/533] Improve getting the query count in Airflow API endpoints (#32630) * Improve the query count getting in Airflow API * Move get_query_count to utils.db * Add a comment to explain why we reset order_by * Update airflow/utils/db.py Co-authored-by: Tzu-ping Chung --------- Co-authored-by: Tzu-ping Chung --- airflow/api_connexion/endpoints/dag_endpoint.py | 7 ++++--- .../api_connexion/endpoints/dag_run_endpoint.py | 12 +++++++++--- .../endpoints/dag_warning_endpoint.py | 5 +++-- .../api_connexion/endpoints/dataset_endpoint.py | 3 ++- .../endpoints/task_instance_endpoint.py | 12 ++++++------ airflow/api_connexion/endpoints/xcom_endpoint.py | 5 +++-- airflow/utils/db.py | 15 +++++++++++++++ airflow/www/views.py | 13 ++++++------- 8 files changed, 48 insertions(+), 24 deletions(-) diff --git a/airflow/api_connexion/endpoints/dag_endpoint.py b/airflow/api_connexion/endpoints/dag_endpoint.py index 55eb971c08b0..a301699b380d 100644 --- a/airflow/api_connexion/endpoints/dag_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_endpoint.py @@ -22,7 +22,7 @@ from connexion import NoContent from flask import g, request from marshmallow import ValidationError -from sqlalchemy import func, select, update +from sqlalchemy import select, update from sqlalchemy.orm import Session from sqlalchemy.sql.expression import or_ @@ -41,6 +41,7 @@ from airflow.models.dag import DagModel, DagTag from airflow.security import permissions from airflow.utils.airflow_flask_app import get_airflow_app +from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session @@ -99,7 +100,7 @@ def get_dags( cond = [DagModel.tags.any(DagTag.name == tag) for tag in tags] dags_query = dags_query.where(or_(*cond)) - total_entries = session.scalar(select(func.count()).select_from(dags_query)) + total_entries = get_query_count(dags_query, session=session) dags_query = apply_sorting(dags_query, order_by, {}, allowed_attrs) dags = session.scalars(dags_query.offset(offset).limit(limit)).all() @@ -159,7 +160,7 @@ def patch_dags(limit, session, offset=0, only_active=True, tags=None, dag_id_pat cond = [DagModel.tags.any(DagTag.name == tag) for tag in tags] dags_query = dags_query.where(or_(*cond)) - total_entries = session.scalar(select(func.count()).select_from(dags_query)) + total_entries = get_query_count(dags_query, session=session) dags = session.scalars(dags_query.order_by(DagModel.dag_id).offset(offset).limit(limit)).all() diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index f62b28273f7b..490923c6c381 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -23,7 +23,7 @@ from flask import g from flask_login import current_user from marshmallow import ValidationError -from sqlalchemy import delete, func, or_, select +from sqlalchemy import delete, or_, select from sqlalchemy.orm import Session from sqlalchemy.sql import Select @@ -35,7 +35,12 @@ from airflow.api_connexion import security from airflow.api_connexion.endpoints.request_dict import get_json_request_dict from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound -from airflow.api_connexion.parameters import apply_sorting, check_limit, format_datetime, format_parameters +from airflow.api_connexion.parameters import ( + apply_sorting, + check_limit, + format_datetime, + format_parameters, +) from airflow.api_connexion.schemas.dag_run_schema import ( DAGRunCollection, clear_dagrun_form_schema, @@ -57,6 +62,7 @@ from airflow.models import DagModel, DagRun from airflow.security import permissions from airflow.utils.airflow_flask_app import get_airflow_app +from airflow.utils.db import get_query_count from airflow.utils.log.action_logger import action_event_from_permission from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState @@ -166,7 +172,7 @@ def _fetch_dag_runs( if updated_at_lte: query = query.where(DagRun.updated_at <= updated_at_lte) - total_entries = session.scalar(select(func.count()).select_from(query)) + total_entries = get_query_count(query, session=session) to_replace = {"dag_run_id": "run_id"} allowed_filter_attrs = [ "id", diff --git a/airflow/api_connexion/endpoints/dag_warning_endpoint.py b/airflow/api_connexion/endpoints/dag_warning_endpoint.py index 66aa1184a192..1031d532a00f 100644 --- a/airflow/api_connexion/endpoints/dag_warning_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_warning_endpoint.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from sqlalchemy import func, select +from sqlalchemy import select from sqlalchemy.orm import Session from airflow.api_connexion import security @@ -28,6 +28,7 @@ from airflow.api_connexion.types import APIResponse from airflow.models.dagwarning import DagWarning as DagWarningModel from airflow.security import permissions +from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session @@ -54,7 +55,7 @@ def get_dag_warnings( query = query.where(DagWarningModel.dag_id == dag_id) if warning_type: query = query.where(DagWarningModel.warning_type == warning_type) - total_entries = session.scalar(select(func.count()).select_from(query)) + total_entries = get_query_count(query, session=session) query = apply_sorting(query=query, order_by=order_by, allowed_attrs=allowed_filter_attrs) dag_warnings = session.scalars(query.offset(offset).limit(limit)).all() return dag_warning_collection_schema.dump( diff --git a/airflow/api_connexion/endpoints/dataset_endpoint.py b/airflow/api_connexion/endpoints/dataset_endpoint.py index 9f4fa443b99d..71012b59a2e7 100644 --- a/airflow/api_connexion/endpoints/dataset_endpoint.py +++ b/airflow/api_connexion/endpoints/dataset_endpoint.py @@ -32,6 +32,7 @@ from airflow.api_connexion.types import APIResponse from airflow.models.dataset import DatasetEvent, DatasetModel from airflow.security import permissions +from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session @@ -112,7 +113,7 @@ def get_dataset_events( query = query.options(subqueryload(DatasetEvent.created_dagruns)) - total_entries = session.scalar(select(func.count()).select_from(query)) + total_entries = get_query_count(query, session=session) query = apply_sorting(query, order_by, {}, allowed_attrs) events = session.scalars(query.offset(offset).limit(limit)).all() return dataset_event_collection_schema.dump( diff --git a/airflow/api_connexion/endpoints/task_instance_endpoint.py b/airflow/api_connexion/endpoints/task_instance_endpoint.py index 2496433a26d7..55db7ef8b937 100644 --- a/airflow/api_connexion/endpoints/task_instance_endpoint.py +++ b/airflow/api_connexion/endpoints/task_instance_endpoint.py @@ -19,7 +19,7 @@ from typing import Any, Iterable, TypeVar from marshmallow import ValidationError -from sqlalchemy import and_, func, or_, select +from sqlalchemy import and_, or_, select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload from sqlalchemy.sql import ClauseElement, Select @@ -48,6 +48,7 @@ from airflow.models.taskinstance import TaskInstance as TI, clear_task_instances from airflow.security import permissions from airflow.utils.airflow_flask_app import get_airflow_app +from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState, TaskInstanceState @@ -196,7 +197,7 @@ def get_mapped_task_instances( ) # 0 can mean a mapped TI that expanded to an empty list, so it is not an automatic 404 - unfiltered_total_count = session.execute(select(func.count("*")).select_from(base_query)).scalar() + unfiltered_total_count = get_query_count(base_query, session=session) if unfiltered_total_count == 0: dag = get_airflow_app().dag_bag.get_dag(dag_id) if not dag: @@ -229,7 +230,7 @@ def get_mapped_task_instances( base_query = _apply_array_filter(base_query, key=TI.queue, values=queue) # Count elements before joining extra columns - total_entries = session.execute(select(func.count("*")).select_from(base_query)).scalar() + total_entries = get_query_count(base_query, session=session) # Add SLA miss entry_query = ( @@ -355,8 +356,7 @@ def get_task_instances( base_query = _apply_array_filter(base_query, key=TI.queue, values=queue) # Count elements before joining extra columns - count_query = select(func.count("*")).select_from(base_query) - total_entries = session.execute(count_query).scalar() + total_entries = get_query_count(base_query, session=session) # Add join entry_query = ( @@ -420,7 +420,7 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse: base_query = _apply_array_filter(base_query, key=TI.queue, values=data["queue"]) # Count elements before joining extra columns - total_entries = session.execute(select(func.count("*")).select_from(base_query)).scalar() + total_entries = get_query_count(base_query, session=session) # Add join base_query = base_query.join( SlaMiss, diff --git a/airflow/api_connexion/endpoints/xcom_endpoint.py b/airflow/api_connexion/endpoints/xcom_endpoint.py index d95a3d15fce6..2ea4db79f7aa 100644 --- a/airflow/api_connexion/endpoints/xcom_endpoint.py +++ b/airflow/api_connexion/endpoints/xcom_endpoint.py @@ -19,7 +19,7 @@ import copy from flask import g -from sqlalchemy import and_, func, select +from sqlalchemy import and_, select from sqlalchemy.orm import Session from airflow.api_connexion import security @@ -31,6 +31,7 @@ from airflow.security import permissions from airflow.settings import conf from airflow.utils.airflow_flask_app import get_airflow_app +from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session @@ -75,7 +76,7 @@ def get_xcom_entries( if xcom_key is not None: query = query.where(XCom.key == xcom_key) query = query.order_by(DR.execution_date, XCom.task_id, XCom.dag_id, XCom.key) - total_entries = session.execute(select(func.count()).select_from(query)).scalar() + total_entries = get_query_count(query, session=session) query = session.scalars(query.offset(offset).limit(limit)) return xcom_collection_schema.dump(XComCollection(xcom_entries=query, total_entries=total_entries)) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index d6c5664a96f4..8e8c4b7fa93b 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -45,6 +45,7 @@ from alembic.runtime.environment import EnvironmentContext from alembic.script import ScriptDirectory from sqlalchemy.orm import Query, Session + from sqlalchemy.sql.selectable import Select from airflow.models.base import Base from airflow.models.connection import Connection @@ -1872,3 +1873,17 @@ def get_sqla_model_classes(): return [mapper.class_ for mapper in Base.registry.mappers] except AttributeError: return Base._decl_class_registry.values() + + +def get_query_count(query_stmt: Select, session: Session) -> int: + """Get count of query. + + A SELECT COUNT() FROM is issued against the subquery built from the + given statement. The ORDER BY clause is stripped from the statement + since it's unnecessary for COUNT, and can impact query planning and + degrade performance. + + :meta private: + """ + count_stmt = select(func.count()).select_from(query_stmt.order_by(None).subquery()) + return session.scalar(count_stmt) diff --git a/airflow/www/views.py b/airflow/www/views.py index 85375a518b3f..d175b7be5079 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -117,6 +117,7 @@ from airflow.utils.airflow_flask_app import get_airflow_app from airflow.utils.dag_edges import dag_edges from airflow.utils.dates import infer_time_unit, scale_time_units +from airflow.utils.db import get_query_count from airflow.utils.docs import get_doc_url_for_provider, get_docs_url from airflow.utils.helpers import alchemy_to_dict, exactly_one from airflow.utils.log import secrets_masker @@ -759,7 +760,7 @@ def index(self): dags_query = dags_query.where(DagModel.tags.any(DagTag.name.in_(arg_tags_filter))) dags_query = dags_query.where(DagModel.dag_id.in_(filter_dag_ids)) - filtered_dag_count = session.scalar(select(func.count()).select_from(dags_query)) + filtered_dag_count = get_query_count(dags_query, session=session) if filtered_dag_count == 0 and len(arg_tags_filter): flash( "No matching DAG tags found.", @@ -811,8 +812,8 @@ def index(self): status_count_active = is_paused_count.get(False, 0) status_count_paused = is_paused_count.get(True, 0) - status_count_running = session.scalar(select(func.count()).select_from(running_dags)) - status_count_failed = session.scalar(select(func.count()).select_from(failed_dags)) + status_count_running = get_query_count(running_dags, session=session) + status_count_failed = get_query_count(failed_dags, session=session) all_dags_count = status_count_active + status_count_paused if arg_status_filter == "active": @@ -951,9 +952,7 @@ def _iter_parsed_moved_data_table_names(): .where(Log.event == "robots") .where(Log.dttm > (utcnow() - datetime.timedelta(days=7))) ) - robots_file_access_count = session.scalar( - select(func.count()).select_from(robots_file_access_count) - ) + robots_file_access_count = get_query_count(robots_file_access_count, session=session) if robots_file_access_count > 0: flash( Markup( @@ -4171,7 +4170,7 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): arg_sorting_direction = request.args.get("sorting_direction", default="desc") logs_per_page = PAGE_SIZE - audit_logs_count = session.scalar(select(func.count()).select_from(query)) + audit_logs_count = get_query_count(query, session=session) num_of_pages = int(math.ceil(audit_logs_count / float(logs_per_page))) start = current_page * logs_per_page From 3e467ba510d29e912d89115769726111b8bce891 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Sat, 22 Jul 2023 18:23:12 +0800 Subject: [PATCH 270/533] Fix prefix group false graph (#32764) * Fix edge rendering for prefix_group_id=false task groups * Ensure nested prefix_group_id=false works --- .../www/static/js/dag/details/graph/Node.tsx | 1 + .../www/static/js/dag/details/graph/utils.ts | 30 +++++++- airflow/www/static/js/utils/graph.ts | 72 +++++++++---------- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/airflow/www/static/js/dag/details/graph/Node.tsx b/airflow/www/static/js/dag/details/graph/Node.tsx index 3655dd27c7e9..01c2d038f717 100644 --- a/airflow/www/static/js/dag/details/graph/Node.tsx +++ b/airflow/www/static/js/dag/details/graph/Node.tsx @@ -47,6 +47,7 @@ export interface CustomNodeProps { isOpen?: boolean; isActive?: boolean; setupTeardownType?: "setup" | "teardown"; + fullParentNode?: string; } export const BaseNode = ({ diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index 459632024692..b1e27fb0eda3 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -51,10 +51,18 @@ export const flattenNodes = ({ }: FlattenNodesProps) => { let nodes: ReactFlowNode[] = []; const parentNode = parent ? { parentNode: parent.id } : undefined; + + let fullParentNode: string | undefined; + if (parent) { + fullParentNode = + parent.parentNode && !parent.id.startsWith(parent.parentNode) + ? `${parent.parentNode}.${parent.id}` + : parent.id; + } children.forEach((node) => { let instance: TaskInstance | undefined; const group = getTask({ taskId: node.id, task: groups }); - if (!node.id.includes("join_id") && selected.runId) { + if (!node.id.endsWith("join_id") && selected.runId) { instance = group?.instances.find((ti) => ti.runId === selected.runId); } const isSelected = node.id === selected.taskId && !!instance; @@ -82,6 +90,7 @@ export const flattenNodes = ({ } onToggleGroups(newGroupIds); }, + fullParentNode, ...node.value, }, type: "custom", @@ -155,8 +164,23 @@ export const buildEdges = ({ type: "custom", })) .map((e) => { - const sourceIds = e.source.split("."); - const targetIds = e.target.split("."); + const sourceNode = nodes.find((n) => n.id === e.source); + const targetNode = nodes.find((n) => n.id === e.target); + + // Before finding the depth of the edge, append the parentNode in case prefix_group_id is false + const sourceIds = ( + sourceNode?.data.fullParentNode && + e.source.startsWith(sourceNode.data.fullParentNode) + ? e.source + : `${sourceNode?.data.fullParentNode}.${e.source}` + ).split("."); + const targetIds = ( + targetNode?.data.fullParentNode && + e.target.startsWith(targetNode.data.fullParentNode) + ? e.target + : `${targetNode?.data.fullParentNode}.${e.target}` + ).split("."); + const isSelected = selectedTaskId && (e.source === selectedTaskId || e.target === selectedTaskId); diff --git a/airflow/www/static/js/utils/graph.ts b/airflow/www/static/js/utils/graph.ts index abcf0c548257..ec38f87cf915 100644 --- a/airflow/www/static/js/utils/graph.ts +++ b/airflow/www/static/js/utils/graph.ts @@ -76,6 +76,19 @@ const generateGraph = ({ arrange, }: GenerateProps) => { const closedGroupIds: string[] = []; + let filteredEdges = unformattedEdges; + + const getNestedChildIds = (children: DepNode[]) => { + let childIds: string[] = []; + children.forEach((c) => { + childIds.push(c.id); + if (c.children) { + const nestedChildIds = getNestedChildIds(c.children); + childIds = [...childIds, ...nestedChildIds]; + } + }); + return childIds; + }; const formatChildNode = ( node: DepNode @@ -106,7 +119,25 @@ const generateGraph = ({ }; } const isJoinNode = id.includes("join_id"); - if (children?.length) closedGroupIds.push(value.label); + if (!isOpen && children?.length) { + const childIds = getNestedChildIds(children); + filteredEdges = filteredEdges + // Filter out internal group edges + .filter( + (e) => + !( + childIds.indexOf(e.sourceId) > -1 && + childIds.indexOf(e.targetId) > -1 + ) + ) + // For external group edges, point to the group itself instead of a child node + .map((e) => ({ + ...e, + sourceId: childIds.indexOf(e.sourceId) > -1 ? node.id : e.sourceId, + targetId: childIds.indexOf(e.targetId) > -1 ? node.id : e.targetId, + })); + closedGroupIds.push(value.label); + } return { id, label: value.label, @@ -119,29 +150,10 @@ const generateGraph = ({ height: isJoinNode ? 10 : 60, }; }; - const children = nodes.map(formatChildNode); - const edges = unformattedEdges - .map((edge) => { - let { sourceId, targetId } = edge; - const splitSource = sourceId.split("."); - const splitTarget = targetId.split("."); + const children = nodes.map(formatChildNode); - if (closedGroupIds.includes(splitSource[splitSource.length - 2])) { - splitSource.pop(); - sourceId = splitSource.join("."); - } - if (closedGroupIds.includes(splitTarget[splitTarget.length - 2])) { - splitTarget.pop(); - targetId = splitTarget.join("."); - } - return { - ...edge, - targetId, - sourceId, - }; - }) - // Deduplicate edges + const edges = filteredEdges .filter( (value, index, self) => index === @@ -149,22 +161,6 @@ const generateGraph = ({ (t) => t.sourceId === value.sourceId && t.targetId === value.targetId ) ) - .filter((edge) => { - const splitSource = edge.sourceId.split("."); - const splitTarget = edge.targetId.split("."); - if ( - splitSource - .slice(0, splitSource.length - 1) - .some((id) => closedGroupIds.includes(id)) || - splitTarget - .slice(0, splitTarget.length - 1) - .some((id) => closedGroupIds.includes(id)) - ) { - return false; - } - if (edge.sourceId === edge.targetId) return false; - return true; - }) .map((e) => ({ id: `${e.sourceId}-${e.targetId}`, sources: [e.sourceId], From 4c878798ef88a1fa45956163630d71b6fc4f401f Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sat, 22 Jul 2023 15:54:03 +0530 Subject: [PATCH 271/533] Sort extra_links for predictable order in UI. (#32762) --- airflow/models/abstractoperator.py | 2 +- tests/serialization/test_dag_serialization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 0c6d89fff3b5..97e3941f0f7d 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -469,7 +469,7 @@ def global_operator_extra_link_dict(self) -> dict[str, Any]: @cached_property def extra_links(self) -> list[str]: - return list(set(self.operator_extra_link_dict).union(self.global_operator_extra_link_dict)) + return sorted(set(self.operator_extra_link_dict).union(self.global_operator_extra_link_dict)) def get_extra_links(self, ti: TaskInstance, link_name: str) -> str | None: """For an operator, gets the URLs that the ``extra_links`` entry points to. diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index c0c0f2f0cde8..c449a6f998e8 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -986,7 +986,7 @@ def test_extra_serialized_field_and_operator_links( assert serialized_dag["dag"]["tasks"][0]["_operator_extra_links"] == serialized_links # Test all the extra_links are set - assert set(simple_task.extra_links) == {*links, "airflow", "github", "google"} + assert simple_task.extra_links == sorted({*links, "airflow", "github", "google"}) dr = dag_maker.create_dagrun(execution_date=test_date) (ti,) = dr.task_instances From fcc6f284c742bdc554edecc5a83d9eaa7d9d7ba4 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sat, 22 Jul 2023 13:32:53 +0200 Subject: [PATCH 272/533] Update the watcher resource version in SparkK8SOp when it's too old (#32768) --- .../kubernetes/operators/spark_kubernetes.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index ccc13469faae..42913a23de65 100644 --- a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -20,6 +20,7 @@ import datetime from typing import TYPE_CHECKING, Sequence +from kubernetes.client import ApiException from kubernetes.watch import Watch from airflow import AirflowException @@ -27,6 +28,8 @@ from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook, _load_body_to_dict if TYPE_CHECKING: + from kubernetes.client.models import CoreV1EventList + from airflow.utils.context import Context @@ -85,6 +88,25 @@ def __init__( cluster_context=self.cluster_context, ) + def _get_namespace_event_stream(self, namespace, query_kwargs=None): + try: + return Watch().stream( + self.hook.core_v1_client.list_namespaced_event, + namespace=namespace, + watch=True, + **(query_kwargs or {}), + ) + except ApiException as e: + if e.status == 410: # Resource version is too old + events: CoreV1EventList = self.hook.core_v1_client.list_namespaced_event( + namespace=namespace, watch=False + ) + resource_version = events.metadata.resource_version + query_kwargs["resource_version"] = resource_version + return self._get_namespace_event_stream(namespace, query_kwargs) + else: + raise + def execute(self, context: Context): body = _load_body_to_dict(self.application_file) name = body["metadata"]["name"] @@ -94,11 +116,11 @@ def execute(self, context: Context): is_job_created = False if self.watch: try: - namespace_event_stream = Watch().stream( - self.hook.core_v1_client.list_namespaced_event, + namespace_event_stream = self._get_namespace_event_stream( namespace=namespace, - watch=True, - field_selector=f"involvedObject.kind=SparkApplication,involvedObject.name={name}", + query_kwargs={ + "field_selector": f"involvedObject.kind=SparkApplication,involvedObject.name={name}" + }, ) response = self.hook.create_custom_object( From 43a5b4750590bf43bb59cc7bd8377934737f63e8 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Sat, 22 Jul 2023 18:17:24 +0530 Subject: [PATCH 273/533] Improving the instructions for updating example dag paths (#32771) --- dev/example_dags/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/dev/example_dags/README.md b/dev/example_dags/README.md index 369cf460d693..aef1dddcaa0b 100644 --- a/dev/example_dags/README.md +++ b/dev/example_dags/README.md @@ -17,5 +17,18 @@ under the License. --> -Run this script in a `docs-archive` folder of checked out `github.com:apache/airflow-site.git` repo +Run this script in a `docs-archive` folder of checked out `airflow-site` repo to refresh links to example dags to the right versions. + +Instructions: + +```shell script +git clone https://github.com/apache/airflow-site.git airflow-site +cd airflow-site +export AIRFLOW_SITE_DIRECTORY="$(pwd)" + +cd - +cp update_example_dags_paths.py $AIRFLOW_SITE_DIRECTORY/docs-archive +cd $AIRFLOW_SITE_DIRECTORY/docs-archive +python update_example_dags_paths.py +``` From 57f203251b223550d6e7bb717910109af9aeed29 Mon Sep 17 00:00:00 2001 From: Rishi Kulkarni Date: Sat, 22 Jul 2023 13:28:30 -0400 Subject: [PATCH 274/533] FIX AWS deferrable operators by using AioCredentials when using `assume_role` (#32733) * FIX: deferrable operators now use AioCredentials Fixes airflow.providers.amazong.aws.hooks.base_aws.BaseSessionFactory feeds synchronous credentials to aiobotocore when using `assume_role` #32732 * formatting * use dict unpacking * TEST: add test that checks that credentials._refresh is a coroutine * TEST: assert that `credentials.get_frozen_credentials` is a coroutine --- .../providers/amazon/aws/hooks/base_aws.py | 35 +++++++++++++------ .../amazon/aws/hooks/test_base_aws.py | 4 +++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py index aaec87dd0911..94782401ac13 100644 --- a/airflow/providers/amazon/aws/hooks/base_aws.py +++ b/airflow/providers/amazon/aws/hooks/base_aws.py @@ -197,22 +197,35 @@ def _create_basic_session(self, session_kwargs: dict[str, Any]) -> boto3.session def _create_session_with_assume_role( self, session_kwargs: dict[str, Any], deferrable: bool = False ) -> boto3.session.Session: - if self.conn.assume_role_method == "assume_role_with_web_identity": # Deferred credentials have no initial credentials credential_fetcher = self._get_web_identity_credential_fetcher() - credentials = botocore.credentials.DeferredRefreshableCredentials( - method="assume-role-with-web-identity", - refresh_using=credential_fetcher.fetch_credentials, - time_fetcher=lambda: datetime.datetime.now(tz=tzlocal()), - ) + + params = { + "method": "assume-role-with-web-identity", + "refresh_using": credential_fetcher.fetch_credentials, + "time_fetcher": lambda: datetime.datetime.now(tz=tzlocal()), + } + + if deferrable: + from aiobotocore.credentials import AioDeferredRefreshableCredentials + + credentials = AioDeferredRefreshableCredentials(**params) + else: + credentials = botocore.credentials.DeferredRefreshableCredentials(**params) else: # Refreshable credentials do have initial credentials - credentials = botocore.credentials.RefreshableCredentials.create_from_metadata( - metadata=self._refresh_credentials(), - refresh_using=self._refresh_credentials, - method="sts-assume-role", - ) + params = { + "metadata": self._refresh_credentials(), + "refresh_using": self._refresh_credentials, + "method": "sts-assume-role", + } + if deferrable: + from aiobotocore.credentials import AioRefreshableCredentials + + credentials = AioRefreshableCredentials.create_from_metadata(**params) + else: + credentials = botocore.credentials.RefreshableCredentials.create_from_metadata(**params) if deferrable: from aiobotocore.session import get_session as async_get_session diff --git a/tests/providers/amazon/aws/hooks/test_base_aws.py b/tests/providers/amazon/aws/hooks/test_base_aws.py index 6f84e4ad2402..8ba8b6b81266 100644 --- a/tests/providers/amazon/aws/hooks/test_base_aws.py +++ b/tests/providers/amazon/aws/hooks/test_base_aws.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import inspect import json import os from base64 import b64encode @@ -332,6 +333,9 @@ def side_effect(): # Validate method of botocore credentials provider. # It shouldn't be 'explicit' which refers in this case to initial credentials. credentials = await session.get_credentials() + + assert inspect.iscoroutinefunction(credentials.get_frozen_credentials) + assert credentials.method == "sts-assume-role" From accdb0bf64fd85fc825a7aa09fd5cb307cc86291 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 22 Jul 2023 20:31:24 +0200 Subject: [PATCH 275/533] Update PIP to 23.2.1 - released today (#32774) --- Dockerfile | 8 ++++---- Dockerfile.ci | 6 +++--- IMAGES.rst | 2 +- dev/breeze/src/airflow_breeze/global_constants.py | 2 +- docs/docker-stack/build-arg-ref.rst | 2 +- scripts/docker/common.sh | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14f02ec90377..15ee45f7e3bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.6.3" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bullseye" -ARG AIRFLOW_PIP_VERSION=23.2 +ARG AIRFLOW_PIP_VERSION=23.2.1 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -471,7 +471,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.2" + export AIRFLOW_PIP_VERSION="23.2.1" fi fi } @@ -1197,7 +1197,7 @@ COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scrip # THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED -ARG AIRFLOW_PIP_VERSION=23.2 +ARG AIRFLOW_PIP_VERSION=23.2.1 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} COPY --from=scripts common.sh /scripts/docker/ @@ -1387,7 +1387,7 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ # THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED -ARG AIRFLOW_PIP_VERSION=23.2 +ARG AIRFLOW_PIP_VERSION=23.2.1 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} COPY --from=scripts common.sh /scripts/docker/ diff --git a/Dockerfile.ci b/Dockerfile.ci index bbea500d8f6a..5aa9d72ec0d9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -431,7 +431,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.2" + export AIRFLOW_PIP_VERSION="23.2.1" fi fi } @@ -1262,7 +1262,7 @@ RUN bash /scripts/docker/install_os_dependencies.sh dev # THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED -ARG AIRFLOW_PIP_VERSION=23.2 +ARG AIRFLOW_PIP_VERSION=23.2.1 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} COPY --from=scripts common.sh /scripts/docker/ @@ -1319,7 +1319,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="4"0 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" -ARG AIRFLOW_PIP_VERSION=23.2 +ARG AIRFLOW_PIP_VERSION=23.2.1 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" diff --git a/IMAGES.rst b/IMAGES.rst index 2fa95b4a9c0f..7360e43648f4 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -462,7 +462,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_DEV_APT_ENV`` | | Additional env variables defined | | | | when installing dev deps | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``23.2`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``23.2.1`` | PIP version used. | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 18ff5d7828f6..9cdca2edb3bb 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -87,7 +87,7 @@ ALLOWED_MYSQL_VERSIONS = ["5.7", "8"] ALLOWED_MSSQL_VERSIONS = ["2017-latest", "2019-latest"] -PIP_VERSION = "23.2" +PIP_VERSION = "23.2.1" @lru_cache(maxsize=None) diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 8544beff6dad..c954cbf69c7e 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. | +------------------------------------------+------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``23.2`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``23.2.1`` | PIP version used. | +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``ADDITIONAL_PIP_INSTALL_FLAGS`` | | additional ``pip`` flags passed to the | | | | installation commands (except when | diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index f6066c3806af..f4fd2e75f76f 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -42,7 +42,7 @@ function common::get_airflow_version_specification() { function common::override_pip_version_if_needed() { if [[ -n ${AIRFLOW_VERSION} ]]; then if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION="23.2" + export AIRFLOW_PIP_VERSION="23.2.1" fi fi } From 56c41d460c3f2a4e871c7834033c3152e71f71d2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 22 Jul 2023 21:38:58 +0200 Subject: [PATCH 276/533] Introduce decorator to load providers configuration (#32765) A number of commands in Airflow relies on the fact that providers configuration is loaded. This is a rather fast operation as it does not involve any importing of provider classes, just discovering entrypoints, running them and parsing yaml configuration, so it is a very low sub-second time to do it. We cannot do it once in settings/config because we actually need settings/config to be pre-initialized without providers in order to be able to bootstrap airflow, therefore we need to run it individually in each command that can be run with the "airflow" entrypoint. Decorator seems to be best suited to do the job: * easy to apply and not easy to forget when you create another command and look at other commands * nicely wraps around local ProvidersManager import There are exceptions for the "version" and "providers lazy-loaded" commands because they are NOT supposed to initialize configuration of providers. --- airflow/__main__.py | 11 ++-- airflow/cli/cli_config.py | 6 +- airflow/cli/commands/celery_command.py | 5 ++ airflow/cli/commands/config_command.py | 6 +- airflow/cli/commands/connection_command.py | 8 +++ airflow/cli/commands/dag_command.py | 23 +++++++- airflow/cli/commands/dag_processor_command.py | 2 + airflow/cli/commands/db_command.py | 11 ++++ airflow/cli/commands/info_command.py | 2 + airflow/cli/commands/internal_api_command.py | 2 + airflow/cli/commands/jobs_command.py | 2 + airflow/cli/commands/kerberos_command.py | 2 + airflow/cli/commands/kubernetes_command.py | 3 + airflow/cli/commands/plugins_command.py | 2 + airflow/cli/commands/pool_command.py | 7 +++ airflow/cli/commands/provider_command.py | 17 +++++- airflow/cli/commands/role_command.py | 7 +++ .../cli/commands/rotate_fernet_key_command.py | 2 + airflow/cli/commands/scheduler_command.py | 2 + airflow/cli/commands/standalone_command.py | 2 + airflow/cli/commands/sync_perm_command.py | 2 + airflow/cli/commands/task_command.py | 7 +++ airflow/cli/commands/triggerer_command.py | 2 + airflow/cli/commands/user_command.py | 7 +++ airflow/cli/commands/variable_command.py | 7 +++ airflow/cli/commands/webserver_command.py | 2 + .../celery/executors/celery_executor_utils.py | 22 +++++++- .../executors/celery_kubernetes_executor.py | 7 +-- .../celery/executors/default_celery.py | 24 +++++--- .../utils/providers_configuration_loader.py | 55 +++++++++++++++++++ tests/cli/test_cli_parser.py | 4 +- .../executors/test_celery_executor.py | 16 +++++- .../celery/executors/test_celery_executor.py | 8 ++- 33 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 airflow/utils/providers_configuration_loader.py diff --git a/airflow/__main__.py b/airflow/__main__.py index 893c6bde099d..c15f75b4b917 100644 --- a/airflow/__main__.py +++ b/airflow/__main__.py @@ -46,12 +46,13 @@ def main(): argcomplete.autocomplete(parser) args = parser.parse_args() - # Here we ensure that the default configuration is written if needed before running any command - # that might need it. This used to be done during configuration initialization but having it - # in main ensures that it is not done during tests and other ways airflow imports are used - from airflow.configuration import write_default_airflow_configuration_if_needed + if args.subcommand not in ["lazy_loaded", "version"]: + # Here we ensure that the default configuration is written if needed before running any command + # that might need it. This used to be done during configuration initialization but having it + # in main ensures that it is not done during tests and other ways airflow imports are used + from airflow.configuration import write_default_airflow_configuration_if_needed - write_default_airflow_configuration_if_needed() + write_default_airflow_configuration_if_needed() args.func(args) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index a57a8dae1dec..30e71302832c 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -1872,9 +1872,9 @@ class GroupCommand(NamedTuple): args=(ARG_OUTPUT, ARG_VERBOSE), ), ActionCommand( - name="status", - help="Get information about provider initialization status", - func=lazy_load_command("airflow.cli.commands.provider_command.status"), + name="lazy-loaded", + help="Checks that provider configuration is lazy loaded", + func=lazy_load_command("airflow.cli.commands.provider_command.lazy_loaded"), args=(ARG_VERBOSE,), ), ) diff --git a/airflow/cli/commands/celery_command.py b/airflow/cli/commands/celery_command.py index 7adbea36e685..af7dcbc6077a 100644 --- a/airflow/cli/commands/celery_command.py +++ b/airflow/cli/commands/celery_command.py @@ -36,12 +36,14 @@ from airflow.configuration import conf from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.serve_logs import serve_logs WORKER_PROCESS_NAME = "worker" @cli_utils.action_cli +@providers_configuration_loaded def flower(args): """Starts Flower, Celery monitoring tool.""" # This needs to be imported locally to not trigger Providers Manager initialization @@ -103,6 +105,7 @@ def _serve_logs(skip_serve_logs: bool = False): @after_setup_logger.connect() +@providers_configuration_loaded def logger_setup_handler(logger, **kwargs): """ Reconfigure the logger. @@ -132,6 +135,7 @@ def filter(self, record): @cli_utils.action_cli +@providers_configuration_loaded def worker(args): """Starts Airflow Celery worker.""" # This needs to be imported locally to not trigger Providers Manager initialization @@ -239,6 +243,7 @@ def worker(args): @cli_utils.action_cli +@providers_configuration_loaded def stop_worker(args): """Sends SIGTERM to Celery worker.""" # Read PID from file diff --git a/airflow/cli/commands/config_command.py b/airflow/cli/commands/config_command.py index 86130dab102c..4855c7dd9ff4 100644 --- a/airflow/cli/commands/config_command.py +++ b/airflow/cli/commands/config_command.py @@ -25,8 +25,10 @@ from airflow.configuration import conf from airflow.utils.cli import should_use_colors from airflow.utils.code_utils import get_terminal_formatter +from airflow.utils.providers_configuration_loader import providers_configuration_loaded +@providers_configuration_loaded def show_config(args): """Show current application configuration.""" with io.StringIO() as output: @@ -47,6 +49,7 @@ def show_config(args): print(code) +@providers_configuration_loaded def get_value(args): """Get one value from configuration.""" # while this will make get_value quite a bit slower we must initialize configuration @@ -54,9 +57,6 @@ def get_value(args): # providers are initialized. Theoretically Providers might add new sections and options # but also override defaults for existing options, so without loading all providers we # cannot be sure what is the final value of the option. - from airflow.providers_manager import ProvidersManager - - ProvidersManager().initialize_providers_configuration() if not conf.has_option(args.section, args.option): raise SystemExit(f"The option [{args.section}/{args.option}] is not found in config.") diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py index e7b83e342e29..5db123f004a8 100644 --- a/airflow/cli/commands/connection_command.py +++ b/airflow/cli/commands/connection_command.py @@ -39,6 +39,7 @@ from airflow.secrets.local_filesystem import load_connections_dict from airflow.utils import cli as cli_utils, helpers, yaml from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import create_session @@ -61,6 +62,7 @@ def _connection_mapper(conn: Connection) -> dict[str, Any]: @suppress_logs_and_warning +@providers_configuration_loaded def connections_get(args): """Get a connection.""" try: @@ -75,6 +77,7 @@ def connections_get(args): @suppress_logs_and_warning +@providers_configuration_loaded def connections_list(args): """Lists all connections at the command line.""" with create_session() as session: @@ -150,6 +153,7 @@ def _get_connection_types() -> list[str]: return _connection_types +@providers_configuration_loaded def connections_export(args): """Exports all connections to a file.""" file_formats = [".yaml", ".json", ".env"] @@ -200,6 +204,7 @@ def connections_export(args): @cli_utils.action_cli +@providers_configuration_loaded def connections_add(args): """Adds new connection.""" has_uri = bool(args.conn_uri) @@ -291,6 +296,7 @@ def connections_add(args): @cli_utils.action_cli +@providers_configuration_loaded def connections_delete(args): """Deletes connection from DB.""" with create_session() as session: @@ -306,6 +312,7 @@ def connections_delete(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def connections_import(args): """Imports connections from a file.""" if os.path.exists(args.file): @@ -343,6 +350,7 @@ def _import_helper(file_path: str, overwrite: bool) -> None: @suppress_logs_and_warning +@providers_configuration_loaded def connections_test(args) -> None: """Test an Airflow connection.""" console = AirflowConsole() diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index 66decad78b16..f965e24b5ef7 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -45,6 +45,7 @@ from airflow.utils import cli as cli_utils, timezone from airflow.utils.cli import get_dag, get_dags, process_subdir, sigint_handler, suppress_logs_and_warning from airflow.utils.dot_renderer import render_dag, render_dag_dependencies +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import NEW_SESSION, create_session, provide_session from airflow.utils.state import DagRunState @@ -120,6 +121,7 @@ def _run_dag_backfill(dags: list[DAG], args) -> None: @cli_utils.action_cli +@providers_configuration_loaded def dag_backfill(args, dag: list[DAG] | DAG | None = None) -> None: """Creates backfill job or dry run for a DAG or list of DAGs using regex.""" logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT) @@ -150,6 +152,7 @@ def dag_backfill(args, dag: list[DAG] | DAG | None = None) -> None: @cli_utils.action_cli +@providers_configuration_loaded def dag_trigger(args) -> None: """Creates a dag run for the specified dag.""" api_client = get_current_api_client() @@ -170,6 +173,7 @@ def dag_trigger(args) -> None: @cli_utils.action_cli +@providers_configuration_loaded def dag_delete(args) -> None: """Deletes all DB records related to the specified dag.""" api_client = get_current_api_client() @@ -188,17 +192,20 @@ def dag_delete(args) -> None: @cli_utils.action_cli +@providers_configuration_loaded def dag_pause(args) -> None: """Pauses a DAG.""" set_is_paused(True, args) @cli_utils.action_cli +@providers_configuration_loaded def dag_unpause(args) -> None: """Unpauses a DAG.""" set_is_paused(False, args) +@providers_configuration_loaded def set_is_paused(is_paused: bool, args) -> None: """Sets is_paused for DAG by a given dag_id.""" dag = DagModel.get_dagmodel(args.dag_id) @@ -211,6 +218,7 @@ def set_is_paused(is_paused: bool, args) -> None: print(f"Dag: {args.dag_id}, paused: {is_paused}") +@providers_configuration_loaded def dag_dependencies_show(args) -> None: """Displays DAG dependencies, save to file or show as imgcat image.""" dot = render_dag_dependencies(SerializedDagModel.get_dag_dependencies()) @@ -230,6 +238,7 @@ def dag_dependencies_show(args) -> None: print(dot.source) +@providers_configuration_loaded def dag_show(args) -> None: """Displays DAG or saves it's graphic representation to the file.""" dag = get_dag(args.subdir, args.dag_id) @@ -273,6 +282,7 @@ def _save_dot_to_file(dot: Dot, filename: str) -> None: @cli_utils.action_cli +@providers_configuration_loaded @provide_session def dag_state(args, session: Session = NEW_SESSION) -> None: """ @@ -296,6 +306,7 @@ def dag_state(args, session: Session = NEW_SESSION) -> None: @cli_utils.action_cli +@providers_configuration_loaded def dag_next_execution(args) -> None: """ Returns the next execution datetime of a DAG at the command line. @@ -335,6 +346,7 @@ def print_execution_interval(interval: DataInterval | None): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def dag_list_dags(args) -> None: """Displays dags with or without stats at the command line.""" dagbag = DagBag(process_subdir(args.subdir)) @@ -360,6 +372,7 @@ def dag_list_dags(args) -> None: @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded @provide_session def dag_details(args, session=NEW_SESSION): """Get DAG details given a DAG id.""" @@ -381,6 +394,7 @@ def dag_details(args, session=NEW_SESSION): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def dag_list_import_errors(args) -> None: """Displays dags with import errors on the command line.""" dagbag = DagBag(process_subdir(args.subdir)) @@ -395,6 +409,7 @@ def dag_list_import_errors(args) -> None: @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def dag_report(args) -> None: """Displays dagbag stats at the command line.""" dagbag = DagBag(process_subdir(args.subdir)) @@ -413,6 +428,7 @@ def dag_report(args) -> None: @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded @provide_session def dag_list_jobs(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> None: """Lists latest n jobs.""" @@ -443,6 +459,7 @@ def dag_list_jobs(args, dag: DAG | None = None, session: Session = NEW_SESSION) @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded @provide_session def dag_list_dag_runs(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> None: """Lists dag runs for a given DAG.""" @@ -479,8 +496,9 @@ def dag_list_dag_runs(args, dag: DAG | None = None, session: Session = NEW_SESSI ) -@provide_session @cli_utils.action_cli +@providers_configuration_loaded +@provide_session def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> None: """Execute one single DagRun for a given DAG and execution date.""" run_conf = None @@ -513,8 +531,9 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No print(dot_graph.source) -@provide_session @cli_utils.action_cli +@providers_configuration_loaded +@provide_session def dag_reserialize(args, session: Session = NEW_SESSION) -> None: """Serialize a DAG instance.""" session.execute(delete(SerializedDagModel).execution_options(synchronize_session=False)) diff --git a/airflow/cli/commands/dag_processor_command.py b/airflow/cli/commands/dag_processor_command.py index 70bebf285b5e..757bd778cca5 100644 --- a/airflow/cli/commands/dag_processor_command.py +++ b/airflow/cli/commands/dag_processor_command.py @@ -31,6 +31,7 @@ from airflow.jobs.job import Job, run_job from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging +from airflow.utils.providers_configuration_loader import providers_configuration_loaded log = logging.getLogger(__name__) @@ -53,6 +54,7 @@ def _create_dag_processor_job_runner(args: Any) -> DagProcessorJobRunner: @cli_utils.action_cli +@providers_configuration_loaded def dag_processor(args): """Starts Airflow Dag Processor Job.""" if not conf.getboolean("scheduler", "standalone_dag_processor"): diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py index 64d54cc22ea7..e703f6271e87 100644 --- a/airflow/cli/commands/db_command.py +++ b/airflow/cli/commands/db_command.py @@ -31,10 +31,12 @@ from airflow.utils.db import REVISION_HEADS_MAP from airflow.utils.db_cleanup import config_dict, drop_archived_tables, export_archived_records, run_cleanup from airflow.utils.process_utils import execute_interactive +from airflow.utils.providers_configuration_loader import providers_configuration_loaded log = logging.getLogger(__name__) +@providers_configuration_loaded def initdb(args): """Initializes the metadata database.""" print("DB: " + repr(settings.engine.url)) @@ -42,6 +44,7 @@ def initdb(args): print("Initialization done") +@providers_configuration_loaded def resetdb(args): """Resets the metadata database.""" print("DB: " + repr(settings.engine.url)) @@ -51,6 +54,7 @@ def resetdb(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def upgradedb(args): """Upgrades the metadata database.""" print("DB: " + repr(settings.engine.url)) @@ -96,6 +100,7 @@ def upgradedb(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def downgrade(args): """Downgrades the metadata database.""" if args.to_revision and args.to_version: @@ -142,12 +147,14 @@ def downgrade(args): raise SystemExit("Cancelled") +@providers_configuration_loaded def check_migrations(args): """Function to wait for all airflow migrations to complete. Used for launching airflow in k8s.""" db.check_migrations(timeout=args.migration_wait_timeout) @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def shell(args): """Run a shell that allows to access metadata database.""" url = settings.engine.url @@ -191,6 +198,7 @@ def shell(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def check(args): """Runs a check command that checks if db is available.""" retries: int = args.retry @@ -215,6 +223,7 @@ def _warn_remaining_retries(retrystate: RetryCallState): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def cleanup_tables(args): """Purges old records in metadata database.""" run_cleanup( @@ -228,6 +237,7 @@ def cleanup_tables(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def export_archived(args): """Exports archived records from metadata database.""" export_archived_records( @@ -240,6 +250,7 @@ def export_archived(args): @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def drop_archived(args): """Drops archived tables from metadata database.""" drop_archived_tables( diff --git a/airflow/cli/commands/info_command.py b/airflow/cli/commands/info_command.py index 7261dfc48415..2e60d80b271d 100644 --- a/airflow/cli/commands/info_command.py +++ b/airflow/cli/commands/info_command.py @@ -35,6 +35,7 @@ from airflow.typing_compat import Protocol from airflow.utils.cli import suppress_logs_and_warning from airflow.utils.platform import getuser +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.version import version as airflow_version log = logging.getLogger(__name__) @@ -378,6 +379,7 @@ def _send_report_to_fileio(info): @suppress_logs_and_warning +@providers_configuration_loaded def show_info(args): """Show information related to Airflow, system and other.""" # Enforce anonymization, when file_io upload is tuned on. diff --git a/airflow/cli/commands/internal_api_command.py b/airflow/cli/commands/internal_api_command.py index 3d8f205bd1e7..72fe57c206a0 100644 --- a/airflow/cli/commands/internal_api_command.py +++ b/airflow/cli/commands/internal_api_command.py @@ -47,6 +47,7 @@ from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging from airflow.utils.process_utils import check_if_pidfile_process_is_running +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.www.extensions.init_dagbag import init_dagbag from airflow.www.extensions.init_jinja_globals import init_jinja_globals from airflow.www.extensions.init_manifest_files import configure_manifest_files @@ -58,6 +59,7 @@ @cli_utils.action_cli +@providers_configuration_loaded def internal_api(args): """Starts Airflow Internal API.""" print(settings.HEADER) diff --git a/airflow/cli/commands/jobs_command.py b/airflow/cli/commands/jobs_command.py index bcdd6df475d6..b6509ea6424e 100644 --- a/airflow/cli/commands/jobs_command.py +++ b/airflow/cli/commands/jobs_command.py @@ -21,10 +21,12 @@ from airflow.jobs.job import Job from airflow.utils.net import get_hostname +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import JobState +@providers_configuration_loaded @provide_session def check(args, session: Session = NEW_SESSION) -> None: """Checks if job(s) are still alive.""" diff --git a/airflow/cli/commands/kerberos_command.py b/airflow/cli/commands/kerberos_command.py index 4bbe3f6df919..4dd63d52ebdd 100644 --- a/airflow/cli/commands/kerberos_command.py +++ b/airflow/cli/commands/kerberos_command.py @@ -24,9 +24,11 @@ from airflow.security import kerberos as krb from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations +from airflow.utils.providers_configuration_loader import providers_configuration_loaded @cli_utils.action_cli +@providers_configuration_loaded def kerberos(args): """Start a kerberos ticket renewer.""" print(settings.HEADER) diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py index c367d4be87b6..1555f7be9258 100644 --- a/airflow/cli/commands/kubernetes_command.py +++ b/airflow/cli/commands/kubernetes_command.py @@ -33,9 +33,11 @@ from airflow.models import DagRun, TaskInstance from airflow.utils import cli as cli_utils, yaml from airflow.utils.cli import get_dag +from airflow.utils.providers_configuration_loader import providers_configuration_loaded @cli_utils.action_cli +@providers_configuration_loaded def generate_pod_yaml(args): """Generates yaml files for each task in the DAG. Used for testing output of KubernetesExecutor.""" execution_date = args.execution_date @@ -71,6 +73,7 @@ def generate_pod_yaml(args): @cli_utils.action_cli +@providers_configuration_loaded def cleanup_pods(args): """Clean up k8s pods in evicted/failed/succeeded/pending states.""" namespace = args.namespace diff --git a/airflow/cli/commands/plugins_command.py b/airflow/cli/commands/plugins_command.py index 50ee58309911..29dd75674afe 100644 --- a/airflow/cli/commands/plugins_command.py +++ b/airflow/cli/commands/plugins_command.py @@ -23,6 +23,7 @@ from airflow.cli.simple_table import AirflowConsole from airflow.plugins_manager import PluginsDirectorySource, get_plugin_info from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded def _get_name(class_like_object) -> str: @@ -39,6 +40,7 @@ def _join_plugins_names(value: list[Any] | Any) -> str: @suppress_logs_and_warning +@providers_configuration_loaded def dump_plugins(args): """Dump plugins information.""" plugins_info: list[dict[str, str]] = get_plugin_info() diff --git a/airflow/cli/commands/pool_command.py b/airflow/cli/commands/pool_command.py index aa56ba8fea7d..8d9e206f1bb2 100644 --- a/airflow/cli/commands/pool_command.py +++ b/airflow/cli/commands/pool_command.py @@ -27,6 +27,7 @@ from airflow.exceptions import PoolNotFound from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded def _show_pools(pools, output): @@ -42,6 +43,7 @@ def _show_pools(pools, output): @suppress_logs_and_warning +@providers_configuration_loaded def pool_list(args): """Displays info of all the pools.""" api_client = get_current_api_client() @@ -50,6 +52,7 @@ def pool_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def pool_get(args): """Displays pool info by a given name.""" api_client = get_current_api_client() @@ -62,6 +65,7 @@ def pool_get(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def pool_set(args): """Creates new pool with a given name and slots.""" api_client = get_current_api_client() @@ -71,6 +75,7 @@ def pool_set(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def pool_delete(args): """Deletes pool by a given name.""" api_client = get_current_api_client() @@ -83,6 +88,7 @@ def pool_delete(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def pool_import(args): """Imports pools from the file.""" if not os.path.exists(args.file): @@ -93,6 +99,7 @@ def pool_import(args): print(f"Uploaded {len(pools)} pool(s)") +@providers_configuration_loaded def pool_export(args): """Exports all the pools to the file.""" pools = pool_export_helper(args.file) diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py index adcb1d18fa14..a55032d9f210 100644 --- a/airflow/cli/commands/provider_command.py +++ b/airflow/cli/commands/provider_command.py @@ -24,6 +24,7 @@ from airflow.cli.simple_table import AirflowConsole from airflow.providers_manager import ProvidersManager from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded ERROR_IMPORTING_HOOK = "Error when importing hook!" @@ -33,6 +34,7 @@ def _remove_rst_syntax(value: str) -> str: @suppress_logs_and_warning +@providers_configuration_loaded def provider_get(args): """Get a provider info.""" providers = ProvidersManager().providers @@ -54,6 +56,7 @@ def provider_get(args): @suppress_logs_and_warning +@providers_configuration_loaded def providers_list(args): """Lists all providers at the command line.""" AirflowConsole().print_as( @@ -68,6 +71,7 @@ def providers_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def hooks_list(args): """Lists all hooks at the command line.""" AirflowConsole().print_as( @@ -84,6 +88,7 @@ def hooks_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def triggers_list(args): AirflowConsole().print_as( data=ProvidersManager().trigger, @@ -97,6 +102,7 @@ def triggers_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def connection_form_widget_list(args): """Lists all custom connection form fields at the command line.""" AirflowConsole().print_as( @@ -112,6 +118,7 @@ def connection_form_widget_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def connection_field_behaviours(args): """Lists field behaviours.""" AirflowConsole().print_as( @@ -124,6 +131,7 @@ def connection_field_behaviours(args): @suppress_logs_and_warning +@providers_configuration_loaded def extra_links_list(args): """Lists all extra links at the command line.""" AirflowConsole().print_as( @@ -136,6 +144,7 @@ def extra_links_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def logging_list(args): """Lists all log task handlers at the command line.""" AirflowConsole().print_as( @@ -148,6 +157,7 @@ def logging_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def secrets_backends_list(args): """Lists all secrets backends at the command line.""" AirflowConsole().print_as( @@ -160,6 +170,7 @@ def secrets_backends_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def auth_backend_list(args): """Lists all API auth backend modules at the command line.""" AirflowConsole().print_as( @@ -172,6 +183,7 @@ def auth_backend_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def executors_list(args): """Lists all executors at the command line.""" AirflowConsole().print_as( @@ -184,6 +196,7 @@ def executors_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def config_list(args): """Lists all configurations at the command line.""" AirflowConsole().print_as( @@ -196,8 +209,8 @@ def config_list(args): @suppress_logs_and_warning -def status(args): - """Informs if providers manager has been initialized. +def lazy_loaded(args): + """Informs if providers manager has been initialized too early. If provider is initialized, shows the stack trace and exit with error code 1. """ diff --git a/airflow/cli/commands/role_command.py b/airflow/cli/commands/role_command.py index 91fa26742941..db11d69dd2f8 100644 --- a/airflow/cli/commands/role_command.py +++ b/airflow/cli/commands/role_command.py @@ -26,11 +26,13 @@ from airflow.cli.simple_table import AirflowConsole from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.www.fab_security.sqla.models import Action, Permission, Resource, Role from airflow.www.security import EXISTING_ROLES @suppress_logs_and_warning +@providers_configuration_loaded def roles_list(args): """Lists all existing roles.""" from airflow.utils.cli_app_builder import get_application_builder @@ -58,6 +60,7 @@ def roles_list(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def roles_create(args): """Creates new empty role in DB.""" from airflow.utils.cli_app_builder import get_application_builder @@ -70,6 +73,7 @@ def roles_create(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def roles_delete(args): """Deletes role in DB.""" from airflow.utils.cli_app_builder import get_application_builder @@ -138,6 +142,7 @@ def __roles_add_or_remove_permissions(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def roles_add_perms(args): """Adds permissions to role in DB.""" __roles_add_or_remove_permissions(args) @@ -145,12 +150,14 @@ def roles_add_perms(args): @cli_utils.action_cli @suppress_logs_and_warning +@providers_configuration_loaded def roles_del_perms(args): """Deletes permissions from role in DB.""" __roles_add_or_remove_permissions(args) @suppress_logs_and_warning +@providers_configuration_loaded def roles_export(args): """ Exports all the roles from the database to a file. diff --git a/airflow/cli/commands/rotate_fernet_key_command.py b/airflow/cli/commands/rotate_fernet_key_command.py index e9973978e0a1..1a47a29e3149 100644 --- a/airflow/cli/commands/rotate_fernet_key_command.py +++ b/airflow/cli/commands/rotate_fernet_key_command.py @@ -21,10 +21,12 @@ from airflow.models import Connection, Variable from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import create_session @cli_utils.action_cli +@providers_configuration_loaded def rotate_fernet_key(args): """Rotates all encrypted connection credentials and variables.""" with create_session() as session: diff --git a/airflow/cli/commands/scheduler_command.py b/airflow/cli/commands/scheduler_command.py index 22f9a758088a..808645282cac 100644 --- a/airflow/cli/commands/scheduler_command.py +++ b/airflow/cli/commands/scheduler_command.py @@ -32,6 +32,7 @@ from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.utils import cli as cli_utils from airflow.utils.cli import process_subdir, setup_locations, setup_logging, sigint_handler, sigquit_handler +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.scheduler_health import serve_health_check @@ -43,6 +44,7 @@ def _run_scheduler_job(job_runner: SchedulerJobRunner, *, skip_serve_logs: bool) @cli_utils.action_cli +@providers_configuration_loaded def scheduler(args): """Starts Airflow Scheduler.""" print(settings.HEADER) diff --git a/airflow/cli/commands/standalone_command.py b/airflow/cli/commands/standalone_command.py index 9bdd49a82a68..68abfdd0002b 100644 --- a/airflow/cli/commands/standalone_command.py +++ b/airflow/cli/commands/standalone_command.py @@ -35,6 +35,7 @@ from airflow.jobs.scheduler_job_runner import SchedulerJobRunner from airflow.jobs.triggerer_job_runner import TriggererJobRunner from airflow.utils import db +from airflow.utils.providers_configuration_loader import providers_configuration_loaded class StandaloneCommand: @@ -56,6 +57,7 @@ def __init__(self): self.ready_time = None self.ready_delay = 3 + @providers_configuration_loaded def run(self): """Main run loop.""" self.print_output("standalone", "Starting Airflow Standalone") diff --git a/airflow/cli/commands/sync_perm_command.py b/airflow/cli/commands/sync_perm_command.py index 7ae340696788..ab458b2d93b4 100644 --- a/airflow/cli/commands/sync_perm_command.py +++ b/airflow/cli/commands/sync_perm_command.py @@ -19,9 +19,11 @@ from __future__ import annotations from airflow.utils import cli as cli_utils +from airflow.utils.providers_configuration_loader import providers_configuration_loaded @cli_utils.action_cli +@providers_configuration_loaded def sync_perm(args): """Updates permissions for existing roles and DAGs.""" from airflow.utils.cli_app_builder import get_application_builder diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index 796d3a8cf120..da47e8dd8b37 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -65,6 +65,7 @@ from airflow.utils.log.logging_mixin import StreamLogWriter from airflow.utils.log.secrets_masker import RedactedIO from airflow.utils.net import get_hostname +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import NEW_SESSION, create_session, provide_session from airflow.utils.state import DagRunState @@ -438,6 +439,7 @@ def task_run(args, dag: DAG | None = None) -> TaskReturnCode | None: @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def task_failed_deps(args) -> None: """ Get task instance dependencies that were not met. @@ -468,6 +470,7 @@ def task_failed_deps(args) -> None: @cli_utils.action_cli(check_db=False) @suppress_logs_and_warning +@providers_configuration_loaded def task_state(args) -> None: """ Returns the state of a TaskInstance at the command line. @@ -483,6 +486,7 @@ def task_state(args) -> None: @cli_utils.action_cli(check_db=False) @suppress_logs_and_warning +@providers_configuration_loaded def task_list(args, dag: DAG | None = None) -> None: """Lists the tasks within a DAG at the command line.""" dag = dag or get_dag(args.subdir, args.dag_id) @@ -530,6 +534,7 @@ def _guess_debugger() -> _SupportedDebugger: @cli_utils.action_cli(check_db=False) @suppress_logs_and_warning +@providers_configuration_loaded @provide_session def task_states_for_dag_run(args, session: Session = NEW_SESSION) -> None: """Get the status of all task instances in a DagRun.""" @@ -631,6 +636,7 @@ def task_test(args, dag: DAG | None = None) -> None: @cli_utils.action_cli(check_db=False) @suppress_logs_and_warning +@providers_configuration_loaded def task_render(args, dag: DAG | None = None) -> None: """Renders and displays templated fields for a given task.""" if not dag: @@ -653,6 +659,7 @@ def task_render(args, dag: DAG | None = None) -> None: @cli_utils.action_cli(check_db=False) +@providers_configuration_loaded def task_clear(args) -> None: """Clears all task instances or only those matched by regex for a DAG(s).""" logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT) diff --git a/airflow/cli/commands/triggerer_command.py b/airflow/cli/commands/triggerer_command.py index aa06d641c760..c7d0827bd8e5 100644 --- a/airflow/cli/commands/triggerer_command.py +++ b/airflow/cli/commands/triggerer_command.py @@ -32,6 +32,7 @@ from airflow.jobs.triggerer_job_runner import TriggererJobRunner from airflow.utils import cli as cli_utils from airflow.utils.cli import setup_locations, setup_logging, sigint_handler, sigquit_handler +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.serve_logs import serve_logs @@ -51,6 +52,7 @@ def _serve_logs(skip_serve_logs: bool = False) -> Generator[None, None, None]: @cli_utils.action_cli +@providers_configuration_loaded def triggerer(args): """Starts Airflow Triggerer.""" settings.MASK_SECRETS_IN_LOGS = True diff --git a/airflow/cli/commands/user_command.py b/airflow/cli/commands/user_command.py index a5a5be978756..1553d27a0156 100644 --- a/airflow/cli/commands/user_command.py +++ b/airflow/cli/commands/user_command.py @@ -32,6 +32,7 @@ from airflow.cli.simple_table import AirflowConsole from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded class UserSchema(Schema): @@ -46,6 +47,7 @@ class UserSchema(Schema): @suppress_logs_and_warning +@providers_configuration_loaded def users_list(args): """Lists users at the command line.""" from airflow.utils.cli_app_builder import get_application_builder @@ -60,6 +62,7 @@ def users_list(args): @cli_utils.action_cli(check_db=True) +@providers_configuration_loaded def users_create(args): """Creates new user in the DB.""" from airflow.utils.cli_app_builder import get_application_builder @@ -108,6 +111,7 @@ def _find_user(args): @cli_utils.action_cli +@providers_configuration_loaded def users_delete(args): """Deletes user from DB.""" user = _find_user(args) @@ -125,6 +129,7 @@ def users_delete(args): @cli_utils.action_cli +@providers_configuration_loaded def users_manage_role(args, remove=False): """Deletes or appends user roles.""" user = _find_user(args) @@ -153,6 +158,7 @@ def users_manage_role(args, remove=False): print(f'User "{user.username}" added to role "{args.role}"') +@providers_configuration_loaded def users_export(args): """Exports all users to the json file.""" from airflow.utils.cli_app_builder import get_application_builder @@ -182,6 +188,7 @@ def remove_underscores(s): @cli_utils.action_cli +@providers_configuration_loaded def users_import(args): """Imports users from the json file.""" json_file = getattr(args, "import") diff --git a/airflow/cli/commands/variable_command.py b/airflow/cli/commands/variable_command.py index 32f6b0c1987c..34b46530d562 100644 --- a/airflow/cli/commands/variable_command.py +++ b/airflow/cli/commands/variable_command.py @@ -28,10 +28,12 @@ from airflow.models import Variable from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.session import create_session @suppress_logs_and_warning +@providers_configuration_loaded def variables_list(args): """Displays all the variables.""" with create_session() as session: @@ -40,6 +42,7 @@ def variables_list(args): @suppress_logs_and_warning +@providers_configuration_loaded def variables_get(args): """Displays variable by a given name.""" try: @@ -54,6 +57,7 @@ def variables_get(args): @cli_utils.action_cli +@providers_configuration_loaded def variables_set(args): """Creates new variable with a given name and value.""" Variable.set(args.key, args.value, serialize_json=args.json) @@ -61,6 +65,7 @@ def variables_set(args): @cli_utils.action_cli +@providers_configuration_loaded def variables_delete(args): """Deletes variable by a given name.""" Variable.delete(args.key) @@ -68,6 +73,7 @@ def variables_delete(args): @cli_utils.action_cli +@providers_configuration_loaded def variables_import(args): """Imports variables from a given file.""" if os.path.exists(args.file): @@ -76,6 +82,7 @@ def variables_import(args): raise SystemExit("Missing variables file.") +@providers_configuration_loaded def variables_export(args): """Exports all the variables to the file.""" _variable_export_helper(args.file) diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/webserver_command.py index f5d44e505869..5399f8ba6404 100644 --- a/airflow/cli/commands/webserver_command.py +++ b/airflow/cli/commands/webserver_command.py @@ -42,6 +42,7 @@ from airflow.utils.hashlib_wrapper import md5 from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.process_utils import check_if_pidfile_process_is_running +from airflow.utils.providers_configuration_loader import providers_configuration_loaded log = logging.getLogger(__name__) @@ -320,6 +321,7 @@ def _check_workers(self) -> None: @cli_utils.action_cli +@providers_configuration_loaded def webserver(args): """Starts Airflow Webserver.""" print(settings.HEADER) diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 1c02dbb57ba6..e970174a658c 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -57,18 +57,36 @@ TaskInstanceInCelery = Tuple[TaskInstanceKey, CommandType, Optional[str], Task] +# IMPORTANT NOTE! Celery Executor has initialization done dynamically and it performs initialization when +# it is imported, so we need fallbacks here in order to be able to import the class directly without +# having configuration initialized before. Do not remove those fallbacks! +# +# This is not strictly needed for production: +# +# * for Airflow 2.6 and before the defaults will come from the core defaults +# * for Airflow 2.7+ the defaults will be loaded via ProvidersManager +# +# But it helps in our tests to import the executor class and validate if the celery code can be imported +# in the current and older versions of Airflow. + OPERATION_TIMEOUT = conf.getfloat("celery", "operation_timeout", fallback=1.0) # Make it constant for unit test. CELERY_FETCH_ERR_MSG_HEADER = "Error fetching Celery task state" if conf.has_option("celery", "celery_config_options"): - celery_configuration = conf.getimport("celery", "celery_config_options") + celery_configuration = conf.getimport( + "celery", + "celery_config_options", + fallback="airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG", + ) else: celery_configuration = DEFAULT_CELERY_CONFIG -celery_app_name = conf.get("celery", "CELERY_APP_NAME") +celery_app_name = conf.get( + "celery", "CELERY_APP_NAME", fallback="airflow.providers.celery.executors.celery_executor" +) if celery_app_name == "airflow.executors.celery_executor": warnings.warn( "The celery.CELERY_APP_NAME configuration uses deprecated package name: " diff --git a/airflow/providers/celery/executors/celery_kubernetes_executor.py b/airflow/providers/celery/executors/celery_kubernetes_executor.py index 72c037b684ea..d79b3cd1db67 100644 --- a/airflow/providers/celery/executors/celery_kubernetes_executor.py +++ b/airflow/providers/celery/executors/celery_kubernetes_executor.py @@ -26,6 +26,7 @@ from airflow.executors.kubernetes_executor import KubernetesExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.utils.log.logging_mixin import LoggingMixin +from airflow.utils.providers_configuration_loader import providers_configuration_loaded if TYPE_CHECKING: from airflow.executors.base_executor import CommandType, EventBufferValueType, QueuedTaskInstanceType @@ -57,12 +58,8 @@ class CeleryKubernetesExecutor(LoggingMixin): callback_sink: BaseCallbackSink | None = None @cached_property + @providers_configuration_loaded def kubernetes_queue(self) -> str: - # lazily retrieve the value of kubernetes_queue from the configuration - # because it might need providers - from airflow.providers_manager import ProvidersManager - - ProvidersManager().initialize_providers_configuration() return conf.get("celery_kubernetes_executor", "kubernetes_queue") def __init__(self, celery_executor: CeleryExecutor, kubernetes_executor: KubernetesExecutor): diff --git a/airflow/providers/celery/executors/default_celery.py b/airflow/providers/celery/executors/default_celery.py index f5c23cbb81a3..6af4c1cd201c 100644 --- a/airflow/providers/celery/executors/default_celery.py +++ b/airflow/providers/celery/executors/default_celery.py @@ -25,7 +25,6 @@ from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException -from airflow.providers_manager import ProvidersManager def _broker_supports_visibility_timeout(url): @@ -34,8 +33,19 @@ def _broker_supports_visibility_timeout(url): log = logging.getLogger(__name__) -ProvidersManager().initialize_providers_configuration() -broker_url = conf.get("celery", "BROKER_URL") +# IMPORTANT NOTE! Celery Executor has initialization done dynamically and it performs initialization when +# it is imported, so we need fallbacks here in order to be able to import the class directly without +# having configuration initialized before. Do not remove those fallbacks! +# +# This is not strictly needed for production: +# +# * for Airflow 2.6 and before the defaults will come from the core defaults +# * for Airflow 2.7+ the defaults will be loaded via ProvidersManager +# +# But it helps in our tests to import the executor class and validate if the celery code can be imported +# in the current and older versions of Airflow. + +broker_url = conf.get("celery", "BROKER_URL", fallback="redis://redis:6379/0") broker_transport_options = conf.getsection("celery_broker_transport_options") or {} if "visibility_timeout" not in broker_transport_options: @@ -61,19 +71,19 @@ def _broker_supports_visibility_timeout(url): DEFAULT_CELERY_CONFIG = { "accept_content": ["json"], "event_serializer": "json", - "worker_prefetch_multiplier": conf.getint("celery", "worker_prefetch_multiplier"), + "worker_prefetch_multiplier": conf.getint("celery", "worker_prefetch_multiplier", fallback=1), "task_acks_late": True, "task_default_queue": conf.get("operators", "DEFAULT_QUEUE"), "task_default_exchange": conf.get("operators", "DEFAULT_QUEUE"), - "task_track_started": conf.getboolean("celery", "task_track_started"), + "task_track_started": conf.getboolean("celery", "task_track_started", fallback=True), "broker_url": broker_url, "broker_transport_options": broker_transport_options_for_celery, "result_backend": result_backend, "database_engine_options": conf.getjson( "celery", "result_backend_sqlalchemy_engine_options", fallback={} ), - "worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY"), - "worker_enable_remote_control": conf.getboolean("celery", "worker_enable_remote_control"), + "worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY", fallback=16), + "worker_enable_remote_control": conf.getboolean("celery", "worker_enable_remote_control", fallback=True), } diff --git a/airflow/utils/providers_configuration_loader.py b/airflow/utils/providers_configuration_loader.py new file mode 100644 index 000000000000..df7f9d31755f --- /dev/null +++ b/airflow/utils/providers_configuration_loader.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from functools import wraps +from typing import Callable, TypeVar, cast + +T = TypeVar("T", bound=Callable) + + +def providers_configuration_loaded(func: T) -> T: + """ + Decorator that makes sure that providers configuration is loaded before actually calling + the decorated function. + + ProvidersManager initialization of configuration is relatively inexpensive - it walks through + all providers's entrypoints, retrieve the provider_info and loads config yaml parts of the get_info. + Unlike initialization of hooks and operators it does not import any of the provider's code, so it can + be run quickly by all commands that need to access providers configuration. We cannot even import + ProvidersManager while importing any of the commands, so we need to locally import it here. + + We cannot initialize the configuration in settings/conf because of the way how conf/settings are used + internally - they are loaded while importing airflow, and we need to access airflow version conf in the + ProvidesManager initialization, so instead we opt for decorating all the methods that need it with this + decorator. + + The decorator should be placed below @suppress_logs_and_warning but above @provide_session in order to + avoid spoiling the output of formatted options with some warnings ar infos, and to be prepared that + session creation might need some configuration defaults from the providers configuration. + + :param func: function to makes sure that providers configuration is loaded before actually calling + """ + + @wraps(func) + def wrapped_function(*args, **kwargs): + from airflow.providers_manager import ProvidersManager + + ProvidersManager().initialize_providers_configuration() + return func(*args, **kwargs) + + return cast(T, wrapped_function) diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index e770e19416eb..6089961de94b 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -324,7 +324,7 @@ def test_cli_parsing_does_not_initialize_providers_manager(self): CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) CONFIG_FILE.touch(exist_ok=True) result = subprocess.run( - [sys.executable, "-m", "airflow", "providers", "status"], + [sys.executable, "-m", "airflow", "providers", "lazy-loaded"], env={"PYTHONPATH": os.pathsep.join(sys.path)}, check=False, text=True, @@ -339,7 +339,7 @@ def test_airflow_config_contains_providers(self): """ CONFIG_FILE.unlink(missing_ok=True) result = subprocess.run( - [sys.executable, "-m", "airflow", "version"], + [sys.executable, "-m", "airflow", "config", "list"], env={"PYTHONPATH": os.pathsep.join(sys.path)}, check=False, text=True, diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index c4c3e5d6f04e..9e8d365c5ce8 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -39,7 +39,6 @@ from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.operators.bash import BashOperator -from airflow.providers.celery.executors import celery_executor, celery_executor_utils from airflow.utils.state import State from tests.test_utils import db @@ -61,6 +60,8 @@ def task_id(self): @contextlib.contextmanager def _prepare_app(broker_url=None, execute=None): + from airflow.providers.celery.executors import celery_executor_utils + broker_url = broker_url or conf.get("celery", "BROKER_URL") execute = execute or celery_executor_utils.execute_command.__wrapped__ @@ -106,6 +107,8 @@ def teardown_method(self) -> None: @pytest.mark.flaky(reruns=3) @pytest.mark.parametrize("broker_url", _prepare_test_bodies()) def test_celery_integration(self, broker_url): + from airflow.providers.celery.executors import celery_executor, celery_executor_utils + success_command = ["airflow", "tasks", "run", "true", "some_parameter"] fail_command = ["airflow", "version"] @@ -163,6 +166,8 @@ def fake_execute_command(command): assert executor.queued_tasks == {} def test_error_sending_task(self): + from airflow.providers.celery.executors import celery_executor + def fake_execute_command(): pass @@ -189,6 +194,7 @@ def fake_execute_command(): def test_retry_on_error_sending_task(self, caplog): """Test that Airflow retries publishing tasks to Celery Broker at least 3 times""" + from airflow.providers.celery.executors import celery_executor, celery_executor_utils with _prepare_app(), caplog.at_level(logging.INFO), mock.patch.object( # Mock `with timeout()` to _instantly_ fail. @@ -268,6 +274,8 @@ class TestBulkStateFetcher: return_value=[json.dumps({"status": "SUCCESS", "task_id": "123"})], ) def test_should_support_kv_backend(self, mock_mget, caplog): + from airflow.providers.celery.executors import celery_executor, celery_executor_utils + caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) with _prepare_app(): mock_backend = BaseKeyValueStoreBackend(app=celery_executor.app) @@ -293,6 +301,8 @@ def test_should_support_kv_backend(self, mock_mget, caplog): @mock.patch("celery.backends.database.DatabaseBackend.ResultSession") def test_should_support_db_backend(self, mock_session, caplog): + from airflow.providers.celery.executors import celery_executor, celery_executor_utils + caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) with _prepare_app(): mock_backend = DatabaseBackend(app=celery_executor.app, url="sqlite3://") @@ -318,6 +328,8 @@ def test_should_support_db_backend(self, mock_session, caplog): @mock.patch("celery.backends.database.DatabaseBackend.ResultSession") def test_should_retry_db_backend(self, mock_session, caplog): + from airflow.providers.celery.executors import celery_executor, celery_executor_utils + caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) from sqlalchemy.exc import DatabaseError @@ -352,6 +364,8 @@ def test_should_retry_db_backend(self, mock_session, caplog): ] def test_should_support_base_backend(self, caplog): + from airflow.providers.celery.executors import celery_executor_utils + caplog.set_level(logging.DEBUG, logger=self.bulk_state_fetcher_logger) with _prepare_app(): mock_backend = mock.MagicMock(autospec=BaseBackend) diff --git a/tests/providers/celery/executors/test_celery_executor.py b/tests/providers/celery/executors/test_celery_executor.py index fd00d6a08357..c8ddee8c4ee8 100644 --- a/tests/providers/celery/executors/test_celery_executor.py +++ b/tests/providers/celery/executors/test_celery_executor.py @@ -37,7 +37,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.taskinstance import TaskInstance, TaskInstanceKey -from airflow.providers.celery.executors import celery_executor, celery_executor_utils +from airflow.providers.celery.executors import celery_executor, celery_executor_utils, default_celery from airflow.providers.celery.executors.celery_executor import CeleryExecutor from airflow.utils import timezone from airflow.utils.state import State @@ -65,6 +65,7 @@ def task_id(self): @contextlib.contextmanager def _prepare_app(broker_url=None, execute=None): broker_url = broker_url or conf.get("celery", "BROKER_URL") + execute = execute or celery_executor_utils.execute_command.__wrapped__ test_config = dict(celery_executor_utils.celery_configuration) @@ -185,6 +186,7 @@ def test_try_adopt_task_instances_none(self): key1 = TaskInstance(task=task_1, run_id=None) tis = [key1] + executor = celery_executor.CeleryExecutor() assert executor.try_adopt_task_instances(tis) == tis @@ -208,6 +210,7 @@ def test_try_adopt_task_instances(self): ti2.state = State.QUEUED tis = [ti1, ti2] + executor = celery_executor.CeleryExecutor() assert executor.running == set() assert executor.tasks == {} @@ -243,6 +246,7 @@ def test_cleanup_stuck_queued_tasks(self, mock_fail): tis = [ti] with _prepare_app() as app: app.control.revoke = mock.MagicMock() + executor = celery_executor.CeleryExecutor() executor.job_id = 1 executor.running = {ti.key} @@ -258,8 +262,6 @@ def test_cleanup_stuck_queued_tasks(self, mock_fail): def test_result_backend_sqlalchemy_engine_options(self, mock_celery): import importlib - from airflow.providers.celery.executors import celery_executor_utils, default_celery - # reload celery conf to apply the new config importlib.reload(default_celery) # reload celery_executor_utils to recreate the celery app with new config From 82e6226738bcf7f3981c5b8db714a849c26a6962 Mon Sep 17 00:00:00 2001 From: VladaZakharova <80038284+VladaZakharova@users.noreply.github.com> Date: Sun, 23 Jul 2023 07:55:23 +0200 Subject: [PATCH 277/533] Fix Datafusion system tests (#32749) --- .../google/cloud/hooks/datafusion.py | 23 +++++-- .../cloud/datafusion/example_datafusion.py | 60 +++++++++------- .../datafusion/example_datafusion_async.py | 68 ++++++++++--------- 3 files changed, 87 insertions(+), 64 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/airflow/providers/google/cloud/hooks/datafusion.py index f74958cc1607..770e3da39169 100644 --- a/airflow/providers/google/cloud/hooks/datafusion.py +++ b/airflow/providers/google/cloud/hooks/datafusion.py @@ -319,7 +319,7 @@ def create_pipeline( namespace: str = "default", ) -> None: """ - Creates a Cloud Data Fusion pipeline. + Creates a batch Cloud Data Fusion pipeline. :param pipeline_name: Your pipeline name. :param pipeline: The pipeline definition. For more information check: @@ -343,12 +343,12 @@ def delete_pipeline( namespace: str = "default", ) -> None: """ - Deletes a Cloud Data Fusion pipeline. + Deletes a batch Cloud Data Fusion pipeline. :param pipeline_name: Your pipeline name. :param version_id: Version of pipeline to delete :param instance_url: Endpoint on which the REST APIs is accessible for the instance. - :param namespace: f your pipeline belongs to a Basic edition instance, the namespace ID + :param namespace: if your pipeline belongs to a Basic edition instance, the namespace ID is always default. If your pipeline belongs to an Enterprise edition instance, you can create a namespace. """ @@ -357,9 +357,20 @@ def delete_pipeline( url = os.path.join(url, "versions", version_id) response = self._cdap_request(url=url, method="DELETE", body=None) - self._check_response_status_and_data( - response, f"Deleting a pipeline failed with code {response.status}" - ) + # Check for 409 error: the previous step for starting/stopping pipeline could still be in progress. + # Waiting some time before retry. + for time_to_wait in exponential_sleep_generator(initial=10, maximum=120): + try: + self._check_response_status_and_data( + response, f"Deleting a pipeline failed with code {response.status}: {response.data}" + ) + break + except AirflowException as exc: + if "409" in str(exc): + sleep(time_to_wait) + response = self._cdap_request(url=url, method="DELETE", body=None) + else: + raise def list_pipelines( self, diff --git a/tests/system/providers/google/cloud/datafusion/example_datafusion.py b/tests/system/providers/google/cloud/datafusion/example_datafusion.py index 4ff221c92852..4d3bb7418dd4 100644 --- a/tests/system/providers/google/cloud/datafusion/example_datafusion.py +++ b/tests/system/providers/google/cloud/datafusion/example_datafusion.py @@ -23,7 +23,6 @@ from datetime import datetime from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datafusion import ( CloudDataFusionCreateInstanceOperator, CloudDataFusionCreatePipelineOperator, @@ -36,34 +35,36 @@ CloudDataFusionStopPipelineOperator, CloudDataFusionUpdateInstanceOperator, ) -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator +from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.sensors.datafusion import CloudDataFusionPipelineStateSensor +from airflow.utils.trigger_rule import TriggerRule # [START howto_data_fusion_env_variables] SERVICE_ACCOUNT = os.environ.get("GCP_DATAFUSION_SERVICE_ACCOUNT") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") LOCATION = "europe-north1" -INSTANCE_NAME = "airflow-test-instance" +DAG_ID = "example_data_fusion" +INSTANCE_NAME = "test-instance" INSTANCE = { "type": "BASIC", "displayName": INSTANCE_NAME, "dataprocServiceAccount": SERVICE_ACCOUNT, } -BUCKET_1 = "test-datafusion-1" -BUCKET_2 = "test-datafusion-2" -BUCKET_1_URI = f"gs://{BUCKET_1}" -BUCKET_2_URI = f"gs://{BUCKET_2}" +BUCKET_NAME_1 = "test-datafusion-1" +BUCKET_NAME_2 = "test-datafusion-2" +BUCKET_NAME_1_URI = f"gs://{BUCKET_NAME_1}" +BUCKET_NAME_2_URI = f"gs://{BUCKET_NAME_2}" PIPELINE_NAME = "test-pipe" PIPELINE = { "artifact": { "name": "cdap-data-pipeline", - "version": "6.7.2", + "version": "6.8.3", "scope": "SYSTEM", }, "description": "Data Pipeline Application", - "name": "test-pipe", + "name": PIPELINE_NAME, "config": { "resources": {"memoryMB": 2048, "virtualCores": 1}, "driverResources": {"memoryMB": 2048, "virtualCores": 1}, @@ -80,7 +81,7 @@ "name": "GCSFile", "type": "batchsource", "label": "GCS", - "artifact": {"name": "google-cloud", "version": "0.20.3", "scope": "SYSTEM"}, + "artifact": {"name": "google-cloud", "version": "0.21.2", "scope": "SYSTEM"}, "properties": { "project": "auto-detect", "format": "text", @@ -91,7 +92,7 @@ "encrypted": "false", "schema": '{"type":"record","name":"textfile","fields":[{"name"\ :"offset","type":"long"},{"name":"body","type":"string"}]}', - "path": BUCKET_1_URI, + "path": BUCKET_NAME_1_URI, "referenceName": "foo_bucket", "useConnection": "false", "serviceAccountType": "filePath", @@ -109,7 +110,7 @@ "name": "GCS", "type": "batchsink", "label": "GCS2", - "artifact": {"name": "google-cloud", "version": "0.20.3", "scope": "SYSTEM"}, + "artifact": {"name": "google-cloud", "version": "0.21.2", "scope": "SYSTEM"}, "properties": { "project": "auto-detect", "suffix": "yyyy-MM-dd-HH-mm", @@ -119,7 +120,7 @@ "schema": '{"type":"record","name":"textfile","fields":[{"name"\ :"offset","type":"long"},{"name":"body","type":"string"}]}', "referenceName": "bar", - "path": BUCKET_2_URI, + "path": BUCKET_NAME_2_URI, "serviceAccountType": "filePath", "contentType": "application/octet-stream", }, @@ -146,19 +147,20 @@ # [END howto_data_fusion_env_variables] with models.DAG( - "example_data_fusion", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, + tags=["example", "datafusion"], ) as dag: create_bucket1 = GCSCreateBucketOperator( task_id="create_bucket1", - bucket_name=BUCKET_1, + bucket_name=BUCKET_NAME_1, project_id=PROJECT_ID, ) create_bucket2 = GCSCreateBucketOperator( task_id="create_bucket2", - bucket_name=BUCKET_2, + bucket_name=BUCKET_NAME_2, project_id=PROJECT_ID, ) @@ -255,38 +257,44 @@ pipeline_name=PIPELINE_NAME, instance_name=INSTANCE_NAME, task_id="delete_pipeline", + trigger_rule=TriggerRule.ALL_DONE, ) # [END howto_cloud_data_fusion_delete_pipeline] # [START howto_cloud_data_fusion_delete_instance_operator] delete_instance = CloudDataFusionDeleteInstanceOperator( - location=LOCATION, instance_name=INSTANCE_NAME, task_id="delete_instance" + location=LOCATION, + instance_name=INSTANCE_NAME, + task_id="delete_instance", + trigger_rule=TriggerRule.ALL_DONE, ) # [END howto_cloud_data_fusion_delete_instance_operator] - # Add sleep before creating pipeline - sleep = BashOperator(task_id="sleep", bash_command="sleep 60") - - # Add sleep before creating pipeline - sleep_30 = BashOperator(task_id="sleep_30", bash_command="sleep 30") + delete_bucket1 = GCSDeleteBucketOperator( + task_id="delete_bucket1", bucket_name=BUCKET_NAME_1, trigger_rule=TriggerRule.ALL_DONE + ) + delete_bucket2 = GCSDeleteBucketOperator( + task_id="delete_bucket2", bucket_name=BUCKET_NAME_1, trigger_rule=TriggerRule.ALL_DONE + ) ( - create_bucket1 - >> create_bucket2 + # TEST SETUP + [create_bucket1, create_bucket2] + # TEST BODY >> create_instance >> get_instance >> restart_instance >> update_instance - >> sleep >> create_pipeline >> list_pipelines >> start_pipeline_async >> start_pipeline_sensor >> start_pipeline >> stop_pipeline - >> sleep_30 >> delete_pipeline >> delete_instance + # TEST TEARDOWN + >> [delete_bucket1, delete_bucket2] ) from tests.system.utils.watcher import watcher diff --git a/tests/system/providers/google/cloud/datafusion/example_datafusion_async.py b/tests/system/providers/google/cloud/datafusion/example_datafusion_async.py index f8c345c7350d..43b1241e59e1 100644 --- a/tests/system/providers/google/cloud/datafusion/example_datafusion_async.py +++ b/tests/system/providers/google/cloud/datafusion/example_datafusion_async.py @@ -23,7 +23,6 @@ from datetime import datetime from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datafusion import ( CloudDataFusionCreateInstanceOperator, CloudDataFusionCreatePipelineOperator, @@ -36,33 +35,35 @@ CloudDataFusionStopPipelineOperator, CloudDataFusionUpdateInstanceOperator, ) -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator +from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.utils.trigger_rule import TriggerRule # [START howto_data_fusion_env_variables] SERVICE_ACCOUNT = os.environ.get("GCP_DATAFUSION_SERVICE_ACCOUNT") PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") +DAG_ID = "example_data_fusion_async" LOCATION = "europe-north1" -INSTANCE_NAME = "airflow-test-instance" +INSTANCE_NAME = "test-instance-async" INSTANCE = { "type": "BASIC", "displayName": INSTANCE_NAME, "dataprocServiceAccount": SERVICE_ACCOUNT, } -BUCKET_1 = "test-datafusion-1" -BUCKET_2 = "test-datafusion-2" -BUCKET_1_URI = f"gs://{BUCKET_1}" -BUCKET_2_URI = f"gs://{BUCKET_2}" +BUCKET_NAME_1 = "test-datafusion-async-1" +BUCKET_NAME_2 = "test-datafusion-async-2" +BUCKET_NAME_1_URI = f"gs://{BUCKET_NAME_1}" +BUCKET_NAME_2_URI = f"gs://{BUCKET_NAME_2}" PIPELINE_NAME = "test-pipe" PIPELINE = { "artifact": { "name": "cdap-data-pipeline", - "version": "6.7.2", + "version": "6.8.3", "scope": "SYSTEM", }, "description": "Data Pipeline Application", - "name": "test-pipe", + "name": PIPELINE_NAME, "config": { "resources": {"memoryMB": 2048, "virtualCores": 1}, "driverResources": {"memoryMB": 2048, "virtualCores": 1}, @@ -79,7 +80,7 @@ "name": "GCSFile", "type": "batchsource", "label": "GCS", - "artifact": {"name": "google-cloud", "version": "0.20.3", "scope": "SYSTEM"}, + "artifact": {"name": "google-cloud", "version": "0.21.2", "scope": "SYSTEM"}, "properties": { "project": "auto-detect", "format": "text", @@ -90,7 +91,7 @@ "encrypted": "false", "schema": '{"type":"record","name":"textfile","fields":[{"name"\ :"offset","type":"long"},{"name":"body","type":"string"}]}', - "path": BUCKET_1_URI, + "path": BUCKET_NAME_1_URI, "referenceName": "foo_bucket", "useConnection": "false", "serviceAccountType": "filePath", @@ -108,7 +109,7 @@ "name": "GCS", "type": "batchsink", "label": "GCS2", - "artifact": {"name": "google-cloud", "version": "0.20.3", "scope": "SYSTEM"}, + "artifact": {"name": "google-cloud", "version": "0.21.2", "scope": "SYSTEM"}, "properties": { "project": "auto-detect", "suffix": "yyyy-MM-dd-HH-mm", @@ -118,7 +119,7 @@ "schema": '{"type":"record","name":"textfile","fields":[{"name"\ :"offset","type":"long"},{"name":"body","type":"string"}]}', "referenceName": "bar", - "path": BUCKET_2_URI, + "path": BUCKET_NAME_2_URI, "serviceAccountType": "filePath", "contentType": "application/octet-stream", }, @@ -145,19 +146,20 @@ # [END howto_data_fusion_env_variables] with models.DAG( - "example_data_fusion_async", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, + tags=["example", "datafusion", "deferrable"], ) as dag: create_bucket1 = GCSCreateBucketOperator( task_id="create_bucket1", - bucket_name=BUCKET_1, + bucket_name=BUCKET_NAME_1, project_id=PROJECT_ID, ) create_bucket2 = GCSCreateBucketOperator( task_id="create_bucket2", - bucket_name=BUCKET_2, + bucket_name=BUCKET_NAME_2, project_id=PROJECT_ID, ) @@ -209,7 +211,7 @@ # [END howto_cloud_data_fusion_list_pipelines] # [START howto_cloud_data_fusion_start_pipeline_def] - start_pipeline_async = CloudDataFusionStartPipelineOperator( + start_pipeline_def = CloudDataFusionStartPipelineOperator( location=LOCATION, pipeline_name=PIPELINE_NAME, instance_name=INSTANCE_NAME, @@ -233,40 +235,42 @@ pipeline_name=PIPELINE_NAME, instance_name=INSTANCE_NAME, task_id="delete_pipeline", + trigger_rule=TriggerRule.ALL_DONE, ) # [END howto_cloud_data_fusion_delete_pipeline] # [START howto_cloud_data_fusion_delete_instance_operator] delete_instance = CloudDataFusionDeleteInstanceOperator( - location=LOCATION, instance_name=INSTANCE_NAME, task_id="delete_instance" + location=LOCATION, + instance_name=INSTANCE_NAME, + task_id="delete_instance", + trigger_rule=TriggerRule.ALL_DONE, ) # [END howto_cloud_data_fusion_delete_instance_operator] - # - # Add sleep before creating pipeline - sleep_30_1 = BashOperator(task_id="sleep_30_1", bash_command="sleep 30") - - # Add sleep before deleting pipeline - sleep_30 = BashOperator(task_id="sleep_30", bash_command="sleep 30") - # Add sleep before starting pipeline - sleep_20 = BashOperator(task_id="sleep_20", bash_command="sleep 40") + delete_bucket1 = GCSDeleteBucketOperator( + task_id="delete_bucket1", bucket_name=BUCKET_NAME_1, trigger_rule=TriggerRule.ALL_DONE + ) + delete_bucket2 = GCSDeleteBucketOperator( + task_id="delete_bucket2", bucket_name=BUCKET_NAME_1, trigger_rule=TriggerRule.ALL_DONE + ) ( - create_bucket1 - >> create_bucket2 + # TEST SETUP + [create_bucket1, create_bucket2] + # TEST BODY >> create_instance >> get_instance >> restart_instance >> update_instance - >> sleep_30_1 >> create_pipeline >> list_pipelines - >> sleep_20 - >> start_pipeline_async + >> start_pipeline_def >> stop_pipeline - >> sleep_30 >> delete_pipeline >> delete_instance + # TEST TEARDOWN + >> [delete_bucket1, delete_bucket2] ) from tests.system.utils.watcher import watcher From 6362ba5ab45a38008814616df4e17717cc3726c3 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 23 Jul 2023 08:26:54 +0200 Subject: [PATCH 278/533] Move writing configuration for webserver to main (webserver limited) (#32766) So far configuration was automatically written no matter what command has been invoked. This means tha webserver configuration is stored for all components, where it was only really needed for webserver. This small change moves the place where configuration writing is invoked - webserver configuration will only be automatically checked for and written when webserver command has been invoked via __main__ entrypoint. This has also the nice side-effect that it is not written during tests, or when airflow is just imported. --- airflow/__main__.py | 7 ++++++- airflow/configuration.py | 12 +++++++++--- tests/www/views/test_views.py | 5 +++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/airflow/__main__.py b/airflow/__main__.py index c15f75b4b917..c88f655bee2b 100644 --- a/airflow/__main__.py +++ b/airflow/__main__.py @@ -34,6 +34,7 @@ # any possible import cycles with settings downstream. from airflow import configuration from airflow.cli import cli_parser +from airflow.configuration import write_webserver_configuration_if_needed def main(): @@ -52,7 +53,11 @@ def main(): # in main ensures that it is not done during tests and other ways airflow imports are used from airflow.configuration import write_default_airflow_configuration_if_needed - write_default_airflow_configuration_if_needed() + conf = write_default_airflow_configuration_if_needed() + + if args.subcommand == "webserver": + write_webserver_configuration_if_needed(conf) + args.func(args) diff --git a/airflow/configuration.py b/airflow/configuration.py index 317c10c01310..cca5588e543b 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -47,6 +47,7 @@ from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend from airflow.utils import yaml from airflow.utils.module_loading import import_string +from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.weight_rule import WeightRule log = logging.getLogger(__name__) @@ -1928,7 +1929,7 @@ def create_default_config_parser(configuration_description: dict[str, dict[str, return parser -def write_default_airflow_configuration_if_needed(): +def write_default_airflow_configuration_if_needed() -> AirflowConfigParser: if not os.path.isfile(AIRFLOW_CONFIG): log.debug("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True) @@ -1949,6 +1950,7 @@ def write_default_airflow_configuration_if_needed(): only_defaults=True, ) make_group_other_inaccessible(AIRFLOW_CONFIG) + return conf def load_standard_airflow_configuration(airflow_config_parser: AirflowConfigParser): @@ -1992,7 +1994,6 @@ def initialize_config() -> AirflowConfigParser: Called for you automatically as part of the Airflow boot process. """ - global WEBSERVER_CONFIG airflow_config_parser = AirflowConfigParser() if airflow_config_parser.getboolean("core", "unit_test_mode"): airflow_config_parser.load_test_config() @@ -2004,6 +2005,12 @@ def initialize_config() -> AirflowConfigParser: if airflow_config_parser.getboolean("core", "unit_test_mode"): airflow_config_parser.load_test_config() + return airflow_config_parser + + +@providers_configuration_loaded +def write_webserver_configuration_if_needed(airflow_config_parser: AirflowConfigParser): + global WEBSERVER_CONFIG WEBSERVER_CONFIG = airflow_config_parser.get("webserver", "config_file") if not os.path.isfile(WEBSERVER_CONFIG): import shutil @@ -2011,7 +2018,6 @@ def initialize_config() -> AirflowConfigParser: pathlib.Path(WEBSERVER_CONFIG).parent.mkdir(parents=True, exist_ok=True) log.info("Creating new FAB webserver config file in: %s", WEBSERVER_CONFIG) shutil.copy(_default_config_file_path("default_webserver_config.py"), WEBSERVER_CONFIG) - return airflow_config_parser def make_group_other_inaccessible(file_path: str): diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index 755df3b7aa7f..5dbb70edbd7e 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -24,7 +24,7 @@ import pytest -from airflow.configuration import initialize_config +from airflow.configuration import initialize_config, write_webserver_configuration_if_needed from airflow.plugins_manager import AirflowPlugin, EntryPointSource from airflow.utils.task_group import TaskGroup from airflow.www import views @@ -68,7 +68,8 @@ def test_webserver_configuration_config_file(mock_webserver_config_global, admin config_file = str(tmp_path / "my_custom_webserver_config.py") with mock.patch.dict(os.environ, {"AIRFLOW__WEBSERVER__CONFIG_FILE": config_file}): - initialize_config() + conf = initialize_config() + write_webserver_configuration_if_needed(conf) assert airflow.configuration.WEBSERVER_CONFIG == config_file assert os.path.isfile(config_file) From 42b4b43c4c2ccf0b6e7eaa105c982df495768d01 Mon Sep 17 00:00:00 2001 From: Michal Charemza Date: Sun, 23 Jul 2023 07:43:35 +0100 Subject: [PATCH 279/533] Add Redis task handler (#31855) * Allow FileTaskHandler to delegate to instances of logging.Handler FileTaskHandler is the base class for logging handlers, including those that don't log to files via delegating to logging.FileHandler, e.g. in the CloudwatchTaskHandler at https://github.com/apache/airflow/blob/2940b9fa55a6a72c60c2162e541631addec3d6b8/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py#L67 It is suspected that type checking is not enabled in this part of the CloudwatchTaskHandler otherwise it would have already been failing. This change adjusts the base class so if type checking is enabled in the task handler, if it delegates to a logging.Handler that is not a logging.FileHandler as the CloudWatchHandler, then the type checking should pass. This was originally part of https://github.com/apache/airflow/pull/31855 and split out. related: https://github.com/apache/airflow/issues/31834 * Add Redis task handler This stores log lines in Redis up to a configured maximum log lines, always keeping the most recent, up to a configured TTL. This deviates from other existing task handlers in that it accepts a connection ID. This allows it to be used in addition to other handlers, and so allows a graceful/reversible transition from one logging system to another. This is particularly useful in situations that use Redis as a message broker, where additional infrastructure isn't desired. closes: https://github.com/apache/airflow/issues/31834 --- .../microsoft/azure/log/wasb_task_handler.py | 2 + airflow/providers/redis/log/__init__.py | 17 +++ .../providers/redis/log/redis_task_handler.py | 105 ++++++++++++++++++ airflow/providers/redis/provider.yaml | 3 + airflow/utils/log/file_task_handler.py | 2 +- docs/apache-airflow-providers-redis/index.rst | 7 ++ .../logging/index.rst | 24 ++++ tests/providers/redis/log/__init__.py | 17 +++ .../redis/log/test_redis_task_handler.py | 94 ++++++++++++++++ 9 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 airflow/providers/redis/log/__init__.py create mode 100644 airflow/providers/redis/log/redis_task_handler.py create mode 100644 docs/apache-airflow-providers-redis/logging/index.rst create mode 100644 tests/providers/redis/log/__init__.py create mode 100644 tests/providers/redis/log/test_redis_task_handler.py diff --git a/airflow/providers/microsoft/azure/log/wasb_task_handler.py b/airflow/providers/microsoft/azure/log/wasb_task_handler.py index 96c87219ca13..97a8af5ae1d6 100644 --- a/airflow/providers/microsoft/azure/log/wasb_task_handler.py +++ b/airflow/providers/microsoft/azure/log/wasb_task_handler.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import logging import os import shutil from functools import cached_property @@ -62,6 +63,7 @@ def __init__( **kwargs, ) -> None: super().__init__(base_log_folder, filename_template) + self.handler: logging.FileHandler | None = None self.wasb_container = wasb_container self.remote_base = wasb_log_folder self.log_relative_path = "" diff --git a/airflow/providers/redis/log/__init__.py b/airflow/providers/redis/log/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/providers/redis/log/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/redis/log/redis_task_handler.py b/airflow/providers/redis/log/redis_task_handler.py new file mode 100644 index 000000000000..b2e4a8fc16de --- /dev/null +++ b/airflow/providers/redis/log/redis_task_handler.py @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +from redis import Redis + +from airflow.configuration import conf +from airflow.models import TaskInstance +from airflow.providers.redis.hooks.redis import RedisHook +from airflow.utils.log.file_task_handler import FileTaskHandler +from airflow.utils.log.logging_mixin import LoggingMixin + + +class RedisTaskHandler(FileTaskHandler, LoggingMixin): + """ + RedisTaskHandler is a Python log handler that handles and reads task instance logs. + It extends airflow FileTaskHandler and uploads to and reads from Redis. + + :param base_log_folder: + base folder to store logs locally + :param max_lines: + Maximum number of lines of log to store + If omitted, this is 10000. + :param ttl_seconds: + Maximum number of seconds to store logs + If omitted, this is the equivalent of 28 days. + :param conn_id: + Airflow connection ID for the Redis hook to use + If omitted or None, the ID specified in the option logging.remote_log_conn_id is used. + """ + + trigger_should_wrap = True + + def __init__( + self, + base_log_folder: str, + max_lines: int = 10000, + ttl_seconds: int = 60 * 60 * 24 * 28, + conn_id: str | None = None, + ): + super().__init__(base_log_folder) + self.handler: _RedisHandler | None = None + self.max_lines = max_lines + self.ttl_seconds = ttl_seconds + self.conn_id = conn_id if conn_id is not None else conf.get("logging", "REMOTE_LOG_CONN_ID") + + @cached_property + def conn(self): + return RedisHook(redis_conn_id=self.conn_id).get_conn() + + def _read( + self, + ti: TaskInstance, + try_number: int, + metadata: dict[str, Any] | None = None, + ): + log_str = b"\n".join( + self.conn.lrange(self._render_filename(ti, try_number), start=0, end=-1) + ).decode() + return log_str, {"end_of_log": True} + + def set_context(self, ti: TaskInstance): + super().set_context(ti) + self.handler = _RedisHandler( + self.conn, + key=self._render_filename(ti, ti.try_number), + max_lines=self.max_lines, + ttl_seconds=self.ttl_seconds, + ) + self.handler.setFormatter(self.formatter) + + +class _RedisHandler(logging.Handler): + def __init__(self, conn: Redis, key: str, max_lines: int, ttl_seconds: int): + super().__init__() + self.conn = conn + self.key = key + self.max_lines = max_lines + self.ttl_seconds = ttl_seconds + + def emit(self, record): + p = self.conn.pipeline() + p.rpush(self.key, self.format(record)) + p.ltrim(self.key, start=-self.max_lines, end=-1) + p.expire(self.key, time=self.ttl_seconds) + p.execute() diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index c679abe00e01..59becb7a6936 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -64,3 +64,6 @@ hooks: connection-types: - hook-class-name: airflow.providers.redis.hooks.redis.RedisHook connection-type: redis + +logging: + - airflow.providers.redis.redis_task_handler.RedisTaskHandler diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 5d791aaa0cca..fb540ed9ecb0 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -147,7 +147,7 @@ class FileTaskHandler(logging.Handler): def __init__(self, base_log_folder: str, filename_template: str | None = None): super().__init__() - self.handler: logging.FileHandler | None = None + self.handler: logging.Handler | None = None self.local_base = base_log_folder if filename_template is not None: warnings.warn( diff --git a/docs/apache-airflow-providers-redis/index.rst b/docs/apache-airflow-providers-redis/index.rst index 02bc13257f5d..d33b628b1a3e 100644 --- a/docs/apache-airflow-providers-redis/index.rst +++ b/docs/apache-airflow-providers-redis/index.rst @@ -29,6 +29,13 @@ Changelog Security +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Logging + .. toctree:: :hidden: :maxdepth: 1 diff --git a/docs/apache-airflow-providers-redis/logging/index.rst b/docs/apache-airflow-providers-redis/logging/index.rst new file mode 100644 index 000000000000..6dea5fff2b99 --- /dev/null +++ b/docs/apache-airflow-providers-redis/logging/index.rst @@ -0,0 +1,24 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _write-logs-redis: + +Writing logs to Redis +--------------------- + +Airflow can be configured to store log lines in Redis up to a configured maximum log lines, always keeping the most recent, up to a configured TTL. This deviates from other existing task handlers in that it accepts a connection ID. +This allows it to be used in addition to other handlers, and so allows a graceful/reversible transition from one logging system to another. This is particularly useful in situations that use Redis as a message broker, where additional infrastructure isn't desired. diff --git a/tests/providers/redis/log/__init__.py b/tests/providers/redis/log/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/tests/providers/redis/log/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/redis/log/test_redis_task_handler.py b/tests/providers/redis/log/test_redis_task_handler.py new file mode 100644 index 000000000000..3dc481074dd3 --- /dev/null +++ b/tests/providers/redis/log/test_redis_task_handler.py @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from unittest.mock import patch + +import pytest + +from airflow.models import DAG, DagRun, TaskInstance +from airflow.operators.empty import EmptyOperator +from airflow.providers.redis.log.redis_task_handler import RedisTaskHandler +from airflow.utils.session import create_session +from airflow.utils.state import State +from airflow.utils.timezone import datetime +from tests.test_utils.config import conf_vars + + +class TestRedisTaskHandler: + @pytest.fixture + def ti(self): + date = datetime(2020, 1, 1) + dag = DAG(dag_id="dag_for_testing_redis_task_handler", start_date=date) + task = EmptyOperator(task_id="task_for_testing_redis_log_handler", dag=dag) + dag_run = DagRun(dag_id=dag.dag_id, execution_date=date, run_id="test", run_type="scheduled") + + with create_session() as session: + session.add(dag_run) + session.commit() + session.refresh(dag_run) + + ti = TaskInstance(task=task, run_id=dag_run.run_id) + ti.dag_run = dag_run + ti.try_number = 1 + ti.state = State.RUNNING + + yield ti + + with create_session() as session: + session.query(DagRun).delete() + + @conf_vars({("logging", "remote_log_conn_id"): "redis_default"}) + def test_write(self, ti): + handler = RedisTaskHandler("any", max_lines=5, ttl_seconds=2) + handler.set_context(ti) + logger = logging.getLogger(__name__) + logger.addHandler(handler) + + key = ( + "dag_id=dag_for_testing_redis_task_handler/run_id=test" + + "/task_id=task_for_testing_redis_log_handler/attempt=1.log" + ) + + with patch("redis.Redis.pipeline") as pipeline: + logger.info("Test log event") + + pipeline.return_value.rpush.assert_called_once_with(key, "Test log event") + pipeline.return_value.ltrim.assert_called_once_with(key, start=-5, end=-1) + pipeline.return_value.expire.assert_called_once_with(key, time=2) + pipeline.return_value.execute.assert_called_once_with() + + @conf_vars({("logging", "remote_log_conn_id"): "redis_default"}) + def test_read(self, ti): + handler = RedisTaskHandler("any") + handler.set_context(ti) + logger = logging.getLogger(__name__) + logger.addHandler(handler) + + key = ( + "dag_id=dag_for_testing_redis_task_handler/run_id=test" + + "/task_id=task_for_testing_redis_log_handler/attempt=1.log" + ) + + with patch("redis.Redis.lrange") as lrange: + lrange.return_value = [b"Line 1", b"Line 2"] + logs = handler.read(ti) + + assert logs == ([[("", "Line 1\nLine 2")]], [{"end_of_log": True}]) + lrange.assert_called_once_with(key, start=0, end=-1) From 01a61b0e7cb37ab24ede2537387e5ce77a7bd133 Mon Sep 17 00:00:00 2001 From: Vijayasarathi Balasubramanian Date: Sun, 23 Jul 2023 08:24:10 -0400 Subject: [PATCH 280/533] Documentation Update to enhance readability (#32730) small readability change Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/apache-airflow/authoring-and-scheduling/plugins.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/plugins.rst b/docs/apache-airflow/authoring-and-scheduling/plugins.rst index c8e8293b6421..2a9314f39eb7 100644 --- a/docs/apache-airflow/authoring-and-scheduling/plugins.rst +++ b/docs/apache-airflow/authoring-and-scheduling/plugins.rst @@ -57,7 +57,6 @@ Examples: * An auditing tool, helping understand who accesses what * A config-driven SLA monitoring tool, allowing you to set monitored tables and at what time they should land, alert people, and expose visualizations of outages -* ... Why build on top of Airflow? ---------------------------- From 505fe854e295035000129737e0009c919d0f68ef Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Sun, 23 Jul 2023 14:34:39 +0200 Subject: [PATCH 281/533] Remove unnecessary cast (#32782) --- airflow/utils/providers_configuration_loader.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/airflow/utils/providers_configuration_loader.py b/airflow/utils/providers_configuration_loader.py index df7f9d31755f..543a25c05e6c 100644 --- a/airflow/utils/providers_configuration_loader.py +++ b/airflow/utils/providers_configuration_loader.py @@ -17,12 +17,15 @@ from __future__ import annotations from functools import wraps -from typing import Callable, TypeVar, cast +from typing import Callable, TypeVar -T = TypeVar("T", bound=Callable) +from airflow.typing_compat import ParamSpec +PS = ParamSpec("PS") +RT = TypeVar("RT") -def providers_configuration_loaded(func: T) -> T: + +def providers_configuration_loaded(func: Callable[PS, RT]) -> Callable[PS, RT]: """ Decorator that makes sure that providers configuration is loaded before actually calling the decorated function. @@ -46,10 +49,10 @@ def providers_configuration_loaded(func: T) -> T: """ @wraps(func) - def wrapped_function(*args, **kwargs): + def wrapped_function(*args, **kwargs) -> RT: from airflow.providers_manager import ProvidersManager ProvidersManager().initialize_providers_configuration() return func(*args, **kwargs) - return cast(T, wrapped_function) + return wrapped_function From 685328e3572043fba6db432edcaacf8d06cf88d0 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 23 Jul 2023 21:56:46 +0200 Subject: [PATCH 282/533] Quarantine test_backfill_integration in dask executor (#32780) The test has been recently failing with deadlock (see #32778) and needs thorough looking at if we want to find the root cause/remedium. In the meantime it looks like a niche case connected with Dask Executor that is rather obsure and we have no expertise in solving problems with and diagnosing, so until the problem is diagnosed it might be a long time (and maybe even we decide not to care about it and let Dask community take a look and either fix or ignore it. We aim to have a very low number of those Quarantined tests (currently we have 1 and we have not run it for a while as this was a mysql test run on Postgres) but we have now the opportunity to also improve the quarantined tests framework. This test will be run together with other (1) quarantined test and: * they will not be run in our regular tests * they are run sequentially not in parallel with all other tests * they are run for all 4 backends but only for the default versions of those backends * failure of the quarantined tests will not cause failure of the whole job or limit constraints from being generated and updated --- .github/workflows/ci.yml | 47 +++++++- .../commands/testing_commands.py | 2 + images/breeze/output-commands-hash.txt | 4 +- images/breeze/output_testing.svg | 24 ++-- images/breeze/output_testing_tests.svg | 114 +++++++++--------- tests/executors/test_dask_executor.py | 4 + 6 files changed, 118 insertions(+), 77 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc0c1838252b..cda3148c59a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1308,15 +1308,13 @@ jobs: needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "Quarantined" + TEST_TYPE: "Quarantined" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" - BACKEND: "sqlite" - BACKEND_VERSION: "" JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" COVERAGE: "${{needs.build-info.outputs.run-coverage}}" - if: needs.build-info.outputs.run-tests == 'true' + if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.runs-on == 'self-hosted' steps: - name: Cleanup repo shell: bash @@ -1329,8 +1327,45 @@ jobs: Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image - name: > - Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined - run: breeze testing tests --run-in-parallel || true + Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "postgres" + BACKEND_VERSION: "11" + POSTGRES_VERSION: "11" + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "mysql" + BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}} + MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "mssql" + BACKEND_VERSION: ${{needs.build-info.outputs.default-mssql-version}} + MSSQL_VERSION: ${{needs.build-info.outputs.default-mssql-version}} + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down + - name: > + Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze testing tests || true + env: + BACKEND: "sqlite" + BACKEND_VERSION: "" + MSSQL_VERSION: "" + - name: > + Cleaning up ${{needs.build-info.outputs.default-python-version}}:Quarantined + run: breeze down - name: > Post Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined" uses: ./.github/actions/post_tests diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 12e7ad98b02a..90109b90f481 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -331,12 +331,14 @@ def run_tests_in_parallel( "should be run: `Providers[airbyte,http]` or " "excluded from the full test suite: `Providers[-amazon,google]`", default="All", + envvar="TEST_TYPE", type=NotVerifiedBetterChoice(ALLOWED_TEST_TYPE_CHOICES), ) @click.option( "--test-timeout", help="Test timeout. Set the pytest setup, execution and teardown timeouts to this value", default=60, + envvar="TEST_TIMEOUT", type=IntRange(min=0), show_default=True, ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index a678af7996c6..b3fc1ec2265f 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -66,5 +66,5 @@ static-checks:f9ec0d7edaba84180403d95469d94ea0 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 -testing:tests:e18fbd845ca5783879244bf0f9f9c51e -testing:f360a66d2b1b4c39880117ae6166007f +testing:tests:3c202e65824e405269e78f58936980e0 +testing:68a089bc30e0e60f834f205df1e9f086 diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index 82c6f8e344e3..7d14280b5273 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -96,19 +96,19 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integration tests.                                     │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│integration-tests                        Run the specified integration tests.                                     │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index ec3da499fa7f..9cdc350da969 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -35,8 +35,8 @@ .breeze-testing-tests-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-tests-r2 { fill: #c5c8c6 } .breeze-testing-tests-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-tests-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-tests-r5 { fill: #868887 } +.breeze-testing-tests-r4 { fill: #868887 } +.breeze-testing-tests-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-tests-r6 { fill: #98a84b;font-weight: bold } .breeze-testing-tests-r7 { fill: #8d7b39 } @@ -232,64 +232,64 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--test-typeType of test to run. With Providers, you can specify tests of which providers should be    │ -│run: `Providers[airbyte,http]` or excluded from the full test suite:                       │ -│`Providers[-amazon,google]`                                                                │ -│(All | API | Always | CLI | Core | Other | Providers | WWW | PlainAsserts | Postgres |     │ -│MySQL | Quarantine)                                                                        │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--collect-onlyCollect tests only, do not run them.│ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot | statsd│ -│| statsd | trino)                                                                          │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--parallel-test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Other Providers WWW PlainAsserts] │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, │ -│or `sdist` if Airflow should be removed, installed from wheel packages or sdist     │ -│packages available in dist folder respectively. Implies --mount-sources `remove`.   │ -│(none | wheel | sdist | <airflow_version>)                                          │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =│ -│selected).                                                                          │ -│(selected | all | skip | remove)                                                    │ -│[default: selected]                                                                 │ -│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ -│--remove-arm-packagesRemoves arm packages from the image to test if ARM collection works│ -│--skip-docker-compose-downSkips running docker-compose down after tests│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--test-typeType of test to run. With Providers, you can specify tests of which providers should be    │ +│run: `Providers[airbyte,http]` or excluded from the full test suite:                       │ +│`Providers[-amazon,google]`                                                                │ +│(All | API | Always | CLI | Core | Other | Providers | WWW | PlainAsserts | Postgres |     │ +│MySQL | Quarantine)                                                                        │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--collect-onlyCollect tests only, do not run them.│ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot | statsd│ +│| statsd | trino)                                                                          │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--parallel-test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Other Providers WWW PlainAsserts] │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, │ +│or `sdist` if Airflow should be removed, installed from wheel packages or sdist     │ +│packages available in dist folder respectively. Implies --mount-sources `remove`.   │ +│(none | wheel | sdist | <airflow_version>)                                          │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =│ +│selected).                                                                          │ +│(selected | all | skip | remove)                                                    │ +│[default: selected]                                                                 │ +│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ +│--remove-arm-packagesRemoves arm packages from the image to test if ARM collection works│ +│--skip-docker-compose-downSkips running docker-compose down after tests│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/tests/executors/test_dask_executor.py b/tests/executors/test_dask_executor.py index f64f2a7dfdc7..d8ded41e158b 100644 --- a/tests/executors/test_dask_executor.py +++ b/tests/executors/test_dask_executor.py @@ -101,6 +101,10 @@ def test_dask_executor_functions(self): executor = DaskExecutor(cluster_address=self.cluster.scheduler_address) self.assert_tasks_on_executor(executor, timeout_executor=120) + # This test is quarantined because it became rather flaky on our CI in July 2023 and reason for this + # is unknown. An issue for that was created: https://github.com/apache/airflow/issues/32778 and the + # marker should be removed while (possibly) the reason for flaky behaviour is found and fixed. + @pytest.mark.quarantined @pytest.mark.execution_timeout(180) def test_backfill_integration(self): """ From 73bc49adb17957e5bb8dee357c04534c6b41f9dd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 24 Jul 2023 07:53:52 +0800 Subject: [PATCH 283/533] Fix depends_on_past work for dynamic tasks (#32397) Co-authored-by: Zhyhimont Dmitry --- airflow/ti_deps/deps/prev_dagrun_dep.py | 129 +++++++++++++++++---- tests/ti_deps/deps/test_prev_dagrun_dep.py | 123 +++++++++++--------- 2 files changed, 174 insertions(+), 78 deletions(-) diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py b/airflow/ti_deps/deps/prev_dagrun_dep.py index 62acdbca331b..a21944d410dd 100644 --- a/airflow/ti_deps/deps/prev_dagrun_dep.py +++ b/airflow/ti_deps/deps/prev_dagrun_dep.py @@ -17,13 +17,22 @@ # under the License. from __future__ import annotations -from sqlalchemy import func +from typing import TYPE_CHECKING + +from sqlalchemy import func, literal, or_, select +from sqlalchemy.orm import Session from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI from airflow.ti_deps.deps.base_ti_dep import BaseTIDep from airflow.utils.session import provide_session from airflow.utils.state import TaskInstanceState +if TYPE_CHECKING: + from airflow.models.dagrun import DagRun + from airflow.models.operator import Operator + +_SUCCESSFUL_STATES = (TaskInstanceState.SKIPPED, TaskInstanceState.SUCCESS) + class PrevDagrunDep(BaseTIDep): """ @@ -42,8 +51,87 @@ def _push_past_deps_met_xcom_if_needed(ti: TI, dep_context): if dep_context.wait_for_past_depends_before_skipping: ti.xcom_push(key=PAST_DEPENDS_MET, value=True) + @staticmethod + def _has_tis(dagrun: DagRun, task_id: str, *, session: Session) -> bool: + """Check if a task has presence in the specified DAG run. + + This function exists for easy mocking in tests. + """ + return ( + session.scalar( + select(literal(True)) + .where(TI.dag_id == dagrun.dag_id, TI.task_id == task_id, TI.run_id == dagrun.run_id) + .limit(1) + ) + is not None + ) + + @staticmethod + def _has_any_prior_tis(ti: TI, *, session: Session) -> bool: + """Check if a task has ever been run before. + + This function exists for easy mocking in tests. + """ + return ( + session.scalar( + select(literal(True)) + .where( + TI.dag_id == ti.dag_id, + TI.task_id == ti.task_id, + TI.execution_date < ti.execution_date, + ) + .limit(1) + ) + is not None + ) + + @staticmethod + def _count_unsuccessful_tis(dagrun: DagRun, task_id: str, *, session: Session) -> int: + """Get a count of unsuccessful task instances in a given run. + + Due to historical design considerations, "unsuccessful" here means the + task instance is not in either SUCCESS or SKIPPED state. This means that + unfinished states such as RUNNING are considered unsuccessful. + + This function exists for easy mocking in tests. + """ + return session.scalar( + select(func.count()).where( + TI.dag_id == dagrun.dag_id, + TI.task_id == task_id, + TI.run_id == dagrun.run_id, + or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)), + ) + ) + + @staticmethod + def _has_unsuccessful_dependants(dagrun: DagRun, task: Operator, *, session: Session) -> bool: + """Check if any of the task's dependants are unsuccessful in a given run. + + Due to historical design considerations, "unsuccessful" here means the + task instance is not in either SUCCESS or SKIPPED state. This means that + unfinished states such as RUNNING are considered unsuccessful. + + This function exists for easy mocking in tests. + """ + if not task.downstream_task_ids: + return False + return ( + session.scalar( + select(literal(True)) + .where( + TI.dag_id == dagrun.dag_id, + TI.task_id.in_(task.downstream_task_ids), + TI.run_id == dagrun.run_id, + or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)), + ) + .limit(1) + ) + is not None + ) + @provide_session - def _get_dep_statuses(self, ti: TI, session, dep_context): + def _get_dep_statuses(self, ti: TI, session: Session, dep_context): if dep_context.ignore_depends_on_past: self._push_past_deps_met_xcom_if_needed(ti, dep_context) reason = "The context specified that the state of past DAGs could be ignored." @@ -80,20 +168,9 @@ def _get_dep_statuses(self, ti: TI, session, dep_context): yield self._passing_status(reason="This task instance was the first task instance for its task.") return - previous_ti = last_dagrun.get_task_instance(ti.task_id, map_index=ti.map_index, session=session) - if not previous_ti: + if not self._has_tis(last_dagrun, ti.task_id, session=session): if ti.task.ignore_first_depends_on_past: - has_historical_ti = ( - session.query(func.count(TI.dag_id)) - .filter( - TI.dag_id == ti.dag_id, - TI.task_id == ti.task_id, - TI.execution_date < ti.execution_date, - ) - .scalar() - > 0 - ) - if not has_historical_ti: + if not self._has_any_prior_tis(ti, session=session): self._push_past_deps_met_xcom_if_needed(ti, dep_context) yield self._passing_status( reason="ignore_first_depends_on_past is true for this task " @@ -107,22 +184,24 @@ def _get_dep_statuses(self, ti: TI, session, dep_context): ) return - if previous_ti.state not in {TaskInstanceState.SKIPPED, TaskInstanceState.SUCCESS}: - yield self._failing_status( - reason=( - f"depends_on_past is true for this task, but the previous task instance {previous_ti} " - f"is in the state '{previous_ti.state}' which is not a successful state." - ) + unsuccessful_tis_count = self._count_unsuccessful_tis(last_dagrun, ti.task_id, session=session) + if unsuccessful_tis_count > 0: + reason = ( + f"depends_on_past is true for this task, but {unsuccessful_tis_count} " + f"previous task instance(s) are not in a successful state." ) + yield self._failing_status(reason=reason) return - previous_ti.task = ti.task - if ti.task.wait_for_downstream and not previous_ti.are_dependents_done(session=session): + if ti.task.wait_for_downstream and self._has_unsuccessful_dependants( + last_dagrun, ti.task, session=session + ): yield self._failing_status( reason=( - f"The tasks downstream of the previous task instance {previous_ti} haven't completed " - f"(and wait_for_downstream is True)." + "The tasks downstream of the previous task instance(s) " + "haven't completed, and wait_for_downstream is True." ) ) return + self._push_past_deps_met_xcom_if_needed(ti, dep_context) diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py index bf641ad6b5d0..262b90027d62 100644 --- a/tests/ti_deps/deps/test_prev_dagrun_dep.py +++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -from unittest.mock import Mock +from unittest.mock import ANY, Mock, patch import pytest @@ -25,7 +25,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.ti_deps.dep_context import DepContext from airflow.ti_deps.deps.prev_dagrun_dep import PrevDagrunDep -from airflow.utils.state import State +from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.timezone import convert_to_utc, datetime from airflow.utils.types import DagRunType from tests.test_utils.db import clear_db_runs @@ -51,7 +51,7 @@ def test_first_task_run_of_new_task(self): # Old DAG run will include only TaskInstance of old_task dag.create_dagrun( run_id="old_run", - state=State.SUCCESS, + state=TaskInstanceState.SUCCESS, execution_date=old_task.start_date, run_type=DagRunType.SCHEDULED, ) @@ -67,7 +67,7 @@ def test_first_task_run_of_new_task(self): # New DAG run will include 1st TaskInstance of new_task dr = dag.create_dagrun( run_id="new_run", - state=State.RUNNING, + state=DagRunState.RUNNING, execution_date=convert_to_utc(datetime(2016, 1, 2)), run_type=DagRunType.SCHEDULED, ) @@ -75,16 +75,24 @@ def test_first_task_run_of_new_task(self): ti = dr.get_task_instance(new_task.task_id) ti.task = new_task - # this is important, we need to assert there is no previous_ti of this ti - assert ti.previous_ti is None - dep_context = DepContext(ignore_depends_on_past=False) - assert PrevDagrunDep().is_met(ti=ti, dep_context=dep_context) + dep = PrevDagrunDep() + + with patch.object(dep, "_has_any_prior_tis", Mock(return_value=False)) as mock_has_any_prior_tis: + assert dep.is_met(ti=ti, dep_context=dep_context) + mock_has_any_prior_tis.assert_called_once_with(ti, session=ANY) @pytest.mark.parametrize( - "depends_on_past, wait_for_past_depends_before_skipping, wait_for_downstream, prev_ti," - " context_ignore_depends_on_past, dep_met, past_depends_met_xcom_sent", + ( + "depends_on_past", + "wait_for_past_depends_before_skipping", + "wait_for_downstream", + "prev_tis", + "context_ignore_depends_on_past", + "expected_dep_met", + "past_depends_met_xcom_sent", + ), [ # If the task does not set depends_on_past, the previous dagrun should # be ignored, even though previous_ti would otherwise fail the dep. @@ -93,10 +101,7 @@ def test_first_task_run_of_new_task(self): False, False, False, # wait_for_downstream=True overrides depends_on_past=False. - Mock( - state=State.NONE, - **{"are_dependents_done.return_value": False}, - ), + [Mock(state=None, **{"are_dependents_done.return_value": False})], False, True, False, @@ -109,10 +114,7 @@ def test_first_task_run_of_new_task(self): False, True, False, # wait_for_downstream=True overrides depends_on_past=False. - Mock( - state=State.NONE, - **{"are_dependents_done.return_value": False}, - ), + [Mock(state=None, **{"are_dependents_done.return_value": False})], False, True, True, @@ -125,10 +127,7 @@ def test_first_task_run_of_new_task(self): True, False, False, - Mock( - state=State.SUCCESS, - **{"are_dependents_done.return_value": True}, - ), + [Mock(state=TaskInstanceState.SUCCESS, **{"are_dependents_done.return_value": True})], True, True, False, @@ -141,10 +140,7 @@ def test_first_task_run_of_new_task(self): True, True, False, - Mock( - state=State.SUCCESS, - **{"are_dependents_done.return_value": True}, - ), + [Mock(state=TaskInstanceState.SUCCESS, **{"are_dependents_done.return_value": True})], True, True, True, @@ -152,19 +148,16 @@ def test_first_task_run_of_new_task(self): ), # The first task run should pass since it has no previous dagrun. # wait_for_past_depends_before_skipping is False, past_depends_met xcom should not be sent - pytest.param(True, False, False, None, False, True, False, id="first_task_run"), + pytest.param(True, False, False, [], False, True, False, id="first_task_run"), # The first task run should pass since it has no previous dagrun. # wait_for_past_depends_before_skipping is True, past_depends_met xcom should be sent - pytest.param(True, True, False, None, False, True, True, id="first_task_run"), + pytest.param(True, True, False, [], False, True, True, id="first_task_run_wait"), # Previous TI did not complete execution. This dep should fail. pytest.param( True, False, False, - Mock( - state=State.NONE, - **{"are_dependents_done.return_value": True}, - ), + [Mock(state=None, **{"are_dependents_done.return_value": True})], False, False, False, @@ -177,10 +170,7 @@ def test_first_task_run_of_new_task(self): True, False, True, - Mock( - state=State.SUCCESS, - **{"are_dependents_done.return_value": False}, - ), + [Mock(state=TaskInstanceState.SUCCESS, **{"are_dependents_done.return_value": False})], False, False, False, @@ -192,10 +182,7 @@ def test_first_task_run_of_new_task(self): True, False, True, - Mock( - state=State.SUCCESS, - **{"are_dependents_done.return_value": True}, - ), + [Mock(state=TaskInstanceState.SUCCESS, **{"are_dependents_done.return_value": True})], False, True, False, @@ -207,10 +194,7 @@ def test_first_task_run_of_new_task(self): True, True, True, - Mock( - state=State.SUCCESS, - **{"are_dependents_done.return_value": True}, - ), + [Mock(state=TaskInstanceState.SUCCESS, **{"are_dependents_done.return_value": True})], False, True, True, @@ -222,9 +206,9 @@ def test_dagrun_dep( depends_on_past, wait_for_past_depends_before_skipping, wait_for_downstream, - prev_ti, + prev_tis, context_ignore_depends_on_past, - dep_met, + expected_dep_met, past_depends_met_xcom_sent, ): task = BaseOperator( @@ -234,26 +218,59 @@ def test_dagrun_dep( start_date=datetime(2016, 1, 1), wait_for_downstream=wait_for_downstream, ) - if prev_ti: - prev_dagrun = Mock( - execution_date=datetime(2016, 1, 2), - **{"get_task_instance.return_value": prev_ti}, - ) + if prev_tis: + prev_dagrun = Mock(execution_date=datetime(2016, 1, 2)) else: prev_dagrun = None + dagrun = Mock( **{ "get_previous_scheduled_dagrun.return_value": prev_dagrun, "get_previous_dagrun.return_value": prev_dagrun, }, ) - ti = Mock(task=task, **{"get_dagrun.return_value": dagrun, "xcom_push.return_value": None}) + ti = Mock( + task=task, + task_id=task.task_id, + **{"get_dagrun.return_value": dagrun, "xcom_push.return_value": None}, + ) dep_context = DepContext( ignore_depends_on_past=context_ignore_depends_on_past, wait_for_past_depends_before_skipping=wait_for_past_depends_before_skipping, ) - assert PrevDagrunDep().is_met(ti=ti, dep_context=dep_context) == dep_met + unsuccessful_tis_count = sum( + int(ti.state not in {TaskInstanceState.SUCCESS, TaskInstanceState.SKIPPED}) for ti in prev_tis + ) + + mock_has_tis = Mock(return_value=bool(prev_tis)) + mock_has_any_prior_tis = Mock(return_value=bool(prev_tis)) + mock_count_unsuccessful_tis = Mock(return_value=unsuccessful_tis_count) + mock_has_unsuccessful_dependants = Mock(return_value=any(not ti.are_dependents_done() for ti in prev_tis)) + + dep = PrevDagrunDep() + with patch.multiple( + dep, + _has_tis=mock_has_tis, + _has_any_prior_tis=mock_has_any_prior_tis, + _count_unsuccessful_tis=mock_count_unsuccessful_tis, + _has_unsuccessful_dependants=mock_has_unsuccessful_dependants, + ): + actual_dep_met = dep.is_met(ti=ti, dep_context=dep_context) + + mock_has_any_prior_tis.assert_not_called() + if depends_on_past and not context_ignore_depends_on_past and prev_tis: + mock_has_tis.assert_called_once_with(prev_dagrun, "test_task", session=ANY) + mock_count_unsuccessful_tis.assert_called_once_with(prev_dagrun, "test_task", session=ANY) + else: + mock_has_tis.assert_not_called() + mock_count_unsuccessful_tis.assert_not_called() + if depends_on_past and not context_ignore_depends_on_past and prev_tis and not unsuccessful_tis_count: + mock_has_unsuccessful_dependants.assert_called_once_with(prev_dagrun, task, session=ANY) + else: + mock_has_unsuccessful_dependants.assert_not_called() + + assert actual_dep_met == expected_dep_met if past_depends_met_xcom_sent: ti.xcom_push.assert_called_with(key="past_depends_met", value=True) else: From 0339f94a500700813fc51119ae84563cb5793e3e Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 24 Jul 2023 12:11:51 +0100 Subject: [PATCH 284/533] Fix failing dagrun query due to sqlalchemy 2.0 refactor (#32801) --- airflow/www/views.py | 7 +++---- tests/www/views/test_views_tasks.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/airflow/www/views.py b/airflow/www/views.py index d175b7be5079..359f1ca52b77 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2316,9 +2316,8 @@ def clear(self, *, session: Session = NEW_SESSION): # Lock the related dag runs to prevent from possible dead lock. # https://github.com/apache/airflow/pull/26658 - dag_runs_query = session.scalars( - select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() - ) + dag_runs_query = select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() + if start_date is None and end_date is None: dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) else: @@ -2328,7 +2327,7 @@ def clear(self, *, session: Session = NEW_SESSION): if end_date is not None: dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) - locked_dag_run_ids = dag_runs_query.all() + locked_dag_run_ids = session.scalars(dag_runs_query).all() elif task_id: if map_indexes is None: task_ids = [task_id] diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 43b229325051..d8c6a488a92b 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -93,6 +93,14 @@ def init_dagruns(app, reset_dagruns): start_date=timezone.utcnow(), state=State.RUNNING, ) + app.dag_bag.get_dag("example_task_group").create_dagrun( + run_id=DEFAULT_DAGRUN, + run_type=DagRunType.SCHEDULED, + execution_date=DEFAULT_DATE, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + start_date=timezone.utcnow(), + state=State.RUNNING, + ) yield clear_db_runs() @@ -506,12 +514,27 @@ def test_code_from_db_all_example_dags(admin_client): ), "example_bash_operator", ), + ( + "clear", + dict( + group_id="section_1", + dag_id="example_task_group", + execution_date=DEFAULT_DATE, + upstream="false", + downstream="false", + future="false", + past="false", + only_failed="false", + ), + "example_task_group", + ), ], ids=[ "paused", "failed-flash-hint", "success-flash-hint", "clear", + "clear-task-group", ], ) def test_views_post(admin_client, url, data, content): From 6313e5293280773aed7598e1befb8d371e8f5614 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 24 Jul 2023 19:23:12 +0800 Subject: [PATCH 285/533] Add missing execute_complete method for `DatabricksRunNowOperator` (#32689) --- airflow/providers/databricks/operators/databricks.py | 7 ++++--- tests/providers/databricks/operators/test_databricks.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index fb27f0c01a11..b1c51a3f85a4 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -671,6 +671,10 @@ def execute(self, context: Context): else: _handle_databricks_operator_execution(self, hook, self.log, context) + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event: + _handle_deferrable_databricks_operator_completion(event, self.log) + def on_kill(self): if self.run_id: self._hook.cancel_run(self.run_id) @@ -698,6 +702,3 @@ def execute(self, context): hook = self._get_hook(caller="DatabricksRunNowDeferrableOperator") self.run_id = hook.run_now(self.json) _handle_deferrable_databricks_operator_execution(self, hook, self.log, context) - - def execute_complete(self, context: dict | None, event: dict): - _handle_deferrable_databricks_operator_completion(event, self.log) diff --git a/tests/providers/databricks/operators/test_databricks.py b/tests/providers/databricks/operators/test_databricks.py index b7cc1de3048a..32413329f674 100644 --- a/tests/providers/databricks/operators/test_databricks.py +++ b/tests/providers/databricks/operators/test_databricks.py @@ -517,7 +517,7 @@ def test_execute_complete_failure(self, db_mock_class): op.execute_complete(context=None, event=event) def test_execute_complete_incorrect_event_validation_failure(self): - event = {} + event = {"event_id": "no such column"} op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID) with pytest.raises(AirflowException): op.execute_complete(context=None, event=event) @@ -951,7 +951,7 @@ def test_execute_complete_failure(self, db_mock_class): op.execute_complete(context=None, event=event) def test_execute_complete_incorrect_event_validation_failure(self): - event = {} + event = {"event_id": "no such column"} op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID, job_id=JOB_ID) with pytest.raises(AirflowException): op.execute_complete(context=None, event=event) From 6f75a8a9172725f63b14b83b4b57c4a10a4286c8 Mon Sep 17 00:00:00 2001 From: Michael Robinson <68482867+merobi-hub@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:00:57 -0400 Subject: [PATCH 286/533] improve performance of pr script in dev/stats (#32735) * add error handling in case of 404 errors to protm script Signed-off-by: Michael Robinson * add error handling in case of missing score when using pickle file Signed-off-by: Michael Robinson * reformat Signed-off-by: Michael Robinson * add UknownObjectException class to error handling Signed-off-by: Michael Robinson * fix github imports Signed-off-by: Michael Robinson * fix interaction_score Signed-off-by: Michael Robinson * remove attribute error handling Signed-off-by: Michael Robinson * remove second attribute error handling block Signed-off-by: Michael Robinson * change api endpoint for pull requests in pr script in stats Signed-off-by: Michael Robinson * reformat Signed-off-by: Michael Robinson * Update dev/stats/get_important_pr_candidates.py Co-authored-by: Wei Lee * reformat nested loop and add new line Signed-off-by: Michael Robinson * consolidate funs to decrease number of api calls Signed-off-by: Michael Robinson * add rate limit parameter and limit info to output Signed-off-by: Michael Robinson * refactor after linting Signed-off-by: Michael Robinson * refactor after linting contd Signed-off-by: Michael Robinson --------- Signed-off-by: Michael Robinson Co-authored-by: Wei Lee --- dev/stats/get_important_pr_candidates.py | 228 ++++++++++------------- 1 file changed, 100 insertions(+), 128 deletions(-) diff --git a/dev/stats/get_important_pr_candidates.py b/dev/stats/get_important_pr_candidates.py index 3aed16f13f74..738eed53c2fb 100755 --- a/dev/stats/get_important_pr_candidates.py +++ b/dev/stats/get_important_pr_candidates.py @@ -33,7 +33,6 @@ logger = logging.getLogger(__name__) - console = Console(width=400, color_system="standard") option_github_token = click.option( @@ -53,7 +52,6 @@ class PrStat: PROVIDER_SCORE = 0.8 REGULAR_SCORE = 1.0 - REVIEW_INTERACTION_VALUE = 2.0 COMMENT_INTERACTION_VALUE = 1.0 REACTION_INTERACTION_VALUE = 0.5 @@ -61,12 +59,19 @@ class PrStat: def __init__(self, g, pull_request: PullRequest): self.g = g self.pull_request = pull_request + self.title = pull_request.title self._users: set[str] = set() + self.len_comments: int = 0 + self.comment_reactions: int = 0 self.issue_nums: list[int] = [] - self.len_issue_comments = 0 - self.num_issue_comments = 0 - self.num_issue_reactions = 0 - self.protm_score = 0 + self.len_issue_comments: int = 0 + self.num_issue_comments: int = 0 + self.num_issue_reactions: int = 0 + self.num_comments: int = 0 + self.num_conv_comments: int = 0 + self.num_protm: int = 0 + self.conv_comment_reactions: int = 0 + self.interaction_score = 1.0 @property def label_score(self) -> float: @@ -77,53 +82,33 @@ def label_score(self) -> float: return PrStat.PROVIDER_SCORE return PrStat.REGULAR_SCORE - @cached_property - def num_comments(self): - """counts reviewer comments & checks for #protm tag""" - num_comments = 0 - num_protm = 0 + def calc_comments(self): + """counts reviewer comments, checks for #protm tag, counts rxns""" for comment in self.pull_request.get_comments(): self._users.add(comment.user.login) lowercase_body = comment.body.lower() if "protm" in lowercase_body: - num_protm += 1 - num_comments += 1 - self.protm_score = num_protm - return num_comments + self.num_protm += 1 + self.num_comments += 1 + if comment.body is not None: + self.len_comments += len(comment.body) + for reaction in comment.get_reactions(): + self._users.add(reaction.user.login) + self.comment_reactions += 1 - @cached_property - def num_conv_comments(self) -> int: - """counts conversational comments & checks for #protm tag""" - num_conv_comments = 0 - num_protm = 0 + def calc_conv_comments(self): + """counts conversational comments, checks for #protm tag, counts rxns""" for conv_comment in self.pull_request.get_issue_comments(): self._users.add(conv_comment.user.login) lowercase_body = conv_comment.body.lower() if "protm" in lowercase_body: - num_protm += 1 - num_conv_comments += 1 - self.protm_score = num_protm - return num_conv_comments - - @cached_property - def num_reactions(self) -> int: - """counts reactions to reviewer comments""" - reactions = 0 - for comment in self.pull_request.get_comments(): - for reaction in comment.get_reactions(): - self._users.add(reaction.user.login) - reactions += 1 - return reactions - - @cached_property - def num_conv_reactions(self) -> int: - """counts reactions to conversational comments""" - reactions = 0 - for conv_comment in self.pull_request.get_issue_comments(): + self.num_protm += 1 + self.num_conv_comments += 1 for reaction in conv_comment.get_reactions(): self._users.add(reaction.user.login) - reactions += 1 - return reactions + self.conv_comment_reactions += 1 + if conv_comment.body is not None: + self.len_issue_comments += len(conv_comment.body) @cached_property def num_reviews(self) -> int: @@ -134,22 +119,17 @@ def num_reviews(self) -> int: num_reviews += 1 return num_reviews - @cached_property def issues(self): """finds issues in PR""" if self.pull_request.body is not None: regex = r"(?<=closes: #|elated: #)\d{5}" issue_strs = re.findall(regex, self.pull_request.body) - issue_ints = [eval(s) for s in issue_strs] - self.issue_nums = issue_ints - return issue_ints + self.issue_nums = [eval(s) for s in issue_strs] - @cached_property - def issue_reactions(self) -> int: + def issue_reactions(self): """counts reactions to issue comments""" if self.issue_nums: repo = self.g.get_repo("apache/airflow") - issue_reactions = 0 for num in self.issue_nums: try: issue = repo.get_issue(num) @@ -157,43 +137,22 @@ def issue_reactions(self) -> int: continue for reaction in issue.get_reactions(): self._users.add(reaction.user.login) - issue_reactions += 1 - self.num_issue_reactions = issue_reactions - return issue_reactions - return 0 - - @cached_property - def issue_comments(self) -> int: - """counts issue comments and calculates comment length""" - if self.issue_nums: - repo = self.g.get_repo("apache/airflow") - issue_comments = 0 - len_issue_comments = 0 - for num in self.issue_nums: - try: - issue = repo.get_issue(num) - except UnknownObjectException: - continue + self.num_issue_reactions += 1 for issue_comment in issue.get_comments(): - issue_comments += 1 + self.num_issue_comments += 1 self._users.add(issue_comment.user.login) if issue_comment.body is not None: - len_issue_comments += len(issue_comment.body) - self.len_issue_comments = len_issue_comments - self.num_issue_comments = issue_comments - return issue_comments - return 0 + self.len_issue_comments += len(issue_comment.body) - @property - def interaction_score(self) -> float: + def calc_interaction_score(self): + """calculates interaction score""" interactions = ( - self.num_comments + self.num_conv_comments + self.issue_comments + self.num_comments + self.num_conv_comments + self.num_issue_comments ) * PrStat.COMMENT_INTERACTION_VALUE interactions += ( - self.num_reactions + self.num_conv_reactions + self.issue_reactions + self.comment_reactions + self.conv_comment_reactions + self.num_issue_reactions ) * PrStat.REACTION_INTERACTION_VALUE - interactions += self.num_reviews * PrStat.REVIEW_INTERACTION_VALUE - return interactions + self.interaction_score += interactions + self.num_reviews * PrStat.REVIEW_INTERACTION_VALUE @cached_property def num_interacting_users(self) -> int: @@ -232,25 +191,18 @@ def change_score(self) -> float: @cached_property def comment_length(self) -> int: - length = 0 - for comment in self.pull_request.get_comments(): - if comment.body is not None: - length += len(comment.body) + rev_length = 0 for comment in self.pull_request.get_review_comments(): if comment.body is not None: - length += len(comment.body) - for conv_comment in self.pull_request.get_issue_comments(): - if conv_comment.body is not None: - length += len(conv_comment.body) - length += self.len_issue_comments - return length + rev_length += len(comment.body) + return self.len_comments + self.len_issue_comments + rev_length @property def length_score(self) -> float: score = 1.0 - if self.comment_length > 3000: + if self.len_comments > 3000: score *= 1.3 - if self.comment_length < 200: + if self.len_comments < 200: score *= 0.8 if self.body_length > 2000: score *= 1.4 @@ -260,6 +212,9 @@ def length_score(self) -> float: score *= 0.4 return round(score, 3) + def adjust_interaction_score(self): + self.interaction_score *= min(self.num_protm + 1, 3) + @property def score(self): # @@ -281,12 +236,15 @@ def score(self): # # Weight PRs with protm tags more heavily: # If there is at least one protm tag, multiply the interaction score by the number of tags, up to 3. - interaction_score = self.interaction_score - interaction_score *= min(self.protm_score + 1, 3) + # + self.calc_comments() + self.calc_conv_comments() + self.calc_interaction_score() + self.adjust_interaction_score() return round( 1.0 - * interaction_score + * self.interaction_score * self.label_score * self.length_score * self.change_score @@ -295,7 +253,7 @@ def score(self): ) def __str__(self) -> str: - if self.protm_score > 0: + if self.num_protm > 0: return ( "[magenta]##Tagged PR## [/]" f"Score: {self.score:.2f}: PR{self.pull_request.number}" @@ -312,7 +270,7 @@ def __str__(self) -> str: ) def verboseStr(self) -> str: - if self.protm_score > 0: + if self.num_protm > 0: console.print("********************* Tagged with '#protm' *********************", style="magenta") return ( f"-- Created at [bright_blue]{self.pull_request.created_at}[/], " @@ -320,14 +278,14 @@ def verboseStr(self) -> str: f"-- Label score: [green]{self.label_score}[/]\n" f"-- Length score: [green]{self.length_score}[/] " f"(body length: {self.body_length}, " - f"comment length: {self.comment_length})\n" + f"comment length: {self.len_comments})\n" f"-- Interaction score: [green]{self.interaction_score}[/] " f"(users interacting: {self.num_interacting_users}, " f"reviews: {self.num_reviews}, " f"review comments: {self.num_comments}, " - f"review reactions: {self.num_reactions}, " + f"review reactions: {self.comment_reactions}, " f"non-review comments: {self.num_conv_comments}, " - f"non-review reactions: {self.num_conv_reactions}, " + f"non-review reactions: {self.conv_comment_reactions}, " f"issue comments: {self.num_issue_comments}, " f"issue reactions: {self.num_issue_reactions})\n" f"-- Change score: [green]{self.change_score}[/] " @@ -359,6 +317,11 @@ def verboseStr(self) -> str: @click.option("--save", type=click.File("wb"), help="Save PR data to a pickle file") @click.option("--load", type=click.File("rb"), help="Load PR data from a file and recalculate scores") @click.option("--verbose", is_flag="True", help="Print scoring details") +@click.option( + "--rate-limit", + is_flag="True", + help="Print API rate limit reset time using system time, and requests remaining", +) def main( github_token: str, date_start: datetime, @@ -367,67 +330,76 @@ def main( date_end: datetime, top_number: int, verbose: bool, + rate_limit: bool, ): + g = Github(github_token) + + if rate_limit: + r = g.get_rate_limit() + requests_remaining: int = r.core.remaining + console.print( + f"[blue]GitHub API Rate Limit Info\n" + f"[green]Requests remaining: [red]{requests_remaining}\n" + f"[green]Reset time: [blue]{r.core.reset.astimezone()}" + ) + selected_prs: list[PrStat] = [] if load: console.print("Loading PRs from cache and recalculating scores.") selected_prs = pickle.load(load, encoding="bytes") issue_num = 0 - for pr_stat in selected_prs: + for pr in selected_prs: issue_num += 1 console.print( - f"[green]Loading PR: #{pr_stat.pull_request.number} `{pr_stat.pull_request.title}`.[/]" - f" Score: {pr_stat.score}." - f" Url: {pr_stat.pull_request.html_url}" + f"[green]Loading PR: #{pr.pull_request.number} `{pr.pull_request.title}`.[/]" + f" Score: {pr.score}." + f" Url: {pr.pull_request.html_url}" ) if verbose: - console.print(pr_stat.verboseStr()) + console.print(pr.verboseStr()) else: console.print(f"Finding best candidate PRs between {date_start} and {date_end}.") - g = Github(github_token) repo = g.get_repo("apache/airflow") - pulls = repo.get_pulls(state="closed", sort="created", direction="desc") + commits = repo.get_commits(since=date_start, until=date_end) + pulls: list[PullRequest] = [pull for commit in commits for pull in commit.get_pulls()] issue_num = 0 - for pr in pulls: - if not pr.merged: - continue - - if not (date_start < pr.merged_at < date_end): - console.print( - f"[bright_blue]Skipping {pr.number} {pr.title} as it was not " - f"merged between {date_start} and {date_end}]" - ) - continue - - if pr.merged_at < date_start: - console.print("[bright_blue]Completed selecting candidates") - break - + scores: dict = {} + for pull in pulls: + p = PrStat(g=g, pull_request=pull) # type: ignore + scores.update({pull.number: [p.score, pull.title]}) issue_num += 1 - pr_stat = PrStat(pull_request=pr, g=g) # type: ignore console.print( - f"[green]Selecting PR: #{pr.number} `{pr.title}` as candidate.[/]" - f" Score: {pr_stat.score}." - f" Url: {pr.html_url}" + f"[green]Selecting PR: #{pull.number} `{pull.title}` as candidate.[/]" + f" Score: {scores[pull.number][0]}." + f" Url: {pull.html_url}" ) if verbose: - console.print(pr_stat.verboseStr()) + console.print(p.verboseStr()) - selected_prs.append(pr_stat) + selected_prs.append(p) if issue_num == MAX_PR_CANDIDATES: console.print(f"[red]Reached {MAX_PR_CANDIDATES}. Stopping") break console.print(f"Top {top_number} out of {issue_num} PRs:") - for pr_stat in sorted(selected_prs, key=lambda s: -s.score)[:top_number]: - console.print(f" * {pr_stat}") + for pr_scored in sorted(scores.items(), key=lambda s: s[1], reverse=True)[:top_number]: + console.print(f"[green] * PR #{pr_scored[0]}: {pr_scored[1][1]}. Score: [magenta]{pr_scored[1][0]}") if save: pickle.dump(selected_prs, save) + if rate_limit: + r = g.get_rate_limit() + console.print( + f"[blue]GitHub API Rate Limit Info\n" + f"[green]Requests remaining: [red]{r.core.remaining}\n" + f"[green]Requests made: [red]{requests_remaining - r.core.remaining}\n" + f"[green]Reset time: [blue]{r.core.reset.astimezone()}" + ) + if __name__ == "__main__": main() From d05e42e5d2081909c9c33de4bd4dfb759ac860c1 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Mon, 24 Jul 2023 17:13:14 +0300 Subject: [PATCH 287/533] Add `ExternalBranchPythonOperator` (#32787) --- airflow/operators/python.py | 15 +++ tests/operators/test_python.py | 206 +++++++++++++++++++++++++++++++-- 2 files changed, 213 insertions(+), 8 deletions(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 622e901984b2..574fe5b0e34f 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -766,6 +766,21 @@ def _get_airflow_version_from_target_env(self) -> str | None: return None +class ExternalBranchPythonOperator(ExternalPythonOperator, SkipMixin): + """ + A workflow can "branch" or follow a path after the execution of this task, + Extends ExternalPythonOperator, so expects to get Python: + virtualenv that should be used (in ``VENV/bin`` folder). Should be absolute path, + so it can run on separate virtualenv similarly to ExternalPythonOperator. + """ + + def execute(self, context: Context) -> Any: + branch = super().execute(context) + self.log.info("Branch callable return %s", branch) + self.skip_all_except(context["ti"], branch) + return branch + + def get_current_context() -> Context: """ Retrieve the execution context dictionary without altering user method's signature. diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index d262aa0e4694..534a6be5a992 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -41,6 +41,7 @@ from airflow.operators.empty import EmptyOperator from airflow.operators.python import ( BranchPythonOperator, + ExternalBranchPythonOperator, ExternalPythonOperator, PythonOperator, PythonVirtualenvOperator, @@ -308,7 +309,11 @@ def setup_tests(self): def test_with_dag_run(self): with self.dag: - branch_op = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: "branch_1") + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) branch_op >> [self.branch_1, self.branch_2] dr = self.create_dag_run() @@ -319,7 +324,11 @@ def test_with_dag_run(self): def test_with_skip_in_branch_downstream_dependencies(self): with self.dag: - branch_op = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: "branch_1") + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) branch_op >> self.branch_1 >> self.branch_2 branch_op >> self.branch_2 @@ -331,7 +340,11 @@ def test_with_skip_in_branch_downstream_dependencies(self): def test_with_skip_in_branch_downstream_dependencies2(self): with self.dag: - branch_op = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: "branch_2") + + def f(): + return "branch_2" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) branch_op >> self.branch_1 >> self.branch_2 branch_op >> self.branch_2 @@ -343,7 +356,11 @@ def test_with_skip_in_branch_downstream_dependencies2(self): def test_xcom_push(self): with self.dag: - branch_op = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: "branch_1") + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) branch_op >> [self.branch_1, self.branch_2] dr = self.create_dag_run() @@ -361,7 +378,11 @@ def test_clear_skipped_downstream_task(self): should not cause it to be executed. """ with self.dag: - branch_op = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: "branch_1") + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) branches = [self.branch_1, self.branch_2] branch_op >> branches @@ -392,12 +413,18 @@ def test_clear_skipped_downstream_task(self): self.assert_expected_task_states(dr, expected_states) def test_raise_exception_on_no_accepted_type_return(self): - ti = self.create_ti(lambda: 5) + def f(): + return 5 + + ti = self.create_ti(f) with pytest.raises(AirflowException, match="must be either None, a task ID, or an Iterable of IDs"): ti.run() def test_raise_exception_on_invalid_task_id(self): - ti = self.create_ti(lambda: "some_task_id") + def f(): + return "some_task_id" + + ti = self.create_ti(f) with pytest.raises(AirflowException, match="Invalid tasks found: {'some_task_id'}"): ti.run() @@ -413,7 +440,11 @@ def test_empty_branch(self, choice, expected_states): Tests that BranchPythonOperator handles empty branches properly. """ with self.dag: - branch = BranchPythonOperator(task_id=self.task_id, python_callable=lambda: choice) + + def f(): + return choice + + branch = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) task1 = EmptyOperator(task_id="task1") join = EmptyOperator(task_id="join", trigger_rule="none_failed_min_one_success") @@ -1065,6 +1096,165 @@ def f(): task._read_result(path=mock.Mock()) +class TestExternalBranchPythonOperator(BaseTestPythonVirtualenvOperator): + opcls = ExternalBranchPythonOperator + + @pytest.fixture(autouse=True) + def setup_tests(self): + self.branch_1 = EmptyOperator(task_id="branch_1") + self.branch_2 = EmptyOperator(task_id="branch_2") + + @staticmethod + def default_kwargs(*, python_version=sys.version_info[0], **kwargs): + kwargs["python"] = sys.executable + return kwargs + + def test_with_args(self): + def f(a, b, c=False, d=False): + if a == 0 and b == 1 and c and not d: + return True + else: + raise Exception + + with pytest.raises(AirflowException, match="but got 'bool'"): + self.run_as_task(f, op_args=[0, 1], op_kwargs={"c": True}) + + def test_return_false(self): + def f(): + return False + + with pytest.raises(AirflowException, match="but got 'bool'"): + self.run_as_task(f) + + def test_context(self): + def f(templates_dict): + return templates_dict["ds"] + + with pytest.raises(AirflowException, match="Invalid tasks found:"): + self.run_as_task(f, templates_dict={"ds": "{{ ds }}"}) + + def test_with_dag_run(self): + with self.dag: + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) + branch_op >> [self.branch_1, self.branch_2] + + dr = self.create_dag_run() + branch_op.run(start_date=self.default_date, end_date=self.default_date) + self.assert_expected_task_states( + dr, {self.task_id: State.SUCCESS, "branch_1": State.NONE, "branch_2": State.SKIPPED} + ) + + def test_with_skip_in_branch_downstream_dependencies(self): + with self.dag: + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) + branch_op >> self.branch_1 >> self.branch_2 + branch_op >> self.branch_2 + + dr = self.create_dag_run() + branch_op.run(start_date=self.default_date, end_date=self.default_date) + self.assert_expected_task_states( + dr, {self.task_id: State.SUCCESS, "branch_1": State.NONE, "branch_2": State.NONE} + ) + + def test_with_skip_in_branch_downstream_dependencies2(self): + with self.dag: + + def f(): + return "branch_2" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) + branch_op >> self.branch_1 >> self.branch_2 + branch_op >> self.branch_2 + + dr = self.create_dag_run() + branch_op.run(start_date=self.default_date, end_date=self.default_date) + self.assert_expected_task_states( + dr, {self.task_id: State.SUCCESS, "branch_1": State.SKIPPED, "branch_2": State.NONE} + ) + + def test_xcom_push(self): + with self.dag: + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) + branch_op >> [self.branch_1, self.branch_2] + + dr = self.create_dag_run() + branch_op.run(start_date=self.default_date, end_date=self.default_date) + for ti in dr.get_task_instances(): + if ti.task_id == self.task_id: + assert ti.xcom_pull(task_ids=self.task_id) == "branch_1" + break + else: + pytest.fail(f"{self.task_id!r} not found.") + + def test_clear_skipped_downstream_task(self): + """ + After a downstream task is skipped by BranchPythonOperator, clearing the skipped task + should not cause it to be executed. + """ + with self.dag: + + def f(): + return "branch_1" + + branch_op = self.opcls(task_id=self.task_id, python_callable=f, **self.default_kwargs()) + branches = [self.branch_1, self.branch_2] + branch_op >> branches + + dr = self.create_dag_run() + branch_op.run(start_date=self.default_date, end_date=self.default_date) + for task in branches: + task.run(start_date=self.default_date, end_date=self.default_date) + + expected_states = { + self.task_id: State.SUCCESS, + "branch_1": State.SUCCESS, + "branch_2": State.SKIPPED, + } + + self.assert_expected_task_states(dr, expected_states) + + # Clear the children tasks. + tis = dr.get_task_instances() + children_tis = [ti for ti in tis if ti.task_id in branch_op.get_direct_relative_ids()] + with create_session() as session: + clear_task_instances(children_tis, session=session, dag=branch_op.dag) + + # Run the cleared tasks again. + for task in branches: + task.run(start_date=self.default_date, end_date=self.default_date) + + # Check if the states are correct after children tasks are cleared. + self.assert_expected_task_states(dr, expected_states) + + def test_raise_exception_on_no_accepted_type_return(self): + def f(): + return 5 + + ti = self.create_ti(f) + with pytest.raises(AirflowException, match="must be either None, a task ID, or an Iterable of IDs"): + ti.run() + + def test_raise_exception_on_invalid_task_id(self): + def f(): + return "some_task_id" + + ti = self.create_ti(f) + with pytest.raises(AirflowException, match="Invalid tasks found: {'some_task_id'}"): + ti.run() + + class TestCurrentContext: def test_current_context_no_context_raise(self): with pytest.raises(AirflowException): From 8012c9fce64f152b006f88497d65ea81d29571b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivica=20Kolenka=C5=A1?= Date: Mon, 24 Jul 2023 19:09:44 +0200 Subject: [PATCH 288/533] Add support for querying Redshift Serverless clusters (#32785) --- .../amazon/aws/hooks/redshift_data.py | 10 +++ .../amazon/aws/operators/redshift_data.py | 7 ++ .../amazon/aws/hooks/test_redshift_data.py | 67 ++++++++++++++++++- .../aws/operators/test_redshift_data.py | 42 ++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/redshift_data.py b/airflow/providers/amazon/aws/hooks/redshift_data.py index fddd42bd61df..a522d3e8c968 100644 --- a/airflow/providers/amazon/aws/hooks/redshift_data.py +++ b/airflow/providers/amazon/aws/hooks/redshift_data.py @@ -60,6 +60,7 @@ def execute_query( with_event: bool = False, wait_for_completion: bool = True, poll_interval: int = 10, + workgroup_name: str | None = None, ) -> str: """ Execute a statement against Amazon Redshift. @@ -74,6 +75,9 @@ def execute_query( :param with_event: indicates whether to send an event to EventBridge :param wait_for_completion: indicates whether to wait for a result, if True wait, if False don't wait :param poll_interval: how often in seconds to check the query status + :param workgroup_name: name of the Redshift Serverless workgroup. Mutually exclusive with + `cluster_identifier`. Specify this parameter to query Redshift Serverless. More info + https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-serverless.html :returns statement_id: str, the UUID of the statement """ @@ -85,6 +89,7 @@ def execute_query( "WithEvent": with_event, "SecretArn": secret_arn, "StatementName": statement_name, + "WorkgroupName": workgroup_name, } if isinstance(sql, list): kwargs["Sqls"] = sql @@ -95,6 +100,9 @@ def execute_query( statement_id = resp["Id"] + if bool(cluster_identifier) is bool(workgroup_name): + raise ValueError("Either 'cluster_identifier' or 'workgroup_name' must be specified.") + if wait_for_completion: self.wait_for_results(statement_id, poll_interval=poll_interval) @@ -127,6 +135,7 @@ def get_table_primary_key( database: str, schema: str | None = "public", cluster_identifier: str | None = None, + workgroup_name: str | None = None, db_user: str | None = None, secret_arn: str | None = None, statement_name: str | None = None, @@ -168,6 +177,7 @@ def get_table_primary_key( sql=sql, database=database, cluster_identifier=cluster_identifier, + workgroup_name=workgroup_name, db_user=db_user, secret_arn=secret_arn, statement_name=statement_name, diff --git a/airflow/providers/amazon/aws/operators/redshift_data.py b/airflow/providers/amazon/aws/operators/redshift_data.py index bf560c9973ce..126c585e3fcf 100644 --- a/airflow/providers/amazon/aws/operators/redshift_data.py +++ b/airflow/providers/amazon/aws/operators/redshift_data.py @@ -51,6 +51,9 @@ class RedshiftDataOperator(BaseOperator): if False (default) will return statement ID :param aws_conn_id: aws connection to use :param region: aws region to use + :param workgroup_name: name of the Redshift Serverless workgroup. Mutually exclusive with + `cluster_identifier`. Specify this parameter to query Redshift Serverless. More info + https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-serverless.html """ template_fields = ( @@ -62,6 +65,7 @@ class RedshiftDataOperator(BaseOperator): "statement_name", "aws_conn_id", "region", + "workgroup_name", ) template_ext = (".sql",) template_fields_renderers = {"sql": "sql"} @@ -82,12 +86,14 @@ def __init__( return_sql_result: bool = False, aws_conn_id: str = "aws_default", region: str | None = None, + workgroup_name: str | None = None, **kwargs, ) -> None: super().__init__(**kwargs) self.database = database self.sql = sql self.cluster_identifier = cluster_identifier + self.workgroup_name = workgroup_name self.db_user = db_user self.parameters = parameters self.secret_arn = secret_arn @@ -119,6 +125,7 @@ def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str: database=self.database, sql=self.sql, cluster_identifier=self.cluster_identifier, + workgroup_name=self.workgroup_name, db_user=self.db_user, parameters=self.parameters, secret_arn=self.secret_arn, diff --git a/tests/providers/amazon/aws/hooks/test_redshift_data.py b/tests/providers/amazon/aws/hooks/test_redshift_data.py index 92920f704269..cc174a872cbd 100644 --- a/tests/providers/amazon/aws/hooks/test_redshift_data.py +++ b/tests/providers/amazon/aws/hooks/test_redshift_data.py @@ -20,6 +20,8 @@ import logging from unittest import mock +import pytest + from airflow.providers.amazon.aws.hooks.redshift_data import RedshiftDataHook SQL = "sql" @@ -39,22 +41,50 @@ def test_conn_attribute(self): @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") def test_execute_without_waiting(self, mock_conn): mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + cluster_identifier = "cluster_identifier" hook = RedshiftDataHook() hook.execute_query( database=DATABASE, + cluster_identifier=cluster_identifier, sql=SQL, wait_for_completion=False, ) mock_conn.execute_statement.assert_called_once_with( Database=DATABASE, + ClusterIdentifier=cluster_identifier, Sql=SQL, WithEvent=False, ) mock_conn.describe_statement.assert_not_called() + @pytest.mark.parametrize( + "cluster_identifier, workgroup_name", + [ + (None, None), + ("some_cluster", "some_workgroup"), + ], + ) @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") - def test_execute_with_all_parameters(self, mock_conn): + def test_execute_requires_either_cluster_identifier_or_workgroup_name( + self, mock_conn, cluster_identifier, workgroup_name + ): + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + cluster_identifier = "cluster_identifier" + workgroup_name = "workgroup_name" + + with pytest.raises(ValueError): + hook = RedshiftDataHook() + hook.execute_query( + database=DATABASE, + cluster_identifier=cluster_identifier, + workgroup_name=workgroup_name, + sql=SQL, + wait_for_completion=False, + ) + + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute_with_all_parameters_cluster_identifier(self, mock_conn): cluster_identifier = "cluster_identifier" db_user = "db_user" secret_arn = "secret_arn" @@ -88,6 +118,41 @@ def test_execute_with_all_parameters(self, mock_conn): Id=STATEMENT_ID, ) + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") + def test_execute_with_all_parameters_workgroup_name(self, mock_conn): + workgroup_name = "workgroup_name" + db_user = "db_user" + secret_arn = "secret_arn" + statement_name = "statement_name" + parameters = [{"name": "id", "value": "1"}] + mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} + mock_conn.describe_statement.return_value = {"Status": "FINISHED"} + + hook = RedshiftDataHook() + hook.execute_query( + sql=SQL, + database=DATABASE, + workgroup_name=workgroup_name, + db_user=db_user, + secret_arn=secret_arn, + statement_name=statement_name, + parameters=parameters, + ) + + mock_conn.execute_statement.assert_called_once_with( + Database=DATABASE, + Sql=SQL, + WorkgroupName=workgroup_name, + DbUser=db_user, + SecretArn=secret_arn, + StatementName=statement_name, + Parameters=parameters, + WithEvent=False, + ) + mock_conn.describe_statement.assert_called_once_with( + Id=STATEMENT_ID, + ) + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.conn") def test_batch_execute(self, mock_conn): mock_conn.execute_statement.return_value = {"Id": STATEMENT_ID} diff --git a/tests/providers/amazon/aws/operators/test_redshift_data.py b/tests/providers/amazon/aws/operators/test_redshift_data.py index be77f96c307d..e5a851fe737e 100644 --- a/tests/providers/amazon/aws/operators/test_redshift_data.py +++ b/tests/providers/amazon/aws/operators/test_redshift_data.py @@ -32,6 +32,7 @@ class TestRedshiftDataOperator: @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") def test_execute(self, mock_exec_query): cluster_identifier = "cluster_identifier" + workgroup_name = None db_user = "db_user" secret_arn = "secret_arn" statement_name = "statement_name" @@ -57,6 +58,46 @@ def test_execute(self, mock_exec_query): sql=SQL, database=DATABASE, cluster_identifier=cluster_identifier, + workgroup_name=workgroup_name, + db_user=db_user, + secret_arn=secret_arn, + statement_name=statement_name, + parameters=parameters, + with_event=False, + wait_for_completion=wait_for_completion, + poll_interval=poll_interval, + ) + + @mock.patch("airflow.providers.amazon.aws.hooks.redshift_data.RedshiftDataHook.execute_query") + def test_execute_with_workgroup_name(self, mock_exec_query): + cluster_identifier = None + workgroup_name = "workgroup_name" + db_user = "db_user" + secret_arn = "secret_arn" + statement_name = "statement_name" + parameters = [{"name": "id", "value": "1"}] + poll_interval = 5 + wait_for_completion = True + + operator = RedshiftDataOperator( + aws_conn_id=CONN_ID, + task_id=TASK_ID, + sql=SQL, + database=DATABASE, + workgroup_name=workgroup_name, + db_user=db_user, + secret_arn=secret_arn, + statement_name=statement_name, + parameters=parameters, + wait_for_completion=True, + poll_interval=poll_interval, + ) + operator.execute(None) + mock_exec_query.assert_called_once_with( + sql=SQL, + database=DATABASE, + cluster_identifier=cluster_identifier, + workgroup_name=workgroup_name, db_user=db_user, secret_arn=secret_arn, statement_name=statement_name, @@ -85,6 +126,7 @@ def test_on_kill_with_query(self, mock_conn): operator = RedshiftDataOperator( aws_conn_id=CONN_ID, task_id=TASK_ID, + cluster_identifier="cluster_identifier", sql=SQL, database=DATABASE, wait_for_completion=False, From 282854b55fd8b0ef46ae0b9032b67654b4789249 Mon Sep 17 00:00:00 2001 From: ieunea1128 <137749867+ieunea1128@users.noreply.github.com> Date: Tue, 25 Jul 2023 02:10:07 +0900 Subject: [PATCH 289/533] Add endpoint_url in test_connection (#32664) --- .../providers/amazon/aws/hooks/base_aws.py | 6 +++++- .../amazon/aws/hooks/test_base_aws.py | 20 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py index 94782401ac13..01748089c893 100644 --- a/airflow/providers/amazon/aws/hooks/base_aws.py +++ b/airflow/providers/amazon/aws/hooks/base_aws.py @@ -809,7 +809,11 @@ def test_connection(self): """ try: session = self.get_session() - conn_info = session.client("sts").get_caller_identity() + test_endpoint_url = self.conn_config.extra_config.get("test_endpoint_url") + conn_info = session.client( + "sts", + endpoint_url=test_endpoint_url, + ).get_caller_identity() metadata = conn_info.pop("ResponseMetadata", {}) if metadata.get("HTTPStatusCode") != 200: try: diff --git a/tests/providers/amazon/aws/hooks/test_base_aws.py b/tests/providers/amazon/aws/hooks/test_base_aws.py index 8ba8b6b81266..084c28238207 100644 --- a/tests/providers/amazon/aws/hooks/test_base_aws.py +++ b/tests/providers/amazon/aws/hooks/test_base_aws.py @@ -295,6 +295,7 @@ def test_get_credentials_from_role_arn(self, conn_id, conn_extra, region_name): conn = AwsConnectionWrapper.from_connection_metadata(conn_id=conn_id, extra=extra) sf = BaseSessionFactory(conn=conn) session = sf.create_session() + assert session.region_name == region_name # Validate method of botocore credentials provider. # It shouldn't be 'explicit' which refers in this case to initial credentials. @@ -858,7 +859,7 @@ def test_hook_connection_test_failed(self, mock_boto3_session): result, message = hook.test_connection() assert not result assert message == json.dumps(response_metadata) - mock_sts_client.assert_called_once_with("sts") + mock_sts_client.assert_called_once_with("sts", endpoint_url=None) def mock_error(): raise ConnectionError("Test Error") @@ -872,6 +873,23 @@ def mock_error(): assert hook.client_type == "ec2" + @mock_sts + @pytest.mark.parametrize( + "test_endpoint_url, result_url", + [ + (None, "https://sts.amazonaws.com"), + ("https://sts.us-east-1.amazonaws.com", "https://sts.us-east-1.amazonaws.com"), + ], + ) + def test_hook_connection_endpoint_url_valid(self, test_endpoint_url, result_url): + """Test if test_endpoint_url is valid in test connection""" + conn = AwsConnectionWrapper.from_connection_metadata(conn_id=MOCK_AWS_CONN_ID) + sf = BaseSessionFactory(conn=conn) + session = sf.create_session() + client = session.client("sts", endpoint_url=test_endpoint_url) + + assert client._endpoint.host == result_url + @mock.patch.dict(os.environ, {f"AIRFLOW_CONN_{MOCK_AWS_CONN_ID.upper()}": "aws://"}) def test_conn_config_conn_id_exists(self): """Test retrieve connection config if aws_conn_id exists.""" From 1706f058582a0668555eee874bcf4ccdc248acbb Mon Sep 17 00:00:00 2001 From: Syed Hussain <103602455+syedahsn@users.noreply.github.com> Date: Mon, 24 Jul 2023 10:30:37 -0700 Subject: [PATCH 290/533] EMR serverless Create/Start/Stop/Delete Application deferrable mode (#32513) * Minor code refactoring * Add type annotations * update doc string about default value of deferrable --- airflow/providers/amazon/aws/hooks/emr.py | 26 ++- airflow/providers/amazon/aws/operators/emr.py | 172 ++++++++++++++++-- airflow/providers/amazon/aws/triggers/emr.py | 144 ++++++++++++++- .../operators/emr/emr_serverless.rst | 6 + .../aws/operators/test_emr_serverless.py | 73 +++++++- .../aws/triggers/test_emr_serverless.py | 29 +++ 6 files changed, 417 insertions(+), 33 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/emr.py b/airflow/providers/amazon/aws/hooks/emr.py index 2fb35a34a636..dea61c0858d5 100644 --- a/airflow/providers/amazon/aws/hooks/emr.py +++ b/airflow/providers/amazon/aws/hooks/emr.py @@ -256,9 +256,14 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: kwargs["client_type"] = "emr-serverless" super().__init__(*args, **kwargs) - def cancel_running_jobs(self, application_id: str, waiter_config: dict = {}): + def cancel_running_jobs( + self, application_id: str, waiter_config: dict | None = None, wait_for_completion: bool = True + ) -> int: """ - List all jobs in an intermediate state, cancel them, then wait for those jobs to reach terminal state. + Cancel jobs in an intermediate state, and return the number of cancelled jobs. + + If wait_for_completion is True, then the method will wait until all jobs are + cancelled before returning. Note: if new jobs are triggered while this operation is ongoing, it's going to time out and return an error. @@ -284,13 +289,16 @@ def cancel_running_jobs(self, application_id: str, waiter_config: dict = {}): ) for job_id in job_ids: self.conn.cancel_job_run(applicationId=application_id, jobRunId=job_id) - if count > 0: - self.log.info("now waiting for the %s cancelled job(s) to terminate", count) - self.get_waiter("no_job_running").wait( - applicationId=application_id, - states=list(self.JOB_INTERMEDIATE_STATES.union({"CANCELLING"})), - WaiterConfig=waiter_config, - ) + if wait_for_completion: + if count > 0: + self.log.info("now waiting for the %s cancelled job(s) to terminate", count) + self.get_waiter("no_job_running").wait( + applicationId=application_id, + states=list(self.JOB_INTERMEDIATE_STATES.union({"CANCELLING"})), + WaiterConfig=waiter_config or {}, + ) + + return count class EmrContainerHook(AwsBaseHook): diff --git a/airflow/providers/amazon/aws/operators/emr.py b/airflow/providers/amazon/aws/operators/emr.py index c01dbbbe91d6..f75b12327fe7 100644 --- a/airflow/providers/amazon/aws/operators/emr.py +++ b/airflow/providers/amazon/aws/operators/emr.py @@ -33,8 +33,12 @@ EmrAddStepsTrigger, EmrContainerTrigger, EmrCreateJobFlowTrigger, + EmrServerlessCancelJobsTrigger, + EmrServerlessCreateApplicationTrigger, + EmrServerlessDeleteApplicationTrigger, EmrServerlessStartApplicationTrigger, EmrServerlessStartJobTrigger, + EmrServerlessStopApplicationTrigger, EmrTerminateJobFlowTrigger, ) from airflow.providers.amazon.aws.utils.waiter import waiter @@ -974,7 +978,7 @@ class EmrServerlessCreateApplicationOperator(BaseOperator): :param release_label: The EMR release version associated with the application. :param job_type: The type of application you want to start, such as Spark or Hive. :param wait_for_completion: If true, wait for the Application to start before returning. Default to True. - If set to False, ``waiter_countdown`` and ``waiter_check_interval_seconds`` will only be applied when + If set to False, ``waiter_max_attempts`` and ``waiter_delay`` will only be applied when waiting for the application to be in the ``CREATED`` state. :param client_request_token: The client idempotency token of the application to create. Its value must be unique for each request. @@ -987,6 +991,9 @@ class EmrServerlessCreateApplicationOperator(BaseOperator): :waiter_max_attempts: Number of times the waiter should poll the application to check the state. If not set, the waiter will use its default value. :param waiter_delay: Number of seconds between polling the state of the application. + :param deferrable: If True, the operator will wait asynchronously for application to be created. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False, but can be overridden in config file by setting default_deferrable to True) """ def __init__( @@ -1001,6 +1008,7 @@ def __init__( waiter_check_interval_seconds: int | ArgNotSet = NOTSET, waiter_max_attempts: int | ArgNotSet = NOTSET, waiter_delay: int | ArgNotSet = NOTSET, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if waiter_check_interval_seconds is NOTSET: @@ -1032,6 +1040,7 @@ def __init__( self.config = config or {} self.waiter_max_attempts = int(waiter_max_attempts) # type: ignore[arg-type] self.waiter_delay = int(waiter_delay) # type: ignore[arg-type] + self.deferrable = deferrable super().__init__(**kwargs) self.client_request_token = client_request_token or str(uuid4()) @@ -1054,8 +1063,19 @@ def execute(self, context: Context) -> str | None: raise AirflowException(f"Application Creation failed: {response}") self.log.info("EMR serverless application created: %s", application_id) - waiter = self.hook.get_waiter("serverless_app_created") + if self.deferrable: + self.defer( + trigger=EmrServerlessCreateApplicationTrigger( + application_id=application_id, + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="start_application_deferred", + ) + waiter = self.hook.get_waiter("serverless_app_created") wait( waiter=waiter, waiter_delay=self.waiter_delay, @@ -1081,6 +1101,32 @@ def execute(self, context: Context) -> str | None: ) return application_id + def start_application_deferred(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] != "success": + raise AirflowException(f"Application {event['application_id']} failed to create") + self.log.info("Starting application %s", event["application_id"]) + self.hook.conn.start_application(applicationId=event["application_id"]) + self.defer( + trigger=EmrServerlessStartApplicationTrigger( + application_id=event["application_id"], + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="execute_complete", + ) + + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None or event["status"] != "success": + raise AirflowException(f"Trigger error: Application failed to start, event is {event}") + + self.log.info("Application %s started", event["application_id"]) + return event["application_id"] + class EmrServerlessStartJobOperator(BaseOperator): """ @@ -1312,14 +1358,21 @@ class EmrServerlessStopApplicationOperator(BaseOperator): :param application_id: ID of the EMR Serverless application to stop. :param wait_for_completion: If true, wait for the Application to stop before returning. Default to True :param aws_conn_id: AWS connection to use - :param waiter_countdown: Total amount of time, in seconds, the operator will wait for + :param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for the application be stopped. Defaults to 5 minutes. - :param waiter_check_interval_seconds: Number of seconds between polling the state of the application. - Defaults to 30 seconds. + :param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state of the + application. Defaults to 60 seconds. :param force_stop: If set to True, any job for that app that is not in a terminal state will be cancelled. Otherwise, trying to stop an app with running jobs will return an error. If you want to wait for the jobs to finish gracefully, use :class:`airflow.providers.amazon.aws.sensors.emr.EmrServerlessJobSensor` + :waiter_max_attempts: Number of times the waiter should poll the application to check the state. + Default is 25. + :param waiter_delay: Number of seconds between polling the state of the application. + Default is 60 seconds. + :param deferrable: If True, the operator will wait asynchronously for the application to stop. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False, but can be overridden in config file by setting default_deferrable to True) """ template_fields: Sequence[str] = ("application_id",) @@ -1334,6 +1387,7 @@ def __init__( waiter_max_attempts: int | ArgNotSet = NOTSET, waiter_delay: int | ArgNotSet = NOTSET, force_stop: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if waiter_check_interval_seconds is NOTSET: @@ -1359,10 +1413,11 @@ def __init__( ) self.aws_conn_id = aws_conn_id self.application_id = application_id - self.wait_for_completion = wait_for_completion + self.wait_for_completion = False if deferrable else wait_for_completion self.waiter_max_attempts = int(waiter_max_attempts) # type: ignore[arg-type] self.waiter_delay = int(waiter_delay) # type: ignore[arg-type] self.force_stop = force_stop + self.deferrable = deferrable super().__init__(**kwargs) @cached_property @@ -1374,16 +1429,46 @@ def execute(self, context: Context) -> None: self.log.info("Stopping application: %s", self.application_id) if self.force_stop: - self.hook.cancel_running_jobs( - self.application_id, - waiter_config={ - "Delay": self.waiter_delay, - "MaxAttempts": self.waiter_max_attempts, - }, + count = self.hook.cancel_running_jobs( + application_id=self.application_id, + wait_for_completion=False, ) + if count > 0: + self.log.info("now waiting for the %s cancelled job(s) to terminate", count) + if self.deferrable: + self.defer( + trigger=EmrServerlessCancelJobsTrigger( + application_id=self.application_id, + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="stop_application", + ) + self.hook.get_waiter("no_job_running").wait( + applicationId=self.application_id, + states=list(self.hook.JOB_INTERMEDIATE_STATES.union({"CANCELLING"})), + WaiterConfig={ + "Delay": self.waiter_delay, + "MaxAttempts": self.waiter_max_attempts, + }, + ) + else: + self.log.info("no running jobs found with application ID %s", self.application_id) self.hook.conn.stop_application(applicationId=self.application_id) - + if self.deferrable: + self.defer( + trigger=EmrServerlessStopApplicationTrigger( + application_id=self.application_id, + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="execute_complete", + ) if self.wait_for_completion: waiter = self.hook.get_waiter("serverless_app_stopped") wait( @@ -1397,6 +1482,30 @@ def execute(self, context: Context) -> None: ) self.log.info("EMR serverless application %s stopped successfully", self.application_id) + def stop_application(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "success": + self.hook.conn.stop_application(applicationId=self.application_id) + self.defer( + trigger=EmrServerlessStopApplicationTrigger( + application_id=self.application_id, + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="execute_complete", + ) + + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "success": + self.log.info("EMR serverless application %s stopped successfully", self.application_id) + class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperator): """ @@ -1410,10 +1519,17 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato :param wait_for_completion: If true, wait for the Application to be deleted before returning. Defaults to True. Note that this operator will always wait for the application to be STOPPED first. :param aws_conn_id: AWS connection to use - :param waiter_countdown: Total amount of time, in seconds, the operator will wait for each step of first, - the application to be stopped, and then deleted. Defaults to 25 minutes. - :param waiter_check_interval_seconds: Number of seconds between polling the state of the application. + :param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for each + step of first,the application to be stopped, and then deleted. Defaults to 25 minutes. + :param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state + of the application. Defaults to 60 seconds. + :waiter_max_attempts: Number of times the waiter should poll the application to check the state. + Defaults to 25. + :param waiter_delay: Number of seconds between polling the state of the application. Defaults to 60 seconds. + :param deferrable: If True, the operator will wait asynchronously for application to be deleted. + This implies waiting for completion. This mode requires aiobotocore module to be installed. + (default: False, but can be overridden in config file by setting default_deferrable to True) :param force_stop: If set to True, any job for that app that is not in a terminal state will be cancelled. Otherwise, trying to delete an app with running jobs will return an error. If you want to wait for the jobs to finish gracefully, use @@ -1432,6 +1548,7 @@ def __init__( waiter_max_attempts: int | ArgNotSet = NOTSET, waiter_delay: int | ArgNotSet = NOTSET, force_stop: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): if waiter_check_interval_seconds is NOTSET: @@ -1467,6 +1584,8 @@ def __init__( force_stop=force_stop, **kwargs, ) + self.deferrable = deferrable + self.wait_for_delete_completion = False if deferrable else wait_for_completion def execute(self, context: Context) -> None: # super stops the app (or makes sure it's already stopped) @@ -1478,7 +1597,19 @@ def execute(self, context: Context) -> None: if response["ResponseMetadata"]["HTTPStatusCode"] != 200: raise AirflowException(f"Application deletion failed: {response}") - if self.wait_for_delete_completion: + if self.deferrable: + self.defer( + trigger=EmrServerlessDeleteApplicationTrigger( + application_id=self.application_id, + aws_conn_id=self.aws_conn_id, + waiter_delay=self.waiter_delay, + waiter_max_attempts=self.waiter_max_attempts, + ), + timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay), + method_name="execute_complete", + ) + + elif self.wait_for_delete_completion: waiter = self.hook.get_waiter("serverless_app_terminated") wait( @@ -1492,3 +1623,10 @@ def execute(self, context: Context) -> None: ) self.log.info("EMR serverless application deleted") + + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None: + if event is None: + self.log.error("Trigger error: event is None") + raise AirflowException("Trigger error: event is None") + elif event["status"] == "success": + self.log.info("EMR serverless application %s deleted successfully", self.application_id) diff --git a/airflow/providers/amazon/aws/triggers/emr.py b/airflow/providers/amazon/aws/triggers/emr.py index 471a8a747a08..d7d9844af33f 100644 --- a/airflow/providers/amazon/aws/triggers/emr.py +++ b/airflow/providers/amazon/aws/triggers/emr.py @@ -285,6 +285,41 @@ def hook(self) -> AwsGenericHook: return EmrHook(self.aws_conn_id) +class EmrServerlessCreateApplicationTrigger(AwsBaseWaiterTrigger): + """ + Poll an Emr Serverless application and wait for it to be created. + + :param application_id: The ID of the application being polled. + :waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made + :param aws_conn_id: Reference to AWS connection id + """ + + def __init__( + self, + application_id: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, + aws_conn_id: str = "aws_default", + ) -> None: + super().__init__( + serialized_fields={"application_id": application_id}, + waiter_name="serverless_app_created", + waiter_args={"applicationId": application_id}, + failure_message="Application creation failed", + status_message="Application status is", + status_queries=["application.state", "application.stateDetails"], + return_key="application_id", + return_value=application_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return EmrServerlessHook(self.aws_conn_id) + + class EmrServerlessStartApplicationTrigger(AwsBaseWaiterTrigger): """ Poll an Emr Serverless application and wait for it to be started. @@ -301,7 +336,7 @@ def __init__( waiter_delay: int = 30, waiter_max_attempts: int = 60, aws_conn_id: str = "aws_default", - ): + ) -> None: super().__init__( serialized_fields={"application_id": application_id}, waiter_name="serverless_app_started", @@ -320,6 +355,41 @@ def hook(self) -> AwsGenericHook: return EmrServerlessHook(self.aws_conn_id) +class EmrServerlessStopApplicationTrigger(AwsBaseWaiterTrigger): + """ + Poll an Emr Serverless application and wait for it to be stopped. + + :param application_id: The ID of the application being polled. + :waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made + :param aws_conn_id: Reference to AWS connection id. + """ + + def __init__( + self, + application_id: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, + aws_conn_id: str = "aws_default", + ) -> None: + super().__init__( + serialized_fields={"application_id": application_id}, + waiter_name="serverless_app_stopped", + waiter_args={"applicationId": application_id}, + failure_message="Application failed to start", + status_message="Application status is", + status_queries=["application.state", "application.stateDetails"], + return_key="application_id", + return_value=application_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return EmrServerlessHook(self.aws_conn_id) + + class EmrServerlessStartJobTrigger(AwsBaseWaiterTrigger): """ Poll an Emr Serverless job run and wait for it to be completed. @@ -355,3 +425,75 @@ def __init__( def hook(self) -> AwsGenericHook: return EmrServerlessHook(self.aws_conn_id) + + +class EmrServerlessDeleteApplicationTrigger(AwsBaseWaiterTrigger): + """ + Poll an Emr Serverless application and wait for it to be deleted. + + :param application_id: The ID of the application being polled. + :waiter_delay: polling period in seconds to check for the status + :param waiter_max_attempts: The maximum number of attempts to be made + :param aws_conn_id: Reference to AWS connection id + """ + + def __init__( + self, + application_id: str, + waiter_delay: int = 30, + waiter_max_attempts: int = 60, + aws_conn_id: str = "aws_default", + ) -> None: + super().__init__( + serialized_fields={"application_id": application_id}, + waiter_name="serverless_app_terminated", + waiter_args={"applicationId": application_id}, + failure_message="Application failed to start", + status_message="Application status is", + status_queries=["application.state", "application.stateDetails"], + return_key="application_id", + return_value=application_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return EmrServerlessHook(self.aws_conn_id) + + +class EmrServerlessCancelJobsTrigger(AwsBaseWaiterTrigger): + """ + Trigger for canceling a list of jobs in an EMR Serverless application. + + :param application_id: EMR Serverless application ID + :param aws_conn_id: Reference to AWS connection id + :param waiter_delay: Delay in seconds between each attempt to check the status + :param waiter_max_attempts: Maximum number of attempts to check the status + """ + + def __init__( + self, + application_id: str, + aws_conn_id: str, + waiter_delay: int, + waiter_max_attempts: int, + ) -> None: + self.hook_instance = EmrServerlessHook(aws_conn_id) + states = list(self.hook_instance.JOB_INTERMEDIATE_STATES.union({"CANCELLING"})) + super().__init__( + serialized_fields={"application_id": application_id}, + waiter_name="no_job_running", + waiter_args={"applicationId": application_id, "states": states}, + failure_message="Error while waiting for jobs to cancel", + status_message="Currently running jobs", + status_queries=["jobRuns[*].applicationId", "jobRuns[*].state"], + return_key="application_id", + return_value=application_id, + waiter_delay=waiter_delay, + waiter_max_attempts=waiter_max_attempts, + aws_conn_id=aws_conn_id, + ) + + def hook(self) -> AwsGenericHook: + return self.hook_instance diff --git a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst index 76638815b8da..bcd5995e5c90 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst @@ -40,6 +40,8 @@ Create an EMR Serverless Application You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrServerlessCreateApplicationOperator` to create a new EMR Serverless Application. +This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires +the aiobotocore module to be installed. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py :language: python @@ -70,6 +72,8 @@ Stop an EMR Serverless Application You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrServerlessStopApplicationOperator` to stop an EMR Serverless Application. +This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires +the aiobotocore module to be installed. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py :language: python @@ -84,6 +88,8 @@ Delete an EMR Serverless Application You can use :class:`~airflow.providers.amazon.aws.operators.emr.EmrServerlessDeleteApplicationOperator` to delete an EMR Serverless Application. +This operator can be run in deferrable mode by passing ``deferrable=True`` as a parameter. This requires +the aiobotocore module to be installed. .. exampleinclude:: /../../tests/system/providers/amazon/aws/example_emr_serverless.py :language: python diff --git a/tests/providers/amazon/aws/operators/test_emr_serverless.py b/tests/providers/amazon/aws/operators/test_emr_serverless.py index 2d9f37830fd0..5e16313d9b52 100644 --- a/tests/providers/amazon/aws/operators/test_emr_serverless.py +++ b/tests/providers/amazon/aws/operators/test_emr_serverless.py @@ -17,7 +17,7 @@ from __future__ import annotations from unittest import mock -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock from uuid import UUID import pytest @@ -333,6 +333,24 @@ def test_create_application_waiter_params( assert operator.waiter_delay == expected[0] assert operator.waiter_max_attempts == expected[1] + @mock.patch.object(EmrServerlessHook, "conn") + def test_create_application_deferrable(self, mock_conn): + mock_conn.create_application.return_value = { + "applicationId": application_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + operator = EmrServerlessCreateApplicationOperator( + task_id=task_id, + release_label=release_label, + job_type=job_type, + client_request_token=client_request_token, + config=config, + deferrable=True, + ) + + with pytest.raises(TaskDeferred): + operator.execute(None) + class TestEmrServerlessStartJobOperator: @mock.patch.object(EmrServerlessHook, "get_waiter") @@ -851,6 +869,18 @@ def test_delete_application_waiter_params( assert operator.waiter_delay == expected[0] assert operator.waiter_max_attempts == expected[1] + @mock.patch.object(EmrServerlessHook, "conn") + def test_delete_application_deferrable(self, mock_conn): + mock_conn.delete_application.return_value = {"ResponseMetadata": {"HTTPStatusCode": 200}} + + operator = EmrServerlessDeleteApplicationOperator( + task_id=task_id, + application_id=application_id, + deferrable=True, + ) + with pytest.raises(TaskDeferred): + operator.execute(None) + class TestEmrServerlessStopOperator: @mock.patch.object(EmrServerlessHook, "get_waiter") @@ -876,14 +906,45 @@ def test_stop_no_wait(self, mock_conn: MagicMock, mock_get_waiter: MagicMock): mock_get_waiter().wait.assert_not_called() mock_conn.stop_application.assert_called_once() - @mock.patch.object(EmrServerlessStopApplicationOperator, "hook", new_callable=PropertyMock) - def test_force_stop(self, mock_hook: MagicMock): + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch.object(EmrServerlessHook, "cancel_running_jobs") + def test_force_stop(self, mock_cancel_running_jobs, mock_conn, mock_get_waiter): + mock_cancel_running_jobs.return_value = 0 + mock_conn.stop_application.return_value = {} + mock_get_waiter().wait.return_value = True + operator = EmrServerlessStopApplicationOperator( task_id=task_id, application_id="test", force_stop=True ) operator.execute(None) - mock_hook().cancel_running_jobs.assert_called_once() - mock_hook().conn.stop_application.assert_called_once() - mock_hook().get_waiter().wait.assert_called_once() + mock_cancel_running_jobs.assert_called_once() + mock_conn.stop_application.assert_called_once() + mock_get_waiter().wait.assert_called_once() + + @mock.patch.object(EmrServerlessHook, "cancel_running_jobs") + def test_stop_application_deferrable_with_force_stop(self, mock_cancel_running_jobs, caplog): + mock_cancel_running_jobs.return_value = 2 + operator = EmrServerlessStopApplicationOperator( + task_id=task_id, application_id="test", deferrable=True, force_stop=True + ) + with pytest.raises(TaskDeferred): + operator.execute(None) + assert "now waiting for the 2 cancelled job(s) to terminate" in caplog.messages + + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch.object(EmrServerlessHook, "cancel_running_jobs") + def test_stop_application_deferrable_without_force_stop( + self, mock_cancel_running_jobs, mock_conn, caplog + ): + mock_conn.stop_application.return_value = {} + mock_cancel_running_jobs.return_value = 0 + operator = EmrServerlessStopApplicationOperator( + task_id=task_id, application_id="test", deferrable=True, force_stop=True + ) + with pytest.raises(TaskDeferred): + operator.execute(None) + + assert "no running jobs found with application ID test" in caplog.messages diff --git a/tests/providers/amazon/aws/triggers/test_emr_serverless.py b/tests/providers/amazon/aws/triggers/test_emr_serverless.py index 029fc4ccbf73..39f1a9a0abd5 100644 --- a/tests/providers/amazon/aws/triggers/test_emr_serverless.py +++ b/tests/providers/amazon/aws/triggers/test_emr_serverless.py @@ -14,13 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + from __future__ import annotations import pytest from airflow.providers.amazon.aws.triggers.emr import ( + EmrServerlessCancelJobsTrigger, + EmrServerlessCreateApplicationTrigger, + EmrServerlessDeleteApplicationTrigger, EmrServerlessStartApplicationTrigger, EmrServerlessStartJobTrigger, + EmrServerlessStopApplicationTrigger, ) TEST_APPLICATION_ID = "test-application-id" @@ -35,12 +40,36 @@ class TestEmrTriggers: @pytest.mark.parametrize( "trigger", [ + EmrServerlessCreateApplicationTrigger( + application_id=TEST_APPLICATION_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), EmrServerlessStartApplicationTrigger( application_id=TEST_APPLICATION_ID, aws_conn_id=TEST_AWS_CONN_ID, waiter_delay=TEST_WAITER_DELAY, waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, ), + EmrServerlessStopApplicationTrigger( + application_id=TEST_APPLICATION_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + EmrServerlessDeleteApplicationTrigger( + application_id=TEST_APPLICATION_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), + EmrServerlessCancelJobsTrigger( + application_id=TEST_APPLICATION_ID, + aws_conn_id=TEST_AWS_CONN_ID, + waiter_delay=TEST_WAITER_DELAY, + waiter_max_attempts=TEST_WAITER_MAX_ATTEMPTS, + ), EmrServerlessStartJobTrigger( application_id=TEST_APPLICATION_ID, job_id=TEST_JOB_ID, From 031e3945e44030d4f085753ffdc43dc104708f91 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Mon, 24 Jul 2023 13:46:44 -0400 Subject: [PATCH 291/533] Allow auth managers to override the security manager (#32525) Allow auth managers to override the security manager --- airflow/api/auth/backend/session.py | 4 +- airflow/auth/managers/base_auth_manager.py | 12 + airflow/auth/managers/fab/fab_auth_manager.py | 5 + .../managers/fab/security_manager_override.py | 220 ++++++++++++++++++ airflow/configuration.py | 34 ++- airflow/www/auth.py | 5 +- airflow/www/decorators.py | 4 +- airflow/www/extensions/init_appbuilder.py | 7 - airflow/www/extensions/init_auth_manager.py | 40 ++++ airflow/www/extensions/init_jinja_globals.py | 5 +- airflow/www/extensions/init_security.py | 5 +- airflow/www/fab_security/manager.py | 137 +---------- airflow/www/fab_security/sqla/manager.py | 2 +- airflow/www/security.py | 40 +++- airflow/www/views.py | 12 +- .../auh/managers/fab/test_fab_auth_manager.py | 4 + tests/auh/managers/test_base_auth_manager.py | 35 +++ 17 files changed, 409 insertions(+), 162 deletions(-) create mode 100644 airflow/auth/managers/fab/security_manager_override.py create mode 100644 airflow/www/extensions/init_auth_manager.py create mode 100644 tests/auh/managers/test_base_auth_manager.py diff --git a/airflow/api/auth/backend/session.py b/airflow/api/auth/backend/session.py index c55f7484605c..ef914b57e442 100644 --- a/airflow/api/auth/backend/session.py +++ b/airflow/api/auth/backend/session.py @@ -22,7 +22,7 @@ from flask import Response -from airflow.configuration import auth_manager +from airflow.www.extensions.init_auth_manager import get_auth_manager CLIENT_AUTH: tuple[str, str] | Any | None = None @@ -39,7 +39,7 @@ def requires_authentication(function: T): @wraps(function) def decorated(*args, **kwargs): - if not auth_manager.is_logged_in(): + if not get_auth_manager().is_logged_in(): return Response("Unauthorized", 401, {}) return function(*args, **kwargs) diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index 462fe34d6304..ab5356bf8ca3 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -38,3 +38,15 @@ def get_user_name(self) -> str: def is_logged_in(self) -> bool: """Return whether the user is logged in.""" ... + + def get_security_manager_override_class(self) -> type: + """ + Return the security manager override class. + + The security manager override class is responsible for overriding the default security manager + class airflow.www.security.AirflowSecurityManager with a custom implementation. This class is + essentially inherited from airflow.www.security.AirflowSecurityManager. + + By default, return an empty class. + """ + return object diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index b9f0c1e1df5a..f90a9ac06353 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -20,6 +20,7 @@ from flask_login import current_user from airflow.auth.managers.base_auth_manager import BaseAuthManager +from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride class FabAuthManager(BaseAuthManager): @@ -43,3 +44,7 @@ def get_user_name(self) -> str: def is_logged_in(self) -> bool: """Return whether the user is logged in.""" return current_user and not current_user.is_anonymous + + def get_security_manager_override_class(self) -> type: + """Return the security manager override.""" + return FabAirflowSecurityManagerOverride diff --git a/airflow/auth/managers/fab/security_manager_override.py b/airflow/auth/managers/fab/security_manager_override.py new file mode 100644 index 000000000000..5be9ee1f3672 --- /dev/null +++ b/airflow/auth/managers/fab/security_manager_override.py @@ -0,0 +1,220 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from functools import cached_property + +from flask_appbuilder.const import AUTH_DB, AUTH_LDAP, AUTH_OAUTH, AUTH_OID, AUTH_REMOTE_USER +from flask_babel import lazy_gettext + + +class FabAirflowSecurityManagerOverride: + """ + This security manager overrides the default AirflowSecurityManager security manager. + + This security manager is used only if the auth manager FabAuthManager is used. It defines everything in + the security manager that is needed for the FabAuthManager to work. Any operation specific to + the AirflowSecurityManager should be defined here instead of AirflowSecurityManager. + + :param appbuilder: The appbuilder. + :param actionmodelview: The obj instance for action model view. + :param authdbview: The class for auth db view. + :param authldapview: The class for auth ldap view. + :param authoauthview: The class for auth oauth view. + :param authoidview: The class for auth oid view. + :param authremoteuserview: The class for auth remote user view. + :param permissionmodelview: The class for permission model view. + :param registeruser_view: The class for register user view. + :param registeruserdbview: The class for register user db view. + :param registeruseroauthview: The class for register user oauth view. + :param registerusermodelview: The class for register user model view. + :param registeruseroidview: The class for register user oid view. + :param resetmypasswordview: The class for reset my password view. + :param resetpasswordview: The class for reset password view. + :param rolemodelview: The class for role model view. + :param userinfoeditview: The class for user info edit view. + :param userdbmodelview: The class for user db model view. + :param userldapmodelview: The class for user ldap model view. + :param useroauthmodelview: The class for user oauth model view. + :param useroidmodelview: The class for user oid model view. + :param userremoteusermodelview: The class for user remote user model view. + :param userstatschartview: The class for user stats chart view. + """ + + """ The obj instance for authentication view """ + auth_view = None + """ The obj instance for user view """ + user_view = None + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.appbuilder = kwargs["appbuilder"] + self.actionmodelview = kwargs["actionmodelview"] + self.authdbview = kwargs["authdbview"] + self.authldapview = kwargs["authldapview"] + self.authoauthview = kwargs["authoauthview"] + self.authoidview = kwargs["authoidview"] + self.authremoteuserview = kwargs["authremoteuserview"] + self.permissionmodelview = kwargs["permissionmodelview"] + self.registeruser_view = kwargs["registeruser_view"] + self.registeruserdbview = kwargs["registeruserdbview"] + self.registeruseroauthview = kwargs["registeruseroauthview"] + self.registerusermodelview = kwargs["registerusermodelview"] + self.registeruseroidview = kwargs["registeruseroidview"] + self.resetmypasswordview = kwargs["resetmypasswordview"] + self.resetpasswordview = kwargs["resetpasswordview"] + self.rolemodelview = kwargs["rolemodelview"] + self.userinfoeditview = kwargs["userinfoeditview"] + self.userdbmodelview = kwargs["userdbmodelview"] + self.userldapmodelview = kwargs["userldapmodelview"] + self.useroauthmodelview = kwargs["useroauthmodelview"] + self.useroidmodelview = kwargs["useroidmodelview"] + self.userremoteusermodelview = kwargs["userremoteusermodelview"] + self.userstatschartview = kwargs["userstatschartview"] + + def register_views(self): + """Register FAB auth manager related views.""" + if not self.appbuilder.app.config.get("FAB_ADD_SECURITY_VIEWS", True): + return + + if self.auth_user_registration: + if self.auth_type == AUTH_DB: + self.registeruser_view = self.registeruserdbview() + elif self.auth_type == AUTH_OID: + self.registeruser_view = self.registeruseroidview() + elif self.auth_type == AUTH_OAUTH: + self.registeruser_view = self.registeruseroauthview() + if self.registeruser_view: + self.appbuilder.add_view_no_menu(self.registeruser_view) + + self.appbuilder.add_view_no_menu(self.resetpasswordview()) + self.appbuilder.add_view_no_menu(self.resetmypasswordview()) + self.appbuilder.add_view_no_menu(self.userinfoeditview()) + + if self.auth_type == AUTH_DB: + self.user_view = self.userdbmodelview + self.auth_view = self.authdbview() + elif self.auth_type == AUTH_LDAP: + self.user_view = self.userldapmodelview + self.auth_view = self.authldapview() + elif self.auth_type == AUTH_OAUTH: + self.user_view = self.useroauthmodelview + self.auth_view = self.authoauthview() + elif self.auth_type == AUTH_REMOTE_USER: + self.user_view = self.userremoteusermodelview + self.auth_view = self.authremoteuserview() + else: + self.user_view = self.useroidmodelview + self.auth_view = self.authoidview() + + self.appbuilder.add_view_no_menu(self.auth_view) + + # this needs to be done after the view is added, otherwise the blueprint + # is not initialized + if self.is_auth_limited: + self.limiter.limit(self.auth_rate_limit, methods=["POST"])(self.auth_view.blueprint) + + self.user_view = self.appbuilder.add_view( + self.user_view, + "List Users", + icon="fa-user", + label=lazy_gettext("List Users"), + category="Security", + category_icon="fa-cogs", + category_label=lazy_gettext("Security"), + ) + + role_view = self.appbuilder.add_view( + self.rolemodelview, + "List Roles", + icon="fa-group", + label=lazy_gettext("List Roles"), + category="Security", + category_icon="fa-cogs", + ) + role_view.related_views = [self.user_view.__class__] + + if self.userstatschartview: + self.appbuilder.add_view( + self.userstatschartview, + "User's Statistics", + icon="fa-bar-chart-o", + label=lazy_gettext("User's Statistics"), + category="Security", + ) + if self.auth_user_registration: + self.appbuilder.add_view( + self.registerusermodelview, + "User's Statistics", + icon="fa-user-plus", + label=lazy_gettext("User Registrations"), + category="Security", + ) + self.appbuilder.menu.add_separator("Security") + if self.appbuilder.app.config.get("FAB_ADD_SECURITY_PERMISSION_VIEW", True): + self.appbuilder.add_view( + self.actionmodelview, + "Actions", + icon="fa-lock", + label=lazy_gettext("Actions"), + category="Security", + ) + if self.appbuilder.app.config.get("FAB_ADD_SECURITY_VIEW_MENU_VIEW", True): + self.appbuilder.add_view( + self.resourcemodelview, + "Resources", + icon="fa-list-alt", + label=lazy_gettext("Resources"), + category="Security", + ) + if self.appbuilder.app.config.get("FAB_ADD_SECURITY_PERMISSION_VIEWS_VIEW", True): + self.appbuilder.add_view( + self.permissionmodelview, + "Permission Pairs", + icon="fa-link", + label=lazy_gettext("Permissions"), + category="Security", + ) + + @property + def auth_user_registration(self): + """Will user self registration be allowed.""" + return self.appbuilder.get_app.config["AUTH_USER_REGISTRATION"] + + @property + def auth_type(self): + """Get the auth type.""" + return self.appbuilder.get_app.config["AUTH_TYPE"] + + @property + def is_auth_limited(self) -> bool: + """Is the auth rate limited.""" + return self.appbuilder.get_app.config["AUTH_RATE_LIMITED"] + + @property + def auth_rate_limit(self) -> str: + """Get the auth rate limit.""" + return self.appbuilder.get_app.config["AUTH_RATE_LIMIT"] + + @cached_property + def resourcemodelview(self): + """Return the resource model view.""" + from airflow.www.views import ResourceModelView + + return ResourceModelView diff --git a/airflow/configuration.py b/airflow/configuration.py index cca5588e543b..d6492891d3dd 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -2209,6 +2209,39 @@ def initialize_secrets_backends() -> list[BaseSecretsBackend]: return backend_list +@functools.lru_cache(maxsize=None) +def _DEFAULT_CONFIG() -> str: + path = _default_config_file_path("default_airflow.cfg") + with open(path) as fh: + return fh.read() + + +@functools.lru_cache(maxsize=None) +def _TEST_CONFIG() -> str: + path = _default_config_file_path("default_test.cfg") + with open(path) as fh: + return fh.read() + + +_deprecated = { + "DEFAULT_CONFIG": _DEFAULT_CONFIG, + "TEST_CONFIG": _TEST_CONFIG, + "TEST_CONFIG_FILE_PATH": functools.partial(_default_config_file_path, "default_test.cfg"), + "DEFAULT_CONFIG_FILE_PATH": functools.partial(_default_config_file_path, "default_airflow.cfg"), +} + + +def __getattr__(name): + if name in _deprecated: + warnings.warn( + f"{__name__}.{name} is deprecated and will be removed in future", + DeprecationWarning, + stacklevel=2, + ) + return _deprecated[name]() + raise AttributeError(f"module {__name__} has no attribute {name}") + + def initialize_auth_manager() -> BaseAuthManager: """ Initialize auth manager. @@ -2257,5 +2290,4 @@ def initialize_auth_manager() -> BaseAuthManager: conf = initialize_config() secrets_backend_list = initialize_secrets_backends() -auth_manager = initialize_auth_manager() conf.validate() diff --git a/airflow/www/auth.py b/airflow/www/auth.py index 82fb5d34c5c0..54114da1c8a7 100644 --- a/airflow/www/auth.py +++ b/airflow/www/auth.py @@ -21,8 +21,9 @@ from flask import current_app, flash, g, redirect, render_template, request, url_for -from airflow.configuration import auth_manager, conf +from airflow.configuration import conf from airflow.utils.net import get_hostname +from airflow.www.extensions.init_auth_manager import get_auth_manager T = TypeVar("T", bound=Callable) @@ -46,7 +47,7 @@ def decorated(*args, **kwargs): ) if appbuilder.sm.check_authorization(permissions, dag_id): return func(*args, **kwargs) - elif auth_manager.is_logged_in() and not g.user.perms: + elif get_auth_manager().is_logged_in() and not g.user.perms: return ( render_template( "airflow/no_roles_permissions.html", diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index fc386d220f6b..af316e3ed0fd 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -29,10 +29,10 @@ from flask import after_this_request, g, request from pendulum.parsing.exceptions import ParserError -from airflow.configuration import auth_manager from airflow.models import Log from airflow.utils.log import secrets_masker from airflow.utils.session import create_session +from airflow.www.extensions.init_auth_manager import get_auth_manager T = TypeVar("T", bound=Callable) @@ -85,7 +85,7 @@ def wrapper(*args, **kwargs): __tracebackhide__ = True # Hide from pytest traceback. with create_session() as session: - if not auth_manager.is_logged_in(): + if not get_auth_manager().is_logged_in(): user = "anonymous" else: user = f"{g.user.username} ({g.user.get_full_name()})" diff --git a/airflow/www/extensions/init_appbuilder.py b/airflow/www/extensions/init_appbuilder.py index ac9d2c9107df..ae793ca95646 100644 --- a/airflow/www/extensions/init_appbuilder.py +++ b/airflow/www/extensions/init_appbuilder.py @@ -208,13 +208,6 @@ def init_app(self, app, session): if self.update_perms: # default is True, if False takes precedence from config self.update_perms = app.config.get("FAB_UPDATE_PERMS", True) - _security_manager_class_name = app.config.get("FAB_SECURITY_MANAGER_CLASS", None) - if _security_manager_class_name is not None: - self.security_manager_class = dynamic_class_import(_security_manager_class_name) - if self.security_manager_class is None: - from flask_appbuilder.security.sqla.manager import SecurityManager - - self.security_manager_class = SecurityManager self._addon_managers = app.config["ADDON_MANAGERS"] self.session = session diff --git a/airflow/www/extensions/init_auth_manager.py b/airflow/www/extensions/init_auth_manager.py new file mode 100644 index 000000000000..d21139f67069 --- /dev/null +++ b/airflow/www/extensions/init_auth_manager.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.auth.managers.base_auth_manager import BaseAuthManager +from airflow.compat.functools import cache +from airflow.configuration import conf +from airflow.exceptions import AirflowConfigException + + +@cache +def get_auth_manager() -> BaseAuthManager: + """ + Initialize auth manager. + + Import the user manager class, instantiate it and return it. + """ + auth_manager_cls = conf.getimport(section="core", key="auth_manager") + + if not auth_manager_cls: + raise AirflowConfigException( + "No auth manager defined in the config. " + "Please specify one using section/key [core/auth_manager]." + ) + + return auth_manager_cls() diff --git a/airflow/www/extensions/init_jinja_globals.py b/airflow/www/extensions/init_jinja_globals.py index 9ef948084cc9..13baeea7bc67 100644 --- a/airflow/www/extensions/init_jinja_globals.py +++ b/airflow/www/extensions/init_jinja_globals.py @@ -21,10 +21,11 @@ import pendulum import airflow -from airflow.configuration import auth_manager, conf +from airflow.configuration import conf from airflow.settings import IS_K8S_OR_K8SCELERY_EXECUTOR, STATE_COLORS from airflow.utils.net import get_hostname from airflow.utils.platform import get_airflow_git_version +from airflow.www.extensions.init_auth_manager import get_auth_manager def init_jinja_globals(app): @@ -68,7 +69,7 @@ def prepare_jinja_globals(): "git_version": git_version, "k8s_or_k8scelery_executor": IS_K8S_OR_K8SCELERY_EXECUTOR, "rest_api_enabled": False, - "auth_manager": auth_manager, + "auth_manager": get_auth_manager(), "config_test_connection": conf.get("core", "test_connection", fallback="Disabled"), } diff --git a/airflow/www/extensions/init_security.py b/airflow/www/extensions/init_security.py index ba57f99b1448..17f93fc1c5ee 100644 --- a/airflow/www/extensions/init_security.py +++ b/airflow/www/extensions/init_security.py @@ -22,8 +22,9 @@ from flask import g, redirect, url_for from flask_login import logout_user -from airflow.configuration import auth_manager, conf +from airflow.configuration import conf from airflow.exceptions import AirflowConfigException, AirflowException +from airflow.www.extensions.init_auth_manager import get_auth_manager log = logging.getLogger(__name__) @@ -68,6 +69,6 @@ def init_api_experimental_auth(app): def init_check_user_active(app): @app.before_request def check_user_active(): - if auth_manager.is_logged_in() and not g.user.is_active: + if get_auth_manager().is_logged_in() and not g.user.is_active: logout_user() return redirect(url_for(app.appbuilder.sm.auth_view.endpoint + ".login")) diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index 00d38064363e..3c59522a6268 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -22,7 +22,6 @@ import datetime import json import logging -from functools import cached_property from typing import Any from uuid import uuid4 @@ -34,7 +33,6 @@ AUTH_LDAP, AUTH_OAUTH, AUTH_OID, - AUTH_REMOTE_USER, LOGMSG_ERR_SEC_ADD_REGISTER_USER, LOGMSG_ERR_SEC_AUTH_LDAP, LOGMSG_ERR_SEC_AUTH_LDAP_TLS, @@ -66,14 +64,14 @@ UserRemoteUserModelView, UserStatsChartView, ) -from flask_babel import lazy_gettext as _ from flask_jwt_extended import JWTManager, current_user as current_user_jwt from flask_limiter import Limiter from flask_limiter.util import get_remote_address from flask_login import AnonymousUserMixin, LoginManager, current_user from werkzeug.security import check_password_hash, generate_password_hash -from airflow.configuration import auth_manager, conf +from airflow.configuration import conf +from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.fab_security.sqla.models import Action, Permission, RegisterUser, Resource, Role, User # This product contains a modified portion of 'Flask App Builder' developed by Daniel Vaz Gaspar. @@ -208,12 +206,6 @@ def oauth_tokengetter(token=None): userstatschartview = UserStatsChartView permissionmodelview = PermissionModelView - @cached_property - def resourcemodelview(self): - from airflow.www.views import ResourceModelView - - return ResourceModelView - def __init__(self, appbuilder): self.appbuilder = appbuilder app = self.appbuilder.get_app @@ -374,11 +366,6 @@ def builtin_roles(self): def api_login_allow_multiple_providers(self): return self.appbuilder.get_app.config["AUTH_API_LOGIN_ALLOW_MULTIPLE_PROVIDERS"] - @property - def auth_type(self): - """Get the auth type.""" - return self.appbuilder.get_app.config["AUTH_TYPE"] - @property def auth_username_ci(self): """Gets the auth username for CI.""" @@ -529,18 +516,10 @@ def oauth_providers(self): """Oauth providers.""" return self.appbuilder.get_app.config["OAUTH_PROVIDERS"] - @property - def is_auth_limited(self) -> bool: - return self.appbuilder.get_app.config["AUTH_RATE_LIMITED"] - - @property - def auth_rate_limit(self) -> str: - return self.appbuilder.get_app.config["AUTH_RATE_LIMIT"] - @property def current_user(self): """Current user object.""" - if auth_manager.is_logged_in(): + if get_auth_manager().is_logged_in(): return g.user elif current_user_jwt: return current_user_jwt @@ -732,114 +711,6 @@ def _azure_jwt_token_parse(self, id_token): return jwt_decoded_payload - def register_views(self): - if not self.appbuilder.app.config.get("FAB_ADD_SECURITY_VIEWS", True): - return - - if self.auth_user_registration: - if self.auth_type == AUTH_DB: - self.registeruser_view = self.registeruserdbview() - elif self.auth_type == AUTH_OID: - self.registeruser_view = self.registeruseroidview() - elif self.auth_type == AUTH_OAUTH: - self.registeruser_view = self.registeruseroauthview() - if self.registeruser_view: - self.appbuilder.add_view_no_menu(self.registeruser_view) - - self.appbuilder.add_view_no_menu(self.resetpasswordview()) - self.appbuilder.add_view_no_menu(self.resetmypasswordview()) - self.appbuilder.add_view_no_menu(self.userinfoeditview()) - - if self.auth_type == AUTH_DB: - self.user_view = self.userdbmodelview - self.auth_view = self.authdbview() - - elif self.auth_type == AUTH_LDAP: - self.user_view = self.userldapmodelview - self.auth_view = self.authldapview() - elif self.auth_type == AUTH_OAUTH: - self.user_view = self.useroauthmodelview - self.auth_view = self.authoauthview() - elif self.auth_type == AUTH_REMOTE_USER: - self.user_view = self.userremoteusermodelview - self.auth_view = self.authremoteuserview() - else: - self.user_view = self.useroidmodelview - self.auth_view = self.authoidview() - if self.auth_user_registration: - pass - # self.registeruser_view = self.registeruseroidview() - # self.appbuilder.add_view_no_menu(self.registeruser_view) - - self.appbuilder.add_view_no_menu(self.auth_view) - - # this needs to be done after the view is added, otherwise the blueprint - # is not initialized - if self.is_auth_limited: - self.limiter.limit(self.auth_rate_limit, methods=["POST"])(self.auth_view.blueprint) - - self.user_view = self.appbuilder.add_view( - self.user_view, - "List Users", - icon="fa-user", - label=_("List Users"), - category="Security", - category_icon="fa-cogs", - category_label=_("Security"), - ) - - role_view = self.appbuilder.add_view( - self.rolemodelview, - "List Roles", - icon="fa-group", - label=_("List Roles"), - category="Security", - category_icon="fa-cogs", - ) - role_view.related_views = [self.user_view.__class__] - - if self.userstatschartview: - self.appbuilder.add_view( - self.userstatschartview, - "User's Statistics", - icon="fa-bar-chart-o", - label=_("User's Statistics"), - category="Security", - ) - if self.auth_user_registration: - self.appbuilder.add_view( - self.registerusermodelview, - "User's Statistics", - icon="fa-user-plus", - label=_("User Registrations"), - category="Security", - ) - self.appbuilder.menu.add_separator("Security") - if self.appbuilder.app.config.get("FAB_ADD_SECURITY_PERMISSION_VIEW", True): - self.appbuilder.add_view( - self.actionmodelview, - "Actions", - icon="fa-lock", - label=_("Actions"), - category="Security", - ) - if self.appbuilder.app.config.get("FAB_ADD_SECURITY_VIEW_MENU_VIEW", True): - self.appbuilder.add_view( - self.resourcemodelview, - "Resources", - icon="fa-list-alt", - label=_("Resources"), - category="Security", - ) - if self.appbuilder.app.config.get("FAB_ADD_SECURITY_PERMISSION_VIEWS_VIEW", True): - self.appbuilder.add_view( - self.permissionmodelview, - "Permission Pairs", - icon="fa-link", - label=_("Permissions"), - category="Security", - ) - def create_db(self): """Setups the DB, creates admin and public roles if they don't exist.""" roles_mapping = self.appbuilder.get_app.config.get("FAB_ROLES_MAPPING", {}) @@ -1415,7 +1286,7 @@ def _get_user_permission_resources( return result def get_user_menu_access(self, menu_names: list[str] | None = None) -> set[str]: - if auth_manager.is_logged_in(): + if get_auth_manager().is_logged_in(): return self._get_user_permission_resources(g.user, "menu_access", resource_names=menu_names) elif current_user_jwt: return self._get_user_permission_resources( diff --git a/airflow/www/fab_security/sqla/manager.py b/airflow/www/fab_security/sqla/manager.py index 62decfb184b0..c0daf5553ca2 100644 --- a/airflow/www/fab_security/sqla/manager.py +++ b/airflow/www/fab_security/sqla/manager.py @@ -58,7 +58,7 @@ class SecurityManager(BaseSecurityManager): permission_model = Permission registeruser_model = RegisterUser - def __init__(self, appbuilder): + def __init__(self, appbuilder, **kwargs): """ Class constructor. diff --git a/airflow/www/security.py b/airflow/www/security.py index 33188328fc09..d9229c9b1eff 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -18,18 +18,18 @@ from __future__ import annotations import warnings -from typing import Any, Collection, Container, Iterable, Sequence +from typing import TYPE_CHECKING, Any, Collection, Container, Iterable, Sequence from flask import g from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload -from airflow.configuration import auth_manager from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.models import DagBag, DagModel from airflow.security import permissions from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.session import NEW_SESSION, provide_session +from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.fab_security.sqla.manager import SecurityManager from airflow.www.fab_security.sqla.models import Permission, Resource, Role, User from airflow.www.fab_security.views import ( @@ -57,8 +57,14 @@ "Public", } +if TYPE_CHECKING: + SecurityManagerOverride: type = object +else: + # Fetch the security manager override from the auth manager + SecurityManagerOverride = get_auth_manager().get_security_manager_override_class() -class AirflowSecurityManager(SecurityManager, LoggingMixin): + +class AirflowSecurityManager(SecurityManagerOverride, SecurityManager, LoggingMixin): """Custom security manager, which introduces a permission model adapted to Airflow.""" ########################################################################### @@ -193,7 +199,31 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin): userstatschartview = CustomUserStatsChartView def __init__(self, appbuilder) -> None: - super().__init__(appbuilder) + super().__init__( + appbuilder=appbuilder, + actionmodelview=self.actionmodelview, + authdbview=self.authdbview, + authldapview=self.authldapview, + authoauthview=self.authoauthview, + authoidview=self.authoidview, + authremoteuserview=self.authremoteuserview, + permissionmodelview=self.permissionmodelview, + registeruser_view=self.registeruser_view, + registeruserdbview=self.registeruserdbview, + registeruseroauthview=self.registeruseroauthview, + registerusermodelview=self.registerusermodelview, + registeruseroidview=self.registeruseroidview, + resetmypasswordview=self.resetmypasswordview, + resetpasswordview=self.resetpasswordview, + rolemodelview=self.rolemodelview, + userinfoeditview=self.userinfoeditview, + userdbmodelview=self.userdbmodelview, + userldapmodelview=self.userldapmodelview, + useroauthmodelview=self.useroauthmodelview, + useroidmodelview=self.useroidmodelview, + userremoteusermodelview=self.userremoteusermodelview, + userstatschartview=self.userstatschartview, + ) # Go and fix up the SQLAInterface used from the stock one to our subclass. # This is needed to support the "hack" where we had to edit @@ -339,7 +369,7 @@ def get_accessible_dag_ids( if not user_actions: user_actions = [permissions.ACTION_CAN_EDIT, permissions.ACTION_CAN_READ] - if not auth_manager.is_logged_in(): + if not get_auth_manager().is_logged_in(): roles = user.roles else: if (permissions.ACTION_CAN_EDIT in user_actions and self.can_edit_all_dags(user)) or ( diff --git a/airflow/www/views.py b/airflow/www/views.py index 359f1ca52b77..54f46575be35 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -83,7 +83,7 @@ set_dag_run_state_to_success, set_state, ) -from airflow.configuration import AIRFLOW_CONFIG, auth_manager, conf +from airflow.configuration import AIRFLOW_CONFIG, conf from airflow.datasets import Dataset from airflow.exceptions import ( AirflowConfigException, @@ -131,6 +131,7 @@ from airflow.version import version from airflow.www import auth, utils as wwwutils from airflow.www.decorators import action_logging, gzipped +from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.forms import ( DagRunEditForm, DateTimeForm, @@ -622,16 +623,17 @@ def method_not_allowed(error): def show_traceback(error): """Show Traceback for a given error.""" + is_logged_in = get_auth_manager().is_logged_in() return ( render_template( "airflow/traceback.html", - python_version=sys.version.split(" ")[0] if auth_manager.is_logged_in() else "redact", - airflow_version=version if auth_manager.is_logged_in() else "redact", + python_version=sys.version.split(" ")[0] if is_logged_in else "redact", + airflow_version=version if is_logged_in else "redact", hostname=get_hostname() - if conf.getboolean("webserver", "EXPOSE_HOSTNAME") and auth_manager.is_logged_in() + if conf.getboolean("webserver", "EXPOSE_HOSTNAME") and is_logged_in else "redact", info=traceback.format_exc() - if conf.getboolean("webserver", "EXPOSE_STACKTRACE") and auth_manager.is_logged_in() + if conf.getboolean("webserver", "EXPOSE_STACKTRACE") and is_logged_in else "Error! Please contact server admin.", ), 500, diff --git a/tests/auh/managers/fab/test_fab_auth_manager.py b/tests/auh/managers/fab/test_fab_auth_manager.py index 4f24b1297ef7..baaec623f47c 100644 --- a/tests/auh/managers/fab/test_fab_auth_manager.py +++ b/tests/auh/managers/fab/test_fab_auth_manager.py @@ -22,6 +22,7 @@ import pytest from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager +from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride from airflow.www.fab_security.sqla.models import User @@ -55,3 +56,6 @@ def test_is_logged_in(self, mock_current_user, auth_manager): mock_current_user.return_value = user assert auth_manager.is_logged_in() is False + + def test_get_security_manager_override_class_return_fab_security_manager_override(self, auth_manager): + assert auth_manager.get_security_manager_override_class() is FabAirflowSecurityManagerOverride diff --git a/tests/auh/managers/test_base_auth_manager.py b/tests/auh/managers/test_base_auth_manager.py new file mode 100644 index 000000000000..61bee75371a0 --- /dev/null +++ b/tests/auh/managers/test_base_auth_manager.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.auth.managers.base_auth_manager import BaseAuthManager + + +@pytest.fixture +def auth_manager(): + class EmptyAuthManager(BaseAuthManager): + def get_user_name(self) -> str: + raise NotImplementedError() + + return EmptyAuthManager() + + +class TestBaseAuthManager: + def test_get_security_manager_override_class_return_empty_class(self, auth_manager): + assert auth_manager.get_security_manager_override_class() is object From 2ac237b3eba93ed0c5fa15bced690f42d7444897 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 24 Jul 2023 20:32:04 +0200 Subject: [PATCH 292/533] Add pre-Airflow-2-7 hardcoded defaults for config for older providers (#32775) * Quarantine test_backfill_integration in dask executor The test has been recently failing with deadlock (see #32778) and needs thorough looking at if we want to find the root cause/remedium. In the meantime it looks like a niche case connected with Dask Executor that is rather obsure and we have no expertise in solving problems with and diagnosing, so until the problem is diagnosed it might be a long time (and maybe even we decide not to care about it and let Dask community take a look and either fix or ignore it. We aim to have a very low number of those Quarantined tests (currently we have 1 and we have not run it for a while as this was a mysql test run on Postgres) but we have now the opportunity to also improve the quarantined tests framework. This test will be run together with other (1) quarantined test and: * they will not be run in our regular tests * they are run sequentially not in parallel with all other tests * they are run for all 4 backends but only for the default versions of those backends * failure of the quarantined tests will not cause failure of the whole job or limit constraints from being generated and updated * Add pre-Airflow-2-7 hardcoded defaults for config for older providers During thorough testing and review of moving configuration to provoders I realised that there was a case that was not handled properly. In some cases some providers and DAGs could rely on some default values being available as default, but when we move them from core, and use older version of provider those defaults were not available: * they were remove as defaults in core * the old providers did not have "config" section to contribute the defaults This would be a breaking change and old providers (Celery, K8s) could fail - as it happened in some tests. This PR implements a nice solution to that, also allowing to remove some manual fallbacks in Celery and Kubernetes executor code. The solution is to add a hard-coded "pre-2.7" configuration which would only contain "provider" pre-2.7 hard-coded defaults and make it a fallback option if the values are neither set nor defaults contributed by the providers. We do not have to maintain those - the defaults are "frozen" effectively at the values available just before 2.7. The nice side effect is that we can remove a number of fallbacks, because this hard-coded configuration becomes the fallback automatically, That entirely solves the case where you want to install older providers on 2.7 where config.yml does not contain those provider values. * Update airflow/configuration.py Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- airflow/cli/cli_config.py | 10 +- airflow/config_templates/pre_2_7_defaults.cfg | 93 +++++++++++++++++++ airflow/configuration.py | 27 +++++- airflow/executors/kubernetes_executor.py | 2 +- .../kubernetes/kubernetes_helper_functions.py | 2 +- .../celery/executors/celery_executor.py | 2 +- .../celery/executors/celery_executor_utils.py | 24 +---- airflow/settings.py | 2 +- airflow/utils/log/file_task_handler.py | 2 +- tests/core/test_configuration.py | 9 +- 10 files changed, 135 insertions(+), 38 deletions(-) create mode 100644 airflow/config_templates/pre_2_7_defaults.cfg diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 30e71302832c..a68de35b86df 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -843,7 +843,7 @@ def string_lower_type(val): ("-c", "--concurrency"), type=int, help="The number of worker processes", - default=conf.getint("celery", "worker_concurrency", fallback=16), + default=conf.getint("celery", "worker_concurrency"), ) ARG_CELERY_HOSTNAME = Arg( ("-H", "--celery-hostname"), @@ -870,24 +870,24 @@ def string_lower_type(val): ARG_BROKER_API = Arg(("-a", "--broker-api"), help="Broker API") ARG_FLOWER_HOSTNAME = Arg( ("-H", "--hostname"), - default=conf.get("celery", "FLOWER_HOST", fallback="0.0.0.0"), + default=conf.get("celery", "FLOWER_HOST"), help="Set the hostname on which to run the server", ) ARG_FLOWER_PORT = Arg( ("-p", "--port"), - default=conf.getint("celery", "FLOWER_PORT", fallback=5555), + default=conf.getint("celery", "FLOWER_PORT"), type=int, help="The port on which to run the server", ) ARG_FLOWER_CONF = Arg(("-c", "--flower-conf"), help="Configuration file for flower") ARG_FLOWER_URL_PREFIX = Arg( ("-u", "--url-prefix"), - default=conf.get("celery", "FLOWER_URL_PREFIX", fallback=""), + default=conf.get("celery", "FLOWER_URL_PREFIX"), help="URL prefix for Flower", ) ARG_FLOWER_BASIC_AUTH = Arg( ("-A", "--basic-auth"), - default=conf.get("celery", "FLOWER_BASIC_AUTH", fallback=""), + default=conf.get("celery", "FLOWER_BASIC_AUTH"), help=( "Securing Flower with Basic Authentication. " "Accepts user:password pairs separated by a comma. " diff --git a/airflow/config_templates/pre_2_7_defaults.cfg b/airflow/config_templates/pre_2_7_defaults.cfg new file mode 100644 index 000000000000..abdef2ee078e --- /dev/null +++ b/airflow/config_templates/pre_2_7_defaults.cfg @@ -0,0 +1,93 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file contains pre Airflow 2.7, provider defaults for Airflow configuration. +# They are provided as fallback option to older version of the +# providers that might expect them to be present. +# +# NOTE !!!! Please DO NOT modify values in the file even if they change in corresponding +# providers. The values here should be treated as "read only" and should not be modified +# even if defaults in newer versions of corresponding Providers change. +# They are only here so that backwards compatible behaviour for old provider +# versions can be maintained. + +[atlas] +sasl_enabled = False +host = +port = 21000 +username = +password = + +[hive] +default_hive_mapred_queue = + +[local_kubernetes_executor] +kubernetes_queue = kubernetes + +[celery_kubernetes_executor] +kubernetes_queue = kubernetes + +[celery] +celery_app_name = airflow.executors.celery_executor +worker_concurrency = 16 +worker_prefetch_multiplier = 1 +worker_enable_remote_control = true +broker_url = redis://redis:6379/0 +result_backend_sqlalchemy_engine_options = +flower_host = 0.0.0.0 +flower_url_prefix = +flower_port = 5555 +flower_basic_auth = +sync_parallelism = 0 +celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG +ssl_active = False +ssl_key = +ssl_cert = +ssl_cacert = +pool = prefork +operation_timeout = 1.0 +task_track_started = True +task_publish_max_retries = 3 +worker_precheck = False + +[elasticsearch_configs] +use_ssl = False +verify_certs = True + +[kubernetes_executor] +api_client_retry_configuration = +logs_task_metadata = False +pod_template_file = +worker_container_repository = +worker_container_tag = +namespace = default +delete_worker_pods = True +delete_worker_pods_on_failure = False +worker_pods_creation_batch_size = 1 +multi_namespace_mode = False +multi_namespace_mode_namespace_list = +in_cluster = True +kube_client_request_args = +delete_option_kwargs = +enable_tcp_keepalive = True +tcp_keep_idle = 120 +tcp_keep_intvl = 30 +tcp_keep_cnt = 6 +verify_ssl = True +worker_pods_queued_check_interval = 60 +ssl_ca_cert = diff --git a/airflow/configuration.py b/airflow/configuration.py index d6492891d3dd..54ce39225dd8 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -163,7 +163,6 @@ def retrieve_configuration_description( :param include_airflow: Include Airflow configs :param include_providers: Include provider configs :param selected_provider: If specified, include selected provider only - :param config_file_name: name of the file in "config_templates" directory to read default config from :return: Python dictionary containing configs & their info """ base_configuration_description: dict[str, dict[str, Any]] = {} @@ -208,6 +207,7 @@ def __init__( # interpolation placeholders. The _default_values config parser will interpolate them # properly when we call get() on it. self._default_values = create_default_config_parser(self.configuration_description) + self._pre_2_7_default_values = create_pre_2_7_defaults() if default_config is not None: self._update_defaults_from_string(default_config) self._update_logging_deprecated_template_to_one_from_defaults() @@ -287,6 +287,10 @@ def get_default_value(self, section: str, key: str, fallback: Any = None, raw=Fa return value.replace("%", "%%") return value + def get_default_pre_2_7_value(self, section: str, key: str, **kwargs) -> Any: + """Get pre 2.7 default config values.""" + return self._pre_2_7_default_values.get(section, key, fallback=None, **kwargs) + # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}_cmd" pattern, the idea behind this # is to not store password on boxes in text files. @@ -1043,6 +1047,11 @@ def get( # type: ignore[override,misc] # ...then the default config if self.get_default_value(section, key) is not None or "fallback" in kwargs: return expand_env_var(self.get_default_value(section, key, **kwargs)) + + if self.get_default_pre_2_7_value(section, key) is not None: + # no expansion needed + return self.get_default_pre_2_7_value(section, key, **kwargs) + if not suppress_warnings: log.warning("section/key [%s/%s] not found in config", section, key) @@ -1402,7 +1411,10 @@ def as_dict( ) config_sources: ConfigSourcesType = {} + + # We check sequentially all those sources and the last one we saw it in will "win" configs: Iterable[tuple[str, ConfigParser]] = [ + ("default-pre-2-7", self._pre_2_7_default_values), ("default", self._default_values), ("airflow.cfg", self), ] @@ -1735,6 +1747,7 @@ def _replace_section_config_with_display_sources( if display_source: updated_source_name = source_name if source_name == "default": + # defaults can come from other sources (default-) that should be used here source_description_section = configuration_description.get(section, {}) source_description_key = source_description_section.get("options", {}).get(k, {}) if source_description_key is not None: @@ -1929,6 +1942,18 @@ def create_default_config_parser(configuration_description: dict[str, dict[str, return parser +def create_pre_2_7_defaults() -> ConfigParser: + """ + Creates parser using the old defaults from Airflow < 2.7.0, in order to be able to fall-back to those + defaults when old version of provider, not supporting "config contribution" is installed with Airflow + 2.7.0+. This "default" configuration does not support variable expansion, those are pretty much + hard-coded defaults we want to fall-back to in such case. + """ + config_parser = ConfigParser() + config_parser.read(_default_config_file_path("pre_2_7_defaults.cfg")) + return config_parser + + def write_default_airflow_configuration_if_needed() -> AirflowConfigParser: if not os.path.isfile(AIRFLOW_CONFIG): log.debug("Creating new Airflow config file in: %s", AIRFLOW_CONFIG) diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py index c1f8861f895f..d525417a2715 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/executors/kubernetes_executor.py @@ -379,7 +379,7 @@ def _get_pod_namespace(ti: TaskInstance): namespace = None with suppress(Exception): namespace = pod_override.metadata.namespace - return namespace or conf.get("kubernetes_executor", "namespace", fallback="default") + return namespace or conf.get("kubernetes_executor", "namespace") def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]: messages = [] diff --git a/airflow/kubernetes/kubernetes_helper_functions.py b/airflow/kubernetes/kubernetes_helper_functions.py index 4cd3422cb646..27762421ca51 100644 --- a/airflow/kubernetes/kubernetes_helper_functions.py +++ b/airflow/kubernetes/kubernetes_helper_functions.py @@ -128,7 +128,7 @@ def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey: @cache def get_logs_task_metadata() -> bool: - return conf.getboolean("kubernetes_executor", "logs_task_metadata", fallback=False) + return conf.getboolean("kubernetes_executor", "logs_task_metadata") def annotations_for_logging_task_metadata(annotation_set): diff --git a/airflow/providers/celery/executors/celery_executor.py b/airflow/providers/celery/executors/celery_executor.py index 0786c9553853..d587be44db03 100644 --- a/airflow/providers/celery/executors/celery_executor.py +++ b/airflow/providers/celery/executors/celery_executor.py @@ -104,7 +104,7 @@ def __init__(self): self.bulk_state_fetcher = BulkStateFetcher(self._sync_parallelism) self.tasks = {} self.task_publish_retries: Counter[TaskInstanceKey] = Counter() - self.task_publish_max_retries = conf.getint("celery", "task_publish_max_retries", fallback=3) + self.task_publish_max_retries = conf.getint("celery", "task_publish_max_retries") def start(self) -> None: self.log.debug("Starting Celery Executor using %s processes for syncing", self._sync_parallelism) diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index e970174a658c..c414a88b0c22 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -57,36 +57,18 @@ TaskInstanceInCelery = Tuple[TaskInstanceKey, CommandType, Optional[str], Task] -# IMPORTANT NOTE! Celery Executor has initialization done dynamically and it performs initialization when -# it is imported, so we need fallbacks here in order to be able to import the class directly without -# having configuration initialized before. Do not remove those fallbacks! -# -# This is not strictly needed for production: -# -# * for Airflow 2.6 and before the defaults will come from the core defaults -# * for Airflow 2.7+ the defaults will be loaded via ProvidersManager -# -# But it helps in our tests to import the executor class and validate if the celery code can be imported -# in the current and older versions of Airflow. - -OPERATION_TIMEOUT = conf.getfloat("celery", "operation_timeout", fallback=1.0) +OPERATION_TIMEOUT = conf.getfloat("celery", "operation_timeout") # Make it constant for unit test. CELERY_FETCH_ERR_MSG_HEADER = "Error fetching Celery task state" if conf.has_option("celery", "celery_config_options"): - celery_configuration = conf.getimport( - "celery", - "celery_config_options", - fallback="airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG", - ) + celery_configuration = conf.getimport("celery", "celery_config_options") else: celery_configuration = DEFAULT_CELERY_CONFIG -celery_app_name = conf.get( - "celery", "CELERY_APP_NAME", fallback="airflow.providers.celery.executors.celery_executor" -) +celery_app_name = conf.get("celery", "CELERY_APP_NAME") if celery_app_name == "airflow.executors.celery_executor": warnings.warn( "The celery.CELERY_APP_NAME configuration uses deprecated package name: " diff --git a/airflow/settings.py b/airflow/settings.py index f3242def491d..e51cd208d9f7 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -396,7 +396,7 @@ def validate_session(): """Validate ORM Session.""" global engine - worker_precheck = conf.getboolean("celery", "worker_precheck", fallback=False) + worker_precheck = conf.getboolean("celery", "worker_precheck") if not worker_precheck: return True else: diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index fb540ed9ecb0..50938f96cb0d 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -356,7 +356,7 @@ def _get_pod_namespace(ti: TaskInstance): namespace = None with suppress(Exception): namespace = pod_override.metadata.namespace - return namespace or conf.get("kubernetes_executor", "namespace", fallback="default") + return namespace or conf.get("kubernetes_executor", "namespace") def _get_log_retrieval_url( self, ti: TaskInstance, log_relative_path: str, log_type: LogType | None = None diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 6eb50d919f10..91102003a57b 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -1629,8 +1629,8 @@ def test_restore_and_reload_provider_configuration(): assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" conf.restore_core_default_configuration() assert conf.providers_configuration_loaded is False - with pytest.raises(AirflowConfigException, match="not found"): - conf.get("celery", "celery_app_name") + # built-in pre-2-7 celery executor + assert conf.get("celery", "celery_app_name") == "airflow.executors.celery_executor" conf.load_providers_configuration() assert conf.providers_configuration_loaded is True assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" @@ -1639,7 +1639,7 @@ def test_restore_and_reload_provider_configuration(): def test_error_when_contributing_to_existing_section(): from airflow.settings import conf - try: + with conf.make_sure_configuration_loaded(with_providers=True): assert conf.providers_configuration_loaded is True assert conf.get("celery", "celery_app_name") == "airflow.providers.celery.executors.celery_executor" conf.restore_core_default_configuration() @@ -1665,6 +1665,3 @@ def test_error_when_contributing_to_existing_section(): ): conf.load_providers_configuration() assert conf.get("celery", "celery_app_name") == "test" - finally: - conf.restore_core_default_configuration() - conf.load_providers_configuration() From b40df0f938de993e6bd2635c7dfc5b9ce57e4d55 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 24 Jul 2023 21:50:41 +0200 Subject: [PATCH 293/533] Also write webserver_config for worker and internal-api (#32812) We are also using webserver config when we start worker and in the future the internal-api server - which was not happening after #32766. This PR brings the webserver config back for those commands. --- airflow/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/__main__.py b/airflow/__main__.py index c88f655bee2b..20af0ac2744a 100644 --- a/airflow/__main__.py +++ b/airflow/__main__.py @@ -54,8 +54,7 @@ def main(): from airflow.configuration import write_default_airflow_configuration_if_needed conf = write_default_airflow_configuration_if_needed() - - if args.subcommand == "webserver": + if args.subcommand in ["webserver", "internal-api", "worker"]: write_webserver_configuration_if_needed(conf) args.func(args) From 0c0b8a3a0e7d6f23029f76d3fb0d185cdcc671ec Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 24 Jul 2023 21:51:21 +0200 Subject: [PATCH 294/533] Move Hive configuration to Apache Hive provider (#32777) --- airflow/config_templates/config.yml | 18 ----------------- airflow/providers/apache/hive/provider.yaml | 20 +++++++++++++++++++ .../configurations-ref.rst | 18 +++++++++++++++++ .../index.rst | 1 + docs/apache-airflow/configurations-ref.rst | 1 + 5 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 docs/apache-airflow-providers-apache-hive/configurations-ref.rst diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 4cef07dee1de..f87b000aad79 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1380,24 +1380,6 @@ operators: type: string example: ~ default: "False" -hive: - description: ~ - options: - default_hive_mapred_queue: - description: | - Default mapreduce queue for HiveOperator tasks - version_added: ~ - type: string - example: ~ - default: "" - mapred_job_name_template: - description: | - Template for mapred_job_name in HiveOperator, supports the following named parameters - hostname, dag_id, task_id, execution_date - version_added: 2.0.0 - type: string - example: ~ - default: ~ webserver: description: ~ options: diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 79125e2cc3d8..4b260f5b1075 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -127,3 +127,23 @@ connection-types: plugins: - name: hive plugin-class: airflow.providers.apache.hive.plugins.hive.HivePlugin + +config: + hive: + description: ~ + options: + default_hive_mapred_queue: + description: | + Default mapreduce queue for HiveOperator tasks + version_added: ~ + type: string + example: ~ + default: "" + mapred_job_name_template: + description: | + Template for mapred_job_name in HiveOperator, supports the following named parameters + hostname, dag_id, task_id, execution_date + version_added: ~ + type: string + example: ~ + default: ~ diff --git a/docs/apache-airflow-providers-apache-hive/configurations-ref.rst b/docs/apache-airflow-providers-apache-hive/configurations-ref.rst new file mode 100644 index 000000000000..5885c9d91b6e --- /dev/null +++ b/docs/apache-airflow-providers-apache-hive/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-apache-hive/index.rst b/docs/apache-airflow-providers-apache-hive/index.rst index 2676761c73f3..35167cc0b809 100644 --- a/docs/apache-airflow-providers-apache-hive/index.rst +++ b/docs/apache-airflow-providers-apache-hive/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/apache/hive/index> .. toctree:: diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index eb34b10c3592..7a32f1790dea 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -36,6 +36,7 @@ Some of the providers have their own configuration options, you will find detail in the provider's documentation. The pre-installed providers that you may want to configure are: * :doc:`Configuration Reference for Celery Provider ` +* :doc:`Configuration Reference for Apache Hive Provider ` .. note:: For more information see :doc:`/howto/set-config`. From f12b161a882ddbc6b6fda950c4cc5cd277166819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Ducharme?= Date: Mon, 24 Jul 2023 15:57:25 -0400 Subject: [PATCH 295/533] Chart: CeleryExecutor namespace depends on Airflow version (#32753) --- chart/templates/_helpers.yaml | 8 ++++++++ chart/templates/workers/worker-deployment.yaml | 2 +- helm_tests/airflow_core/test_worker.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index e0c158c765b1..742375f21e69 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -371,6 +371,14 @@ If release name contains chart name it will be used as a full name. {{- printf "%s/%s" .Values.kerberos.ccacheMountPath .Values.kerberos.ccacheFileName }} {{- end }} +{{- define "celery_executor_namespace" -}} + {{- if semverCompare ">=2.7.0" .Values.airflowVersion }} + {{- print "airflow.providers.celery.executors.celery_executor.app" -}} + {{- else }} + {{- print "airflow.executors.celery_executor.app" -}} + {{- end }} +{{- end }} + {{- define "pgbouncer_config" -}} {{ $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} {{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 4e306745d7b7..235f3f783f6f 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -205,7 +205,7 @@ spec: {{- else }} - sh - -c - - CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d celery@$(hostname) + - CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app {{ include "celery_executor_namespace" . }} inspect ping -d celery@$(hostname) {{- end }} {{- end }} ports: diff --git a/helm_tests/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py index 20cc851ae917..22d5be18221c 100644 --- a/helm_tests/airflow_core/test_worker.py +++ b/helm_tests/airflow_core/test_worker.py @@ -388,6 +388,24 @@ def test_runtime_class_name_values_are_configurable(self): assert jmespath.search("spec.template.spec.runtimeClassName", docs[0]) == "nvidia" + @pytest.mark.parametrize( + "airflow_version, default_cmd", + [ + ("2.7.0", "airflow.providers.celery.executors.celery_executor.app"), + ("2.6.3", "airflow.executors.celery_executor.app"), + ], + ) + def test_livenessprobe_default_command(self, airflow_version, default_cmd): + docs = render_chart( + values={"airflowVersion": airflow_version}, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + livenessprobe_cmd = jmespath.search( + "spec.template.spec.containers[0].livenessProbe.exec.command", docs[0] + ) + assert default_cmd in livenessprobe_cmd[-1] + def test_livenessprobe_values_are_configurable(self): docs = render_chart( values={ From fd4af0b5c385d876ac59d327a90c6e6bfb312469 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 25 Jul 2023 00:03:47 +0200 Subject: [PATCH 296/533] Remove atlas configuration definition (#32776) Atlas does not have its own provider, the only thing we have with atlas extra is an extra dependency to atlasclient. Abd we also had the atlas configuration for some reason that even the eldest don't know. With hard-coded defaults, we are still keeping backwards compatibility if someone used those values. However we should remove it from "core" configuration because it has absolutely no meaning for Airflow. Users are free to define their own sections and variables following Airflow schema and this squarely fits into this category. --- airflow/config_templates/config.yml | 34 ----------------------------- tests/core/test_configuration.py | 1 - 2 files changed, 35 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index f87b000aad79..47f21ee7369b 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1289,40 +1289,6 @@ openlineage: example: ~ type: boolean version_added: ~ -atlas: - description: ~ - options: - sasl_enabled: - description: ~ - version_added: ~ - type: string - example: ~ - default: "False" - host: - description: ~ - version_added: ~ - type: string - example: ~ - default: "" - port: - description: ~ - version_added: ~ - type: string - example: ~ - default: "21000" - username: - description: ~ - version_added: ~ - type: string - example: ~ - default: "" - password: - description: ~ - version_added: ~ - type: string - sensitive: true - example: ~ - default: "" operators: description: ~ options: diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 91102003a57b..a6229dc07c3a 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -1600,7 +1600,6 @@ def test_sensitive_values(): sensitive_values = { ("database", "sql_alchemy_conn"), ("core", "fernet_key"), - ("atlas", "password"), ("smtp", "smtp_password"), ("webserver", "secret_key"), ("secrets", "backend_kwargs"), From 0c798252efb83844a2e0ba08d74b32eed2e317ad Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 25 Jul 2023 00:05:02 +0200 Subject: [PATCH 297/533] Extract Dask executor to new daskexecutor provider (#32772) --- .../airflow_providers_bug_report.yml | 1 + CONTRIBUTING.rst | 18 +- Dockerfile | 2 +- IMAGES.rst | 2 +- INSTALL | 18 +- airflow/config_templates/config.yml | 33 ---- airflow/executors/__init__.py | 3 + airflow/executors/executor_loader.py | 2 +- .../CREATING_COMMUNITY_PROVIDERS.rst | 74 ++++--- airflow/providers/daskexecutor/CHANGELOG.rst | 33 ++++ airflow/providers/daskexecutor/__init__.py | 41 ++++ .../daskexecutor/executors/__init__.py | 16 ++ .../daskexecutor}/executors/dask_executor.py | 0 airflow/providers/daskexecutor/provider.yaml | 79 ++++++++ .../src/airflow_breeze/global_constants.py | 2 +- .../changelog.rst | 19 ++ .../commits.rst | 27 +++ .../configurations-ref.rst | 18 ++ .../index.rst | 86 ++++++++ .../installing-providers-from-sources.rst | 18 ++ .../security.rst | 38 ++++ .../core-concepts/executor/dask.rst | 4 +- docs/apache-airflow/extra-packages-ref.rst | 10 +- docs/docker-stack/build-arg-ref.rst | 2 +- docs/integration-logos/dask/dask.png | Bin 0 -> 6134 bytes docs/spelling_wordlist.txt | 1 + generated/provider_dependencies.json | 10 + images/breeze/output-commands-hash.txt | 20 +- images/breeze/output_build-docs.svg | 128 ++++++------ images/breeze/output_prod-image.svg | 24 +-- images/breeze/output_prod-image_build.svg | 184 +++++++++--------- images/breeze/output_release-management.svg | 171 ++++++++++++++++ ...ement_generate-issue-content-providers.svg | 72 +++---- ...agement_prepare-provider-documentation.svg | 66 +++---- ...e-management_prepare-provider-packages.svg | 40 ++-- ...output_release-management_publish-docs.svg | 110 +++++------ images/breeze/output_sbom.svg | 20 +- ...ut_sbom_generate-provider-requirements.svg | 54 ++--- scripts/ci/installed_providers.txt | 1 + .../ci/pre_commit/pre_commit_insert_extras.py | 2 +- setup.py | 9 +- tests/providers/daskexecutor/__init__.py | 16 ++ .../daskexecutor}/test_dask_executor.py | 4 +- 43 files changed, 1028 insertions(+), 450 deletions(-) create mode 100644 airflow/providers/daskexecutor/CHANGELOG.rst create mode 100644 airflow/providers/daskexecutor/__init__.py create mode 100644 airflow/providers/daskexecutor/executors/__init__.py rename airflow/{ => providers/daskexecutor}/executors/dask_executor.py (100%) create mode 100644 airflow/providers/daskexecutor/provider.yaml create mode 100644 docs/apache-airflow-providers-daskexecutor/changelog.rst create mode 100644 docs/apache-airflow-providers-daskexecutor/commits.rst create mode 100644 docs/apache-airflow-providers-daskexecutor/configurations-ref.rst create mode 100644 docs/apache-airflow-providers-daskexecutor/index.rst create mode 100644 docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst create mode 100644 docs/apache-airflow-providers-daskexecutor/security.rst create mode 100644 docs/integration-logos/dask/dask.png create mode 100644 images/breeze/output_release-management.svg create mode 100644 tests/providers/daskexecutor/__init__.py rename tests/{executors => providers/daskexecutor}/test_dask_executor.py (98%) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index cb50fcb0c74b..02f6b38a852c 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -49,6 +49,7 @@ body: - cloudant - cncf-kubernetes - common-sql + - daskexecutor - databricks - datadog - dbt-cloud diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 06b9391eb934..ef358cbdc107 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -671,15 +671,15 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, -devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, password, -pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, -sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, -tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk +cloudant, cncf.kubernetes, common.sql, crypto, dask, daskexecutor, databricks, datadog, dbt.cloud, +deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, +ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, +mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, +password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, +segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, +tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk .. END EXTRAS HERE Provider packages diff --git a/Dockerfile b/Dockerfile index 15ee45f7e3bf..16446c0b03e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ # much smaller. # # Use the same builder frontend version for everyone -ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv" +ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv" ARG ADDITIONAL_AIRFLOW_EXTRAS="" ARG ADDITIONAL_PYTHON_DEPS="" diff --git a/IMAGES.rst b/IMAGES.rst index 7360e43648f4..9c34fce80d6e 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -135,7 +135,7 @@ This will build the image using command similar to: .. code-block:: bash pip install \ - apache-airflow[async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv]==2.0.0 \ + apache-airflow[async,amazon,celery,cncf.kubernetes,docker,daskexecutor,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv]==2.0.0 \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.0.0/constraints-3.8.txt" .. note:: diff --git a/INSTALL b/INSTALL index c5bfc7fe55d0..b30f0dbd5f10 100644 --- a/INSTALL +++ b/INSTALL @@ -98,15 +98,15 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, -devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, -elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, -grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, -leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, -neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, password, -pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, -sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, -tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk +cloudant, cncf.kubernetes, common.sql, crypto, dask, daskexecutor, databricks, datadog, dbt.cloud, +deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, +druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, +google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, +ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, +mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, +password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, +segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, +tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 47f21ee7369b..35f16c57a8f4 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1958,39 +1958,6 @@ local_kubernetes_executor: type: string example: ~ default: "kubernetes" -dask: - description: | - This section only applies if you are using the DaskExecutor in - [core] section above - options: - cluster_address: - description: | - The IP address and port of the Dask cluster's scheduler. - version_added: ~ - type: string - example: ~ - default: "127.0.0.1:8786" - tls_ca: - description: | - Path to a CA certificate file encoded in PEM format to access a secured Dask scheduler. - version_added: ~ - type: string - example: ~ - default: "" - tls_cert: - description: | - Path to a certificate file for the client, encoded in PEM format. - version_added: ~ - type: string - example: ~ - default: "" - tls_key: - description: | - Path to a key file for the client, encoded in PEM format. - version_added: ~ - type: string - example: ~ - default: "" scheduler: description: ~ options: diff --git a/airflow/executors/__init__.py b/airflow/executors/__init__.py index 2c342d45f16d..ceeec523397b 100644 --- a/airflow/executors/__init__.py +++ b/airflow/executors/__init__.py @@ -28,6 +28,9 @@ "CeleryKubernetesExecutor": "airflow.providers.celery.executors." "celery_kubernetes_executor.CeleryKubernetesExecutor", }, + "dask_executor": { + "DaskExecutor": "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor", + }, } add_deprecated_classes(__deprecated_classes, __name__) diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 816e172140ae..3553a7118360 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -63,7 +63,7 @@ class ExecutorLoader: CELERY_EXECUTOR: "airflow.providers.celery.executors.celery_executor.CeleryExecutor", CELERY_KUBERNETES_EXECUTOR: "airflow.providers.celery." "executors.celery_kubernetes_executor.CeleryKubernetesExecutor", - DASK_EXECUTOR: "airflow.executors.dask_executor.DaskExecutor", + DASK_EXECUTOR: "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor", KUBERNETES_EXECUTOR: "airflow.executors.kubernetes_executor.KubernetesExecutor", DEBUG_EXECUTOR: "airflow.executors.debug_executor.DebugExecutor", } diff --git a/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst b/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst index be249b53bb04..d4e14e138cf6 100644 --- a/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst +++ b/airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst @@ -55,6 +55,9 @@ the provider may need. Understand that not all providers will need all the compo If you still have doubts about building your provider, we recommend that you read the initial provider guide and open a issue on GitHub so the community can help you. +The folders are optional: example_dags, hooks, links, logs, notifications, operators, secrets, sensors, transfers, +triggers, waiters (and the list changes continuously). + .. code-block:: bash airflow/ @@ -63,20 +66,27 @@ open a issue on GitHub so the community can help you. │ ├── example_dags/ │ │ ├── __init__.py │ │ └── example_.py + │ ├── executors/ + │ │ ├── __init__.py + │ │ └── .py │ ├── hooks/ │ │ ├── __init__.py │ │ └── .py │ ├── operators/ │ │ ├── __init__.py │ │ └── .py - │ ├── sensors/ + .... + │ ├── transfers/ │ │ ├── __init__.py │ │ └── .py - │ └── transfers/ + │ └── triggers/ │ ├── __init__.py │ └── .py └── tests/providers// ├── __init__.py + ├── executors/ + │ ├── __init__.py + │ └── test_.py ├── hooks/ │ ├── __init__.py │ └── test_.py @@ -84,10 +94,11 @@ open a issue on GitHub so the community can help you. │ ├── __init__.py │ ├── test_.py │ └── test__system.py - ├── sensors/ + ... + ├── transfers/ │ ├── __init__.py │ └── test_.py - └── transfers/ + └── triggers/ ├── __init__.py └── test_.py @@ -113,26 +124,27 @@ Some steps for documentation occurs automatically by ``pre-commit`` see `Install .. code-block:: bash - airflow/ - ├── INSTALL - ├── CONTRIBUTING.rst - ├── setup.py - ├── docs/ - │ ├── spelling_wordlist.txt - │ ├── apache-airflow/ - │ │ └── extra-packages-ref.rst - │ ├── integration-logos// - │ │ └── .png - │ └── apache-airflow-providers-/ - │ ├── index.rst - │ ├── commits.rst - │ ├── connections.rst - │ └── operators/ - │ └── .rst - └── providers/ - └── / - ├── provider.yaml - └── CHANGELOG.rst + ├── INSTALL + ├── CONTRIBUTING.rst + ├── setup.py + ├── airflow/ + │ └── providers/ + │ └── / + │ ├── provider.yaml + │ └── CHANGELOG.rst + │ + └── docs/ + ├── spelling_wordlist.txt + ├── apache-airflow/ + │ └── extra-packages-ref.rst + ├── integration-logos// + │ └── .png + └── apache-airflow-providers-/ + ├── index.rst + ├── commits.rst + ├── connections.rst + └── operators/ + └── .rst Files automatically updated by pre-commit: @@ -199,9 +211,19 @@ In the ``docs/apache-airflow-providers-/operators/.r :end-before: [END howto_operator_] -In the ``docs/apache-airflow-providers-new_provider/index.rst``: +Copy from another, similar provider the docs: ``docs/apache-airflow-providers-new_provider/*.rst``: + +At least those docs should be present + +* security.rst +* changelog.rst +* commits.rst +* index.rst +* installing-providers-from-sources.rst +* configurations-ref.rst - if your provider has ``config`` element in provider.yaml with configuration options + specific for your provider -- add all information of the purpose of your provider. It is recommended to check with another provider to help you complete this document as best as possible. +Make sure to update/add all information that are specific for the new provider. In the ``airflow/providers//provider.yaml`` add information of your provider: diff --git a/airflow/providers/daskexecutor/CHANGELOG.rst b/airflow/providers/daskexecutor/CHANGELOG.rst new file mode 100644 index 000000000000..264edd3a3966 --- /dev/null +++ b/airflow/providers/daskexecutor/CHANGELOG.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +``apache-airflow-providers-daskexecutor`` + + +Changelog +--------- + +1.0.0 +..... + +Initial version of the provider. diff --git a/airflow/providers/daskexecutor/__init__.py b/airflow/providers/daskexecutor/__init__.py new file mode 100644 index 000000000000..6548d7b15d61 --- /dev/null +++ b/airflow/providers/daskexecutor/__init__.py @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +__all__ = ["__version__"] + +__version__ = "1.0.0" + +try: + from airflow import __version__ as airflow_version +except ImportError: + from airflow.version import version as airflow_version + +if packaging.version.parse(airflow_version) < packaging.version.parse("2.4.0"): + raise RuntimeError( + f"The package `apache-airflow-providers-daskexecutor:{__version__}` requires Apache Airflow 2.4.0+" # NOQA: E501 + ) diff --git a/airflow/providers/daskexecutor/executors/__init__.py b/airflow/providers/daskexecutor/executors/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/airflow/providers/daskexecutor/executors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/executors/dask_executor.py b/airflow/providers/daskexecutor/executors/dask_executor.py similarity index 100% rename from airflow/executors/dask_executor.py rename to airflow/providers/daskexecutor/executors/dask_executor.py diff --git a/airflow/providers/daskexecutor/provider.yaml b/airflow/providers/daskexecutor/provider.yaml new file mode 100644 index 000000000000..3c6466d46040 --- /dev/null +++ b/airflow/providers/daskexecutor/provider.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-daskexecutor +name: Dask Executor +description: | + `Dask `__ + +suspended: false +versions: + - 1.0.0 + +dependencies: + - apache-airflow>=2.4.0 + # Dask support is limited, we need Dask team to upgrade support for dask if we were to continue + # Supporting it in the future + - cloudpickle>=1.4.1 + # Dask and distributed in version 2023.5.0 break our test + # See https://github.com/dask/dask/issues/10279 + - dask>=2.9.0,!=2022.10.1,!=2023.5.0 + - distributed>=2.11.1,!=2023.5.0 + +integrations: + - integration-name: Dask + external-doc-url: https://www.dask.org/ + logo: /integration-logos/dask/dask.png + tags: [service] + +executors: + - airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor + +config: + dask: + description: | + This section only applies if you are using DaskExecutor. + options: + cluster_address: + description: | + The IP address and port of the Dask cluster's scheduler. + version_added: ~ + type: string + example: ~ + default: "127.0.0.1:8786" + tls_ca: + description: | + Path to a CA certificate file encoded in PEM format to access a secured Dask scheduler. + version_added: ~ + type: string + example: ~ + default: "" + tls_cert: + description: | + Path to a certificate file for the client, encoded in PEM format. + version_added: ~ + type: string + example: ~ + default: "" + tls_key: + description: | + Path to a key file for the client, encoded in PEM format. + version_added: ~ + type: string + example: ~ + default: "" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 9cdca2edb3bb..444bd1e01048 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -341,7 +341,7 @@ def get_airflow_extras(): "async", "celery", "cncf.kubernetes", - "dask", + "daskexecutor", "docker", "elasticsearch", "ftp", diff --git a/docs/apache-airflow-providers-daskexecutor/changelog.rst b/docs/apache-airflow-providers-daskexecutor/changelog.rst new file mode 100644 index 000000000000..a2bf898b3f6a --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/changelog.rst @@ -0,0 +1,19 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../airflow/providers/daskexecutor/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-daskexecutor/commits.rst b/docs/apache-airflow-providers-daskexecutor/commits.rst new file mode 100644 index 000000000000..1b8a85877d0c --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/commits.rst @@ -0,0 +1,27 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Package apache-airflow-providers-daskexecutor +------------------------------------------------------ + +`Dask `__ + + +This is detailed commit list of changes for versions provider package: ``daskexecutor``. +For high-level changelog, see :doc:`package information including changelog `. diff --git a/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst b/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst new file mode 100644 index 000000000000..5885c9d91b6e --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-daskexecutor/index.rst b/docs/apache-airflow-providers-daskexecutor/index.rst new file mode 100644 index 000000000000..be612a695632 --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/index.rst @@ -0,0 +1,86 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-daskexecutor`` +========================================= + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Configuration + Python API <_api/airflow/providers/daskexecutor/index> + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +Package apache-airflow-providers-daskexecutor +------------------------------------------------------ + +`Dask `__ + + +Release: 1.0.0 + +Provider package +---------------- + +This is a provider package for ``daskexecutor`` provider. All classes for this provider package +are in ``airflow.providers.daskexecutor`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below) +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-daskexecutor`` + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider package is ``2.4.0``. + +================== ================================== +PIP package Version required +================== ================================== +``apache-airflow`` ``>=2.4.0`` +``cloudpickle`` ``>=1.4.1`` +``dask`` ``>=2.9.0,!=2022.10.1,!=2023.5.0`` +``distributed`` ``>=2.11.1,!=2023.5.0`` +================== ================================== diff --git a/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst b/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst new file mode 100644 index 000000000000..b4e730f4ff21 --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-daskexecutor/security.rst b/docs/apache-airflow-providers-daskexecutor/security.rst new file mode 100644 index 000000000000..66c6f79a4ecf --- /dev/null +++ b/docs/apache-airflow-providers-daskexecutor/security.rst @@ -0,0 +1,38 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Releasing security patches +-------------------------- + +Airflow providers are released independently from Airflow itself and the information about vulnerabilities +is published separately. You can upgrade providers independently from Airflow itself, following the +instructions found in :doc:`apache-airflow:installation/installing-from-pypi`. + +When we release Provider version, the development is always done from the ``main`` branch where we prepare +the next version. The provider uses strict `SemVer `_ versioning policy. Depending on +the scope of the change, Provider will get ''MAJOR'' version upgrade when there are +breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade +when there are only bug fixes (including security bugfixes) - and this is the only version that receives +security fixes by default, so you should upgrade to latest version of the provider if you want to receive +all released security fixes. + +The only exception to that rule is when we have a critical security fix and good reason to provide an +out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick +and prepare a branch for an older version of the provider following the +`mixed governance model `_ +and requires interested parties to cherry-pick and test the fixes. diff --git a/docs/apache-airflow/core-concepts/executor/dask.rst b/docs/apache-airflow/core-concepts/executor/dask.rst index 7551ffaf8b38..117d39696297 100644 --- a/docs/apache-airflow/core-concepts/executor/dask.rst +++ b/docs/apache-airflow/core-concepts/executor/dask.rst @@ -21,7 +21,7 @@ Dask Executor ============= -:class:`airflow.executors.dask_executor.DaskExecutor` allows you to run Airflow tasks in a Dask Distributed cluster. +:class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` allows you to run Airflow tasks in a Dask Distributed cluster. Dask clusters can be run on a single machine or on remote networks. For complete details, consult the `Distributed documentation `_. @@ -42,7 +42,7 @@ Next start at least one Worker on any machine that can connect to the host: dask-worker $DASK_HOST:$DASK_PORT -Edit your ``airflow.cfg`` to set your executor to :class:`airflow.executors.dask_executor.DaskExecutor` and provide +Edit your ``airflow.cfg`` to set your executor to :class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` and provide the Dask Scheduler address in the ``[dask]`` section. For more information on setting the configuration, see :doc:`../../howto/set-config`. diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 027fbb657e41..ef6c4bf1b212 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -50,7 +50,7 @@ python dependencies for the provided package. +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ | cncf.kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor (also installs the Kubernetes provider package) | | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| dask | ``pip install 'apache-airflow[dask]'`` | DaskExecutor | | +| daskexecutor | ``pip install 'apache-airflow[daskexecutor]'`` | DaskExecutor ((also installs the Daskexecutor provider package) | | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ | deprecated_api | ``pip install 'apache-airflow[deprecated_api]'`` | Deprecated, experimental API that is replaced with the new REST API | | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ @@ -349,10 +349,10 @@ Those are the extras that are needed to generated documentation for Airflow. Thi +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+ -Deprecated 1.10 extras ----------------------- +Deprecated extras +----------------- -These are the extras that have been deprecated in 2.0 and will be removed in Airflow 3.0.0. They were +These are the extras that have been used before and deprecated in 2.0 and will be removed in Airflow 3.0.0. They were all replaced by new extras, which have naming consistent with the names of provider packages. The ``crypto`` extra is not needed any more, because all crypto dependencies are part of airflow package, @@ -371,6 +371,8 @@ so there is no replacement for ``crypto`` extra. +---------------------+-----------------------------+ | crypto | | +---------------------+-----------------------------+ +| dask | daskexecutor | ++---------------------+-----------------------------+ | druid | apache.druid | +---------------------+-----------------------------+ | gcp | google | diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index c954cbf69c7e..a142968b5b4f 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -85,7 +85,7 @@ List of default extras in the production Dockerfile: * async * celery * cncf.kubernetes -* dask +* daskexecutor * docker * elasticsearch * ftp diff --git a/docs/integration-logos/dask/dask.png b/docs/integration-logos/dask/dask.png new file mode 100644 index 0000000000000000000000000000000000000000..1c2b27ee3d364ac288afff556c1cbe411c239eb4 GIT binary patch literal 6134 zcmaKQcQjmG)V}D9E^5>W2?o)L61|tG8KY&0FuGy#>UBnqmceKtdh{Vi36dB+qW4~h z;3bGCL45Q6zTe;9T4&vL*IE1SXP;+3_w2oP!ZQ0NJI}N=6lDi)eVEd12;RhVYPw#uW(L0 z-83Clw)I!Cq}-NjW~6;&^dIiPj2_1yf4yTC)x*d5=5Z>7`nPqdrQ^N+FGTX&#Q{JF(aopRVxAKKbNaMB{9%+KmNa#w9yCNM6YSujp5T# zj#x-6IlMDaDc+q$t2c*bpTA;!R+6Thh6es|j-@9+SZ^;=(FX3C3DQ_3IJHdt(=bOJ z^32aP(V$A6eyo{*lX<7J{0p&}if0;1vASobiQIw-dqlK8sJ)wTRx+H;-;QK3? zbP3t)O!NP6XN0E2oJ<)8@wuRH12vCR7|m*DJTmeXwT$DAUF?bF<&xx7``%d~=6&=EvhzBpI;{IdRZ; zzO3Spj48iZxIC-gY2H1kE#6KhFfZsCo-pI|pX0bo5nx(>IA8iAt&^}obrD5N>T9~> z!P%k!8Ah85Kh@en_7{+3x?jeiJ?jSMfxY{gP5&A;1d+l@U(=ybwjVHXVmsoxNx?0K zL&$s=j+1*k33F7yXqWPim^f&Lfc{JQ^ux&G+CCz5I7Ow~wyLqJ4Gz~BL!jt81tdz1 z4(Tv$N<)A~=fsMBc*GgUGd-OfaDqU(P!RZLcY&zWyRtD17{pk98?aXhH&lv4nXXt5 z&g`f^mhiw0=Doq0JGnm;+;TV_)n)zfVDB}_}jM`u;P z>mTEF-G1S0#<*e%chxR5I!yyDXfN7Kn3zujE^Z$-MOX(u?O5Y>N*!D+HKzDBVzHPg zc`2^Y-@V?Mu(rD2)Ap+mN#9xd)IZp<@o^*Wllph}x5DD9)-CaV2ie2hJbdZ(9!|od zVTiJx32rqVw&)DGy|;TQ*Fs)Ok6+t%(1u2lTzp-w2_u4DA8}k1+W~G>jNh1kL1yJT zkKs=6Kf8m%$+WH_Uw3?_ir~9ms~&R*S+JuDZ@N3OeqZ5l#NJL8)6;9Sr8=S8{H*W2 zt*2OL!Lmn?>GIdnh4{^A@h;EAr-D#_(Yq@a6WS<=j~l83?O131HY~O4$QZ)>b#WiX zkG23JvoO+^r}^R!Euss@&N5><6Tksdp^w8W!;UNvb<` zQcA81m{R+0-?C zt%dMh$X5x0u+5L;A#TeT>*%K02QK%o-AosOQ(H9g=M~_C;*B`de)LP1NOUQvkmqCk z#Yry+5%i%kPFA%|r6ZNz9d5x~SL#ypOq{2CZ`y@lRpm>a&Ncc*Ga{W~ zylD&*&8}3iDOhmOA`$>I+l*RZzuLL zxlKnVECxwuim*d8-$*4|>?#dF1guk_cFVB&kbSrlci8Rgqk>{YJeA!?iPIYr6S8HvRlXk0R z+Mgg2g6y;rb=U1IC@>MRq6GMuje()Z-E3O5$01E`m__yl*K1less;1nBGq^Z`gv8n z&B?{1K1ybGe}VTUJXKxU#-bLp*L66C3Ni@UG&BsX`0Gm5b?>%rjady`vA6at6&N_~ z=z$e6_I;{SX8DbkRdxI~-Vj_{dI1x-J_(~2+cXk_tj|@+7Vi`g4Qx^qbfGh%e5X-r z<(XA+1JM?&OkE*}fN^~kvT*My(C+}UZlMo$W;ianeP1zyKHL1W(*(A1y?>#sg4OzV zm(^ir7C+kb1lmOb)PCP+$25{?80?N1-Y29X$88&+kXdoXwegFg)=N#qRil1SJH|6n z502wR`%J8yLK63l&p&mTR702@SAM&XdQP$zoY76+^cM=23DK8Spbb?1G+LK9E^Jn- z3ief`L`GC=`yb;ADJ2wm{vPfhR7Ax#LpcCVc=VeLp&D&lT-$2Vv|bY1U)tKY9GPWt zSc80-+3?_CCF4gfWDm7<)8!ag?&x!q*m$HFrB|AOE~3hPI^b){`|D-L1HK)jYsd&{ z$CS;r(GfV$T=rdb-l4PMKQjyGNfPz3Lzo-4N*-S>%d3?|Y4FvQ7JAaWati-3I2p0_ zjDl;}TeH0a)?h6ld-QqXSr$fUi;Q4|SS=QW6*Zn>tP=$N5Ac3TG7WbNwmbQ%V%kUB zW0=On4F(ozVZs8_l%hOodaHp+YZUalcIPA^J`Sy5kDp9z5UlRjzUK4s6BZ zg-mP5N3sxS@Kw(>kQFIR3Sim&wyyZq8rDed4}Wc}RmqTc!^JROJQ z`C?1|`rT31lv|Dly9T7TZ&>KlXamVY<-*UddEHON`%f*fy)xiyp83 zVf>+9$Vl(sK)CS_R4(rwqpTK$B-e+I{x&2Z^4<;Sl!>ZEpy^?Lo#cTxI8eaf}-EKHQAPP=s-&^ly~j_u-}> zal5`(!)%asso`MTyc(a?7;3swK~er{%*Is?T{U#gH@7(Juz|k?1N)81mH(ZeL|H#A zC}~uw20AuxkEM@QoE_h8^KEmK&POA90RX?8yWS%okH3h~@z49u{~S9hOS1y~UMs|Y zF&%0>ZBGHK5{RmZn|3=hl71G$VT(^A(3AJX;Og0f9VJ3EONXW)*9Bp?nqZ1`jK_mI{F8EgwSC|0Aa0-SGktE2Nimwgxu zF($*o_96C~c{@0l0uYA^up>0_p7^C3kRbDc!+CL0U}nqfF+wkHG#_~YvG#joS24)wGPPo?>2hn zBw;ox^6Hoe+L`S%=p8wMKr{xk{Z<#GaF+zlCG5dW(cVf0=qk`MtF|vClNdjelA#wz zBeY55dM)n&hJ;h$BP}vI_Mpvm-lV|Gdz!=Uo8rKi{e!dAOH&&4&vohsl%rbRA&{RK zY7oc-0)fBP1}Q|O&T!Z@QdJMXI@Wm~_*1vu*3-BC3$PRY4N2A|iE!!0!KeY%%+CpW zUWLk%=PBddCs*qJMR|*IyritlM0&hxdrS#gzYxm^%E4grak#x6ZBU*6#Kgo(hI6f( zg!(nffY_0%d>@paZEP_cw{js7e}VhX#Y_ z@;*9YDz+w2@5h1%g>^y8YZ@V+?Z_QBM75Qzbd@O1r?x6~A#d)E6J+5RKZJ{CUmVq8 zC4dV%g2V-u!!CUYy77EI>&d(5RC<*-ylKd>ASWDHD9YAal2lZuhzBG^%a?ha_#pRL zZ8~jyWKBftF`wk$!R-f0`UB?ANbMmI#j;RH+bO28E}{VOdoOq5u;tOpqw)hLX4WH6 z=m2usSj@e5zhHl7`e~~b=;;Y1;OX<$2{-vxTlz42urJG^6gv-OJ?kWwFIEf>7;;(3 z<^3z~RREB<{^x*QzhM`$t}MMEE*$jy-htY_kSm5jjeMoO1e?eDjvGgE4GtH_0l@oA z!j9kDR2V;6!?teuj}!PY|BUIYSb+xqLNyj)?&SyOtjB$k_1(8OUFI<{aoNi-C7X96e;4qo~8cuO%jZz?nQxg`}4}lGuW|PF*k;}{AyhU==k}; zAFo%@=Vf*~T)|zuV^VD|Ad2|$k;^O{FZYKB#-@a(K}9T`YPNQ@HW-v;n_DuM_+I== z^|VP!9>%v-qhlywmd|V{t6J~+h^ISfud9gU#+PO0VruE1CoG|>(8bdw1H-LoBtGK= ziJZctV&zx3xbjfvjSRCt%i&kO<#8_Un>2Jp!%4iKP3!YGl_2ZeO$FDU8=%Jfg|Rk( zStmnDHpsZ(+sUJqkC|Vi+M9I1*J~1PIp6=|F`3YfEiU90o^n^mU(eE@a1?-=0XZ z{#!C_Or3(>*UiCcT)s!Ry_?c920vg10xK*9fObrvkZ!SP0hCv>r0$Vnmg_`aGUiCr zV6m(h-c6n#zR?KL@R>kPA7aPl2D^~wjnRfPb7yrMpwP=97FN|^Rjq0iYbbRh8Wr@v z5%M?7zxZ9l)*9j~MuT8GU|Wqzhy%*3GOU7nE2V72??62fEIw>5Zh5&!H%)ICO~dtk z_A>$?{H;8hE8=1;{f4ydk*3=l2}|JGt+^$6QCm>F$LI{bX^CL|e*jM)K`?Mgcbu@? zIpH$BE}Cg}3=LunZ8c@s?CL6ZMN$1=F&UU7GkH_)wKVAn3O|+8Z2y}u`V^eoT0--! z2at-GbtuCg6#krm=~OYW(%HPnPJCQ3Xomimtalh+cf{nYES7$2(A;E+SDe5N5_!?( z65H$z1)R-%xA6PqtR&e%4dEB3w|*QHw;mZf!;Z+va;(bhS@KEQKp+<@-K#sqxw3kL`Bods@T4ER6Ac)Hz8h78vsPe}JY#r9f%|k=5OIC=EOe2)ul_az3&TTH;+&_!;Bot zF==NSuY+qlx+E>QzcH~!o1fwZV!?Q_=UZQQ!F2?$q`a|XpLPdTT`0B!D7SLN%~a^n z5&h${IgboLeznFQwfLMdBbEAax0hRnF5z6{-E zjlO5v3d}=^j)@K%Oyh8p*5aZjFMPSLYd-u;T2{9lQn%FYx^4b!UBHW@_O>WaztMrD zPi?(<n28W%2|O$#?q4F ztE4Blt?}~j^s@Q9*th~dDTj;L+-*fF-?SNb;bDoyK3^yQluvbM9+L}LN$29I9T2zS zItep!%TM>8MGqCMq3e0_fA77t53LGZE~h6CP476O`N7Dol}AV0F18}%FvKGd_No_+ z$o!#45s{T}WCZyLJbD~SoD|Y^JR0R(f0(L$&rsNoqK#SWM#9gO0j=LRPiX{?44|9T zdiieq>Buw(55!(k1|31ueb?|6WN>@aJ^0T%e>k~#%^Htj@lAV#Tr5O~1|P0}A$3|a zVxANt(L(;H=AoK(|Hjn&!(R~gSENx66H(;CIn1(kR*-5P4?Y9n!UP{m>f;?)9n<9E zxIe*0%M4X}AS(%ls#2m;lAw#;RV>yR^MqL4(I`lEP-L!mm^FHeW~Cz7L0h__5`w96AcsY*KDJSW*)s_O_MvIh34t`hcCnCcVfT}Z+341l zvX?_a&mj7^1jHpfhW55`?`xAb?+Ro9c&^tXl}- z({GVHlicKr@EyUx;d#DlFQ_eH)8Gz=n7m#U%14+nc~Xurj3yx|g@@LE@SR7msr`9E xuqYxT#kMoS(wTI5)oxQDgoysn#liTO9+aF8W1v7q5$5?s`r0O1bs8_C{|5jP;!gko literal 0 HcmV?d00001 diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 8effe2cc92ca..6bfca3ef37d5 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -344,6 +344,7 @@ DagRunState DAGs Dask dask +daskexecutor dat Databricks databricks diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 03d821eff15e..091a45cf36ee 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -277,6 +277,16 @@ ], "excluded-python-versions": [] }, + "daskexecutor": { + "deps": [ + "apache-airflow>=2.4.0", + "cloudpickle>=1.4.1", + "dask>=2.9.0,!=2022.10.1,!=2023.5.0", + "distributed>=2.11.1,!=2023.5.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [] + }, "databricks": { "deps": [ "aiohttp>=3.6.3, <4", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b3fc1ec2265f..540a5938a00e 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,7 +2,7 @@ # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:344261ca3aa7ff31e098b1d88280566a -build-docs:927336a331afdb9f7b6797c119d9cc16 +build-docs:1baeefeaba160aef5355ffbb09212cf5 ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96 ci:free-space:49af17b032039c05c41a7a8283f365cc ci:get-workflow-info:8246038093359b9c3c110043419473e2 @@ -31,28 +31,28 @@ k8s:status:1b1b6fb6ccc7a211a3eb44568da71659 k8s:tests:2319c6f5ba0681ff7627e3166a1479ef k8s:upload-k8s-image:a9ac79e2f5e3d6b01fa45e764885913f k8s:db08df55a8200bff5858a3b7549bff47 -prod-image:build:58ab53c2f5e799bbe05d6fac26d001bf +prod-image:build:1a321a1243555ab4e2cc95f44e85c92e prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 -prod-image:a434391e2b5201aaeb5c06601485f7bc +prod-image:ad55d9dce93cbfcc1c5ad8640a007340 release-management:add-back-references:8dab6a30076a55f2d31c6d22a94e0ccb release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485 -release-management:generate-issue-content-providers:9cea234261547208ac812464d3e4a598 +release-management:generate-issue-content-providers:6b0d954cb6dbdec0da0a7988feec58f0 release-management:generate-providers-metadata:d4e8e5cfaa024e3963af02d7a873048d release-management:install-provider-packages:a89493be1ae961c13469b5a25a605069 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5 -release-management:prepare-provider-documentation:bc0b95c6a0f308273ab25f728afdb039 -release-management:prepare-provider-packages:3e19a787c71d5515a2bc05539da0022f -release-management:publish-docs:4a7d78a846f7e58c6af4a55cbdf7104b +release-management:prepare-provider-documentation:b8defe8b94bf790d88d055c3e4c0ffba +release-management:prepare-provider-packages:4599a06e636c6fe8fda41b5a751c1429 +release-management:publish-docs:00eae3f9acc2134c14c9a94d789423ce release-management:release-prod-images:4d85a23a2175bf8894de5aedbdd85614 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:e12492cf058a0c32fc6daecde65a1aef -sbom:generate-provider-requirements:1e7609a0ebafb839d30ec90e4254a628 +release-management:f15d70b8cb924cf33e6d7e2a1f32adf8 +sbom:generate-provider-requirements:9abe53200ea5f40e0bf7c27f6087f27f sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64 -sbom:2ac6b8fc1e84428d5588270eb0d84040 +sbom:935d041028e847d3faf763a95b51063e setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e setup:check-all-params-in-groups:76b3b1d3726ca2a446bab0668b1c50f5 setup:config:38ebaaf93ed42bc7b2a3000eeea2631d diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index f0475ed9c40a..75e92ebb2ccd 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -1,4 +1,4 @@ - + - + @@ -219,9 +219,12 @@ + + + - Command: build-docs + Command: build-docs @@ -232,64 +235,65 @@ -Usage: breeze build-docs [OPTIONS] +Usage: breeze build-docs [OPTIONS] -╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--docs-only-dOnly build documentation.│ -│--spellcheck-only-sOnly run spell checking.│ -│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts    │ -│before the build - useful for a clean build.                                               │ -│--one-pass-onlyBuilds documentation in one pass only. This is useful for debugging sphinx errors.│ -│--package-filterList of packages to consider.                                                              │ -│(apache-airflow | apache-airflow-providers-airbyte | apache-airflow-providers-alibaba |    │ -│apache-airflow-providers-amazon | apache-airflow-providers-apache-beam |                   │ -│apache-airflow-providers-apache-cassandra | apache-airflow-providers-apache-drill |        │ -│apache-airflow-providers-apache-druid | apache-airflow-providers-apache-flink |            │ -│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |              │ -│apache-airflow-providers-apache-impala | apache-airflow-providers-apache-kafka |           │ -│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |             │ -│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |              │ -│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |            │ -│apache-airflow-providers-apprise | apache-airflow-providers-arangodb |                     │ -│apache-airflow-providers-asana | apache-airflow-providers-atlassian-jira |                 │ -│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                      │ -│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |           │ -│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                   │ -│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                   │ -│apache-airflow-providers-discord | apache-airflow-providers-docker |                       │ -│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                 │ -│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                         │ -│apache-airflow-providers-github | apache-airflow-providers-google |                        │ -│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                       │ -│apache-airflow-providers-http | apache-airflow-providers-imap |                            │ -│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                        │ -│apache-airflow-providers-jenkins | apache-airflow-providers-microsoft-azure |              │ -│apache-airflow-providers-microsoft-mssql | apache-airflow-providers-microsoft-psrp |       │ -│apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |                │ -│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                          │ -│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                        │ -│apache-airflow-providers-openlineage | apache-airflow-providers-opsgenie |                 │ -│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |                     │ -│apache-airflow-providers-papermill | apache-airflow-providers-plexus |                     │ -│apache-airflow-providers-postgres | apache-airflow-providers-presto |                      │ -│apache-airflow-providers-qubole | apache-airflow-providers-redis |                         │ -│apache-airflow-providers-salesforce | apache-airflow-providers-samba |                     │ -│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |                     │ -│apache-airflow-providers-sftp | apache-airflow-providers-singularity |                     │ -│apache-airflow-providers-slack | apache-airflow-providers-smtp |                           │ -│apache-airflow-providers-snowflake | apache-airflow-providers-sqlite |                     │ -│apache-airflow-providers-ssh | apache-airflow-providers-tableau |                          │ -│apache-airflow-providers-tabular | apache-airflow-providers-telegram |                     │ -│apache-airflow-providers-trino | apache-airflow-providers-vertica |                        │ -│apache-airflow-providers-zendesk | docker-stack | helm-chart)                              │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--docs-only-dOnly build documentation.│ +│--spellcheck-only-sOnly run spell checking.│ +│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts    │ +│before the build - useful for a clean build.                                               │ +│--one-pass-onlyBuilds documentation in one pass only. This is useful for debugging sphinx errors.│ +│--package-filterList of packages to consider.                                                              │ +│(apache-airflow | apache-airflow-providers-airbyte | apache-airflow-providers-alibaba |    │ +│apache-airflow-providers-amazon | apache-airflow-providers-apache-beam |                   │ +│apache-airflow-providers-apache-cassandra | apache-airflow-providers-apache-drill |        │ +│apache-airflow-providers-apache-druid | apache-airflow-providers-apache-flink |            │ +│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |              │ +│apache-airflow-providers-apache-impala | apache-airflow-providers-apache-kafka |           │ +│apache-airflow-providers-apache-kylin | apache-airflow-providers-apache-livy |             │ +│apache-airflow-providers-apache-pig | apache-airflow-providers-apache-pinot |              │ +│apache-airflow-providers-apache-spark | apache-airflow-providers-apache-sqoop |            │ +│apache-airflow-providers-apprise | apache-airflow-providers-arangodb |                     │ +│apache-airflow-providers-asana | apache-airflow-providers-atlassian-jira |                 │ +│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                      │ +│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |           │ +│apache-airflow-providers-daskexecutor | apache-airflow-providers-databricks |              │ +│apache-airflow-providers-datadog | apache-airflow-providers-dbt-cloud |                    │ +│apache-airflow-providers-dingding | apache-airflow-providers-discord |                     │ +│apache-airflow-providers-docker | apache-airflow-providers-elasticsearch |                 │ +│apache-airflow-providers-exasol | apache-airflow-providers-facebook |                      │ +│apache-airflow-providers-ftp | apache-airflow-providers-github |                           │ +│apache-airflow-providers-google | apache-airflow-providers-grpc |                          │ +│apache-airflow-providers-hashicorp | apache-airflow-providers-http |                       │ +│apache-airflow-providers-imap | apache-airflow-providers-influxdb |                        │ +│apache-airflow-providers-jdbc | apache-airflow-providers-jenkins |                         │ +│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql |      │ +│apache-airflow-providers-microsoft-psrp | apache-airflow-providers-microsoft-winrm |       │ +│apache-airflow-providers-mongo | apache-airflow-providers-mysql |                          │ +│apache-airflow-providers-neo4j | apache-airflow-providers-odbc |                           │ +│apache-airflow-providers-openfaas | apache-airflow-providers-openlineage |                 │ +│apache-airflow-providers-opsgenie | apache-airflow-providers-oracle |                      │ +│apache-airflow-providers-pagerduty | apache-airflow-providers-papermill |                  │ +│apache-airflow-providers-plexus | apache-airflow-providers-postgres |                      │ +│apache-airflow-providers-presto | apache-airflow-providers-qubole |                        │ +│apache-airflow-providers-redis | apache-airflow-providers-salesforce |                     │ +│apache-airflow-providers-samba | apache-airflow-providers-segment |                        │ +│apache-airflow-providers-sendgrid | apache-airflow-providers-sftp |                        │ +│apache-airflow-providers-singularity | apache-airflow-providers-slack |                    │ +│apache-airflow-providers-smtp | apache-airflow-providers-snowflake |                       │ +│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                           │ +│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                      │ +│apache-airflow-providers-telegram | apache-airflow-providers-trino |                       │ +│apache-airflow-providers-vertica | apache-airflow-providers-zendesk | docker-stack |       │ +│helm-chart)                                                                                │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg index 35bf4ae86bb6..cd179a225efe 100644 --- a/images/breeze/output_prod-image.svg +++ b/images/breeze/output_prod-image.svg @@ -35,8 +35,8 @@ .breeze-prod-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-r2 { fill: #c5c8c6 } .breeze-prod-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-r5 { fill: #868887 } +.breeze-prod-image-r4 { fill: #868887 } +.breeze-prod-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage PROD images +Tools that developers can use to manually manage PROD images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ -│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ -│verify Verify Production image.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index 30e4426a06cb..882393e5d0df 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -35,8 +35,8 @@ .breeze-prod-image-build-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-build-r2 { fill: #c5c8c6 } .breeze-prod-image-build-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-build-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-build-r5 { fill: #868887 } +.breeze-prod-image-build-r4 { fill: #868887 } +.breeze-prod-image-build-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-build-r6 { fill: #98a84b;font-weight: bold } .breeze-prod-image-build-r7 { fill: #8d7b39 } @@ -337,99 +337,99 @@ -Usage: breeze prod-image build [OPTIONS] +Usage: breeze prod-image build [OPTIONS] Build Production image. Include building multiple images for all or selected Python versions sequentially. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ -│[default: 3.8]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│aiobotocore,amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,g…│ -│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ -│file. It could be full remote URL to the location file, or local file placed in │ -│`docker-context-files` (in this case it has to start with                       │ -│/opt/airflow/docker-context-files).                                             │ -│(TEXT)                                                                          │ -│--airflow-constraints-modeMode of constraints for PROD image building.                           │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│[default: 3.8]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11]                                                  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsear…│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for PROD image building.                           │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg new file mode 100644 index 000000000000..c25e978e3c16 --- /dev/null +++ b/images/breeze/output_release-management.svg @@ -0,0 +1,171 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management + + + + + + + + + + +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... + +Tools that release managers can use to prepare and manage Airflow releases + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Airflow release commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-airflow-package      Prepare sdist/whl package of Airflow.                                                │ +│create-minor-branch          Create a new version branch and update the default branches in main                  │ +│start-rc-process             Start RC process                                                                     │ +│start-release                Start Airflow release process                                                        │ +│release-prod-images          Release production images to DockerHub (needs DockerHub permissions).                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Providers release commands ─────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-provider-documentation      Prepare CHANGELOG, README and COMMITS information for providers.              │ +│prepare-provider-packages           Prepare sdist/whl packages of Airflow Providers.                              │ +│install-provider-packages           Installs provider packages that can be found in dist.                         │ +│verify-provider-packages            Verifies if all provider code is following expectations for providers.        │ +│generate-providers-metadata         Generates metadata for providers.                                             │ +│generate-issue-content-providers    Generates content for issue to test the release.                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│publish-docs            Command to publish generated documentation to airflow-site                                │ +│generate-constraints    Generates pinned constraint files with all extras from setup.py in parallel.              │ +│add-back-references     Command to add back references for documentation to make it backward compatible           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 12eabb09189c..04fd3c8155f2 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -1,4 +1,4 @@ - + - + @@ -150,9 +150,12 @@ + + + - Command: release-management generate-issue-content-providers + Command: release-management generate-issue-content-providers @@ -163,41 +166,42 @@ -Usage: breeze release-management generate-issue-content-providers [OPTIONS] [airbyte | alibaba | amazon | apache.beam +Usage: breeze release-management generate-issue-content-providers [OPTIONS] [airbyte | alibaba | amazon | apache.beam                                                                   | apache.cassandra | apache.drill | apache.druid |                                                                   apache.flink | apache.hdfs | apache.hive |                                                                   apache.impala | apache.kafka | apache.kylin |                                                                   apache.livy | apache.pig | apache.pinot |                                                                   apache.spark | apache.sqoop | apprise | arangodb |                                                                   asana | atlassian.jira | celery | cloudant | -                                                                  cncf.kubernetes | common.sql | databricks | datadog -                                                                  | dbt.cloud | dingding | discord | docker | -                                                                  elasticsearch | exasol | facebook | ftp | github | -                                                                  google | grpc | hashicorp | http | imap | influxdb | -                                                                  jdbc | jenkins | microsoft.azure | microsoft.mssql | -                                                                  microsoft.psrp | microsoft.winrm | mongo | mysql | -                                                                  neo4j | odbc | openfaas | openlineage | opsgenie | -                                                                  oracle | pagerduty | papermill | plexus | postgres | -                                                                  presto | qubole | redis | salesforce | samba | -                                                                  segment | sendgrid | sftp | singularity | slack | -                                                                  smtp | snowflake | sqlite | ssh | tableau | tabular -                                                                  | telegram | trino | vertica | zendesk]... - -Generates content for issue to test the release. - -╭─ Generate issue content flags ───────────────────────────────────────────────────────────────────────────────────────╮ -│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ -│variable set. Can be generated with:                                                    │ -│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status     │ -│(TEXT)                                                                                  │ -│--suffixSuffix to add to the version prepared(TEXT)│ -│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ -│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ -│--disable-progressDisable progress bar│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                  cncf.kubernetes | common.sql | daskexecutor | +                                                                  databricks | datadog | dbt.cloud | dingding | +                                                                  discord | docker | elasticsearch | exasol | facebook +                                                                  | ftp | github | google | grpc | hashicorp | http | +                                                                  imap | influxdb | jdbc | jenkins | microsoft.azure | +                                                                  microsoft.mssql | microsoft.psrp | microsoft.winrm | +                                                                  mongo | mysql | neo4j | odbc | openfaas | +                                                                  openlineage | opsgenie | oracle | pagerduty | +                                                                  papermill | plexus | postgres | presto | qubole | +                                                                  redis | salesforce | samba | segment | sendgrid | +                                                                  sftp | singularity | slack | smtp | snowflake | +                                                                  sqlite | ssh | tableau | tabular | telegram | trino +                                                                  | vertica | zendesk]... + +Generates content for issue to test the release. + +╭─ Generate issue content flags ───────────────────────────────────────────────────────────────────────────────────────╮ +│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ +│variable set. Can be generated with:                                                    │ +│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status     │ +│(TEXT)                                                                                  │ +│--suffixSuffix to add to the version prepared(TEXT)│ +│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ +│--disable-progressDisable progress bar│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index 69b5e0589ba8..68c55935ffd9 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -35,8 +35,8 @@ .breeze-release-management-prepare-provider-documentation-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r2 { fill: #c5c8c6 } .breeze-release-management-prepare-provider-documentation-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-prepare-provider-documentation-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-prepare-provider-documentation-r5 { fill: #868887 } +.breeze-release-management-prepare-provider-documentation-r4 { fill: #868887 } +.breeze-release-management-prepare-provider-documentation-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r6 { fill: #98a84b;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r7 { fill: #8d7b39 } @@ -175,45 +175,45 @@ -Usage: breeze release-management prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +Usage: breeze release-management prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam |                                                                 apache.cassandra | apache.drill | apache.druid |                                                                 apache.flink | apache.hdfs | apache.hive |                                                                 apache.impala | apache.kafka | apache.kylin |                                                                 apache.livy | apache.pig | apache.pinot | apache.spark                                                                 | apache.sqoop | apprise | arangodb | asana |                                                                 atlassian.jira | celery | cloudant | cncf.kubernetes | -                                                                common.sql | databricks | datadog | dbt.cloud | -                                                                dingding | discord | docker | elasticsearch | exasol | -                                                                facebook | ftp | github | google | grpc | hashicorp | -                                                                http | imap | influxdb | jdbc | jenkins | -                                                                microsoft.azure | microsoft.mssql | microsoft.psrp | -                                                                microsoft.winrm | mongo | mysql | neo4j | odbc | -                                                                openfaas | openlineage | opsgenie | oracle | pagerduty -                                                                | papermill | plexus | postgres | presto | qubole | -                                                                redis | salesforce | samba | segment | sendgrid | sftp -                                                                | singularity | slack | smtp | snowflake | sqlite | -                                                                ssh | tableau | tabular | telegram | trino | vertica | -                                                                zendesk]... +                                                                common.sql | daskexecutor | databricks | datadog | +                                                                dbt.cloud | dingding | discord | docker | +                                                                elasticsearch | exasol | facebook | ftp | github | +                                                                google | grpc | hashicorp | http | imap | influxdb | +                                                                jdbc | jenkins | microsoft.azure | microsoft.mssql | +                                                                microsoft.psrp | microsoft.winrm | mongo | mysql | +                                                                neo4j | odbc | openfaas | openlineage | opsgenie | +                                                                oracle | pagerduty | papermill | plexus | postgres | +                                                                presto | qubole | redis | salesforce | samba | segment +                                                                | sendgrid | sftp | singularity | slack | smtp | +                                                                snowflake | sqlite | ssh | tableau | tabular | +                                                                telegram | trino | vertica | zendesk]... -Prepare CHANGELOG, README and COMMITS information for providers. +Prepare CHANGELOG, README and COMMITS information for providers. -╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--base-branchBase branch to use as diff for documentation generation (used for releasing from old │ -│branch)                                                                              │ -│(TEXT)                                                                               │ -│--only-min-version-updateOnly update minimum version in __init__.py files and regenerate corresponding        │ -│documentation                                                                        │ -│--regenerate-missing-docsOnly regenerate missing documentation, do not bump version. Useful if templates were │ -│added and you need to regenerate documentation.                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--base-branchBase branch to use as diff for documentation generation (used for releasing from old │ +│branch)                                                                              │ +│(TEXT)                                                                               │ +│--only-min-version-updateOnly update minimum version in __init__.py files and regenerate corresponding        │ +│documentation                                                                        │ +│--regenerate-missing-docsOnly regenerate missing documentation, do not bump version. Useful if templates were │ +│added and you need to regenerate documentation.                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index e205331cb588..5a42796427fe 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -35,8 +35,8 @@ .breeze-release-management-prepare-provider-packages-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-prepare-provider-packages-r2 { fill: #c5c8c6 } .breeze-release-management-prepare-provider-packages-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-prepare-provider-packages-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-prepare-provider-packages-r5 { fill: #868887 } +.breeze-release-management-prepare-provider-packages-r4 { fill: #868887 } +.breeze-release-management-prepare-provider-packages-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-prepare-provider-packages-r6 { fill: #8d7b39 } .breeze-release-management-prepare-provider-packages-r7 { fill: #98a84b;font-weight: bold } @@ -154,17 +154,17 @@ -Usage: breeze release-management prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +Usage: breeze release-management prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam |                                                            apache.cassandra | apache.drill | apache.druid |                                                            apache.flink | apache.hdfs | apache.hive | apache.impala |                                                            apache.kafka | apache.kylin | apache.livy | apache.pig |                                                            apache.pinot | apache.spark | apache.sqoop | apprise |                                                            arangodb | asana | atlassian.jira | celery | cloudant | -                                                           cncf.kubernetes | common.sql | databricks | datadog | -                                                           dbt.cloud | dingding | discord | docker | elasticsearch | -                                                           exasol | facebook | ftp | github | google | grpc | -                                                           hashicorp | http | imap | influxdb | jdbc | jenkins | -                                                           microsoft.azure | microsoft.mssql | microsoft.psrp | +                                                           cncf.kubernetes | common.sql | daskexecutor | databricks | +                                                           datadog | dbt.cloud | dingding | discord | docker | +                                                           elasticsearch | exasol | facebook | ftp | github | google | +                                                           grpc | hashicorp | http | imap | influxdb | jdbc | jenkins +                                                           | microsoft.azure | microsoft.mssql | microsoft.psrp |                                                            microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas |                                                            openlineage | opsgenie | oracle | pagerduty | papermill |                                                            plexus | postgres | presto | qubole | redis | salesforce | @@ -174,18 +174,18 @@ Prepare sdist/whl packages of Airflow Providers. -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_publish-docs.svg b/images/breeze/output_release-management_publish-docs.svg index f534f04e953c..a118eae74f03 100644 --- a/images/breeze/output_release-management_publish-docs.svg +++ b/images/breeze/output_release-management_publish-docs.svg @@ -35,8 +35,8 @@ .breeze-release-management-publish-docs-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-publish-docs-r2 { fill: #c5c8c6 } .breeze-release-management-publish-docs-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-publish-docs-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-publish-docs-r5 { fill: #868887 } +.breeze-release-management-publish-docs-r4 { fill: #868887 } +.breeze-release-management-publish-docs-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-publish-docs-r6 { fill: #98a84b;font-weight: bold } .breeze-release-management-publish-docs-r7 { fill: #8d7b39 } .breeze-release-management-publish-docs-r8 { fill: #cc555a } @@ -228,62 +228,62 @@ -Usage: breeze release-management publish-docs [OPTIONS] +Usage: breeze release-management publish-docs [OPTIONS] Command to publish generated documentation to airflow-site -╭─ Publish Docs ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--override-versioned-sOverrides versioned directories.│ -│--package-filterList of packages to consider.                                                      │ -│(apache-airflow | apache-airflow-providers-airbyte |                               │ -│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |               │ -│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra | │ -│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |    │ -│apache-airflow-providers-apache-flink | apache-airflow-providers-apache-hdfs |     │ -│apache-airflow-providers-apache-hive | apache-airflow-providers-apache-impala |    │ -│apache-airflow-providers-apache-kafka | apache-airflow-providers-apache-kylin |    │ -│apache-airflow-providers-apache-livy | apache-airflow-providers-apache-pig |       │ -│apache-airflow-providers-apache-pinot | apache-airflow-providers-apache-spark |    │ -│apache-airflow-providers-apache-sqoop | apache-airflow-providers-apprise |         │ -│apache-airflow-providers-arangodb | apache-airflow-providers-asana |               │ -│apache-airflow-providers-atlassian-jira | apache-airflow-providers-celery |        │ -│apache-airflow-providers-cloudant | apache-airflow-providers-cncf-kubernetes |     │ -│apache-airflow-providers-common-sql | apache-airflow-providers-databricks |        │ -│apache-airflow-providers-datadog | apache-airflow-providers-dbt-cloud |            │ -│apache-airflow-providers-dingding | apache-airflow-providers-discord |             │ -│apache-airflow-providers-docker | apache-airflow-providers-elasticsearch |         │ -│apache-airflow-providers-exasol | apache-airflow-providers-facebook |              │ -│apache-airflow-providers-ftp | apache-airflow-providers-github |                   │ -│apache-airflow-providers-google | apache-airflow-providers-grpc |                  │ -│apache-airflow-providers-hashicorp | apache-airflow-providers-http |               │ -│apache-airflow-providers-imap | apache-airflow-providers-influxdb |                │ -│apache-airflow-providers-jdbc | apache-airflow-providers-jenkins |                 │ -│apache-airflow-providers-microsoft-azure | apache-airflow-providers-microsoft-mssql│ -│| apache-airflow-providers-microsoft-psrp |                                        │ -│apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |        │ -│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                  │ -│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                │ -│apache-airflow-providers-openlineage | apache-airflow-providers-opsgenie |         │ -│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |             │ -│apache-airflow-providers-papermill | apache-airflow-providers-plexus |             │ -│apache-airflow-providers-postgres | apache-airflow-providers-presto |              │ -│apache-airflow-providers-qubole | apache-airflow-providers-redis |                 │ -│apache-airflow-providers-salesforce | apache-airflow-providers-samba |             │ -│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |             │ -│apache-airflow-providers-sftp | apache-airflow-providers-singularity |             │ -│apache-airflow-providers-slack | apache-airflow-providers-smtp |                   │ -│apache-airflow-providers-snowflake | apache-airflow-providers-sqlite |             │ -│apache-airflow-providers-ssh | apache-airflow-providers-tableau |                  │ -│apache-airflow-providers-tabular | apache-airflow-providers-telegram |             │ -│apache-airflow-providers-trino | apache-airflow-providers-vertica |                │ -│apache-airflow-providers-zendesk | docker-stack | helm-chart)                      │ -│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(TEXT)[required]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Publish Docs ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--override-versioned-sOverrides versioned directories.│ +│--package-filterList of packages to consider.                                                      │ +│(apache-airflow | apache-airflow-providers-airbyte |                               │ +│apache-airflow-providers-alibaba | apache-airflow-providers-amazon |               │ +│apache-airflow-providers-apache-beam | apache-airflow-providers-apache-cassandra | │ +│apache-airflow-providers-apache-drill | apache-airflow-providers-apache-druid |    │ +│apache-airflow-providers-apache-flink | apache-airflow-providers-apache-hdfs |     │ +│apache-airflow-providers-apache-hive | apache-airflow-providers-apache-impala |    │ +│apache-airflow-providers-apache-kafka | apache-airflow-providers-apache-kylin |    │ +│apache-airflow-providers-apache-livy | apache-airflow-providers-apache-pig |       │ +│apache-airflow-providers-apache-pinot | apache-airflow-providers-apache-spark |    │ +│apache-airflow-providers-apache-sqoop | apache-airflow-providers-apprise |         │ +│apache-airflow-providers-arangodb | apache-airflow-providers-asana |               │ +│apache-airflow-providers-atlassian-jira | apache-airflow-providers-celery |        │ +│apache-airflow-providers-cloudant | apache-airflow-providers-cncf-kubernetes |     │ +│apache-airflow-providers-common-sql | apache-airflow-providers-daskexecutor |      │ +│apache-airflow-providers-databricks | apache-airflow-providers-datadog |           │ +│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |           │ +│apache-airflow-providers-discord | apache-airflow-providers-docker |               │ +│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |         │ +│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                 │ +│apache-airflow-providers-github | apache-airflow-providers-google |                │ +│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |               │ +│apache-airflow-providers-http | apache-airflow-providers-imap |                    │ +│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                │ +│apache-airflow-providers-jenkins | apache-airflow-providers-microsoft-azure |      │ +│apache-airflow-providers-microsoft-mssql | apache-airflow-providers-microsoft-psrp │ +│| apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |      │ +│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                  │ +│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                │ +│apache-airflow-providers-openlineage | apache-airflow-providers-opsgenie |         │ +│apache-airflow-providers-oracle | apache-airflow-providers-pagerduty |             │ +│apache-airflow-providers-papermill | apache-airflow-providers-plexus |             │ +│apache-airflow-providers-postgres | apache-airflow-providers-presto |              │ +│apache-airflow-providers-qubole | apache-airflow-providers-redis |                 │ +│apache-airflow-providers-salesforce | apache-airflow-providers-samba |             │ +│apache-airflow-providers-segment | apache-airflow-providers-sendgrid |             │ +│apache-airflow-providers-sftp | apache-airflow-providers-singularity |             │ +│apache-airflow-providers-slack | apache-airflow-providers-smtp |                   │ +│apache-airflow-providers-snowflake | apache-airflow-providers-sqlite |             │ +│apache-airflow-providers-ssh | apache-airflow-providers-tableau |                  │ +│apache-airflow-providers-tabular | apache-airflow-providers-telegram |             │ +│apache-airflow-providers-trino | apache-airflow-providers-vertica |                │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                      │ +│*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(TEXT)[required]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_sbom.svg b/images/breeze/output_sbom.svg index 205f0fa24269..0f42e678c179 100644 --- a/images/breeze/output_sbom.svg +++ b/images/breeze/output_sbom.svg @@ -35,8 +35,8 @@ .breeze-sbom-r1 { fill: #c5c8c6;font-weight: bold } .breeze-sbom-r2 { fill: #c5c8c6 } .breeze-sbom-r3 { fill: #d0b344;font-weight: bold } -.breeze-sbom-r4 { fill: #68a0b3;font-weight: bold } -.breeze-sbom-r5 { fill: #868887 } +.breeze-sbom-r4 { fill: #868887 } +.breeze-sbom-r5 { fill: #68a0b3;font-weight: bold } .breeze-sbom-r6 { fill: #98a84b;font-weight: bold } @@ -90,17 +90,17 @@ -Usage: breeze sbom [OPTIONS] COMMAND [ARGS]... +Usage: breeze sbom [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare sbom information -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ SBOM commands ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│update-sbom-information                     Update SBOM information in airflow-site project.                      │ -│generate-provider-requirements              Generate requirements for selected provider.                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ SBOM commands ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│update-sbom-information                     Update SBOM information in airflow-site project.                      │ +│generate-provider-requirements              Generate requirements for selected provider.                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_sbom_generate-provider-requirements.svg b/images/breeze/output_sbom_generate-provider-requirements.svg index a803d93f6d92..9dc8ae664aa2 100644 --- a/images/breeze/output_sbom_generate-provider-requirements.svg +++ b/images/breeze/output_sbom_generate-provider-requirements.svg @@ -35,8 +35,8 @@ .breeze-sbom-generate-provider-requirements-r1 { fill: #c5c8c6;font-weight: bold } .breeze-sbom-generate-provider-requirements-r2 { fill: #c5c8c6 } .breeze-sbom-generate-provider-requirements-r3 { fill: #d0b344;font-weight: bold } -.breeze-sbom-generate-provider-requirements-r4 { fill: #68a0b3;font-weight: bold } -.breeze-sbom-generate-provider-requirements-r5 { fill: #868887 } +.breeze-sbom-generate-provider-requirements-r4 { fill: #868887 } +.breeze-sbom-generate-provider-requirements-r5 { fill: #68a0b3;font-weight: bold } .breeze-sbom-generate-provider-requirements-r6 { fill: #8d7b39 } .breeze-sbom-generate-provider-requirements-r7 { fill: #cc555a } .breeze-sbom-generate-provider-requirements-r8 { fill: #8a4346 } @@ -144,34 +144,34 @@ -Usage: breeze sbom generate-provider-requirements [OPTIONS] +Usage: breeze sbom generate-provider-requirements [OPTIONS] Generate requirements for selected provider. -╭─ Generate provider requirements flags ───────────────────────────────────────────────────────────────────────────────╮ -│--airflow-versionAirflow version to use to generate the requirements(TEXT)│ -│--pythonPython version to generate the requirements for(3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11)│ -│*--provider-idProvider to generate the requirements for                                                  │ -│(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid │ -│| apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | │ -│apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb │ -│| asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook | ftp│ -│| github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |           │ -│microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     │ -│neo4j | odbc | openfaas | openlineage | opsgenie | oracle | pagerduty | papermill | plexus │ -│| postgres | presto | qubole | redis | salesforce | samba | segment | sendgrid | sftp |    │ -│singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |     │ -│trino | vertica | zendesk)                                                                 │ -│[required]                                                                                 │ -│--provider-versionProvider version to generate the requirements for(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Generate provider requirements flags ───────────────────────────────────────────────────────────────────────────────╮ +│--airflow-versionAirflow version to use to generate the requirements(TEXT)│ +│--pythonPython version to generate the requirements for(3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11)│ +│*--provider-idProvider to generate the requirements for                                                  │ +│(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid │ +│| apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb │ +│| asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.sql | daskexecutor │ +│| databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol |│ +│facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc |      │ +│jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo |   │ +│mysql | neo4j | odbc | openfaas | openlineage | opsgenie | oracle | pagerduty | papermill |│ +│plexus | postgres | presto | qubole | redis | salesforce | samba | segment | sendgrid |    │ +│sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram│ +│| trino | vertica | zendesk)                                                               │ +│[required]                                                                                 │ +│--provider-versionProvider version to generate the requirements for(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/installed_providers.txt b/scripts/ci/installed_providers.txt index 3d511abfcc96..9cdcf765eb06 100644 --- a/scripts/ci/installed_providers.txt +++ b/scripts/ci/installed_providers.txt @@ -2,6 +2,7 @@ amazon celery cncf.kubernetes common.sql +daskexecutor docker elasticsearch ftp diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py index 2750e9dbaed8..2ec53c1962c7 100755 --- a/scripts/ci/pre_commit/pre_commit_insert_extras.py +++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py @@ -45,7 +45,7 @@ CONSTANTS_FOOTER = "# END EXTRAS HERE" DEFAULT_EXTRAS = ( - "amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google," + "amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google," "google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas," "postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" ) diff --git a/setup.py b/setup.py index 44998d8747ce..837ef85513eb 100644 --- a/setup.py +++ b/setup.py @@ -481,10 +481,10 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): CORE_EXTRAS_DEPENDENCIES: dict[str, list[str]] = { "aiobotocore": aiobotocore, "async": async_packages, - "celery": celery, + "celery": celery, # TODO: remove and move to a regular provider package in a separate PR "cgroups": cgroups, - "cncf.kubernetes": kubernetes, - "dask": dask, + "cncf.kubernetes": kubernetes, # TODO: remove and move to a regular provider package in a separate PR + "dask": dask, # TODO: remove and move to a provider package in a separate PR "deprecated_api": deprecated_api, "github_enterprise": flask_appbuilder_oauth, "google_auth": flask_appbuilder_oauth, @@ -538,6 +538,7 @@ def add_additional_extras() -> None: "azure": "microsoft.azure", "cassandra": "apache.cassandra", "crypto": "", # this is legacy extra - all dependencies are already "install-requires" + "dask": "daskexecutor", "druid": "apache.druid", "gcp": "google", "gcp_api": "google", @@ -844,7 +845,7 @@ def replace_extra_dependencies_with_provider_packages(extra: str, providers: lis :param extra: Name of the extra to add providers to :param providers: list of provider ids """ - if extra in ["cncf.kubernetes", "kubernetes", "celery"]: + if extra in ["cncf.kubernetes", "kubernetes", "celery", "daskexecutor", "dask"]: EXTRAS_DEPENDENCIES[extra].extend( [get_provider_package_name_from_package_id(package_name) for package_name in providers] ) diff --git a/tests/providers/daskexecutor/__init__.py b/tests/providers/daskexecutor/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/providers/daskexecutor/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/executors/test_dask_executor.py b/tests/providers/daskexecutor/test_dask_executor.py similarity index 98% rename from tests/executors/test_dask_executor.py rename to tests/providers/daskexecutor/test_dask_executor.py index d8ded41e158b..2d559eaa404e 100644 --- a/tests/executors/test_dask_executor.py +++ b/tests/providers/daskexecutor/test_dask_executor.py @@ -24,10 +24,10 @@ from distributed import LocalCluster from airflow.exceptions import AirflowException -from airflow.executors.dask_executor import DaskExecutor from airflow.jobs.backfill_job_runner import BackfillJobRunner from airflow.jobs.job import Job, run_job from airflow.models import DagBag +from airflow.providers.daskexecutor.executors.dask_executor import DaskExecutor from airflow.utils import timezone from tests.test_utils.config import conf_vars @@ -159,7 +159,7 @@ def test_tls(self): # and tasks to have completed. executor.client.close() - @mock.patch("airflow.executors.dask_executor.DaskExecutor.sync") + @mock.patch("airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor.sync") @mock.patch("airflow.executors.base_executor.BaseExecutor.trigger_tasks") @mock.patch("airflow.executors.base_executor.Stats.gauge") def test_gauge_executor_metrics(self, mock_stats_gauge, mock_trigger_tasks, mock_sync): From f57ee511e0407cf47512febe73f23bd59109d586 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Mon, 24 Jul 2023 15:38:46 -0700 Subject: [PATCH 298/533] Remove comment re cli fallbacks, they have been removed (#32818) There was a code comment warning that the fallbacks for celery cli commands should not be removed until some conditions are met, but the fallbacks were removed in #32775 --- airflow/cli/cli_config.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index a68de35b86df..8e4c98445033 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -823,17 +823,6 @@ def string_lower_type(val): action="store_true", ) -# IMPORTANT NOTE! ONLY FOR CELERY ARGUMENTS -# -# Celery configs below have explicit fallback values because celery provider defaults are not yet loaded -# via provider at the time we parse the command line, so in case it is not set, we need to have manual -# fallback. After ProvidersManager.initialize_providers_configuration() is called, the fallbacks are -# not needed anymore and everywhere where you access configuration in provider-specific code and when -# you are sure that providers configuration has been initialized, you can use conf.get() without fallbacks. -# -# DO NOT REMOVE THE FALLBACKS in args parsing even if you are tempted to. -# TODO: possibly move the commands to providers but that could be big performance hit on the CLI -# worker ARG_QUEUES = Arg( ("-q", "--queues"), help="Comma delimited list of queues to serve", From c4b6f06f6e2897b3f1ee06440fc66f191acee9a8 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:50:35 -0400 Subject: [PATCH 299/533] Handle login by auth managers (#32697) --- airflow/auth/managers/base_auth_manager.py | 29 ++++ airflow/auth/managers/fab/auth/__init__.py | 17 +++ .../auth/managers/fab/auth/anonymous_user.py | 48 +++++++ airflow/auth/managers/fab/fab_auth_manager.py | 11 ++ .../managers/fab/security_manager_override.py | 54 ++++++++ airflow/www/auth.py | 9 +- airflow/www/extensions/init_appbuilder.py | 11 +- airflow/www/extensions/init_auth_manager.py | 6 +- airflow/www/extensions/init_security.py | 4 +- airflow/www/fab_security/manager.py | 88 +------------ airflow/www/fab_security/sqla/manager.py | 3 - airflow/www/security.py | 1 + airflow/www/templates/appbuilder/flash.html | 2 +- .../templates/appbuilder/navbar_right.html | 2 +- .../auh/managers/fab/test_fab_auth_manager.py | 24 +++- .../fab/test_security_manager_override.py | 124 ++++++++++++++++++ tests/auh/managers/test_base_auth_manager.py | 17 +++ tests/www/test_security.py | 2 +- 18 files changed, 344 insertions(+), 108 deletions(-) create mode 100644 airflow/auth/managers/fab/auth/__init__.py create mode 100644 airflow/auth/managers/fab/auth/anonymous_user.py create mode 100644 tests/auh/managers/fab/test_security_manager_override.py diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index ab5356bf8ca3..6f148cd31bd5 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -18,9 +18,14 @@ from __future__ import annotations from abc import abstractmethod +from typing import TYPE_CHECKING +from airflow.exceptions import AirflowException from airflow.utils.log.logging_mixin import LoggingMixin +if TYPE_CHECKING: + from airflow.www.security import AirflowSecurityManager + class BaseAuthManager(LoggingMixin): """ @@ -29,6 +34,9 @@ class BaseAuthManager(LoggingMixin): Auth managers are responsible for any user management related operation such as login, logout, authz, ... """ + def __init__(self): + self._security_manager: AirflowSecurityManager | None = None + @abstractmethod def get_user_name(self) -> str: """Return the username associated to the user in session.""" @@ -39,6 +47,11 @@ def is_logged_in(self) -> bool: """Return whether the user is logged in.""" ... + @abstractmethod + def get_url_login(self, **kwargs) -> str: + """Return the login page url.""" + ... + def get_security_manager_override_class(self) -> type: """ Return the security manager override class. @@ -50,3 +63,19 @@ class airflow.www.security.AirflowSecurityManager with a custom implementation. By default, return an empty class. """ return object + + @property + def security_manager(self) -> AirflowSecurityManager: + """Get the security manager.""" + if not self._security_manager: + raise AirflowException("Security manager not defined.") + return self._security_manager + + @security_manager.setter + def security_manager(self, security_manager: AirflowSecurityManager): + """ + Set the security manager. + + :param security_manager: the security manager + """ + self._security_manager = security_manager diff --git a/airflow/auth/managers/fab/auth/__init__.py b/airflow/auth/managers/fab/auth/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/auth/managers/fab/auth/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/fab/auth/anonymous_user.py b/airflow/auth/managers/fab/auth/anonymous_user.py new file mode 100644 index 000000000000..b97ec69e3ef0 --- /dev/null +++ b/airflow/auth/managers/fab/auth/anonymous_user.py @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from flask import current_app +from flask_login import AnonymousUserMixin + + +class AnonymousUser(AnonymousUserMixin): + """User object used when no active user is logged in.""" + + _roles: set[tuple[str, str]] = set() + _perms: set[tuple[str, str]] = set() + + @property + def roles(self): + if not self._roles: + public_role = current_app.appbuilder.get_app.config["AUTH_ROLE_PUBLIC"] + self._roles = {current_app.appbuilder.sm.find_role(public_role)} if public_role else set() + return self._roles + + @roles.setter + def roles(self, roles): + self._roles = roles + self._perms = set() + + @property + def perms(self): + if not self._perms: + self._perms = { + (perm.action.name, perm.resource.name) for role in self.roles for perm in role.permissions + } + return self._perms diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index f90a9ac06353..6dee27c24c36 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -17,8 +17,10 @@ # under the License. from __future__ import annotations +from flask import url_for from flask_login import current_user +from airflow import AirflowException from airflow.auth.managers.base_auth_manager import BaseAuthManager from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride @@ -48,3 +50,12 @@ def is_logged_in(self) -> bool: def get_security_manager_override_class(self) -> type: """Return the security manager override.""" return FabAirflowSecurityManagerOverride + + def get_url_login(self, **kwargs) -> str: + """Return the login page url.""" + if not self.security_manager.auth_view: + raise AirflowException("`auth_view` not defined in the security manager.") + if "next_url" in kwargs and kwargs["next_url"]: + return url_for(f"{self.security_manager.auth_view.endpoint}.login", next=kwargs["next_url"]) + else: + return url_for(f"{self.security_manager.auth_view.endpoint}.login") diff --git a/airflow/auth/managers/fab/security_manager_override.py b/airflow/auth/managers/fab/security_manager_override.py index 5be9ee1f3672..c6649ce6abea 100644 --- a/airflow/auth/managers/fab/security_manager_override.py +++ b/airflow/auth/managers/fab/security_manager_override.py @@ -19,8 +19,14 @@ from functools import cached_property +from flask import g from flask_appbuilder.const import AUTH_DB, AUTH_LDAP, AUTH_OAUTH, AUTH_OID, AUTH_REMOTE_USER from flask_babel import lazy_gettext +from flask_jwt_extended import JWTManager +from flask_login import LoginManager +from werkzeug.security import generate_password_hash + +from airflow.auth.managers.fab.auth.anonymous_user import AnonymousUser class FabAirflowSecurityManagerOverride: @@ -47,6 +53,7 @@ class FabAirflowSecurityManagerOverride: :param resetmypasswordview: The class for reset my password view. :param resetpasswordview: The class for reset password view. :param rolemodelview: The class for role model view. + :param user_model: The user model. :param userinfoeditview: The class for user info edit view. :param userdbmodelview: The class for user db model view. :param userldapmodelview: The class for user ldap model view. @@ -80,6 +87,7 @@ def __init__(self, **kwargs): self.resetmypasswordview = kwargs["resetmypasswordview"] self.resetpasswordview = kwargs["resetpasswordview"] self.rolemodelview = kwargs["rolemodelview"] + self.user_model = kwargs["user_model"] self.userinfoeditview = kwargs["userinfoeditview"] self.userdbmodelview = kwargs["userdbmodelview"] self.userldapmodelview = kwargs["userldapmodelview"] @@ -88,6 +96,12 @@ def __init__(self, **kwargs): self.userremoteusermodelview = kwargs["userremoteusermodelview"] self.userstatschartview = kwargs["userstatschartview"] + # Setup Flask login + self.lm = self.create_login_manager() + + # Setup Flask-Jwt-Extended + self.create_jwt_manager() + def register_views(self): """Register FAB auth manager related views.""" if not self.appbuilder.app.config.get("FAB_ADD_SECURITY_VIEWS", True): @@ -192,6 +206,46 @@ def register_views(self): category="Security", ) + def create_login_manager(self) -> LoginManager: + """Create the login manager.""" + lm = LoginManager(self.appbuilder.app) + lm.anonymous_user = AnonymousUser + lm.login_view = "login" + lm.user_loader(self.load_user) + return lm + + def create_jwt_manager(self): + """Create the JWT manager.""" + jwt_manager = JWTManager() + jwt_manager.init_app(self.appbuilder.app) + jwt_manager.user_lookup_loader(self.load_user_jwt) + + def reset_password(self, userid, password): + """ + Change/Reset a user's password for authdb. + + Password will be hashed and saved. + :param userid: the user id to reset the password + :param password: the clear text password to reset and save hashed on the db + """ + user = self.get_user_by_id(userid) + user.password = generate_password_hash(password) + self.update_user(user) + + def load_user(self, user_id): + """Load user by ID.""" + return self.get_user_by_id(int(user_id)) + + def load_user_jwt(self, _jwt_header, jwt_data): + identity = jwt_data["sub"] + user = self.load_user(identity) + # Set flask g.user to JWT user, we can't do it on before request + g.user = user + return user + + def get_user_by_id(self, pk): + return self.appbuilder.get_session.get(self.user_model, pk) + @property def auth_user_registration(self): """Will user self registration be allowed.""" diff --git a/airflow/www/auth.py b/airflow/www/auth.py index 54114da1c8a7..46b645f19713 100644 --- a/airflow/www/auth.py +++ b/airflow/www/auth.py @@ -19,7 +19,7 @@ from functools import wraps from typing import Callable, Sequence, TypeVar, cast -from flask import current_app, flash, g, redirect, render_template, request, url_for +from flask import current_app, flash, g, redirect, render_template, request from airflow.configuration import conf from airflow.utils.net import get_hostname @@ -61,12 +61,7 @@ def decorated(*args, **kwargs): else: access_denied = "Access is Denied" flash(access_denied, "danger") - return redirect( - url_for( - appbuilder.sm.auth_view.__class__.__name__ + ".login", - next=request.url, - ) - ) + return redirect(get_auth_manager().get_url_login(next=request.url)) return cast(T, decorated) diff --git a/airflow/www/extensions/init_appbuilder.py b/airflow/www/extensions/init_appbuilder.py index ae793ca95646..5a492f9b6662 100644 --- a/airflow/www/extensions/init_appbuilder.py +++ b/airflow/www/extensions/init_appbuilder.py @@ -41,6 +41,7 @@ from airflow import settings from airflow.configuration import conf +from airflow.www.extensions.init_auth_manager import get_auth_manager # This product contains a modified portion of 'Flask App Builder' developed by Daniel Vaz Gaspar. # (https://github.com/dpgaspar/Flask-AppBuilder). @@ -212,6 +213,8 @@ def init_app(self, app, session): self._addon_managers = app.config["ADDON_MANAGERS"] self.session = session self.sm = self.security_manager_class(self) + auth_manager = get_auth_manager() + auth_manager.security_manager = self.sm self.bm = BabelManager(self) self._add_global_static() self._add_global_filters() @@ -583,13 +586,7 @@ def security_converge(self, dry=False) -> dict: return self.sm.security_converge(self.baseviews, self.menu, dry) def get_url_for_login_with(self, next_url: str | None = None) -> str: - if self.sm.auth_view is None: - return "" - return url_for(f"{self.sm.auth_view.endpoint}.{'login'}", next=next_url) - - @property - def get_url_for_login(self): - return url_for(f"{self.sm.auth_view.endpoint}.login") + return get_auth_manager().get_url_login(next_url=next_url) @property def get_url_for_logout(self): diff --git a/airflow/www/extensions/init_auth_manager.py b/airflow/www/extensions/init_auth_manager.py index d21139f67069..a53fdf304bef 100644 --- a/airflow/www/extensions/init_auth_manager.py +++ b/airflow/www/extensions/init_auth_manager.py @@ -16,11 +16,15 @@ # under the License. from __future__ import annotations -from airflow.auth.managers.base_auth_manager import BaseAuthManager +from typing import TYPE_CHECKING + from airflow.compat.functools import cache from airflow.configuration import conf from airflow.exceptions import AirflowConfigException +if TYPE_CHECKING: + from airflow.auth.managers.base_auth_manager import BaseAuthManager + @cache def get_auth_manager() -> BaseAuthManager: diff --git a/airflow/www/extensions/init_security.py b/airflow/www/extensions/init_security.py index 17f93fc1c5ee..cdd4ec886237 100644 --- a/airflow/www/extensions/init_security.py +++ b/airflow/www/extensions/init_security.py @@ -19,7 +19,7 @@ import logging from importlib import import_module -from flask import g, redirect, url_for +from flask import g, redirect from flask_login import logout_user from airflow.configuration import conf @@ -71,4 +71,4 @@ def init_check_user_active(app): def check_user_active(): if get_auth_manager().is_logged_in() and not g.user.is_active: logout_user() - return redirect(url_for(app.appbuilder.sm.auth_view.endpoint + ".login")) + return redirect(get_auth_manager().get_url_login()) diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index 3c59522a6268..dc31a3552221 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -26,7 +26,7 @@ from uuid import uuid4 import re2 -from flask import Flask, current_app, g, session, url_for +from flask import Flask, g, session, url_for from flask_appbuilder import AppBuilder from flask_appbuilder.const import ( AUTH_DB, @@ -64,11 +64,11 @@ UserRemoteUserModelView, UserStatsChartView, ) -from flask_jwt_extended import JWTManager, current_user as current_user_jwt +from flask_jwt_extended import current_user as current_user_jwt from flask_limiter import Limiter from flask_limiter.util import get_remote_address -from flask_login import AnonymousUserMixin, LoginManager, current_user -from werkzeug.security import check_password_hash, generate_password_hash +from flask_login import current_user +from werkzeug.security import check_password_hash from airflow.configuration import conf from airflow.www.extensions.init_auth_manager import get_auth_manager @@ -90,33 +90,6 @@ def _oauth_tokengetter(token=None): return token -class AnonymousUser(AnonymousUserMixin): - """User object used when no active user is logged in.""" - - _roles: set[tuple[str, str]] = set() - _perms: set[tuple[str, str]] = set() - - @property - def roles(self): - if not self._roles: - public_role = current_app.appbuilder.get_app.config["AUTH_ROLE_PUBLIC"] - self._roles = {current_app.appbuilder.sm.find_role(public_role)} if public_role else set() - return self._roles - - @roles.setter - def roles(self, roles): - self._roles = roles - self._perms = set() - - @property - def perms(self): - if not self._perms: - self._perms = set() - for role in self.roles: - self._perms.update({(perm.action.name, perm.resource.name) for perm in role.permissions}) - return self._perms - - class BaseSecurityManager: """Base class to define the Security Manager interface.""" @@ -273,11 +246,6 @@ def __init__(self, appbuilder): self.oauth_remotes[provider_name] = obj_provider self._builtin_roles = self.create_builtin_roles() - # Setup Flask-Login - self.lm = self.create_login_manager(app) - - # Setup Flask-Jwt-Extended - self.jwt_manager = self.create_jwt_manager(app) # Setup Flask-Limiter self.limiter = self.create_limiter(app) @@ -287,29 +255,6 @@ def create_limiter(self, app: Flask) -> Limiter: limiter.init_app(app) return limiter - def create_login_manager(self, app) -> LoginManager: - """ - Override to implement your custom login manager instance. - - :param app: Flask app - """ - lm = LoginManager(app) - lm.anonymous_user = AnonymousUser - lm.login_view = "login" - lm.user_loader(self.load_user) - return lm - - def create_jwt_manager(self, app) -> JWTManager: - """ - Override to implement your custom JWT manager instance. - - :param app: Flask app - """ - jwt_manager = JWTManager() - jwt_manager.init_app(app) - jwt_manager.user_lookup_loader(self.load_user_jwt) - return jwt_manager - def create_builtin_roles(self): """Returns FAB builtin roles.""" return self.appbuilder.get_app.config.get("FAB_ROLES", {}) @@ -724,20 +669,6 @@ def create_db(self): if self.count_users() == 0 and self.auth_role_public != self.auth_role_admin: log.warning(LOGMSG_WAR_SEC_NO_USER) - def reset_password(self, userid, password): - """ - Change/Reset a user's password for authdb. - Password will be hashed and saved. - - :param userid: - the user.id to reset the password - :param password: - The clear text password to reset and save hashed on the db - """ - user = self.get_user_by_id(userid) - user.password = generate_password_hash(password) - self.update_user(user) - def update_user_auth_stat(self, user, success=True): """Update user authentication stats. @@ -1605,17 +1536,6 @@ def remove_permission_from_role(self, role, permission) -> None: """ raise NotImplementedError - def load_user(self, user_id): - """Load user by ID.""" - return self.get_user_by_id(int(user_id)) - - def load_user_jwt(self, _jwt_header, jwt_data): - identity = jwt_data["sub"] - user = self.load_user(identity) - # Set flask g.user to JWT user, we can't do it on before request - g.user = user - return user - @staticmethod def before_request(): """Hook runs before request.""" diff --git a/airflow/www/fab_security/sqla/manager.py b/airflow/www/fab_security/sqla/manager.py index c0daf5553ca2..c90ca52654a0 100644 --- a/airflow/www/fab_security/sqla/manager.py +++ b/airflow/www/fab_security/sqla/manager.py @@ -229,9 +229,6 @@ def update_user(self, user): self.get_session.rollback() return False - def get_user_by_id(self, pk): - return self.get_session.get(self.user_model, pk) - def add_role(self, name: str) -> Role: role = self.find_role(name) if role is None: diff --git a/airflow/www/security.py b/airflow/www/security.py index d9229c9b1eff..101b55df51a8 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -216,6 +216,7 @@ def __init__(self, appbuilder) -> None: resetmypasswordview=self.resetmypasswordview, resetpasswordview=self.resetpasswordview, rolemodelview=self.rolemodelview, + user_model=self.user_model, userinfoeditview=self.userinfoeditview, userdbmodelview=self.userdbmodelview, userldapmodelview=self.userldapmodelview, diff --git a/airflow/www/templates/appbuilder/flash.html b/airflow/www/templates/appbuilder/flash.html index 41e4b675c18f..37df62008517 100644 --- a/airflow/www/templates/appbuilder/flash.html +++ b/airflow/www/templates/appbuilder/flash.html @@ -33,7 +33,7 @@ {% for category, m in messages %} {% if category == 'dag_import_error' %} {{ dag_import_errors.append((category, m)) if dag_import_errors.append((category, m)) != None else '' }} - {% elif not (request.path == appbuilder.get_url_for_login and 'access is denied' in m.lower()) %} + {% elif not (request.path == auth_manager.get_url_login() and 'access is denied' in m.lower()) %} {# Don't show 'Access is Denied' alert if user is logged out and on the login page. #} {{ show_message(m, category) }} {% endif %} diff --git a/airflow/www/templates/appbuilder/navbar_right.html b/airflow/www/templates/appbuilder/navbar_right.html index 3bbf73eee36e..e26b6fa04e28 100644 --- a/airflow/www/templates/appbuilder/navbar_right.html +++ b/airflow/www/templates/appbuilder/navbar_right.html @@ -80,6 +80,6 @@ {% else %}
  • - login{{_("Log In")}} + login{{_("Log In")}}
    • {% endif %} diff --git a/tests/auh/managers/fab/test_fab_auth_manager.py b/tests/auh/managers/fab/test_fab_auth_manager.py index baaec623f47c..ef8baba45546 100644 --- a/tests/auh/managers/fab/test_fab_auth_manager.py +++ b/tests/auh/managers/fab/test_fab_auth_manager.py @@ -21,14 +21,18 @@ import pytest +from airflow import AirflowException from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride from airflow.www.fab_security.sqla.models import User +from airflow.www.security import ApplessAirflowSecurityManager @pytest.fixture def auth_manager(): - return FabAuthManager() + auth_manager = FabAuthManager() + auth_manager.security_manager = ApplessAirflowSecurityManager() + return auth_manager class TestFabAuthManager: @@ -59,3 +63,21 @@ def test_is_logged_in(self, mock_current_user, auth_manager): def test_get_security_manager_override_class_return_fab_security_manager_override(self, auth_manager): assert auth_manager.get_security_manager_override_class() is FabAirflowSecurityManagerOverride + + def test_get_url_login_when_auth_view_not_defined(self, auth_manager): + with pytest.raises(AirflowException, match="`auth_view` not defined in the security manager."): + auth_manager.get_url_login() + + @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + def test_get_url_login(self, mock_url_for, auth_manager): + auth_manager.security_manager.auth_view = Mock() + auth_manager.security_manager.auth_view.endpoint = "test_endpoint" + auth_manager.get_url_login() + mock_url_for.assert_called_once_with("test_endpoint.login") + + @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + def test_get_url_login_with_next(self, mock_url_for, auth_manager): + auth_manager.security_manager.auth_view = Mock() + auth_manager.security_manager.auth_view.endpoint = "test_endpoint" + auth_manager.get_url_login(next_url="next_url") + mock_url_for.assert_called_once_with("test_endpoint.login", next="next_url") diff --git a/tests/auh/managers/fab/test_security_manager_override.py b/tests/auh/managers/fab/test_security_manager_override.py new file mode 100644 index 000000000000..b93c1d866c30 --- /dev/null +++ b/tests/auh/managers/fab/test_security_manager_override.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import Mock + +import pytest + +from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride + +appbuilder = Mock() +actionmodelview = Mock() +authdbview = Mock() +authldapview = Mock() +authoauthview = Mock() +authoidview = Mock() +authremoteuserview = Mock() +permissionmodelview = Mock() +registeruser_view = Mock() +registeruserdbview = Mock() +registeruseroauthview = Mock() +registerusermodelview = Mock() +registeruseroidview = Mock() +resetmypasswordview = Mock() +resetpasswordview = Mock() +rolemodelview = Mock() +user_model = Mock() +userinfoeditview = Mock() +userdbmodelview = Mock() +userldapmodelview = Mock() +useroauthmodelview = Mock() +useroidmodelview = Mock() +userremoteusermodelview = Mock() +userstatschartview = Mock() + + +@pytest.fixture +def security_manager_override(): + class SubSecurityManager: + def __init__(self, **kwargs): + pass + + class EmptySecurityManager(FabAirflowSecurityManagerOverride, SubSecurityManager): + def __init__(self, appbuilder): + super().__init__( + appbuilder=appbuilder, + actionmodelview=actionmodelview, + authdbview=authdbview, + authldapview=authldapview, + authoauthview=authoauthview, + authoidview=authoidview, + authremoteuserview=authremoteuserview, + permissionmodelview=permissionmodelview, + registeruser_view=registeruser_view, + registeruserdbview=registeruserdbview, + registeruseroauthview=registeruseroauthview, + registerusermodelview=registerusermodelview, + registeruseroidview=registeruseroidview, + resetmypasswordview=resetmypasswordview, + resetpasswordview=resetpasswordview, + rolemodelview=rolemodelview, + user_model=user_model, + userinfoeditview=userinfoeditview, + userdbmodelview=userdbmodelview, + userldapmodelview=userldapmodelview, + useroauthmodelview=useroauthmodelview, + useroidmodelview=useroidmodelview, + userremoteusermodelview=userremoteusermodelview, + userstatschartview=userstatschartview, + ) + + with mock.patch( + "airflow.auth.managers.fab.security_manager_override.LoginManager" + ) as mock_login_manager, mock.patch( + "airflow.auth.managers.fab.security_manager_override.JWTManager" + ) as mock_jwt_manager: + mock_login_manager_instance = Mock() + mock_login_manager.return_value = mock_login_manager_instance + mock_jwt_manager_instance = Mock() + mock_jwt_manager.return_value = mock_jwt_manager_instance + + security_manager_override = EmptySecurityManager(appbuilder) + + mock_login_manager.assert_called_once_with(appbuilder.app) + mock_login_manager_instance.user_loader.assert_called_once_with(security_manager_override.load_user) + mock_jwt_manager_instance.init_app.assert_called_once_with(appbuilder.app) + mock_jwt_manager_instance.user_lookup_loader.assert_called_once_with( + security_manager_override.load_user_jwt + ) + + return security_manager_override + + +class TestFabAirflowSecurityManagerOverride: + def test_load_user(self, security_manager_override): + mock_get_user_by_id = Mock() + security_manager_override.get_user_by_id = mock_get_user_by_id + security_manager_override.load_user("123") + mock_get_user_by_id.assert_called_once_with(123) + + @mock.patch("airflow.auth.managers.fab.security_manager_override.g", spec={}) + def test_load_user_jwt(self, mock_g, security_manager_override): + mock_user = Mock() + mock_load_user = Mock(return_value=mock_user) + security_manager_override.load_user = mock_load_user + actual_user = security_manager_override.load_user_jwt(None, {"sub": "test_identity"}) + mock_load_user.assert_called_once_with("test_identity") + assert actual_user is mock_user + assert mock_g.user is mock_user diff --git a/tests/auh/managers/test_base_auth_manager.py b/tests/auh/managers/test_base_auth_manager.py index 61bee75371a0..f339e7d33fa9 100644 --- a/tests/auh/managers/test_base_auth_manager.py +++ b/tests/auh/managers/test_base_auth_manager.py @@ -19,6 +19,8 @@ import pytest from airflow.auth.managers.base_auth_manager import BaseAuthManager +from airflow.exceptions import AirflowException +from airflow.www.security import ApplessAirflowSecurityManager @pytest.fixture @@ -27,9 +29,24 @@ class EmptyAuthManager(BaseAuthManager): def get_user_name(self) -> str: raise NotImplementedError() + def is_logged_in(self) -> bool: + raise NotImplementedError() + + def get_url_login(self, **kwargs) -> str: + raise NotImplementedError() + return EmptyAuthManager() class TestBaseAuthManager: def test_get_security_manager_override_class_return_empty_class(self, auth_manager): assert auth_manager.get_security_manager_override_class() is object + + def test_get_security_manager_not_defined(self, auth_manager): + with pytest.raises(AirflowException, match="Security manager not defined."): + _security_manager = auth_manager.security_manager + + def test_get_security_manager_defined(self, auth_manager): + auth_manager.security_manager = ApplessAirflowSecurityManager() + _security_manager = auth_manager.security_manager + assert type(_security_manager) is ApplessAirflowSecurityManager diff --git a/tests/www/test_security.py b/tests/www/test_security.py index 97583418ee19..58ccc778e14d 100644 --- a/tests/www/test_security.py +++ b/tests/www/test_security.py @@ -29,6 +29,7 @@ from flask_appbuilder.views import BaseView, ModelView from sqlalchemy import Column, Date, Float, Integer, String +from airflow.auth.managers.fab.auth.anonymous_user import AnonymousUser from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager from airflow.exceptions import AirflowException from airflow.models import DagModel @@ -36,7 +37,6 @@ from airflow.models.dag import DAG from airflow.security import permissions from airflow.www import app as application -from airflow.www.fab_security.manager import AnonymousUser from airflow.www.fab_security.sqla.models import User, assoc_permission_role from airflow.www.utils import CustomSQLAInterface from tests.test_utils.api_connexion_utils import ( From 58e21c66fdcc8a416a697b4efa852473ad8bd6fc Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 25 Jul 2023 11:21:41 +0800 Subject: [PATCH 300/533] fix(providers/databricks): remove the execute method from to-be-deprecated DatabricksRunNowDeferrableOperator (#32806) As mentioned in #27837, we should do something similar to DatabricksRunOperator instead of running the original executing logic in DatabricksRunNowDeferrableOperator This is due to a bug that causes issues with empty job IDs. The DatabricksRunNowOperator operates smoothly and supports deferrable mode, so this PR removes the malfunctioning execute method. --- airflow/providers/databricks/operators/databricks.py | 5 ----- tests/providers/databricks/operators/test_databricks.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index b1c51a3f85a4..ec5d00725549 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -697,8 +697,3 @@ def __init__(self, *args, **kwargs): stacklevel=2, ) super().__init__(deferrable=True, *args, **kwargs) - - def execute(self, context): - hook = self._get_hook(caller="DatabricksRunNowDeferrableOperator") - self.run_id = hook.run_now(self.json) - _handle_deferrable_databricks_operator_execution(self, hook, self.log, context) diff --git a/tests/providers/databricks/operators/test_databricks.py b/tests/providers/databricks/operators/test_databricks.py index 32413329f674..81379ea339c9 100644 --- a/tests/providers/databricks/operators/test_databricks.py +++ b/tests/providers/databricks/operators/test_databricks.py @@ -905,7 +905,7 @@ def test_execute_task_deferred(self, db_mock_class): retry_limit=op.databricks_retry_limit, retry_delay=op.databricks_retry_delay, retry_args=None, - caller="DatabricksRunNowDeferrableOperator", + caller="DatabricksRunNowOperator", ) db_mock.run_now.assert_called_once_with(expected) From ed689f2be90cc8899438be66e3c75c3921e156cb Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Tue, 25 Jul 2023 12:23:10 +0530 Subject: [PATCH 301/533] Display execution_date in graph view task instance tooltip. (#32527) * Fix displaying execution_date in graph view task instance tooltip. * Fix tests. --- airflow/www/utils.py | 6 +++++- tests/www/views/test_views_tasks.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index bcf368ea2048..8d5ee60d0d38 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -84,7 +84,10 @@ def get_mapped_instances(task_instance, session): def get_instance_with_map(task_instance, session): if task_instance.map_index == -1: - return alchemy_to_dict(task_instance) + data = alchemy_to_dict(task_instance) + # Fetch execution_date explicitly since it's not a column and a proxy + data["execution_date"] = task_instance.execution_date + return data mapped_instances = get_mapped_instances(task_instance, session) return get_mapped_summary(task_instance, mapped_instances) @@ -137,6 +140,7 @@ def get_mapped_summary(parent_instance, task_instances): "end_date": group_end_date, "mapped_states": mapped_states, "try_number": get_try_count(parent_instance._try_number, parent_instance.state), + "execution_date": parent_instance.execution_date, } diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index d8c6a488a92b..32df99c0476f 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -1013,6 +1013,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1044,6 +1045,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1075,6 +1077,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1106,6 +1109,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1137,6 +1141,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1168,6 +1173,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1199,6 +1205,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, + "execution_date": DEFAULT_DATE.isoformat(), "executor_config": {}, "external_executor_id": None, "hostname": "", From ca202512afa2e90a406ae64b4a2650c8272290c1 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 25 Jul 2023 11:16:46 +0200 Subject: [PATCH 302/533] listener: call on_task_instance_running after rendering templates (#32716) Signed-off-by: Maciej Obuchowski --- airflow/models/taskinstance.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 4abfd94cd085..520d07f092d8 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1498,14 +1498,9 @@ def _run_raw_task( self.task = self.task.prepare_for_execution() context = self.get_template_context(ignore_param_exceptions=False) - # We lose previous state because it's changed in other process in LocalTaskJob. - # We could probably pass it through here though... - get_listener_manager().hook.on_task_instance_running( - previous_state=TaskInstanceState.QUEUED, task_instance=self, session=session - ) try: if not mark_success: - self._execute_task_with_callbacks(context, test_mode) + self._execute_task_with_callbacks(context, test_mode, session=session) if not test_mode: self.refresh_from_db(lock_for_update=True, session=session) self.state = TaskInstanceState.SUCCESS @@ -1601,7 +1596,7 @@ def _register_dataset_changes(self, *, session: Session) -> None: session=session, ) - def _execute_task_with_callbacks(self, context, test_mode=False): + def _execute_task_with_callbacks(self, context, test_mode: bool = False, *, session: Session): """Prepare Task for Execution.""" from airflow.models.renderedtifields import RenderedTaskInstanceFields @@ -1651,16 +1646,23 @@ def signal_handler(signum, frame): ) # Run pre_execute callback - self.task.pre_execute(context=context) + # Is never MappedOperator at this point + self.task.pre_execute(context=context) # type: ignore[union-attr] # Run on_execute callback self._run_execute_callback(context, self.task) + # Run on_task_instance_running event + get_listener_manager().hook.on_task_instance_running( + previous_state=TaskInstanceState.QUEUED, task_instance=self, session=session + ) + # Execute the task with set_current_context(context): result = self._execute_task(context, task_orig) # Run post_execute callback - self.task.post_execute(context=context, result=result) + # Is never MappedOperator at this point + self.task.post_execute(context=context, result=result) # type: ignore[union-attr] Stats.incr(f"operator_successes_{self.task.task_type}", tags=self.stats_tags) # Same metric with tagging From 6b880844ade6036954d5343f6a74a241b3865153 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 25 Jul 2023 11:17:04 +0200 Subject: [PATCH 303/533] openlineage, sftp: add OpenLineage support for sftp provider (#31360) Signed-off-by: Maciej Obuchowski --- airflow/providers/sftp/operators/sftp.py | 82 +++++++++++++++ generated/provider_dependencies.json | 1 + tests/providers/sftp/operators/test_sftp.py | 110 +++++++++++++++++++- 3 files changed, 192 insertions(+), 1 deletion(-) diff --git a/airflow/providers/sftp/operators/sftp.py b/airflow/providers/sftp/operators/sftp.py index 0920387faab4..8da4b3f33293 100644 --- a/airflow/providers/sftp/operators/sftp.py +++ b/airflow/providers/sftp/operators/sftp.py @@ -19,10 +19,13 @@ from __future__ import annotations import os +import socket import warnings from pathlib import Path from typing import Any, Sequence +import paramiko + from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.providers.sftp.hooks.sftp import SFTPHook @@ -188,3 +191,82 @@ def execute(self, context: Any) -> str | list[str] | None: raise AirflowException(f"Error while transferring {file_msg}, error: {str(e)}") return self.local_filepath + + def get_openlineage_facets_on_start(self): + """ + This returns OpenLineage datasets in format: + input: file:///path + output: file://:/path. + """ + from openlineage.client.run import Dataset + + from airflow.providers.openlineage.extractors import OperatorLineage + + scheme = "file" + local_host = socket.gethostname() + try: + local_host = socket.gethostbyname(local_host) + except Exception as e: + self.log.warning( + f"Failed to resolve local hostname. Using the hostname got by socket.gethostbyname() without resolution. {e}", # noqa: E501 + exc_info=True, + ) + + hook = self.sftp_hook or self.ssh_hook or SFTPHook(ssh_conn_id=self.ssh_conn_id) + + if self.remote_host is not None: + remote_host = self.remote_host + else: + remote_host = hook.get_connection(hook.ssh_conn_id).host + + try: + remote_host = socket.gethostbyname(remote_host) + except OSError as e: + self.log.warning( + f"Failed to resolve remote hostname. Using the provided hostname without resolution. {e}", # noqa: E501 + exc_info=True, + ) + + if hasattr(hook, "port"): + remote_port = hook.port + elif hasattr(hook, "ssh_hook"): + remote_port = hook.ssh_hook.port + + # Since v4.1.0, SFTPOperator accepts both a string (single file) and a list of + # strings (multiple files) as local_filepath and remote_filepath, and internally + # keeps them as list in both cases. But before 4.1.0, only single string is + # allowed. So we consider both cases here for backward compatibility. + if isinstance(self.local_filepath, str): + local_filepath = [self.local_filepath] + else: + local_filepath = self.local_filepath + if isinstance(self.remote_filepath, str): + remote_filepath = [self.remote_filepath] + else: + remote_filepath = self.remote_filepath + + local_datasets = [ + Dataset(namespace=self._get_namespace(scheme, local_host, None, path), name=path) + for path in local_filepath + ] + remote_datasets = [ + Dataset(namespace=self._get_namespace(scheme, remote_host, remote_port, path), name=path) + for path in remote_filepath + ] + + if self.operation.lower() == SFTPOperation.GET: + inputs = remote_datasets + outputs = local_datasets + else: + inputs = local_datasets + outputs = remote_datasets + + return OperatorLineage( + inputs=inputs, + outputs=outputs, + ) + + def _get_namespace(self, scheme, host, port, path) -> str: + port = port or paramiko.config.SSH_PORT + authority = f"{host}:{port}" + return f"{scheme}://{authority}" diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 091a45cf36ee..f4eeb8a27341 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -785,6 +785,7 @@ "apache-airflow>=2.4.0" ], "cross-providers-deps": [ + "openlineage", "ssh" ], "excluded-python-versions": [] diff --git a/tests/providers/sftp/operators/test_sftp.py b/tests/providers/sftp/operators/test_sftp.py index 92137e2a5c18..8adb93f7db67 100644 --- a/tests/providers/sftp/operators/test_sftp.py +++ b/tests/providers/sftp/operators/test_sftp.py @@ -18,13 +18,16 @@ from __future__ import annotations import os +import socket from base64 import b64encode from unittest import mock +import paramiko import pytest +from openlineage.client.run import Dataset from airflow.exceptions import AirflowException -from airflow.models import DAG +from airflow.models import DAG, Connection from airflow.providers.sftp.hooks.sftp import SFTPHook from airflow.providers.sftp.operators.sftp import SFTPOperation, SFTPOperator from airflow.providers.ssh.hooks.ssh import SSHHook @@ -36,6 +39,18 @@ DEFAULT_DATE = datetime(2017, 1, 1) TEST_CONN_ID = "conn_id_for_testing" +LOCAL_FILEPATH = "/path/local" +REMOTE_FILEPATH = "/path/remote" +LOCAL_DATASET = [ + Dataset(namespace=f"file://{socket.gethostbyname(socket.gethostname())}:22", name=LOCAL_FILEPATH) +] +REMOTE_DATASET = [Dataset(namespace="file://remotehost:22", name=REMOTE_FILEPATH)] + +TEST_GET_PUT_PARAMS = [ + (SFTPOperation.GET, (REMOTE_DATASET, LOCAL_DATASET)), + (SFTPOperation.PUT, (LOCAL_DATASET, REMOTE_DATASET)), +] + class TestSFTPOperator: def setup_method(self): @@ -478,3 +493,96 @@ def test_return_str_when_local_filepath_was_str(self, mock_get): return_value = sftp_op.execute(None) assert isinstance(return_value, str) assert return_value == local_filepath + + @pytest.mark.parametrize( + "operation, expected", + TEST_GET_PUT_PARAMS, + ) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_conn", spec=paramiko.SSHClient) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_connection", spec=Connection) + def test_extract_ssh_conn_id(self, get_connection, get_conn, operation, expected): + get_connection.return_value = Connection( + conn_id="sftp_conn_id", + conn_type="sftp", + host="remotehost", + port=22, + ) + + dag_id = "sftp_dag" + task_id = "sftp_task" + + task = SFTPOperator( + task_id=task_id, + ssh_conn_id="sftp_conn_id", + dag=DAG(dag_id), + start_date=timezone.utcnow(), + local_filepath="/path/local", + remote_filepath="/path/remote", + operation=operation, + ) + lineage = task.get_openlineage_facets_on_start() + + assert lineage.inputs == expected[0] + assert lineage.outputs == expected[1] + + @pytest.mark.parametrize( + "operation, expected", + TEST_GET_PUT_PARAMS, + ) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_conn", spec=paramiko.SSHClient) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_connection", spec=Connection) + def test_extract_sftp_hook(self, get_connection, get_conn, operation, expected): + get_connection.return_value = Connection( + conn_id="sftp_conn_id", + conn_type="sftp", + host="remotehost", + port=22, + ) + + dag_id = "sftp_dag" + task_id = "sftp_task" + + task = SFTPOperator( + task_id=task_id, + sftp_hook=SFTPHook(ssh_conn_id="sftp_conn_id"), + dag=DAG(dag_id), + start_date=timezone.utcnow(), + local_filepath="/path/local", + remote_filepath="/path/remote", + operation=operation, + ) + lineage = task.get_openlineage_facets_on_start() + + assert lineage.inputs == expected[0] + assert lineage.outputs == expected[1] + + @pytest.mark.parametrize( + "operation, expected", + TEST_GET_PUT_PARAMS, + ) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_conn", spec=paramiko.SSHClient) + @mock.patch("airflow.providers.ssh.hooks.ssh.SSHHook.get_connection", spec=Connection) + def test_extract_ssh_hook(self, get_connection, get_conn, operation, expected): + get_connection.return_value = Connection( + conn_id="sftp_conn_id", + conn_type="sftp", + host="remotehost", + port=22, + ) + + dag_id = "sftp_dag" + task_id = "sftp_task" + + task = SFTPOperator( + task_id=task_id, + ssh_hook=SSHHook(ssh_conn_id="sftp_conn_id"), + dag=DAG(dag_id), + start_date=timezone.utcnow(), + local_filepath="/path/local", + remote_filepath="/path/remote", + operation=operation, + ) + lineage = task.get_openlineage_facets_on_start() + + assert lineage.inputs == expected[0] + assert lineage.outputs == expected[1] From a23bf4c06a12b4c91f6f0501a8a624c612fd4455 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Tue, 25 Jul 2023 11:17:37 +0200 Subject: [PATCH 304/533] plugins: load plugins from providers (#32692) Signed-off-by: Maciej Obuchowski --- airflow/__init__.py | 9 +++---- airflow/plugins_manager.py | 32 ++++++++++++++++++++++- airflow/providers_manager.py | 36 ++++++++++++++++++++++++++ tests/always/test_providers_manager.py | 30 ++++++++++++++++++++- tests/plugins/test_plugins_manager.py | 33 +++++++++++++++++++++++ 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index d330b9332a22..2e8e088ebc25 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -106,11 +106,6 @@ def __getattr__(name: str): return val -if not settings.LAZY_LOAD_PLUGINS: - from airflow import plugins_manager - - plugins_manager.ensure_plugins_loaded() - if not settings.LAZY_LOAD_PROVIDERS: from airflow import providers_manager @@ -118,6 +113,10 @@ def __getattr__(name: str): manager.initialize_providers_list() manager.initialize_providers_hooks() manager.initialize_providers_extra_links() +if not settings.LAZY_LOAD_PLUGINS: + from airflow import plugins_manager + + plugins_manager.ensure_plugins_loaded() # This is never executed, but tricks static analyzers (PyDev, PyCharm,) diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 946f0642692b..17acdd245c86 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -38,7 +38,7 @@ from airflow import settings from airflow.utils.entry_points import entry_points_with_dist from airflow.utils.file import find_path_from_directory -from airflow.utils.module_loading import qualname +from airflow.utils.module_loading import import_string, qualname if TYPE_CHECKING: from airflow.hooks.base import BaseHook @@ -50,6 +50,7 @@ import_errors: dict[str, str] = {} plugins: list[AirflowPlugin] | None = None +loaded_plugins: set[str] = set() # Plugin components to integrate as modules registered_hooks: list[BaseHook] | None = None @@ -205,10 +206,16 @@ def is_valid_plugin(plugin_obj): def register_plugin(plugin_instance): """ Start plugin load and register it after success initialization. + If plugin is already registered, do nothing. :param plugin_instance: subclass of AirflowPlugin """ global plugins + + if plugin_instance.name in loaded_plugins: + return + + loaded_plugins.add(plugin_instance.name) plugin_instance.on_load() plugins.append(plugin_instance) @@ -267,6 +274,26 @@ def load_plugins_from_plugin_directory(): import_errors[file_path] = str(e) +def load_providers_plugins(): + from airflow.providers_manager import ProvidersManager + + log.debug("Loading plugins from providers") + providers_manager = ProvidersManager() + providers_manager.initialize_providers_plugins() + for plugin in providers_manager.plugins: + log.debug("Importing plugin %s from class %s", plugin.name, plugin.plugin_class) + + try: + plugin_instance = import_string(plugin.plugin_class) + if not is_valid_plugin(plugin_instance): + log.warning("Plugin %s is not a valid plugin", plugin.name) + continue + register_plugin(plugin_instance) + except ImportError: + log.exception("Failed to load plugin %s from class name %s", plugin.name, plugin.plugin_class) + continue + + def make_module(name: str, objects: list[Any]): """Creates new module.""" if not objects: @@ -306,6 +333,9 @@ def ensure_plugins_loaded(): load_plugins_from_plugin_directory() load_entrypoint_plugins() + if not settings.LAZY_LOAD_PROVIDERS: + load_providers_plugins() + # We don't do anything with these for now, but we want to keep track of # them so we can integrate them in to the UI's Connection screens for plugin in plugins: diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index bda6df3441e8..b7689a92da61 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -217,6 +217,14 @@ class TriggerInfo(NamedTuple): integration_name: str +class PluginInfo(NamedTuple): + """Plugin class, name and provider it comes from.""" + + name: str + plugin_class: str + provider_name: str + + class HookInfo(NamedTuple): """Hook information.""" @@ -421,6 +429,8 @@ def __init__(self): self._customized_form_fields_schema_validator = ( _create_customized_form_field_behaviours_schema_validator() ) + # Set of plugins contained in providers + self._plugins_set: set[PluginInfo] = set() @provider_info_cache("list") def initialize_providers_list(self): @@ -516,6 +526,11 @@ def initialize_providers_auth_backends(self): self.initialize_providers_list() self._discover_auth_backends() + @provider_info_cache("plugins") + def initialize_providers_plugins(self): + self.initialize_providers_list() + self._discover_plugins() + def _discover_all_providers_from_packages(self) -> None: """ Discover all providers by scanning packages installed. @@ -1024,6 +1039,21 @@ def _discover_config(self) -> None: if provider.data.get("config"): self._provider_configs[provider_package] = provider.data.get("config") + def _discover_plugins(self) -> None: + """Retrieve all plugins defined in the providers.""" + for provider_package, provider in self._provider_dict.items(): + for plugin_dict in provider.data.get("plugins", ()): + if not _correctness_check(provider_package, plugin_dict["plugin-class"], provider): + log.warning("Plugin not loaded due to above correctness check problem.") + continue + self._plugins_set.add( + PluginInfo( + name=plugin_dict["name"], + plugin_class=plugin_dict["plugin-class"], + provider_name=provider_package, + ) + ) + @provider_info_cache("triggers") def initialize_providers_triggers(self): """Initialization of providers triggers.""" @@ -1062,6 +1092,12 @@ def hooks(self) -> MutableMapping[str, HookInfo | None]: # When we return hooks here it will only be used to retrieve hook information return self._hooks_lazy_dict + @property + def plugins(self) -> list[PluginInfo]: + """Returns information about plugins available in providers.""" + self.initialize_providers_plugins() + return sorted(self._plugins_set, key=lambda x: x.plugin_class) + @property def taskflow_decorators(self) -> dict[str, TaskDecorator]: self.initialize_providers_taskflow_decorator() diff --git a/tests/always/test_providers_manager.py b/tests/always/test_providers_manager.py index b99dbcb84fa3..7e05d1cfb122 100644 --- a/tests/always/test_providers_manager.py +++ b/tests/always/test_providers_manager.py @@ -28,7 +28,13 @@ from wtforms import BooleanField, Field, StringField from airflow.exceptions import AirflowOptionalProviderFeatureException -from airflow.providers_manager import HookClassProvider, LazyDictWithCache, ProviderInfo, ProvidersManager +from airflow.providers_manager import ( + HookClassProvider, + LazyDictWithCache, + PluginInfo, + ProviderInfo, + ProvidersManager, +) class TestProviderManager: @@ -157,6 +163,28 @@ def test_already_registered_conn_type_in_provide(self): " and 'airflow.providers.dummy.hooks.dummy.DummyHook2'." ) in self._caplog.records[0].message + def test_providers_manager_register_plugins(self): + providers_manager = ProvidersManager() + providers_manager._provider_dict["apache-airflow-providers-apache-hive"] = ProviderInfo( + version="0.0.1", + data={ + "plugins": [ + { + "name": "plugin1", + "plugin-class": "airflow.providers.apache.hive.plugins.hive.HivePlugin", + } + ] + }, + package_or_source="package", + ) + providers_manager._discover_plugins() + assert len(providers_manager._plugins_set) == 1 + assert providers_manager._plugins_set.pop() == PluginInfo( + name="plugin1", + plugin_class="airflow.providers.apache.hive.plugins.hive.HivePlugin", + provider_name="apache-airflow-providers-apache-hive", + ) + def test_hooks(self): with pytest.warns(expected_warning=None) as warning_records: with self._caplog.at_level(logging.WARNING): diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py index 019e2a69a89c..bf74863407c1 100644 --- a/tests/plugins/test_plugins_manager.py +++ b/tests/plugins/test_plugins_manager.py @@ -160,6 +160,13 @@ class AirflowNoMenuViewsPlugin(AirflowPlugin): class TestPluginsManager: + @pytest.fixture(autouse=True, scope="function") + def clean_plugins(self): + from airflow import plugins_manager + + plugins_manager.loaded_plugins = set() + plugins_manager.plugins = [] + def test_no_log_when_no_plugins(self, caplog): with mock_plugin_manager(plugins=[]): @@ -378,6 +385,32 @@ def test_registering_plugin_listeners(self): assert get_listener_manager().has_listeners assert get_listener_manager().pm.get_plugins().pop().__name__ == "tests.listeners.empty_listener" + def test_should_import_plugin_from_providers(self): + from airflow import plugins_manager + + with mock.patch("airflow.plugins_manager.plugins", []): + assert len(plugins_manager.plugins) == 0 + plugins_manager.load_providers_plugins() + assert len(plugins_manager.plugins) >= 2 + + def test_does_not_double_import_entrypoint_provider_plugins(self): + from airflow import plugins_manager + + mock_entrypoint = mock.Mock() + mock_entrypoint.name = "test-entrypoint-plugin" + mock_entrypoint.module = "module_name_plugin" + + mock_dist = mock.Mock() + mock_dist.metadata = {"Name": "test-entrypoint-plugin"} + mock_dist.version = "1.0.0" + mock_dist.entry_points = [mock_entrypoint] + + with mock.patch("airflow.plugins_manager.plugins", []): + assert len(plugins_manager.plugins) == 0 + plugins_manager.load_entrypoint_plugins() + plugins_manager.load_providers_plugins() + assert len(plugins_manager.plugins) == 2 + class TestPluginsDirectorySource: def test_should_return_correct_path_name(self): From ce2841bf6ab609f31cb04aea9a39473de281bf24 Mon Sep 17 00:00:00 2001 From: JDarDagran Date: Tue, 25 Jul 2023 12:44:22 +0200 Subject: [PATCH 305/533] Add default port to Openlineage authority method. (#32828) Signed-off-by: Jakub Dardzinski --- airflow/providers/common/sql/hooks/sql.py | 10 ++++++++-- airflow/providers/mysql/hooks/mysql.py | 2 +- .../common/sql/operators/test_sql_execute.py | 13 +++++++++---- tests/providers/mysql/operators/test_mysql.py | 7 ++++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/airflow/providers/common/sql/hooks/sql.py b/airflow/providers/common/sql/hooks/sql.py index d444a952871c..ade3c6a95d64 100644 --- a/airflow/providers/common/sql/hooks/sql.py +++ b/airflow/providers/common/sql/hooks/sql.py @@ -598,13 +598,19 @@ def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLi """ @staticmethod - def get_openlineage_authority_part(connection) -> str: + def get_openlineage_authority_part(connection, default_port: int | None = None) -> str: """ This method serves as common method for several hooks to get authority part from Airflow Connection. The authority represents the hostname and port of the connection and conforms OpenLineage naming convention for a number of databases (e.g. MySQL, Postgres, Trino). + + :param default_port: (optional) used if no port parsed from connection URI """ parsed = urlparse(connection.get_uri()) - authority = f"{parsed.hostname}:{parsed.port}" + port = parsed.port or default_port + if port: + authority = f"{parsed.hostname}:{port}" + else: + authority = parsed.hostname return authority diff --git a/airflow/providers/mysql/hooks/mysql.py b/airflow/providers/mysql/hooks/mysql.py index e5023aac1af3..fa011ed35bc1 100644 --- a/airflow/providers/mysql/hooks/mysql.py +++ b/airflow/providers/mysql/hooks/mysql.py @@ -305,7 +305,7 @@ def get_openlineage_database_info(self, connection): return DatabaseInfo( scheme=self.get_openlineage_database_dialect(connection), - authority=DbApiHook.get_openlineage_authority_part(connection), + authority=DbApiHook.get_openlineage_authority_part(connection, default_port=3306), information_schema_columns=[ "table_schema", "table_name", diff --git a/tests/providers/common/sql/operators/test_sql_execute.py b/tests/providers/common/sql/operators/test_sql_execute.py index ddd137248540..5dc2e6e30cfe 100644 --- a/tests/providers/common/sql/operators/test_sql_execute.py +++ b/tests/providers/common/sql/operators/test_sql_execute.py @@ -281,7 +281,11 @@ def _process_output( ) -def test_execute_openlineage_events(): +@pytest.mark.parametrize( + "connection_port, default_port, expected_port", + [(None, 4321, 4321), (1234, None, 1234), (1234, 4321, 1234)], +) +def test_execute_openlineage_events(connection_port, default_port, expected_port): class DBApiHookForTests(DbApiHook): conn_name_attr = "sql_default" get_conn = MagicMock(name="conn") @@ -291,7 +295,8 @@ def get_openlineage_database_info(self, connection): from airflow.providers.openlineage.sqlparser import DatabaseInfo return DatabaseInfo( - scheme="sqlscheme", authority=DbApiHook.get_openlineage_authority_part(connection) + scheme="sqlscheme", + authority=DbApiHook.get_openlineage_authority_part(connection, default_port=default_port), ) def get_openlineage_database_specific_lineage(self, task_instance): @@ -317,7 +322,7 @@ def get_db_hook(self): (DB_SCHEMA_NAME, "popular_orders_day_of_week", "orders_placed", 3, "int4"), ] dbapi_hook.get_connection.return_value = Connection( - conn_id="sql_default", conn_type="postgresql", host="host", port=1234 + conn_id="sql_default", conn_type="postgresql", host="host", port=connection_port ) dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [rows, []] @@ -325,7 +330,7 @@ def get_db_hook(self): assert len(lineage.inputs) == 0 assert lineage.outputs == [ Dataset( - namespace="sqlscheme://host:1234", + namespace=f"sqlscheme://host:{expected_port}", name="PUBLIC.popular_orders_day_of_week", facets={ "schema": SchemaDatasetFacet( diff --git a/tests/providers/mysql/operators/test_mysql.py b/tests/providers/mysql/operators/test_mysql.py index 9202767f9d93..44aed629177f 100644 --- a/tests/providers/mysql/operators/test_mysql.py +++ b/tests/providers/mysql/operators/test_mysql.py @@ -137,7 +137,8 @@ def test_mysql_operator_openlineage(self, client): assert len(lineage_on_complete.outputs) == 1 -def test_execute_openlineage_events(): +@pytest.mark.parametrize("connection_port", [None, 1234]) +def test_execute_openlineage_events(connection_port): class MySqlHookForTests(MySqlHook): conn_name_attr = "sql_default" get_conn = MagicMock(name="conn") @@ -163,7 +164,7 @@ def get_db_hook(self): (DB_SCHEMA_NAME, "popular_orders_day_of_week", "orders_placed", 3, "int4"), ] dbapi_hook.get_connection.return_value = Connection( - conn_id="mysql_default", conn_type="mysql", host="host", port=1234 + conn_id="mysql_default", conn_type="mysql", host="host", port=connection_port ) dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect = [rows, []] @@ -171,7 +172,7 @@ def get_db_hook(self): assert len(lineage.inputs) == 0 assert lineage.outputs == [ Dataset( - namespace="mysql://host:1234", + namespace=f"mysql://host:{connection_port or 3306}", name="PUBLIC.popular_orders_day_of_week", facets={ "schema": SchemaDatasetFacet( From 9f3af9c862faec73cd477e8398b7607d2d045b37 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 25 Jul 2023 15:27:10 +0100 Subject: [PATCH 306/533] Fix misspelled auth test package (#32830) --- tests/{auh => auth}/__init__.py | 0 tests/{auh => auth}/managers/__init__.py | 0 tests/{auh => auth}/managers/fab/__init__.py | 0 tests/{auh => auth}/managers/fab/test_fab_auth_manager.py | 0 .../{auh => auth}/managers/fab/test_security_manager_override.py | 0 tests/{auh => auth}/managers/test_base_auth_manager.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tests/{auh => auth}/__init__.py (100%) rename tests/{auh => auth}/managers/__init__.py (100%) rename tests/{auh => auth}/managers/fab/__init__.py (100%) rename tests/{auh => auth}/managers/fab/test_fab_auth_manager.py (100%) rename tests/{auh => auth}/managers/fab/test_security_manager_override.py (100%) rename tests/{auh => auth}/managers/test_base_auth_manager.py (100%) diff --git a/tests/auh/__init__.py b/tests/auth/__init__.py similarity index 100% rename from tests/auh/__init__.py rename to tests/auth/__init__.py diff --git a/tests/auh/managers/__init__.py b/tests/auth/managers/__init__.py similarity index 100% rename from tests/auh/managers/__init__.py rename to tests/auth/managers/__init__.py diff --git a/tests/auh/managers/fab/__init__.py b/tests/auth/managers/fab/__init__.py similarity index 100% rename from tests/auh/managers/fab/__init__.py rename to tests/auth/managers/fab/__init__.py diff --git a/tests/auh/managers/fab/test_fab_auth_manager.py b/tests/auth/managers/fab/test_fab_auth_manager.py similarity index 100% rename from tests/auh/managers/fab/test_fab_auth_manager.py rename to tests/auth/managers/fab/test_fab_auth_manager.py diff --git a/tests/auh/managers/fab/test_security_manager_override.py b/tests/auth/managers/fab/test_security_manager_override.py similarity index 100% rename from tests/auh/managers/fab/test_security_manager_override.py rename to tests/auth/managers/fab/test_security_manager_override.py diff --git a/tests/auh/managers/test_base_auth_manager.py b/tests/auth/managers/test_base_auth_manager.py similarity index 100% rename from tests/auh/managers/test_base_auth_manager.py rename to tests/auth/managers/test_base_auth_manager.py From 9570cb1482d25f288e607aaa1210b2457bc5ed12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Tue, 25 Jul 2023 11:46:08 -0700 Subject: [PATCH 307/533] Make Start and Stop SageMaker Pipelines operators deferrable (#32683) --- .../providers/amazon/aws/hooks/sagemaker.py | 38 ++++++--- .../amazon/aws/operators/sagemaker.py | 82 ++++++++++++++++-- airflow/providers/amazon/aws/triggers/ecs.py | 1 + .../amazon/aws/triggers/sagemaker.py | 85 ++++++++++++++++++- .../amazon/aws/waiters/sagemaker.json | 46 ++++++++++ .../aws/operators/test_sagemaker_pipeline.py | 48 +++++++++-- 6 files changed, 269 insertions(+), 31 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py b/airflow/providers/amazon/aws/hooks/sagemaker.py index 72354c9c8bfe..758ba7e9c5d5 100644 --- a/airflow/providers/amazon/aws/hooks/sagemaker.py +++ b/airflow/providers/amazon/aws/hooks/sagemaker.py @@ -23,6 +23,7 @@ import tarfile import tempfile import time +import warnings from collections import Counter from datetime import datetime from functools import partial @@ -30,7 +31,7 @@ from botocore.exceptions import ClientError -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook @@ -1061,7 +1062,7 @@ def start_pipeline( display_name: str = "airflow-triggered-execution", pipeline_params: dict | None = None, wait_for_completion: bool = False, - check_interval: int = 30, + check_interval: int | None = None, verbose: bool = True, ) -> str: """Start a new execution for a SageMaker pipeline. @@ -1073,14 +1074,19 @@ def start_pipeline( :param display_name: The name this pipeline execution will have in the UI. Doesn't need to be unique. :param pipeline_params: Optional parameters for the pipeline. All parameters supplied need to already be present in the pipeline definition. - :param wait_for_completion: Will only return once the pipeline is complete if true. - :param check_interval: How long to wait between checks for pipeline status when waiting for - completion. - :param verbose: Whether to print steps details when waiting for completion. - Defaults to true, consider turning off for pipelines that have thousands of steps. :return: the ARN of the pipeline execution launched. """ + if wait_for_completion or check_interval is not None: + warnings.warn( + "parameter `wait_for_completion` and `check_interval` are deprecated, " + "remove them and call check_status yourself if you want to wait for completion", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + if check_interval is None: + check_interval = 30 + formatted_params = format_tags(pipeline_params, key_label="Name") try: @@ -1108,7 +1114,7 @@ def stop_pipeline( self, pipeline_exec_arn: str, wait_for_completion: bool = False, - check_interval: int = 10, + check_interval: int | None = None, verbose: bool = True, fail_if_not_running: bool = False, ) -> str: @@ -1119,12 +1125,6 @@ def stop_pipeline( :param pipeline_exec_arn: Amazon Resource Name (ARN) of the pipeline execution. It's the ARN of the pipeline itself followed by "/execution/" and an id. - :param wait_for_completion: Whether to wait for the pipeline to reach a final state. - (i.e. either 'Stopped' or 'Failed') - :param check_interval: How long to wait between checks for pipeline status when waiting for - completion. - :param verbose: Whether to print steps details when waiting for completion. - Defaults to true, consider turning off for pipelines that have thousands of steps. :param fail_if_not_running: This method will raise an exception if the pipeline we're trying to stop is not in an "Executing" state when the call is sent (which would mean that the pipeline is already either stopping or stopped). @@ -1133,6 +1133,16 @@ def stop_pipeline( :return: Status of the pipeline execution after the operation. One of 'Executing'|'Stopping'|'Stopped'|'Failed'|'Succeeded'. """ + if wait_for_completion or check_interval is not None: + warnings.warn( + "parameter `wait_for_completion` and `check_interval` are deprecated, " + "remove them and call check_status yourself if you want to wait for completion", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + if check_interval is None: + check_interval = 10 + retries = 2 # i.e. 3 calls max, 1 initial + 2 retries while True: try: diff --git a/airflow/providers/amazon/aws/operators/sagemaker.py b/airflow/providers/amazon/aws/operators/sagemaker.py index ac1b7a73d2de..83a1e4f3d2f9 100644 --- a/airflow/providers/amazon/aws/operators/sagemaker.py +++ b/airflow/providers/amazon/aws/operators/sagemaker.py @@ -30,7 +30,10 @@ from airflow.models import BaseOperator from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook -from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger +from airflow.providers.amazon.aws.triggers.sagemaker import ( + SageMakerPipelineTrigger, + SageMakerTrigger, +) from airflow.providers.amazon.aws.utils import trim_none_values from airflow.providers.amazon.aws.utils.sagemaker import ApprovalStatus from airflow.providers.amazon.aws.utils.tags import format_tags @@ -998,8 +1001,10 @@ class SageMakerStartPipelineOperator(SageMakerBaseOperator): All parameters supplied need to already be present in the pipeline definition. :param wait_for_completion: If true, this operator will only complete once the pipeline is complete. :param check_interval: How long to wait between checks for pipeline status when waiting for completion. + :param waiter_max_attempts: How many times to check the status before failing. :param verbose: Whether to print steps details when waiting for completion. Defaults to true, consider turning off for pipelines that have thousands of steps. + :param deferrable: Run operator in the deferrable mode. :return str: Returns The ARN of the pipeline execution created in Amazon SageMaker. """ @@ -1015,7 +1020,9 @@ def __init__( pipeline_params: dict | None = None, wait_for_completion: bool = False, check_interval: int = CHECK_INTERVAL_SECOND, + waiter_max_attempts: int = 9999, verbose: bool = True, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config={}, aws_conn_id=aws_conn_id, **kwargs) @@ -1024,22 +1031,46 @@ def __init__( self.pipeline_params = pipeline_params self.wait_for_completion = wait_for_completion self.check_interval = check_interval + self.waiter_max_attempts = waiter_max_attempts self.verbose = verbose + self.deferrable = deferrable def execute(self, context: Context) -> str: arn = self.hook.start_pipeline( pipeline_name=self.pipeline_name, display_name=self.display_name, pipeline_params=self.pipeline_params, - wait_for_completion=self.wait_for_completion, - check_interval=self.check_interval, - verbose=self.verbose, ) self.log.info( "Starting a new execution for pipeline %s, running with ARN %s", self.pipeline_name, arn ) + if self.deferrable: + self.defer( + trigger=SageMakerPipelineTrigger( + waiter_type=SageMakerPipelineTrigger.Type.COMPLETE, + pipeline_execution_arn=arn, + waiter_delay=self.check_interval, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + ), + method_name="execute_complete", + ) + elif self.wait_for_completion: + self.hook.check_status( + arn, + "PipelineExecutionStatus", + lambda p: self.hook.describe_pipeline_exec(p, self.verbose), + self.check_interval, + non_terminal_states=self.hook.pipeline_non_terminal_states, + max_ingestion_time=self.waiter_max_attempts * self.check_interval, + ) return arn + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: + if event is None or event["status"] != "success": + raise AirflowException(f"Failure during pipeline execution: {event}") + return event["value"] + class SageMakerStopPipelineOperator(SageMakerBaseOperator): """ @@ -1057,6 +1088,7 @@ class SageMakerStopPipelineOperator(SageMakerBaseOperator): :param verbose: Whether to print steps details when waiting for completion. Defaults to true, consider turning off for pipelines that have thousands of steps. :param fail_if_not_running: raises an exception if the pipeline stopped or succeeded before this was run + :param deferrable: Run operator in the deferrable mode. :return str: Returns the status of the pipeline execution after the operation has been done. """ @@ -1073,23 +1105,24 @@ def __init__( pipeline_exec_arn: str, wait_for_completion: bool = False, check_interval: int = CHECK_INTERVAL_SECOND, + waiter_max_attempts: int = 9999, verbose: bool = True, fail_if_not_running: bool = False, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ): super().__init__(config={}, aws_conn_id=aws_conn_id, **kwargs) self.pipeline_exec_arn = pipeline_exec_arn self.wait_for_completion = wait_for_completion self.check_interval = check_interval + self.waiter_max_attempts = waiter_max_attempts self.verbose = verbose self.fail_if_not_running = fail_if_not_running + self.deferrable = deferrable def execute(self, context: Context) -> str: status = self.hook.stop_pipeline( pipeline_exec_arn=self.pipeline_exec_arn, - wait_for_completion=self.wait_for_completion, - check_interval=self.check_interval, - verbose=self.verbose, fail_if_not_running=self.fail_if_not_running, ) self.log.info( @@ -1097,8 +1130,43 @@ def execute(self, context: Context) -> str: self.pipeline_exec_arn, status, ) + + if status not in self.hook.pipeline_non_terminal_states: + # pipeline already stopped + return status + + # else, eventually wait for completion + if self.deferrable: + self.defer( + trigger=SageMakerPipelineTrigger( + waiter_type=SageMakerPipelineTrigger.Type.STOPPED, + pipeline_execution_arn=self.pipeline_exec_arn, + waiter_delay=self.check_interval, + waiter_max_attempts=self.waiter_max_attempts, + aws_conn_id=self.aws_conn_id, + ), + method_name="execute_complete", + ) + elif self.wait_for_completion: + status = self.hook.check_status( + self.pipeline_exec_arn, + "PipelineExecutionStatus", + lambda p: self.hook.describe_pipeline_exec(p, self.verbose), + self.check_interval, + non_terminal_states=self.hook.pipeline_non_terminal_states, + max_ingestion_time=self.waiter_max_attempts * self.check_interval, + )["PipelineExecutionStatus"] + return status + def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: + if event is None or event["status"] != "success": + raise AirflowException(f"Failure during pipeline execution: {event}") + else: + # theoretically we should do a `describe` call to know this, + # but if we reach this point, this is the only possible status + return "Stopped" + class SageMakerRegisterModelVersionOperator(SageMakerBaseOperator): """ diff --git a/airflow/providers/amazon/aws/triggers/ecs.py b/airflow/providers/amazon/aws/triggers/ecs.py index d0fdfb63b6b6..cc203207d158 100644 --- a/airflow/providers/amazon/aws/triggers/ecs.py +++ b/airflow/providers/amazon/aws/triggers/ecs.py @@ -173,6 +173,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: ) # we reach this point only if the waiter met a success criteria yield TriggerEvent({"status": "success", "task_arn": self.task_arn}) + return except WaiterError as error: if "terminal failure" in str(error): raise diff --git a/airflow/providers/amazon/aws/triggers/sagemaker.py b/airflow/providers/amazon/aws/triggers/sagemaker.py index ca511a4a46d0..ec11323d426e 100644 --- a/airflow/providers/amazon/aws/triggers/sagemaker.py +++ b/airflow/providers/amazon/aws/triggers/sagemaker.py @@ -17,9 +17,15 @@ from __future__ import annotations +import asyncio +from collections import Counter +from enum import IntEnum from functools import cached_property -from typing import Any +from typing import Any, AsyncIterator +from botocore.exceptions import WaiterError + +from airflow import AirflowException from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait from airflow.triggers.base import BaseTrigger, TriggerEvent @@ -115,3 +121,80 @@ async def run(self): status_args=[self._get_response_status_key(self.job_type)], ) yield TriggerEvent({"status": "success", "message": "Job completed."}) + + +class SageMakerPipelineTrigger(BaseTrigger): + """Trigger to wait for a sagemaker pipeline execution to finish.""" + + class Type(IntEnum): + """Type of waiter to use.""" + + COMPLETE = 1 + STOPPED = 2 + + def __init__( + self, + waiter_type: Type, + pipeline_execution_arn: str, + waiter_delay: int, + waiter_max_attempts: int, + aws_conn_id: str, + ): + self.waiter_type = waiter_type + self.pipeline_execution_arn = pipeline_execution_arn + self.waiter_delay = waiter_delay + self.waiter_max_attempts = waiter_max_attempts + self.aws_conn_id = aws_conn_id + + def serialize(self) -> tuple[str, dict[str, Any]]: + return ( + self.__class__.__module__ + "." + self.__class__.__qualname__, + { + "waiter_type": self.waiter_type.value, # saving the int value here + "pipeline_execution_arn": self.pipeline_execution_arn, + "waiter_delay": self.waiter_delay, + "waiter_max_attempts": self.waiter_max_attempts, + "aws_conn_id": self.aws_conn_id, + }, + ) + + _waiter_name = { + Type.COMPLETE: "PipelineExecutionComplete", + Type.STOPPED: "PipelineExecutionStopped", + } + + async def run(self) -> AsyncIterator[TriggerEvent]: + attempts = 0 + hook = SageMakerHook(aws_conn_id=self.aws_conn_id) + async with hook.async_conn as conn: + waiter = hook.get_waiter(self._waiter_name[self.waiter_type], deferrable=True, client=conn) + while attempts < self.waiter_max_attempts: + attempts = attempts + 1 + try: + await waiter.wait( + PipelineExecutionArn=self.pipeline_execution_arn, WaiterConfig={"MaxAttempts": 1} + ) + # we reach this point only if the waiter met a success criteria + yield TriggerEvent({"status": "success", "value": self.pipeline_execution_arn}) + return + except WaiterError as error: + if "terminal failure" in str(error): + raise + + self.log.info( + "Status of the pipeline execution: %s", error.last_response["PipelineExecutionStatus"] + ) + + res = await conn.list_pipeline_execution_steps( + PipelineExecutionArn=self.pipeline_execution_arn + ) + count_by_state = Counter(s["StepStatus"] for s in res["PipelineExecutionSteps"]) + running_steps = [ + s["StepName"] for s in res["PipelineExecutionSteps"] if s["StepStatus"] == "Executing" + ] + self.log.info("State of the pipeline steps: %s", count_by_state) + self.log.info("Steps currently in progress: %s", running_steps) + + await asyncio.sleep(int(self.waiter_delay)) + + raise AirflowException("Waiter error: max attempts reached") diff --git a/airflow/providers/amazon/aws/waiters/sagemaker.json b/airflow/providers/amazon/aws/waiters/sagemaker.json index 2c2760982c5f..eba60c7266b7 100644 --- a/airflow/providers/amazon/aws/waiters/sagemaker.json +++ b/airflow/providers/amazon/aws/waiters/sagemaker.json @@ -104,6 +104,52 @@ "state": "failure" } ] + }, + "PipelineExecutionComplete": { + "delay": 30, + "operation": "DescribePipelineExecution", + "maxAttempts": 60, + "description": "Wait until pipeline execution is Succeeded", + "acceptors": [ + { + "matcher": "path", + "argument": "PipelineExecutionStatus", + "expected": "Succeeded", + "state": "success" + }, + { + "matcher": "path", + "argument": "PipelineExecutionStatus", + "expected": "Failed", + "state": "failure" + }, + { + "matcher": "path", + "argument": "PipelineExecutionStatus", + "expected": "Stopped", + "state": "failure" + } + ] + }, + "PipelineExecutionStopped": { + "delay": 10, + "operation": "DescribePipelineExecution", + "maxAttempts": 120, + "description": "Wait until pipeline execution is Stopped", + "acceptors": [ + { + "matcher": "path", + "argument": "PipelineExecutionStatus", + "expected": "Stopped", + "state": "success" + }, + { + "matcher": "path", + "argument": "PipelineExecutionStatus", + "expected": "Failed", + "state": "failure" + } + ] } } } diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py b/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py index dde36274587e..2d509625c5db 100644 --- a/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py +++ b/tests/providers/amazon/aws/operators/test_sagemaker_pipeline.py @@ -18,17 +18,23 @@ from __future__ import annotations from unittest import mock +from unittest.mock import MagicMock +import pytest + +from airflow.exceptions import TaskDeferred from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook from airflow.providers.amazon.aws.operators.sagemaker import ( SageMakerStartPipelineOperator, SageMakerStopPipelineOperator, ) +from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerPipelineTrigger class TestSageMakerStartPipelineOperator: @mock.patch.object(SageMakerHook, "start_pipeline") - def test_execute(self, start_pipeline): + @mock.patch.object(SageMakerHook, "check_status") + def test_execute(self, check_status, start_pipeline): op = SageMakerStartPipelineOperator( task_id="test_sagemaker_operator", pipeline_name="my_pipeline", @@ -39,17 +45,29 @@ def test_execute(self, start_pipeline): verbose=False, ) - op.execute(None) + op.execute({}) start_pipeline.assert_called_once_with( pipeline_name="my_pipeline", display_name="test_disp_name", pipeline_params={"is_a_test": "yes"}, - wait_for_completion=True, - check_interval=12, - verbose=False, + ) + check_status.assert_called_once() + + @mock.patch.object(SageMakerHook, "start_pipeline") + def test_defer(self, start_mock): + op = SageMakerStartPipelineOperator( + task_id="test_sagemaker_operator", + pipeline_name="my_pipeline", + deferrable=True, ) + with pytest.raises(TaskDeferred) as defer: + op.execute({}) + + assert isinstance(defer.value.trigger, SageMakerPipelineTrigger) + assert defer.value.trigger.waiter_type == SageMakerPipelineTrigger.Type.COMPLETE + class TestSageMakerStopPipelineOperator: @mock.patch.object(SageMakerHook, "stop_pipeline") @@ -58,12 +76,24 @@ def test_execute(self, stop_pipeline): task_id="test_sagemaker_operator", pipeline_exec_arn="pipeline_arn" ) - op.execute(None) + op.execute({}) stop_pipeline.assert_called_once_with( pipeline_exec_arn="pipeline_arn", - wait_for_completion=False, - check_interval=30, fail_if_not_running=False, - verbose=True, ) + + @mock.patch.object(SageMakerHook, "stop_pipeline") + def test_defer(self, stop_mock: MagicMock): + stop_mock.return_value = "Stopping" + op = SageMakerStopPipelineOperator( + task_id="test_sagemaker_operator", + pipeline_exec_arn="my_pipeline_arn", + deferrable=True, + ) + + with pytest.raises(TaskDeferred) as defer: + op.execute({}) + + assert isinstance(defer.value.trigger, SageMakerPipelineTrigger) + assert defer.value.trigger.waiter_type == SageMakerPipelineTrigger.Type.STOPPED From f2108892e89085f695f8a3f52e076b39288497c6 Mon Sep 17 00:00:00 2001 From: Bruce Date: Tue, 25 Jul 2023 15:02:01 -0700 Subject: [PATCH 308/533] Catch arbitrary exception from run_job to prevent zombie scheduler (#32707) --- airflow/cli/commands/scheduler_command.py | 8 ++++++- tests/cli/commands/test_scheduler_command.py | 24 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/airflow/cli/commands/scheduler_command.py b/airflow/cli/commands/scheduler_command.py index 808645282cac..ad0063ffa3d0 100644 --- a/airflow/cli/commands/scheduler_command.py +++ b/airflow/cli/commands/scheduler_command.py @@ -17,6 +17,7 @@ """Scheduler command.""" from __future__ import annotations +import logging import signal from contextlib import contextmanager from multiprocessing import Process @@ -35,12 +36,17 @@ from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.scheduler_health import serve_health_check +log = logging.getLogger(__name__) + def _run_scheduler_job(job_runner: SchedulerJobRunner, *, skip_serve_logs: bool) -> None: InternalApiConfig.force_database_direct_access() enable_health_check = conf.getboolean("scheduler", "ENABLE_HEALTH_CHECK") with _serve_logs(skip_serve_logs), _serve_health_check(enable_health_check): - run_job(job=job_runner.job, execute_callable=job_runner._execute) + try: + run_job(job=job_runner.job, execute_callable=job_runner._execute) + except Exception: + log.exception("Exception when running scheduler job") @cli_utils.action_cli diff --git a/tests/cli/commands/test_scheduler_command.py b/tests/cli/commands/test_scheduler_command.py index 2404b3b27c33..1b66cc7b85e1 100644 --- a/tests/cli/commands/test_scheduler_command.py +++ b/tests/cli/commands/test_scheduler_command.py @@ -139,6 +139,30 @@ def test_disable_scheduler_health( with pytest.raises(AssertionError): mock_process.assert_has_calls([mock.call(target=serve_health_check)]) + @mock.patch("airflow.cli.commands.scheduler_command.SchedulerJobRunner") + @mock.patch("airflow.cli.commands.scheduler_command.Process") + @mock.patch("airflow.cli.commands.scheduler_command.run_job", side_effect=Exception("run_job failed")) + @mock.patch("airflow.cli.commands.scheduler_command.log") + def test_run_job_exception_handling( + self, + mock_log, + mock_run_job, + mock_process, + mock_scheduler_job, + ): + args = self.parser.parse_args(["scheduler"]) + scheduler_command.scheduler(args) + + # Make sure that run_job is called, that the exception has been logged, and that the serve_logs + # sub-process has been terminated + mock_run_job.assert_called_once_with( + job=mock_scheduler_job().job, + execute_callable=mock_scheduler_job()._execute, + ) + mock_log.exception.assert_called_once_with("Exception when running scheduler job") + mock_process.assert_called_once_with(target=serve_logs) + mock_process().terminate.assert_called_once_with() + # Creating MockServer subclass of the HealthServer handler so that we can test the do_GET logic class MockServer(HealthServer): From 8809478f444da794254a52b19c4090b2d5726102 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 26 Jul 2023 09:48:56 +0800 Subject: [PATCH 309/533] Refactor existence-checking SQL to helper (#32790) --- airflow/models/abstractoperator.py | 17 ++++----- airflow/models/pool.py | 10 ++--- airflow/models/xcom_arg.py | 6 ++- airflow/ti_deps/deps/prev_dagrun_dep.py | 49 +++++++++---------------- airflow/utils/db.py | 30 ++++++++++++++- 5 files changed, 62 insertions(+), 50 deletions(-) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 97e3941f0f7d..908cb3258074 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -31,6 +31,7 @@ from airflow.models.taskmixin import DAGNode, DependencyMixin from airflow.template.templater import Templater from airflow.utils.context import Context +from airflow.utils.db import exists_query from airflow.utils.log.secrets_masker import redact from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.sqlalchemy import skip_locked, with_row_locks @@ -593,16 +594,12 @@ def expand_mapped_task(self, run_id: str, *, session: Session) -> tuple[Sequence ) unmapped_ti.state = TaskInstanceState.SKIPPED else: - zero_index_ti_exists = ( - session.scalar( - select(func.count(TaskInstance.task_id)).where( - TaskInstance.dag_id == self.dag_id, - TaskInstance.task_id == self.task_id, - TaskInstance.run_id == run_id, - TaskInstance.map_index == 0, - ) - ) - > 0 + zero_index_ti_exists = exists_query( + TaskInstance.dag_id == self.dag_id, + TaskInstance.task_id == self.task_id, + TaskInstance.run_id == run_id, + TaskInstance.map_index == 0, + session=session, ) if not zero_index_ti_exists: # Otherwise convert this into the first mapped index, and create diff --git a/airflow/models/pool.py b/airflow/models/pool.py index 0aab6fb634ca..3b5708b12129 100644 --- a/airflow/models/pool.py +++ b/airflow/models/pool.py @@ -26,6 +26,7 @@ from airflow.models.base import Base from airflow.ti_deps.dependencies_states import EXECUTION_STATES from airflow.typing_compat import TypedDict +from airflow.utils.db import exists_query from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.sqlalchemy import nowait, with_row_locks from airflow.utils.state import TaskInstanceState @@ -95,11 +96,10 @@ def is_default_pool(id: int, session: Session = NEW_SESSION) -> bool: :param session: SQLAlchemy ORM Session :return: True if id is default_pool, otherwise False """ - return ( - session.scalar( - select(func.count(Pool.id)).where(Pool.id == id, Pool.pool == Pool.DEFAULT_POOL_NAME) - ) - > 0 + return exists_query( + Pool.id == id, + Pool.pool == Pool.DEFAULT_POOL_NAME, + session=session, ) @staticmethod diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py index 154878a4dbe6..2383861483ab 100644 --- a/airflow/models/xcom_arg.py +++ b/airflow/models/xcom_arg.py @@ -30,6 +30,7 @@ from airflow.models.mappedoperator import MappedOperator from airflow.models.taskmixin import DAGNode, DependencyMixin from airflow.utils.context import Context +from airflow.utils.db import exists_query from airflow.utils.edgemodifier import EdgeModifier from airflow.utils.mixins import ResolveMixin from airflow.utils.session import NEW_SESSION, provide_session @@ -377,7 +378,7 @@ def get_task_map_length(self, run_id: str, *, session: Session) -> int | None: task = self.operator if isinstance(task, MappedOperator): - unfinished_ti_count_query = session.query(func.count(TaskInstance.map_index)).filter( + unfinished_ti_exists = exists_query( TaskInstance.dag_id == task.dag_id, TaskInstance.run_id == run_id, TaskInstance.task_id == task.task_id, @@ -388,8 +389,9 @@ def get_task_map_length(self, run_id: str, *, session: Session) -> int | None: TaskInstance.state.is_(None), TaskInstance.state.in_(s.value for s in State.unfinished if s is not None), ), + session=session, ) - if unfinished_ti_count_query.scalar(): + if unfinished_ti_exists: return None # Not all of the expanded tis are done yet. query = session.query(func.count(XCom.map_index)).filter( XCom.dag_id == task.dag_id, diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py b/airflow/ti_deps/deps/prev_dagrun_dep.py index a21944d410dd..0ea192c708e0 100644 --- a/airflow/ti_deps/deps/prev_dagrun_dep.py +++ b/airflow/ti_deps/deps/prev_dagrun_dep.py @@ -19,11 +19,12 @@ from typing import TYPE_CHECKING -from sqlalchemy import func, literal, or_, select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI from airflow.ti_deps.deps.base_ti_dep import BaseTIDep +from airflow.utils.db import exists_query from airflow.utils.session import provide_session from airflow.utils.state import TaskInstanceState @@ -57,13 +58,11 @@ def _has_tis(dagrun: DagRun, task_id: str, *, session: Session) -> bool: This function exists for easy mocking in tests. """ - return ( - session.scalar( - select(literal(True)) - .where(TI.dag_id == dagrun.dag_id, TI.task_id == task_id, TI.run_id == dagrun.run_id) - .limit(1) - ) - is not None + return exists_query( + TI.dag_id == dagrun.dag_id, + TI.task_id == task_id, + TI.run_id == dagrun.run_id, + session=session, ) @staticmethod @@ -72,17 +71,11 @@ def _has_any_prior_tis(ti: TI, *, session: Session) -> bool: This function exists for easy mocking in tests. """ - return ( - session.scalar( - select(literal(True)) - .where( - TI.dag_id == ti.dag_id, - TI.task_id == ti.task_id, - TI.execution_date < ti.execution_date, - ) - .limit(1) - ) - is not None + return exists_query( + TI.dag_id == ti.dag_id, + TI.task_id == ti.task_id, + TI.execution_date < ti.execution_date, + session=session, ) @staticmethod @@ -116,18 +109,12 @@ def _has_unsuccessful_dependants(dagrun: DagRun, task: Operator, *, session: Ses """ if not task.downstream_task_ids: return False - return ( - session.scalar( - select(literal(True)) - .where( - TI.dag_id == dagrun.dag_id, - TI.task_id.in_(task.downstream_task_ids), - TI.run_id == dagrun.run_id, - or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)), - ) - .limit(1) - ) - is not None + return exists_query( + TI.dag_id == dagrun.dag_id, + TI.task_id.in_(task.downstream_task_ids), + TI.run_id == dagrun.run_id, + or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)), + session=session, ) @provide_session diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 8e8c4b7fa93b..9ef504e73946 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -29,7 +29,21 @@ from tempfile import gettempdir from typing import TYPE_CHECKING, Callable, Generator, Iterable -from sqlalchemy import Table, and_, column, delete, exc, func, inspect, or_, select, table, text, tuple_ +from sqlalchemy import ( + Table, + and_, + column, + delete, + exc, + func, + inspect, + literal, + or_, + select, + table, + text, + tuple_, +) import airflow from airflow import settings @@ -45,6 +59,7 @@ from alembic.runtime.environment import EnvironmentContext from alembic.script import ScriptDirectory from sqlalchemy.orm import Query, Session + from sqlalchemy.sql.elements import ClauseElement from sqlalchemy.sql.selectable import Select from airflow.models.base import Base @@ -1875,7 +1890,7 @@ def get_sqla_model_classes(): return Base._decl_class_registry.values() -def get_query_count(query_stmt: Select, session: Session) -> int: +def get_query_count(query_stmt: Select, *, session: Session) -> int: """Get count of query. A SELECT COUNT() FROM is issued against the subquery built from the @@ -1887,3 +1902,14 @@ def get_query_count(query_stmt: Select, session: Session) -> int: """ count_stmt = select(func.count()).select_from(query_stmt.order_by(None).subquery()) return session.scalar(count_stmt) + + +def exists_query(*where: ClauseElement, session: Session) -> bool: + """Check whether there is at least one row matching given clause. + + This does a SELECT 1 WHERE ... LIMIT 1 and check the result. + + :meta private: + """ + stmt = select(literal(True)).where(*where).limit(1) + return session.scalar(stmt) is not None From e93460383f287f9b2af4b6bda3ea6ba17ba3c08b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 26 Jul 2023 08:25:02 +0200 Subject: [PATCH 310/533] Move all k8S classes to cncf.kubernetes provider (#32767) * Move all k8S classes to cncf.kubernetes provider This is the big move of all Kubenetes classes to go to provider. The changes that are implemented in this move: * replaced all imports from airflow.kubernetes to cncf.kubernetes Swith PEP-563 dynamic import rediretion and deprecation messages those messages now support overriding the "replacement" hints to make K8s deprecations more accurate * pre_7_4_0_compatibility package with classes used by past providerrs have been "frozen" and stored in the package with import redirections from airflow.kubernetes(with deprecation warnings) * kubernetes configuration is moved to kubernetes provider * mypy started complaining about conf and set used in configuration. so better solution to handle deprecations and hinting conf returning AirlfowConfigParsing was added. * example_kuberntes_executor uses configuration reading not in top level but in execute method * PodMutationHookException and PodReconciliationError have been moved to cncf.kubernetes provider and they are imported from there with fallback to an airflow.exception ones in case old provider is used in Airflow 2.7.0 * k8s methods in task_instance have been deprecated and reolaced with functions in "cncf.kubernetes` template_rendering module the old way still works but raise deprecaton warnings. * added extras with versions for celery and k8s * raise AirflowOptionalProviderFeatureException in case there is attempt to use CeleryK8sExecutor and cncf.k8s is not installed. * added few "new" core utils to k8s (hashlib_wrapper etc) * both warnings and errors indicate minimum versions for both cncf.k8s and Celery providers. * Update newsfragments/32767.significant.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- .github/CODEOWNERS | 1 - .github/boring-cyborg.yml | 6 - .pre-commit-config.yaml | 16 + STATIC_CODE_CHECKS.rst | 4 + airflow/cli/commands/kubernetes_command.py | 10 +- airflow/config_templates/__init__.py | 2 +- airflow/config_templates/config.yml | 210 ------ airflow/configuration.py | 9 +- airflow/decorators/__init__.pyi | 2 +- .../example_kubernetes_executor.py | 5 +- airflow/exceptions.py | 28 +- airflow/executors/__init__.py | 30 +- airflow/executors/executor_loader.py | 9 +- airflow/kubernetes/__init__.py | 125 ++++ .../__init__.py} | 23 +- .../k8s_model.py | 0 .../kube_client.py | 0 .../pre_7_4_0_compatibility/pod_generator.py | 666 ++++++++++++++++++ .../pod_generator_deprecated.py | 0 .../pre_7_4_0_compatibility/secret.py | 124 ++++ airflow/models/base.py | 2 +- airflow/models/renderedtifields.py | 6 +- airflow/models/taskinstance.py | 80 ++- .../amazon/aws/notifications/chime.py | 2 +- airflow/providers/apache/spark/CHANGELOG.rst | 19 + airflow/providers/apache/spark/__init__.py | 2 +- .../apache/spark/hooks/spark_submit.py | 2 +- airflow/providers/apache/spark/provider.yaml | 6 + airflow/providers/celery/CHANGELOG.rst | 17 +- airflow/providers/celery/__init__.py | 2 +- .../executors/celery_kubernetes_executor.py | 9 +- airflow/providers/celery/provider.yaml | 7 +- .../providers/cncf/kubernetes/CHANGELOG.rst | 17 +- .../cncf/kubernetes/executors}/__init__.py | 0 .../executors/kubernetes_executor.py | 48 +- .../executors/kubernetes_executor_types.py | 0 .../executors/kubernetes_executor_utils.py | 38 +- .../executors/local_kubernetes_executor.py | 2 +- .../cncf/kubernetes/hooks/kubernetes.py | 2 +- .../providers/cncf/kubernetes/k8s_model.py | 59 ++ .../providers/cncf/kubernetes/kube_client.py | 146 ++++ .../cncf}/kubernetes/kube_config.py | 1 - .../__init__.py | 0 .../basic_template.yaml | 0 .../kubernetes/kubernetes_helper_functions.py | 10 +- .../cncf/kubernetes/operators/pod.py | 6 +- .../cncf}/kubernetes/pod_generator.py | 20 +- .../kubernetes/pod_generator_deprecated.py | 309 ++++++++ .../kubernetes/pod_launcher_deprecated.py | 4 +- .../pod_template_file_examples/__init__.py} | 17 - .../dags_in_image_template.yaml | 0 .../dags_in_volume_template.yaml | 0 .../git_sync_template.yaml | 0 .../providers/cncf/kubernetes/provider.yaml | 210 ++++++ .../{ => providers/cncf}/kubernetes/secret.py | 2 +- .../cncf/kubernetes/template_rendering.py | 67 ++ .../kubernetes/utils/k8s_hashlib_wrapper.py | 42 ++ .../cncf/kubernetes/utils/pod_manager.py | 4 +- airflow/serialization/serialized_objects.py | 9 +- .../serialization/serializers/kubernetes.py | 7 +- airflow/timetables/_cron.py | 4 +- airflow/utils/deprecation_tools.py | 68 +- .../volume.py => utils/empty_set.py} | 22 +- airflow/utils/hashlib_wrapper.py | 5 +- airflow/utils/sqlalchemy.py | 8 +- airflow/www/views.py | 6 +- .../commands/kubernetes_commands.py | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 2 + dev/breeze/tests/test_selective_checks.py | 3 +- .../changelog.rst | 2 - .../commits.rst | 16 + .../index.rst | 30 +- .../changelog.rst | 2 - .../commits.rst | 21 + .../apache-airflow-providers-celery/index.rst | 30 +- .../configurations-ref.rst | 18 + .../index.rst | 1 + .../operators.rst | 2 +- docs/apache-airflow/configurations-ref.rst | 1 + .../core-concepts/executor/celery.rst | 7 + .../executor/celery_kubernetes.rst | 8 + .../core-concepts/executor/kubernetes.rst | 14 +- .../executor/local_kubernetes.rst | 9 +- docs/apache-airflow/extra-packages-ref.rst | 90 +-- .../howto/upgrading-from-1-10/index.rst | 4 +- docs/spelling_wordlist.txt | 5 + generated/provider_dependencies.json | 8 +- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_static-checks.svg | 142 ++-- .../conftest.py | 15 +- .../test_kubernetes_pod_operator.py | 8 +- newsfragments/32767.significant.rst | 7 + .../pre_commit_check_airflow_k8s_not_used.py | 81 +++ ...eck_cncf_k8s_used_for_k8s_executor_only.py | 84 +++ .../run_provider_yaml_files_check.py | 2 + setup.py | 3 - tests/cli/commands/test_kubernetes_command.py | 6 +- tests/cli/commands/test_task_command.py | 8 +- tests/models/test_renderedtifields.py | 48 -- tests/models/test_taskinstance.py | 81 --- .../apache/spark/hooks/test_spark_submit.py | 2 +- .../test_celery_kubernetes_executor.py | 2 +- tests/providers/cncf/kubernetes/__init__.py | 1 + .../cncf}/kubernetes/basic_pod.yaml | 0 .../cncf/kubernetes/executors/__init__.py | 17 - .../__init__.py | 16 + .../basic_template.yaml | 0 .../executors/test_kubernetes_executor.py | 158 +++-- .../test_local_kubernetes_executor.py | 4 +- .../cncf}/kubernetes/kube_config | 0 .../cncf}/kubernetes/models/__init__.py | 0 .../cncf}/kubernetes/models/test_secret.py | 10 +- .../cncf/kubernetes/operators/test_pod.py | 2 +- .../{ => providers/cncf}/kubernetes/pod.yaml | 0 .../cncf}/kubernetes/pod_generator_base.yaml | 0 .../pod_generator_base_with_secrets.yaml | 0 .../cncf}/kubernetes/test_client.py | 18 +- .../test_kubernetes_helper_functions.py | 2 +- .../cncf}/kubernetes/test_pod_generator.py | 33 +- .../kubernetes/test_template_rendering.py | 156 ++++ tests/sensors/test_base.py | 6 +- tests/serialization/test_dag_serialization.py | 4 +- .../cncf/kubernetes/example_kubernetes.py | 2 +- .../kubernetes/example_kubernetes_async.py | 2 +- tests/utils/test_log_handlers.py | 6 +- 125 files changed, 2960 insertions(+), 800 deletions(-) rename airflow/kubernetes/{pod.py => pre_7_4_0_compatibility/__init__.py} (57%) rename airflow/kubernetes/{ => pre_7_4_0_compatibility}/k8s_model.py (100%) rename airflow/kubernetes/{ => pre_7_4_0_compatibility}/kube_client.py (100%) create mode 100644 airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py rename airflow/kubernetes/{ => pre_7_4_0_compatibility}/pod_generator_deprecated.py (100%) create mode 100644 airflow/kubernetes/pre_7_4_0_compatibility/secret.py rename {tests/kubernetes => airflow/providers/cncf/kubernetes/executors}/__init__.py (100%) rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor.py (93%) rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_types.py (100%) rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_utils.py (93%) rename airflow/{ => providers/cncf/kubernetes}/executors/local_kubernetes_executor.py (99%) create mode 100644 airflow/providers/cncf/kubernetes/k8s_model.py create mode 100644 airflow/providers/cncf/kubernetes/kube_client.py rename airflow/{ => providers/cncf}/kubernetes/kube_config.py (99%) rename {tests/executors/kubernetes_executor_template_files => airflow/providers/cncf/kubernetes/kubernetes_executor_templates}/__init__.py (100%) rename airflow/{ => providers/cncf/kubernetes}/kubernetes_executor_templates/basic_template.yaml (100%) rename airflow/{ => providers/cncf}/kubernetes/kubernetes_helper_functions.py (94%) rename airflow/{ => providers/cncf}/kubernetes/pod_generator.py (97%) create mode 100644 airflow/providers/cncf/kubernetes/pod_generator_deprecated.py rename airflow/{ => providers/cncf}/kubernetes/pod_launcher_deprecated.py (98%) rename airflow/{kubernetes/pod_runtime_info_env.py => providers/cncf/kubernetes/pod_template_file_examples/__init__.py} (58%) rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/dags_in_image_template.yaml (100%) rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml (100%) rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/git_sync_template.yaml (100%) rename airflow/{ => providers/cncf}/kubernetes/secret.py (98%) create mode 100644 airflow/providers/cncf/kubernetes/template_rendering.py create mode 100644 airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py rename airflow/{kubernetes/volume.py => utils/empty_set.py} (62%) create mode 100644 docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst rename airflow/kubernetes/pod_launcher.py => kubernetes_tests/conftest.py (79%) create mode 100644 newsfragments/32767.significant.rst create mode 100755 scripts/ci/pre_commit/pre_commit_check_airflow_k8s_not_used.py create mode 100755 scripts/ci/pre_commit/pre_commit_check_cncf_k8s_used_for_k8s_executor_only.py rename tests/{ => providers/cncf}/kubernetes/basic_pod.yaml (100%) rename airflow/kubernetes/volume_mount.py => tests/providers/cncf/kubernetes/executors/__init__.py (58%) create mode 100644 tests/providers/cncf/kubernetes/executors/kubernetes_executor_template_files/__init__.py rename tests/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_template_files/basic_template.yaml (100%) rename tests/{ => providers/cncf/kubernetes}/executors/test_kubernetes_executor.py (89%) rename tests/{ => providers/cncf/kubernetes}/executors/test_local_kubernetes_executor.py (97%) rename tests/{ => providers/cncf}/kubernetes/kube_config (100%) rename tests/{ => providers/cncf}/kubernetes/models/__init__.py (100%) rename tests/{ => providers/cncf}/kubernetes/models/test_secret.py (93%) rename tests/{ => providers/cncf}/kubernetes/pod.yaml (100%) rename tests/{ => providers/cncf}/kubernetes/pod_generator_base.yaml (100%) rename tests/{ => providers/cncf}/kubernetes/pod_generator_base_with_secrets.yaml (100%) rename tests/{ => providers/cncf}/kubernetes/test_client.py (87%) rename tests/{ => providers/cncf}/kubernetes/test_kubernetes_helper_functions.py (98%) rename tests/{ => providers/cncf}/kubernetes/test_pod_generator.py (95%) create mode 100644 tests/providers/cncf/kubernetes/test_template_rendering.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dfc0e13a3a7e..aa036aeac58e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,7 +11,6 @@ # Kubernetes /airflow/kubernetes/ @dstandish @jedcunningham -/airflow/kubernetes_executor_templates/ @dstandish @jedcunningham /airflow/executors/celery_kubernetes_executor.py @dstandish @jedcunningham /airflow/executors/kubernetes_executor.py @dstandish @jedcunningham diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 989bb055df74..e2f7c037ecb8 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -164,9 +164,6 @@ labelPRBasedOnFilePath: - airflow/**/kubernetes_*.py - airflow/example_dags/example_kubernetes_executor.py - airflow/providers/cncf/kubernetes/**/* - - airflow/kubernetes/**/* - - airflow/kubernetes_executor_templates/**/* - - airflow/executors/kubernetes_executor.py - airflow/providers/celery/executors/celery_kubernetes_executor.py - docs/apache-airflow/core-concepts/executor/kubernetes.rst - docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst @@ -174,9 +171,6 @@ labelPRBasedOnFilePath: - kubernetes_tests/**/* - tests/providers/cncf/kubernetes/**/* - tests/system/providers/cncf/kubernetes/**/* - - tests/kubernetes/**/* - - tests/executors/kubernetes_executor_template_files/**/* - - tests/executors/*kubernetes*.py area:API: - airflow/api/**/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff8085d03033..d289ef97c489 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -333,6 +333,22 @@ repos: pass_filenames: false entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py additional_dependencies: ['rich>=12.4.4'] + - id: check-airflow-k8s-not-used + name: Check airflow.kubernetes imports are not used + language: python + files: ^airflow/.*\.py$ + require_serial: true + exclude: ^airflow/kubernetes/ + entry: ./scripts/ci/pre_commit/pre_commit_check_airflow_k8s_not_used.py + additional_dependencies: ['rich>=12.4.4'] + - id: check-cncf-k8s-only-for-executors + name: Check cncf.kubernetes imports used for executors only + language: python + files: ^airflow/.*\.py$ + require_serial: true + exclude: ^airflow/kubernetes/|^airflow/providers/ + entry: ./scripts/ci/pre_commit/pre_commit_check_cncf_k8s_used_for_k8s_executor_only.py + additional_dependencies: ['rich>=12.4.4'] - id: check-extra-packages-references name: Checks setup extra packages description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 1729c8ba8e61..bdb77e5c83c9 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -146,6 +146,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-aiobotocore-optional | Check if aiobotocore is an optional dependency only | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-airflow-k8s-not-used | Check airflow.kubernetes imports are not used | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-airflow-provider-compatibility | Check compatibility of Providers with Airflow | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-apache-license-rat | Check if licenses are OK for Apache | | @@ -163,6 +165,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-changelog-has-no-duplicates | Check changelogs for duplicate entries | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-cncf-k8s-only-for-executors | Check cncf.kubernetes imports used for executors only | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-core-deprecation-classes | Verify usage of Airflow deprecation classes in core | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-daysago-import-from-utils | Make sure days_ago is imported from airflow.utils.dates | | diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py index 1555f7be9258..056465577fcd 100644 --- a/airflow/cli/commands/kubernetes_command.py +++ b/airflow/cli/commands/kubernetes_command.py @@ -25,12 +25,12 @@ from kubernetes.client.api_client import ApiClient from kubernetes.client.rest import ApiException -from airflow.executors.kubernetes_executor import KubeConfig -from airflow.kubernetes import pod_generator -from airflow.kubernetes.kube_client import get_kube_client -from airflow.kubernetes.kubernetes_helper_functions import create_pod_id -from airflow.kubernetes.pod_generator import PodGenerator from airflow.models import DagRun, TaskInstance +from airflow.providers.cncf.kubernetes import pod_generator +from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig +from airflow.providers.cncf.kubernetes.kube_client import get_kube_client +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_pod_id +from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.utils import cli as cli_utils, yaml from airflow.utils.cli import get_dag from airflow.utils.providers_configuration_loader import providers_configuration_loaded diff --git a/airflow/config_templates/__init__.py b/airflow/config_templates/__init__.py index 6dd06760cdfd..4bdf46b9a970 100644 --- a/airflow/config_templates/__init__.py +++ b/airflow/config_templates/__init__.py @@ -25,4 +25,4 @@ }, } -add_deprecated_classes(__deprecated_classes, __name__) +add_deprecated_classes(__deprecated_classes, __name__, {}, "The `celery` provider must be >= 3.3.0 for that.") diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 35f16c57a8f4..b550c25932ed 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1943,21 +1943,6 @@ sentry: type: string example: ~ default: ~ -local_kubernetes_executor: - description: | - This section only applies if you are using the ``LocalKubernetesExecutor`` in - ``[core]`` section above - options: - kubernetes_queue: - description: | - Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``. - When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), - the task is executed via ``KubernetesExecutor``, - otherwise via ``LocalExecutor`` - version_added: 2.3.0 - type: string - example: ~ - default: "kubernetes" scheduler: description: ~ options: @@ -2442,201 +2427,6 @@ elasticsearch_configs: type: string example: ~ default: "True" -kubernetes_executor: - description: ~ - renamed: - previous_name: kubernetes - version: 2.5.0 - options: - api_client_retry_configuration: - description: | - Kwargs to override the default urllib3 Retry used in the kubernetes API client - version_added: 2.6.0 - type: string - example: '{ "total": 3, "backoff_factor": 0.5 }' - default: "" - logs_task_metadata: - description: | - Flag to control the information added to kubernetes executor logs for better traceability - version_added: 2.7.0 - type: boolean - example: ~ - default: "False" - pod_template_file: - description: | - Path to the YAML pod file that forms the basis for KubernetesExecutor workers. - version_added: 1.10.11 - type: string - example: ~ - default: "" - see_also: ":ref:`concepts:pod_template_file`" - worker_container_repository: - description: | - The repository of the Kubernetes Image for the Worker to Run - version_added: ~ - type: string - example: ~ - default: "" - worker_container_tag: - description: | - The tag of the Kubernetes Image for the Worker to Run - version_added: ~ - type: string - example: ~ - default: "" - namespace: - description: | - The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` - version_added: ~ - type: string - example: ~ - default: "default" - delete_worker_pods: - description: | - If True, all worker pods will be deleted upon termination - version_added: ~ - type: string - example: ~ - default: "True" - delete_worker_pods_on_failure: - description: | - If False (and delete_worker_pods is True), - failed worker pods will not be deleted so users can investigate them. - This only prevents removal of worker pods where the worker itself failed, - not when the task it ran failed. - version_added: 1.10.11 - type: string - example: ~ - default: "False" - worker_pods_creation_batch_size: - description: | - Number of Kubernetes Worker Pod creation calls per scheduler loop. - Note that the current default of "1" will only launch a single pod - per-heartbeat. It is HIGHLY recommended that users increase this - number to match the tolerance of their kubernetes cluster for - better performance. - version_added: 1.10.3 - type: string - example: ~ - default: "1" - multi_namespace_mode: - description: | - Allows users to launch pods in multiple namespaces. - Will require creating a cluster-role for the scheduler, - or use multi_namespace_mode_namespace_list configuration. - version_added: 1.10.12 - type: boolean - example: ~ - default: "False" - multi_namespace_mode_namespace_list: - description: | - If multi_namespace_mode is True while scheduler does not have a cluster-role, - give the list of namespaces where the scheduler will schedule jobs - Scheduler needs to have the necessary permissions in these namespaces. - version_added: 2.6.0 - type: string - example: ~ - default: "" - in_cluster: - description: | - Use the service account kubernetes gives to pods to connect to kubernetes cluster. - It's intended for clients that expect to be running inside a pod running on kubernetes. - It will raise an exception if called from a process not running in a kubernetes environment. - version_added: ~ - type: string - example: ~ - default: "True" - cluster_context: - description: | - When running with in_cluster=False change the default cluster_context or config_file - options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. - version_added: 1.10.3 - type: string - example: ~ - default: ~ - config_file: - description: | - Path to the kubernetes configfile to be used when ``in_cluster`` is set to False - version_added: 1.10.3 - type: string - example: ~ - default: ~ - kube_client_request_args: - description: | - Keyword parameters to pass while calling a kubernetes client core_v1_api methods - from Kubernetes Executor provided as a single line formatted JSON dictionary string. - List of supported params are similar for all core_v1_apis, hence a single config - variable for all apis. See: - https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py - version_added: 1.10.4 - type: string - example: ~ - default: "" - delete_option_kwargs: - description: | - Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client - ``core_v1_api`` method when using the Kubernetes Executor. - This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` - class defined here: - https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 - version_added: 1.10.12 - type: string - example: '{"grace_period_seconds": 10}' - default: "" - enable_tcp_keepalive: - description: | - Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely - when idle connection is time-outed on services like cloud load balancers or firewalls. - version_added: 2.0.0 - type: boolean - example: ~ - default: "True" - tcp_keep_idle: - description: | - When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has - been idle for `tcp_keep_idle` seconds. - version_added: 2.0.0 - type: integer - example: ~ - default: "120" - tcp_keep_intvl: - description: | - When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond - to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. - version_added: 2.0.0 - type: integer - example: ~ - default: "30" - tcp_keep_cnt: - description: | - When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond - to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before - a connection is considered to be broken. - version_added: 2.0.0 - type: integer - example: ~ - default: "6" - verify_ssl: - description: | - Set this to false to skip verifying SSL certificate of Kubernetes python client. - version_added: 2.1.0 - type: boolean - example: ~ - default: "True" - worker_pods_queued_check_interval: - description: | - How often in seconds to check for task instances stuck in "queued" status without a pod - version_added: 2.2.0 - type: integer - example: ~ - default: "60" - ssl_ca_cert: - description: | - Path to a CA certificate to be used by the Kubernetes client to verify the server's SSL certificate. - version_added: 2.6.0 - type: string - example: ~ - default: "" sensors: description: ~ options: diff --git a/airflow/configuration.py b/airflow/configuration.py index 54ce39225dd8..afbb02183415 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -46,6 +46,7 @@ from airflow.exceptions import AirflowConfigException from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend from airflow.utils import yaml +from airflow.utils.empty_set import _get_empty_set_for_configuration from airflow.utils.module_loading import import_string from airflow.utils.providers_configuration_loader import providers_configuration_loaded from airflow.utils.weight_rule import WeightRule @@ -66,8 +67,6 @@ ENV_VAR_PREFIX = "AIRFLOW__" -EMPTY_SET: Set[tuple[str, str]] = set() # noqa: UP006 - def _parse_sqlite_version(s: str) -> tuple[int, ...]: match = _SQLITE3_VERSION_PATTERN.match(s) @@ -299,7 +298,9 @@ def get_default_pre_2_7_value(self, section: str, key: str, **kwargs) -> Any: @functools.cached_property def sensitive_config_values(self) -> Set[tuple[str, str]]: # noqa: UP006 if self.configuration_description is None: - return EMPTY_SET.copy() # we can't use set() here because set is defined below # ¯\_(ツ)_/¯ + return ( + _get_empty_set_for_configuration() + ) # we can't use set() here because set is defined below # ¯\_(ツ)_/¯ flattened = { (s, k): item for s, s_c in self.configuration_description.items() @@ -2313,6 +2314,6 @@ def initialize_auth_manager() -> BaseAuthManager: FERNET_KEY = "" # Set only if needed when generating a new file WEBSERVER_CONFIG = "" # Set by initialize_config -conf = initialize_config() +conf: AirflowConfigParser = initialize_config() secrets_backend_list = initialize_secrets_backends() conf.validate() diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi index 92fa4dda5b1e..7b2ac1c9ce62 100644 --- a/airflow/decorators/__init__.pyi +++ b/airflow/decorators/__init__.pyi @@ -33,8 +33,8 @@ from airflow.decorators.python_virtualenv import virtualenv_task from airflow.decorators.sensor import sensor_task from airflow.decorators.short_circuit import short_circuit_task from airflow.decorators.task_group import task_group -from airflow.kubernetes.secret import Secret from airflow.models.dag import dag +from airflow.providers.cncf.kubernetes.secret import Secret # Please keep this in sync with __init__.py's __all__. __all__ = [ diff --git a/airflow/example_dags/example_kubernetes_executor.py b/airflow/example_dags/example_kubernetes_executor.py index b9e6bdba3544..a3d6570ac898 100644 --- a/airflow/example_dags/example_kubernetes_executor.py +++ b/airflow/example_dags/example_kubernetes_executor.py @@ -32,9 +32,6 @@ log = logging.getLogger(__name__) -worker_container_repository = conf.get("kubernetes_executor", "worker_container_repository") -worker_container_tag = conf.get("kubernetes_executor", "worker_container_tag") - try: from kubernetes.client import models as k8s except ImportError: @@ -163,6 +160,8 @@ def other_namespace_task(): print_stuff() other_ns_task = other_namespace_task() + worker_container_repository = conf.get("kubernetes_executor", "worker_container_repository") + worker_container_tag = conf.get("kubernetes_executor", "worker_container_tag") # You can also change the base image, here we used the worker image for demonstration. # Note that the image must have the same configuration as the diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 8c65a1f66f97..ea162fe8db15 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -375,12 +375,28 @@ class TaskDeferralError(AirflowException): """Raised when a task failed during deferral for some reason.""" -class PodMutationHookException(AirflowException): - """Raised when exception happens during Pod Mutation Hook execution.""" - - -class PodReconciliationError(AirflowException): - """Raised when an error is encountered while trying to merge pod configs.""" +# The try/except handling is needed after we moved all k8s classes to cncf.kubernetes provider +# These two exceptions are used internally by Kubernetes Executor but also by PodGenerator, so we need +# to leave them here in case older version of cncf.kubernetes provider is used to run KubernetesPodOperator +# and it raises one of those exceptions. The code should be backwards compatible even if you import +# and try/except the exception using direct imports from airflow.exceptions. +# 1) if you have old provider, both provider and pod generator will throw the "airflow.exceptions" exception. +# 2) if you have new provider, both provider and pod generator will throw the +# "airflow.providers.cncf.kubernetes" as it will be imported here from the provider. +try: + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import PodMutationHookException +except ImportError: + + class PodMutationHookException(AirflowException): # type: ignore[no-redef] + """Raised when exception happens during Pod Mutation Hook execution.""" + + +try: + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import PodReconciliationError +except ImportError: + + class PodReconciliationError(AirflowException): # type: ignore[no-redef] + """Raised when an error is encountered while trying to merge pod configs.""" class RemovedInAirflow3Warning(DeprecationWarning): diff --git a/airflow/executors/__init__.py b/airflow/executors/__init__.py index ceeec523397b..f6e18b20a81a 100644 --- a/airflow/executors/__init__.py +++ b/airflow/executors/__init__.py @@ -31,6 +31,34 @@ "dask_executor": { "DaskExecutor": "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor", }, + "kubernetes_executor": { + "KubernetesExecutor": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor.KubernetesExecutor", + }, + "kubernetes_executor_types": { + "ALL_NAMESPACES": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor_types.ALL_NAMESPACES", + "POD_EXECUTOR_DONE_KEY": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY", + }, + "kubernetes_executor_utils": { + "AirflowKubernetesScheduler": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor_utils.AirflowKubernetesScheduler", + "KubernetesJobWatcher": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor_utils.KubernetesJobWatcher", + "ResourceVersion": "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor_utils.ResourceVersion", + }, + "local_kubernetes_executor": { + "LocalKubernetesExecutor": "airflow.providers.cncf.kubernetes.executors.LocalKubernetesExecutor", + }, } -add_deprecated_classes(__deprecated_classes, __name__) +add_deprecated_classes( + __deprecated_classes, + __name__, + {}, + "For Celery executors, the `celery` provider should be >= 3.3.0. " + "For Kubernetes executors, the `cncf.kubernetes` provider should be >= 7.4.0 for that. " + "For Dask executors, any version of `daskexecutor` provider is needed.", +) diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 3553a7118360..ca21bdf05ac8 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -58,13 +58,15 @@ class ExecutorLoader: _default_executor: BaseExecutor | None = None executors = { LOCAL_EXECUTOR: "airflow.executors.local_executor.LocalExecutor", - LOCAL_KUBERNETES_EXECUTOR: "airflow.executors.local_kubernetes_executor.LocalKubernetesExecutor", + LOCAL_KUBERNETES_EXECUTOR: "airflow.providers.cncf.kubernetes." + "executors.local_kubernetes_executor.LocalKubernetesExecutor", SEQUENTIAL_EXECUTOR: "airflow.executors.sequential_executor.SequentialExecutor", CELERY_EXECUTOR: "airflow.providers.celery.executors.celery_executor.CeleryExecutor", CELERY_KUBERNETES_EXECUTOR: "airflow.providers.celery." "executors.celery_kubernetes_executor.CeleryKubernetesExecutor", DASK_EXECUTOR: "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor", - KUBERNETES_EXECUTOR: "airflow.executors.kubernetes_executor.KubernetesExecutor", + KUBERNETES_EXECUTOR: "airflow.providers.cncf.kubernetes." + "executors.kubernetes_executor.KubernetesExecutor", DEBUG_EXECUTOR: "airflow.executors.debug_executor.DebugExecutor", } @@ -98,9 +100,6 @@ def load_executor(cls, executor_name: str) -> BaseExecutor: :return: an instance of executor class via executor_name """ - from airflow.providers_manager import ProvidersManager - - ProvidersManager().initialize_providers_configuration() if executor_name == CELERY_KUBERNETES_EXECUTOR: return cls.__load_celery_kubernetes_executor() elif executor_name == LOCAL_KUBERNETES_EXECUTOR: diff --git a/airflow/kubernetes/__init__.py b/airflow/kubernetes/__init__.py index 13a83393a912..cf4e9a9591ae 100644 --- a/airflow/kubernetes/__init__.py +++ b/airflow/kubernetes/__init__.py @@ -14,3 +14,128 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from airflow.utils.deprecation_tools import add_deprecated_classes + +__deprecated_classes: dict[str, dict[str, str]] = { + "kubernetes_helper_functions": { + "add_pod_suffix": "airflow.providers.cncf.kubernetes.kubernetes_helper_functions.add_pod_suffix.", + "annotations_for_logging_task_metadata": "airflow.providers.cncf.kubernetes." + "kubernetes_helper_functions." + "annotations_for_logging_task_metadata.", + "annotations_to_key": "airflow.providers.cncf.kubernetes." + "kubernetes_helper_functions.annotations_to_key", + "create_pod_id": "airflow.providers.cncf.kubernetes.kubernetes_helper_functions.create_pod_id", + "get_logs_task_metadata": "airflow.providers.cncf.kubernetes." + "kubernetes_helper_functions.get_logs_task_metadata", + "rand_str": "airflow.providers.cncf.kubernetes.kubernetes_helper_functions.rand_str", + }, + "pod": { + "Port": "airflow.providers.cncf.kubernetes.backcompat.pod.Port", + "Resources": "airflow.providers.cncf.kubernetes.backcompat.pod.Resources", + }, + "pod_launcher": { + "PodLauncher": "airflow.providers.cncf.kubernetes.pod_launcher.PodLauncher", + "PodStatus": "airflow.providers.cncf.kubernetes.pod_launcher.PodStatus", + }, + "pod_launcher_deprecated": { + "PodLauncher": "airflow.providers.cncf.kubernetes.pod_launcher_deprecated.PodLauncher", + "PodStatus": "airflow.providers.cncf.kubernetes.pod_launcher_deprecated.PodStatus", + }, + "pod_runtime_info_env": { + "PodRuntimeInfoEnv": "airflow.providers.cncf.kubernetes.backcompat." + "pod_runtime_info_env.PodRuntimeInfoEnv", + }, + "volume": { + "Volume": "airflow.providers.cncf.kubernetes.backcompat.volume.Volume", + }, + "volume_mount": { + "VolumeMount": "airflow.providers.cncf.kubernetes.backcompat.volume_mount.VolumeMount", + }, + # the below classes are not served from provider but from internal pre_7_4_0_compatibility package + "k8s_model": { + "K8SModel": "airflow.kubernetes.pre_7_4_0_compatibility.k8s_model.K8SModel", + "append_to_pod": "airflow.kubernetes.pre_7_4_0_compatibility.k8s_model.append_to_pod", + }, + "kube_client": { + "_disable_verify_ssl": "airflow.kubernetes.pre_7_4_0_compatibility.kube_client._disable_verify_ssl", + "_enable_tcp_keepalive": "airflow.kubernetes.pre_7_4_0_compatibility.kube_client." + "_enable_tcp_keepalive", + "get_kube_client": "airflow.kubernetes.pre_7_4_0_compatibility.kube_client.get_kube_client", + }, + "pod_generator": { + "datetime_to_label_safe_datestring": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator" + ".datetime_to_label_safe_datestring", + "extend_object_field": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator." + "extend_object_field", + "label_safe_datestring_to_datetime": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator." + "label_safe_datestring_to_datetime", + "make_safe_label_value": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator." + "make_safe_label_value", + "merge_objects": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator.merge_objects", + "PodGenerator": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator.PodGenerator", + }, + "pod_generator_deprecated": { + "make_safe_label_value": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator_deprecated." + "make_safe_label_value", + "PodDefaults": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator_deprecated.PodDefaults", + "PodGenerator": "airflow.kubernetes.pre_7_4_0_compatibility.pod_generator_deprecated.PodGenerator", + }, + "secret": { + "Secret": "airflow.kubernetes.pre_7_4_0_compatibility.secret.Secret", + }, +} + +__override_deprecated_names: dict[str, dict[str, str]] = { + "pod": { + "Port": "kubernetes.client.models.V1ContainerPort", + "Resources": "kubernetes.client.models.V1ResourceRequirements", + }, + "pod_runtime_info_env": { + "PodRuntimeInfoEnv": "kubernetes.client.models.V1EnvVar", + }, + "volume": { + "Volume": "kubernetes.client.models.V1Volume", + }, + "volume_mount": { + "VolumeMount": "kubernetes.client.models.V1VolumeMount", + }, + "k8s_model": { + "K8SModel": "airflow.airflow.providers.cncf.kubernetes.k8s_model.K8SModel", + "append_to_pod": "airflow.airflow.providers.cncf.kubernetes.k8s_model.append_to_pod", + }, + "kube_client": { + "_disable_verify_ssl": "airflow.kubernetes.airflow.providers.cncf.kubernetes." + "kube_client._disable_verify_ssl", + "_enable_tcp_keepalive": "airflow.kubernetes.airflow.providers.cncf.kubernetes.kube_client." + "_enable_tcp_keepalive", + "get_kube_client": "airflow.kubernetes.airflow.providers.cncf.kubernetes.kube_client.get_kube_client", + }, + "pod_generator": { + "datetime_to_label_safe_datestring": "airflow.providers.cncf.kubernetes.pod_generator" + ".datetime_to_label_safe_datestring", + "extend_object_field": "airflow.kubernetes.airflow.providers.cncf.kubernetes.pod_generator." + "extend_object_field", + "label_safe_datestring_to_datetime": "airflow.providers.cncf.kubernetes.pod_generator." + "label_safe_datestring_to_datetime", + "make_safe_label_value": "airflow.providers.cncf.kubernetes.pod_generator.make_safe_label_value", + "merge_objects": "airflow.providers.cncf.kubernetes.pod_generator.merge_objects", + "PodGenerator": "airflow.providers.cncf.kubernetes.pod_generator.PodGenerator", + }, + "pod_generator_deprecated": { + "make_safe_label_value": "airflow.providers.cncf.kubernetes.pod_generator_deprecated." + "make_safe_label_value", + "PodDefaults": "airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodDefaults", + "PodGenerator": "airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodGenerator", + }, + "secret": { + "Secret": "airflow.providers.cncf.kubernetes.secret.Secret", + }, +} +add_deprecated_classes( + __deprecated_classes, + __name__, + __override_deprecated_names, + "The `cncf.kubernetes` provider must be >= 7.4.0 for that.", +) diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pre_7_4_0_compatibility/__init__.py similarity index 57% rename from airflow/kubernetes/pod.py rename to airflow/kubernetes/pre_7_4_0_compatibility/__init__.py index 629cbad17c78..18c84b6d03e5 100644 --- a/airflow/kubernetes/pod.py +++ b/airflow/kubernetes/pre_7_4_0_compatibility/__init__.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,24 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module is deprecated. -Please use :mod:`kubernetes.client.models` for `V1ResourceRequirements` and `Port`. -""" from __future__ import annotations +# All the classes in this module should only be kept for backwards-compatibility reasons. +# old cncf.kubernetes providers will use those in their frozen version for pre-7.4.0 release import warnings -from airflow.exceptions import RemovedInAirflow3Warning - -# flake8: noqa - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", RemovedInAirflow3Warning) - from airflow.providers.cncf.kubernetes.backcompat.pod import Port, Resources - warnings.warn( - "This module is deprecated. Please use `kubernetes.client.models` for `V1ResourceRequirements` and `Port`.", - RemovedInAirflow3Warning, + "This module is deprecated. The `cncf.kubernetes` provider before version 7.4.0 uses this module - " + "you should migrate to a newer version of `cncf.kubernetes` to get rid of this warning. If you " + "import the module via `airflow.kubernetes` import, please use `cncf.kubernetes' " + "provider 7.4.0+ and switch all your imports to use `apache.airflow.providers.cncf.kubernetes` " + "to get rid of the warning.", + DeprecationWarning, stacklevel=2, ) diff --git a/airflow/kubernetes/k8s_model.py b/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py similarity index 100% rename from airflow/kubernetes/k8s_model.py rename to airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py diff --git a/airflow/kubernetes/kube_client.py b/airflow/kubernetes/pre_7_4_0_compatibility/kube_client.py similarity index 100% rename from airflow/kubernetes/kube_client.py rename to airflow/kubernetes/pre_7_4_0_compatibility/kube_client.py diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py new file mode 100644 index 000000000000..aaacc8ce45fb --- /dev/null +++ b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py @@ -0,0 +1,666 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Pod generator compatible with cncf-providers released before 2.7.0 of airflow (so pre-7.4.0 of +the cncf.kubernetes provider). + +This module provides an interface between the previous Pod +API and outputs a kubernetes.client.models.V1Pod. +The advantage being that the full Kubernetes API +is supported and no serialization need be written. +""" +from __future__ import annotations + +import copy +import datetime +import logging +import os +import secrets +import string +import warnings +from functools import reduce + +import re2 +from dateutil import parser +from kubernetes.client import models as k8s +from kubernetes.client.api_client import ApiClient + +from airflow.exceptions import ( + AirflowConfigException, + PodMutationHookException, + PodReconciliationError, + RemovedInAirflow3Warning, +) +from airflow.kubernetes.pre_7_4_0_compatibility.pod_generator_deprecated import ( + PodDefaults, + PodGenerator as PodGeneratorDeprecated, +) +from airflow.utils import yaml +from airflow.utils.hashlib_wrapper import md5 +from airflow.version import version as airflow_version + +log = logging.getLogger(__name__) + +MAX_LABEL_LEN = 63 + +alphanum_lower = string.ascii_lowercase + string.digits + + +def rand_str(num): + """Generate random lowercase alphanumeric string of length num. + + :meta private: + """ + return "".join(secrets.choice(alphanum_lower) for _ in range(num)) + + +def add_pod_suffix(pod_name: str, rand_len: int = 8, max_len: int = 80) -> str: + """Add random string to pod name while staying under max length. + + :param pod_name: name of the pod + :param rand_len: length of the random string to append + :max_len: maximum length of the pod name + :meta private: + """ + suffix = "-" + rand_str(rand_len) + return pod_name[: max_len - len(suffix)].strip("-.") + suffix + + +def make_safe_label_value(string: str) -> str: + """ + Normalize a provided label to be of valid length and characters. + + Valid label values must be 63 characters or less and must be empty or begin and + end with an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_), + dots (.), and alphanumerics between. + + If the label value is greater than 63 chars once made safe, or differs in any + way from the original value sent to this function, then we need to truncate to + 53 chars, and append it with a unique hash. + """ + safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) + + if len(safe_label) > MAX_LABEL_LEN or string != safe_label: + safe_hash = md5(string.encode()).hexdigest()[:9] + safe_label = safe_label[: MAX_LABEL_LEN - len(safe_hash) - 1] + "-" + safe_hash + + return safe_label + + +def datetime_to_label_safe_datestring(datetime_obj: datetime.datetime) -> str: + """ + Transform a datetime string to use as a label. + + Kubernetes doesn't like ":" in labels, since ISO datetime format uses ":" but + not "_" let's + replace ":" with "_" + + :param datetime_obj: datetime.datetime object + :return: ISO-like string representing the datetime + """ + return datetime_obj.isoformat().replace(":", "_").replace("+", "_plus_") + + +def label_safe_datestring_to_datetime(string: str) -> datetime.datetime: + """ + Transform a label back to a datetime object. + + Kubernetes doesn't permit ":" in labels. ISO datetime format uses ":" but not + "_", let's + replace ":" with "_" + + :param string: str + :return: datetime.datetime object + """ + return parser.parse(string.replace("_plus_", "+").replace("_", ":")) + + +class PodGenerator: + """ + Contains Kubernetes Airflow Worker configuration logic. + + Represents a kubernetes pod and manages execution of a single pod. + Any configuration that is container specific gets applied to + the first container in the list of containers. + + :param pod: The fully specified pod. Mutually exclusive with `pod_template_file` + :param pod_template_file: Path to YAML file. Mutually exclusive with `pod` + :param extract_xcom: Whether to bring up a container for xcom + """ + + def __init__( + self, + pod: k8s.V1Pod | None = None, + pod_template_file: str | None = None, + extract_xcom: bool = True, + ): + if not pod_template_file and not pod: + raise AirflowConfigException( + "Podgenerator requires either a `pod` or a `pod_template_file` argument" + ) + if pod_template_file and pod: + raise AirflowConfigException("Cannot pass both `pod` and `pod_template_file` arguments") + + if pod_template_file: + self.ud_pod = self.deserialize_model_file(pod_template_file) + else: + self.ud_pod = pod + + # Attach sidecar + self.extract_xcom = extract_xcom + + def gen_pod(self) -> k8s.V1Pod: + """Generates pod.""" + warnings.warn("This function is deprecated. ", RemovedInAirflow3Warning) + result = self.ud_pod + + result.metadata.name = add_pod_suffix(pod_name=result.metadata.name) + + if self.extract_xcom: + result = self.add_xcom_sidecar(result) + + return result + + @staticmethod + def add_xcom_sidecar(pod: k8s.V1Pod) -> k8s.V1Pod: + """Adds sidecar.""" + warnings.warn( + "This function is deprecated. " + "Please use airflow.providers.cncf.kubernetes.utils.xcom_sidecar.add_xcom_sidecar instead" + ) + pod_cp = copy.deepcopy(pod) + pod_cp.spec.volumes = pod.spec.volumes or [] + pod_cp.spec.volumes.insert(0, PodDefaults.VOLUME) + pod_cp.spec.containers[0].volume_mounts = pod_cp.spec.containers[0].volume_mounts or [] + pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaults.VOLUME_MOUNT) + pod_cp.spec.containers.append(PodDefaults.SIDECAR_CONTAINER) + + return pod_cp + + @staticmethod + def from_obj(obj) -> dict | k8s.V1Pod | None: + """Converts to pod from obj.""" + if obj is None: + return None + + k8s_legacy_object = obj.get("KubernetesExecutor", None) + k8s_object = obj.get("pod_override", None) + + if k8s_legacy_object and k8s_object: + raise AirflowConfigException( + "Can not have both a legacy and new" + "executor_config object. Please delete the KubernetesExecutor" + "dict and only use the pod_override kubernetes.client.models.V1Pod" + "object." + ) + if not k8s_object and not k8s_legacy_object: + return None + + if isinstance(k8s_object, k8s.V1Pod): + return k8s_object + elif isinstance(k8s_legacy_object, dict): + warnings.warn( + "Using a dictionary for the executor_config is deprecated and will soon be removed." + 'please use a `kubernetes.client.models.V1Pod` class with a "pod_override" key' + " instead. ", + category=RemovedInAirflow3Warning, + ) + return PodGenerator.from_legacy_obj(obj) + else: + raise TypeError( + "Cannot convert a non-kubernetes.client.models.V1Pod object into a KubernetesExecutorConfig" + ) + + @staticmethod + def from_legacy_obj(obj) -> k8s.V1Pod | None: + """Converts to pod from obj.""" + if obj is None: + return None + + # We do not want to extract constant here from ExecutorLoader because it is just + # A name in dictionary rather than executor selection mechanism and it causes cyclic import + namespaced = obj.get("KubernetesExecutor", {}) + + if not namespaced: + return None + + resources = namespaced.get("resources") + + if resources is None: + requests = { + "cpu": namespaced.pop("request_cpu", None), + "memory": namespaced.pop("request_memory", None), + "ephemeral-storage": namespaced.get("ephemeral-storage"), # We pop this one in limits + } + limits = { + "cpu": namespaced.pop("limit_cpu", None), + "memory": namespaced.pop("limit_memory", None), + "ephemeral-storage": namespaced.pop("ephemeral-storage", None), + } + all_resources = list(requests.values()) + list(limits.values()) + if all(r is None for r in all_resources): + resources = None + else: + # remove None's so they don't become 0's + requests = {k: v for k, v in requests.items() if v is not None} + limits = {k: v for k, v in limits.items() if v is not None} + resources = k8s.V1ResourceRequirements(requests=requests, limits=limits) + namespaced["resources"] = resources + return PodGeneratorDeprecated(**namespaced).gen_pod() + + @staticmethod + def reconcile_pods(base_pod: k8s.V1Pod, client_pod: k8s.V1Pod | None) -> k8s.V1Pod: + """ + Merge Kubernetes Pod objects. + + :param base_pod: has the base attributes which are overwritten if they exist + in the client pod and remain if they do not exist in the client_pod + :param client_pod: the pod that the client wants to create. + :return: the merged pods + + This can't be done recursively as certain fields are overwritten and some are concatenated. + """ + if client_pod is None: + return base_pod + + client_pod_cp = copy.deepcopy(client_pod) + client_pod_cp.spec = PodGenerator.reconcile_specs(base_pod.spec, client_pod_cp.spec) + client_pod_cp.metadata = PodGenerator.reconcile_metadata(base_pod.metadata, client_pod_cp.metadata) + client_pod_cp = merge_objects(base_pod, client_pod_cp) + + return client_pod_cp + + @staticmethod + def reconcile_metadata(base_meta, client_meta): + """ + Merge Kubernetes Metadata objects. + + :param base_meta: has the base attributes which are overwritten if they exist + in the client_meta and remain if they do not exist in the client_meta + :param client_meta: the spec that the client wants to create. + :return: the merged specs + """ + if base_meta and not client_meta: + return base_meta + if not base_meta and client_meta: + return client_meta + elif client_meta and base_meta: + client_meta.labels = merge_objects(base_meta.labels, client_meta.labels) + client_meta.annotations = merge_objects(base_meta.annotations, client_meta.annotations) + extend_object_field(base_meta, client_meta, "managed_fields") + extend_object_field(base_meta, client_meta, "finalizers") + extend_object_field(base_meta, client_meta, "owner_references") + return merge_objects(base_meta, client_meta) + + return None + + @staticmethod + def reconcile_specs( + base_spec: k8s.V1PodSpec | None, client_spec: k8s.V1PodSpec | None + ) -> k8s.V1PodSpec | None: + """ + Merge Kubernetes PodSpec objects. + + :param base_spec: has the base attributes which are overwritten if they exist + in the client_spec and remain if they do not exist in the client_spec + :param client_spec: the spec that the client wants to create. + :return: the merged specs + """ + if base_spec and not client_spec: + return base_spec + if not base_spec and client_spec: + return client_spec + elif client_spec and base_spec: + client_spec.containers = PodGenerator.reconcile_containers( + base_spec.containers, client_spec.containers + ) + merged_spec = extend_object_field(base_spec, client_spec, "init_containers") + merged_spec = extend_object_field(base_spec, merged_spec, "volumes") + return merge_objects(base_spec, merged_spec) + + return None + + @staticmethod + def reconcile_containers( + base_containers: list[k8s.V1Container], client_containers: list[k8s.V1Container] + ) -> list[k8s.V1Container]: + """ + Merge Kubernetes Container objects. + + :param base_containers: has the base attributes which are overwritten if they exist + in the client_containers and remain if they do not exist in the client_containers + :param client_containers: the containers that the client wants to create. + :return: the merged containers + + The runs recursively over the list of containers. + """ + if not base_containers: + return client_containers + if not client_containers: + return base_containers + + client_container = client_containers[0] + base_container = base_containers[0] + client_container = extend_object_field(base_container, client_container, "volume_mounts") + client_container = extend_object_field(base_container, client_container, "env") + client_container = extend_object_field(base_container, client_container, "env_from") + client_container = extend_object_field(base_container, client_container, "ports") + client_container = extend_object_field(base_container, client_container, "volume_devices") + client_container = merge_objects(base_container, client_container) + + return [client_container] + PodGenerator.reconcile_containers( + base_containers[1:], client_containers[1:] + ) + + @classmethod + def construct_pod( + cls, + dag_id: str, + task_id: str, + pod_id: str, + try_number: int, + kube_image: str, + date: datetime.datetime | None, + args: list[str], + pod_override_object: k8s.V1Pod | None, + base_worker_pod: k8s.V1Pod, + namespace: str, + scheduler_job_id: str, + run_id: str | None = None, + map_index: int = -1, + *, + with_mutation_hook: bool = False, + ) -> k8s.V1Pod: + """ + Create a Pod. + + Construct a pod by gathering and consolidating the configuration from 3 places: + - airflow.cfg + - executor_config + - dynamic arguments + """ + if len(pod_id) > 253: + warnings.warn( + "pod_id supplied is longer than 253 characters; truncating and adding unique suffix." + ) + pod_id = add_pod_suffix(pod_name=pod_id, max_len=253) + try: + image = pod_override_object.spec.containers[0].image # type: ignore + if not image: + image = kube_image + except Exception: + image = kube_image + + annotations = { + "dag_id": dag_id, + "task_id": task_id, + "try_number": str(try_number), + } + if map_index >= 0: + annotations["map_index"] = str(map_index) + if date: + annotations["execution_date"] = date.isoformat() + if run_id: + annotations["run_id"] = run_id + + dynamic_pod = k8s.V1Pod( + metadata=k8s.V1ObjectMeta( + namespace=namespace, + annotations=annotations, + name=pod_id, + labels=cls.build_labels_for_k8s_executor_pod( + dag_id=dag_id, + task_id=task_id, + try_number=try_number, + airflow_worker=scheduler_job_id, + map_index=map_index, + execution_date=date, + run_id=run_id, + ), + ), + spec=k8s.V1PodSpec( + containers=[ + k8s.V1Container( + name="base", + args=args, + image=image, + env=[k8s.V1EnvVar(name="AIRFLOW_IS_K8S_EXECUTOR_POD", value="True")], + ) + ] + ), + ) + + # Reconcile the pods starting with the first chronologically, + # Pod from the pod_template_File -> Pod from executor_config arg -> Pod from the K8s executor + pod_list = [base_worker_pod, pod_override_object, dynamic_pod] + + try: + pod = reduce(PodGenerator.reconcile_pods, pod_list) + except Exception as e: + raise PodReconciliationError from e + + if with_mutation_hook: + from airflow.settings import pod_mutation_hook + + try: + pod_mutation_hook(pod) + except Exception as e: + raise PodMutationHookException from e + + return pod + + @classmethod + def build_selector_for_k8s_executor_pod( + cls, + *, + dag_id, + task_id, + try_number, + map_index=None, + execution_date=None, + run_id=None, + airflow_worker=None, + ): + """ + Generate selector for kubernetes executor pod. + + :meta private: + """ + labels = cls.build_labels_for_k8s_executor_pod( + dag_id=dag_id, + task_id=task_id, + try_number=try_number, + map_index=map_index, + execution_date=execution_date, + run_id=run_id, + airflow_worker=airflow_worker, + ) + label_strings = [f"{label_id}={label}" for label_id, label in sorted(labels.items())] + selector = ",".join(label_strings) + if not airflow_worker: # this filters out KPO pods even when we don't know the scheduler job id + selector += ",airflow-worker" + return selector + + @classmethod + def build_labels_for_k8s_executor_pod( + cls, + *, + dag_id, + task_id, + try_number, + airflow_worker=None, + map_index=None, + execution_date=None, + run_id=None, + ): + """ + Generate labels for kubernetes executor pod. + + :meta private: + """ + labels = { + "dag_id": make_safe_label_value(dag_id), + "task_id": make_safe_label_value(task_id), + "try_number": str(try_number), + "kubernetes_executor": "True", + "airflow_version": airflow_version.replace("+", "-"), + } + if airflow_worker is not None: + labels["airflow-worker"] = make_safe_label_value(str(airflow_worker)) + if map_index is not None and map_index >= 0: + labels["map_index"] = str(map_index) + if execution_date: + labels["execution_date"] = datetime_to_label_safe_datestring(execution_date) + if run_id: + labels["run_id"] = make_safe_label_value(run_id) + return labels + + @staticmethod + def serialize_pod(pod: k8s.V1Pod) -> dict: + """ + Convert a k8s.V1Pod into a json serializable dictionary. + + :param pod: k8s.V1Pod object + :return: Serialized version of the pod returned as dict + """ + api_client = ApiClient() + return api_client.sanitize_for_serialization(pod) + + @staticmethod + def deserialize_model_file(path: str) -> k8s.V1Pod: + """ + Generate a Pod from a file. + + :param path: Path to the file + :return: a kubernetes.client.models.V1Pod + """ + if os.path.exists(path): + with open(path) as stream: + pod = yaml.safe_load(stream) + else: + pod = None + log.warning("Model file %s does not exist", path) + + return PodGenerator.deserialize_model_dict(pod) + + @staticmethod + def deserialize_model_dict(pod_dict: dict | None) -> k8s.V1Pod: + """ + Deserializes a Python dictionary to k8s.V1Pod. + + Unfortunately we need access to the private method + ``_ApiClient__deserialize_model`` from the kubernetes client. + This issue is tracked here; https://github.com/kubernetes-client/python/issues/977. + + :param pod_dict: Serialized dict of k8s.V1Pod object + :return: De-serialized k8s.V1Pod + """ + api_client = ApiClient() + return api_client._ApiClient__deserialize_model(pod_dict, k8s.V1Pod) + + @staticmethod + def make_unique_pod_id(pod_id: str) -> str | None: + r""" + Generate a unique Pod name. + + Kubernetes pod names must consist of one or more lowercase + rfc1035/rfc1123 labels separated by '.' with a maximum length of 253 + characters. + + Name must pass the following regex for validation + ``^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`` + + For more details, see: + https://github.com/kubernetes/kubernetes/blob/release-1.1/docs/design/identifiers.md + + :param pod_id: requested pod name + :return: ``str`` valid Pod name of appropriate length + """ + warnings.warn( + "This function is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.", + RemovedInAirflow3Warning, + ) + + if not pod_id: + return None + + max_pod_id_len = 100 # arbitrarily chosen + suffix = rand_str(8) # 8 seems good enough + base_pod_id_len = max_pod_id_len - len(suffix) - 1 # -1 for separator + trimmed_pod_id = pod_id[:base_pod_id_len].rstrip("-.") + return f"{trimmed_pod_id}-{suffix}" + + +def merge_objects(base_obj, client_obj): + """ + Merge objects. + + :param base_obj: has the base attributes which are overwritten if they exist + in the client_obj and remain if they do not exist in the client_obj + :param client_obj: the object that the client wants to create. + :return: the merged objects + """ + if not base_obj: + return client_obj + if not client_obj: + return base_obj + + client_obj_cp = copy.deepcopy(client_obj) + + if isinstance(base_obj, dict) and isinstance(client_obj_cp, dict): + base_obj_cp = copy.deepcopy(base_obj) + base_obj_cp.update(client_obj_cp) + return base_obj_cp + + for base_key in base_obj.to_dict().keys(): + base_val = getattr(base_obj, base_key, None) + if not getattr(client_obj, base_key, None) and base_val: + if not isinstance(client_obj_cp, dict): + setattr(client_obj_cp, base_key, base_val) + else: + client_obj_cp[base_key] = base_val + return client_obj_cp + + +def extend_object_field(base_obj, client_obj, field_name): + """ + Add field values to existing objects. + + :param base_obj: an object which has a property `field_name` that is a list + :param client_obj: an object which has a property `field_name` that is a list. + A copy of this object is returned with `field_name` modified + :param field_name: the name of the list field + :return: the client_obj with the property `field_name` being the two properties appended + """ + client_obj_cp = copy.deepcopy(client_obj) + base_obj_field = getattr(base_obj, field_name, None) + client_obj_field = getattr(client_obj, field_name, None) + + if (not isinstance(base_obj_field, list) and base_obj_field is not None) or ( + not isinstance(client_obj_field, list) and client_obj_field is not None + ): + raise ValueError("The chosen field must be a list.") + + if not base_obj_field: + return client_obj_cp + if not client_obj_field: + setattr(client_obj_cp, field_name, base_obj_field) + return client_obj_cp + + appended_fields = base_obj_field + client_obj_field + setattr(client_obj_cp, field_name, appended_fields) + return client_obj_cp diff --git a/airflow/kubernetes/pod_generator_deprecated.py b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator_deprecated.py similarity index 100% rename from airflow/kubernetes/pod_generator_deprecated.py rename to airflow/kubernetes/pre_7_4_0_compatibility/pod_generator_deprecated.py diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/secret.py b/airflow/kubernetes/pre_7_4_0_compatibility/secret.py new file mode 100644 index 000000000000..14295f5c7a89 --- /dev/null +++ b/airflow/kubernetes/pre_7_4_0_compatibility/secret.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Classes for interacting with Kubernetes API.""" +from __future__ import annotations + +import copy +import uuid + +from kubernetes.client import models as k8s + +from airflow.exceptions import AirflowConfigException +from airflow.kubernetes.pre_7_4_0_compatibility.k8s_model import K8SModel + + +class Secret(K8SModel): + """Defines Kubernetes Secret Volume.""" + + def __init__(self, deploy_type, deploy_target, secret, key=None, items=None): + """ + Initialize a Kubernetes Secret Object. + + Used to track requested secrets from the user. + + :param deploy_type: The type of secret deploy in Kubernetes, either `env` or + `volume` + :param deploy_target: (Optional) The environment variable when + `deploy_type` `env` or file path when `deploy_type` `volume` where + expose secret. If `key` is not provided deploy target should be None. + :param secret: Name of the secrets object in Kubernetes + :param key: (Optional) Key of the secret within the Kubernetes Secret + if not provided in `deploy_type` `env` it will mount all secrets in object + :param items: (Optional) items that can be added to a volume secret for specifying projects of + secret keys to paths + https://kubernetes.io/docs/concepts/configuration/secret/#projection-of-secret-keys-to-specific-paths + """ + if deploy_type not in ("env", "volume"): + raise AirflowConfigException("deploy_type must be env or volume") + + self.deploy_type = deploy_type + self.deploy_target = deploy_target + self.items = items or [] + + if deploy_target is not None and deploy_type == "env": + # if deploying to env, capitalize the deploy target + self.deploy_target = deploy_target.upper() + + if key is not None and deploy_target is None: + raise AirflowConfigException("If `key` is set, `deploy_target` should not be None") + + self.secret = secret + self.key = key + + def to_env_secret(self) -> k8s.V1EnvVar: + """Stores es environment secret.""" + return k8s.V1EnvVar( + name=self.deploy_target, + value_from=k8s.V1EnvVarSource( + secret_key_ref=k8s.V1SecretKeySelector(name=self.secret, key=self.key) + ), + ) + + def to_env_from_secret(self) -> k8s.V1EnvFromSource: + """Reads from environment to secret.""" + return k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource(name=self.secret)) + + def to_volume_secret(self) -> tuple[k8s.V1Volume, k8s.V1VolumeMount]: + """Converts to volume secret.""" + vol_id = f"secretvol{uuid.uuid4()}" + volume = k8s.V1Volume(name=vol_id, secret=k8s.V1SecretVolumeSource(secret_name=self.secret)) + if self.items: + volume.secret.items = self.items + return (volume, k8s.V1VolumeMount(mount_path=self.deploy_target, name=vol_id, read_only=True)) + + def attach_to_pod(self, pod: k8s.V1Pod) -> k8s.V1Pod: + """Attaches to pod.""" + cp_pod = copy.deepcopy(pod) + + if self.deploy_type == "volume": + volume, volume_mount = self.to_volume_secret() + if cp_pod.spec.volumes is None: + cp_pod.spec.volumes = [] + cp_pod.spec.volumes.append(volume) + if cp_pod.spec.containers[0].volume_mounts is None: + cp_pod.spec.containers[0].volume_mounts = [] + cp_pod.spec.containers[0].volume_mounts.append(volume_mount) + + if self.deploy_type == "env" and self.key is not None: + env = self.to_env_secret() + if cp_pod.spec.containers[0].env is None: + cp_pod.spec.containers[0].env = [] + cp_pod.spec.containers[0].env.append(env) + + if self.deploy_type == "env" and self.key is None: + env_from = self.to_env_from_secret() + if cp_pod.spec.containers[0].env_from is None: + cp_pod.spec.containers[0].env_from = [] + cp_pod.spec.containers[0].env_from.append(env_from) + + return cp_pod + + def __eq__(self, other): + return ( + self.deploy_type == other.deploy_type + and self.deploy_target == other.deploy_target + and self.secret == other.secret + and self.key == other.key + ) + + def __repr__(self): + return f"Secret({self.deploy_type}, {self.deploy_target}, {self.secret}, {self.key})" diff --git a/airflow/models/base.py b/airflow/models/base.py index 9965de7ec707..5f6b7e9893dc 100644 --- a/airflow/models/base.py +++ b/airflow/models/base.py @@ -74,7 +74,7 @@ def get_id_collation_args(): return {} -COLLATION_ARGS = get_id_collation_args() +COLLATION_ARGS: dict[str, Any] = get_id_collation_args() def StringID(*, length=ID_LEN, **kwargs) -> String: diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 4b2f80e47d1b..269af8276abe 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -101,7 +101,11 @@ def __init__(self, ti: TaskInstance, render_templates=True): ti.render_templates() self.task = ti.task if os.environ.get("AIRFLOW_IS_K8S_EXECUTOR_POD", None): - self.k8s_pod_yaml = ti.render_k8s_pod_yaml() + # we can safely import it here from provider. In Airflow 2.7.0+ you need to have new version + # of kubernetes provider installed to reach this place + from airflow.providers.cncf.kubernetes.template_rendering import render_k8s_pod_yaml + + self.k8s_pod_yaml = render_k8s_pod_yaml(ti) self.rendered_fields = { field: serialize_template_field(getattr(self.task, field)) for field in self.task.template_fields } diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 520d07f092d8..6eafbab4a42c 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -2251,19 +2251,6 @@ def get_rendered_template_fields(self, session: Session = NEW_SESSION) -> None: "rendering of template_fields." ) from e - @provide_session - def get_rendered_k8s_spec(self, session: Session = NEW_SESSION): - """Fetch rendered template fields from DB.""" - from airflow.models.renderedtifields import RenderedTaskInstanceFields - - rendered_k8s_spec = RenderedTaskInstanceFields.get_k8s_pod_yaml(self, session=session) - if not rendered_k8s_spec: - try: - rendered_k8s_spec = self.render_k8s_pod_yaml() - except (TemplateAssertionError, UndefinedError) as e: - raise AirflowException(f"Unable to render a k8s spec for this taskinstance: {e}") from e - return rendered_k8s_spec - def overwrite_params_with_dag_run_conf(self, params, dag_run): """Overwrite Task Params with DagRun.conf.""" if dag_run and dag_run.conf: @@ -2290,32 +2277,51 @@ def render_templates(self, context: Context | None = None) -> Operator: return original_task def render_k8s_pod_yaml(self) -> dict | None: - """Render k8s pod yaml.""" - from kubernetes.client.api_client import ApiClient - - from airflow.kubernetes.kube_config import KubeConfig - from airflow.kubernetes.kubernetes_helper_functions import create_pod_id # Circular import - from airflow.kubernetes.pod_generator import PodGenerator + """Render the k8s pod yaml.""" + try: + from airflow.providers.cncf.kubernetes.template_rendering import ( + render_k8s_pod_yaml as render_k8s_pod_yaml_from_provider, + ) + except ImportError: + raise RuntimeError( + "You need to have the `cncf.kubernetes` provider installed to use this feature. " + "Also rather than calling it directly you should import " + "render_k8s_pod_yaml from airflow.providers.cncf.kubernetes.template_rendering " + "and call it with TaskInstance as the first argument." + ) + warnings.warn( + "You should not call `task_instance.render_k8s_pod_yaml` directly. This method will be removed" + "in Airflow 3. Rather than calling it directly you should import " + "`render_k8s_pod_yaml` from `airflow.providers.cncf.kubernetes.template_rendering` " + "and call it with `TaskInstance` as the first argument.", + DeprecationWarning, + stacklevel=2, + ) + return render_k8s_pod_yaml_from_provider(self) - kube_config = KubeConfig() - pod = PodGenerator.construct_pod( - dag_id=self.dag_id, - run_id=self.run_id, - task_id=self.task_id, - map_index=self.map_index, - date=None, - pod_id=create_pod_id(self.dag_id, self.task_id), - try_number=self.try_number, - kube_image=kube_config.kube_image, - args=self.command_as_list(), - pod_override_object=PodGenerator.from_obj(self.executor_config), - scheduler_job_id="0", - namespace=kube_config.executor_namespace, - base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file), - with_mutation_hook=True, + @provide_session + def get_rendered_k8s_spec(self, session: Session = NEW_SESSION): + """Render the k8s pod yaml.""" + try: + from airflow.providers.cncf.kubernetes.template_rendering import ( + get_rendered_k8s_spec as get_rendered_k8s_spec_from_provider, + ) + except ImportError: + raise RuntimeError( + "You need to have the `cncf.kubernetes` provider installed to use this feature. " + "Also rather than calling it directly you should import " + "`get_rendered_k8s_spec` from `airflow.providers.cncf.kubernetes.template_rendering` " + "and call it with `TaskInstance` as the first argument." + ) + warnings.warn( + "You should not call `task_instance.render_k8s_pod_yaml` directly. This method will be removed" + "in Airflow 3. Rather than calling it directly you should import " + "`get_rendered_k8s_spec` from `airflow.providers.cncf.kubernetes.template_rendering` " + "and call it with `TaskInstance` as the first argument.", + DeprecationWarning, + stacklevel=2, ) - sanitized_pod = ApiClient().sanitize_for_serialization(pod) - return sanitized_pod + return get_rendered_k8s_spec_from_provider(self, session=session) def get_email_subject_content( self, exception: BaseException, task: BaseOperator | None = None diff --git a/airflow/providers/amazon/aws/notifications/chime.py b/airflow/providers/amazon/aws/notifications/chime.py index c505b3e227ea..d62b4b21d6fa 100644 --- a/airflow/providers/amazon/aws/notifications/chime.py +++ b/airflow/providers/amazon/aws/notifications/chime.py @@ -53,7 +53,7 @@ def hook(self): """To reduce overhead cache the hook for the notifier.""" return ChimeWebhookHook(chime_conn_id=self.chime_conn_id) - def notify(self, context: Context) -> None: + def notify(self, context: Context) -> None: # type: ignore[override] """Send a message to a Chime Chat Room.""" self.hook.send_message(message=self.message) diff --git a/airflow/providers/apache/spark/CHANGELOG.rst b/airflow/providers/apache/spark/CHANGELOG.rst index 8129b380a320..c855333665e9 100644 --- a/airflow/providers/apache/spark/CHANGELOG.rst +++ b/airflow/providers/apache/spark/CHANGELOG.rst @@ -29,6 +29,25 @@ Changelog --------- +4.1.2 +..... + +.. note:: + + The provider now expects ``apache-airflow-providers-cncf-kubernetes`` in version 7.4.0+ installed + in order to run Spark on Kubernetes jobs. You can install the provider with ``cncf.kubernetes`` extra with + ``pip install apache-airflow-providers-spark[cncf.kubernetes]`` to get the right version of the + ``cncf.kubernetes`` provider installed. + + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` + 4.1.1 ..... diff --git a/airflow/providers/apache/spark/__init__.py b/airflow/providers/apache/spark/__init__.py index 888104a68bf7..298c8a54f929 100644 --- a/airflow/providers/apache/spark/__init__.py +++ b/airflow/providers/apache/spark/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "4.1.1" +__version__ = "4.1.2" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/apache/spark/hooks/spark_submit.py b/airflow/providers/apache/spark/hooks/spark_submit.py index 5687df8ac851..22fd6d14363b 100644 --- a/airflow/providers/apache/spark/hooks/spark_submit.py +++ b/airflow/providers/apache/spark/hooks/spark_submit.py @@ -31,7 +31,7 @@ from airflow.utils.log.logging_mixin import LoggingMixin with contextlib.suppress(ImportError, NameError): - from airflow.kubernetes import kube_client + from airflow.providers.cncf.kubernetes import kube_client ALLOWED_SPARK_BINARIES = ["spark-submit", "spark2-submit", "spark3-submit"] diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 134e7b709e07..30878d49139e 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 4.1.2 - 4.1.1 - 4.1.0 - 4.0.1 @@ -76,3 +77,8 @@ connection-types: connection-type: spark_sql - hook-class-name: airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook connection-type: spark + +additional-extras: + - name: cncf.kubernetes + dependencies: + - apache-airflow-providers-cncf-kubernetes>=7.4.0 diff --git a/airflow/providers/celery/CHANGELOG.rst b/airflow/providers/celery/CHANGELOG.rst index 0d0038cc366a..807f7db9eabc 100644 --- a/airflow/providers/celery/CHANGELOG.rst +++ b/airflow/providers/celery/CHANGELOG.rst @@ -33,7 +33,22 @@ Changelog .. note:: This provider release is the first release that has Celery Executor and Celery Kubernetes Executor moved from the core ``apache-airflow`` package to a Celery - provider package. + provider package. It also expects ``apache-airflow-providers-cncf-kubernetes`` in version 7.4.0+ installed + in order to use ``CeleryKubernetesExecutor``. You can install the provider with ``cncf.kubernetes`` extra + with ``pip install apache-airflow-providers-celery[cncf.kubernetes]`` to get the right version of the + ``cncf.kubernetes`` provider installed. + +.. Review and move the new changes to one of the sections above: + * ``Introduce decorator to load providers configuration (#32765)`` + * ``Allow configuration to be contributed by providers (#32604)`` + * ``Move default_celery.py to inside the provider (#32628)`` + * ``Add Executors discovery and documentation (#32532)`` + * ``Move CeleryExecutor to the celery provider (#32526)`` + * ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` + * ``Remove spurious headers for provider changelogs (#32373)`` + * ``Prepare docs for July 2023 wave of Providers (#32298)`` + * ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` + * ``Improve provider documentation and README structure (#32125)`` 3.2.1 ..... diff --git a/airflow/providers/celery/__init__.py b/airflow/providers/celery/__init__.py index 80c2021e933a..1ff2700b3b04 100644 --- a/airflow/providers/celery/__init__.py +++ b/airflow/providers/celery/__init__.py @@ -28,7 +28,7 @@ __all__ = ["__version__"] -__version__ = "3.2.1" +__version__ = "3.3.0" try: from airflow import __version__ as airflow_version diff --git a/airflow/providers/celery/executors/celery_kubernetes_executor.py b/airflow/providers/celery/executors/celery_kubernetes_executor.py index d79b3cd1db67..c5bbaac08139 100644 --- a/airflow/providers/celery/executors/celery_kubernetes_executor.py +++ b/airflow/providers/celery/executors/celery_kubernetes_executor.py @@ -23,8 +23,15 @@ from airflow.callbacks.base_callback_sink import BaseCallbackSink from airflow.callbacks.callback_requests import CallbackRequest from airflow.configuration import conf -from airflow.executors.kubernetes_executor import KubernetesExecutor from airflow.providers.celery.executors.celery_executor import CeleryExecutor + +try: + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubernetesExecutor +except ImportError as e: + from airflow.exceptions import AirflowOptionalProviderFeatureException + + raise AirflowOptionalProviderFeatureException(e) + from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.providers_configuration_loader import providers_configuration_loaded diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 26787d4694ff..0eccce68904f 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -308,8 +308,13 @@ config: type is string, it is required to pass a string that conforms to the dictionary format. See: https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#configuration - version_added: 3.3.0 + version_added: ~ type: string sensitive: true example: '{"password": "password_for_redis_server"}' default: ~ + +additional-extras: + - name: cncf.kubernetes + dependencies: + - apache-airflow-providers-cncf-kubernetes>=7.4.0 diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index a356a6c6dcfe..92ab38bdd742 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -27,6 +27,15 @@ Changelog --------- +7.4.0 +..... + +.. note:: + This provider release is the first release that has Kubernetes Executor and + Local Kubernetes Executor moved from the core ``apache-airflow`` package to the ``cncf.kubernetes`` + provider package. + + 7.3.0 ..... @@ -310,10 +319,10 @@ Param ``node_selectors`` has been removed in ``KubernetesPodOperator``; use ``no The following backcompat modules for KubernetesPodOperator are removed and you must now use the corresponding objects from the kubernetes library: -* ``airflow.providers.cncf.kubernetes.backcompat.pod`` -* ``airflow.providers.cncf.kubernetes.backcompat.pod_runtime_info_env`` -* ``airflow.providers.cncf.kubernetes.backcompat.volume`` -* ``airflow.providers.cncf.kubernetes.backcompat.volume_mount`` +* ``airflow.kubernetes.backcompat.pod`` +* ``airflow.kubernetes.backcompat.pod_runtime_info_env`` +* ``airflow.kubernetes.backcompat.volume`` +* ``airflow.kubernetes.backcompat.volume_mount`` * ``Remove deprecated backcompat objects for KPO (#27518)`` * ``Remove support for node_selectors param in KPO (#27515)`` diff --git a/tests/kubernetes/__init__.py b/airflow/providers/cncf/kubernetes/executors/__init__.py similarity index 100% rename from tests/kubernetes/__init__.py rename to airflow/providers/cncf/kubernetes/executors/__init__.py diff --git a/airflow/executors/kubernetes_executor.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py similarity index 93% rename from airflow/executors/kubernetes_executor.py rename to airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index d525417a2715..e57ca205fe3b 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -35,12 +35,12 @@ from sqlalchemy.orm import Session +from airflow import AirflowException from airflow.configuration import conf -from airflow.exceptions import PodMutationHookException, PodReconciliationError from airflow.executors.base_executor import BaseExecutor -from airflow.executors.kubernetes_executor_types import POD_EXECUTOR_DONE_KEY -from airflow.kubernetes.kube_config import KubeConfig -from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key +from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import POD_EXECUTOR_DONE_KEY +from airflow.providers.cncf.kubernetes.kube_config import KubeConfig +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import annotations_to_key from airflow.utils.event_scheduler import EventScheduler from airflow.utils.log.logging_mixin import remove_escape_codes from airflow.utils.session import NEW_SESSION, provide_session @@ -51,13 +51,23 @@ from kubernetes.client import models as k8s from airflow.executors.base_executor import CommandType - from airflow.executors.kubernetes_executor_types import ( + from airflow.models.taskinstance import TaskInstance + from airflow.models.taskinstancekey import TaskInstanceKey + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import ( KubernetesJobType, KubernetesResultsType, ) - from airflow.executors.kubernetes_executor_utils import AirflowKubernetesScheduler - from airflow.models.taskinstance import TaskInstance - from airflow.models.taskinstancekey import TaskInstanceKey + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ( + AirflowKubernetesScheduler, + ) + + +class PodMutationHookException(AirflowException): + """Raised when exception happens during Pod Mutation Hook execution.""" + + +class PodReconciliationError(AirflowException): + """Raised when an error is encountered while trying to merge pod configs.""" class KubernetesExecutor(BaseExecutor): @@ -98,11 +108,11 @@ def _list_pods(self, query_kwargs): def _make_safe_label_value(self, input_value: str | datetime) -> str: """ Normalize a provided label to be of valid length and characters. - See airflow.kubernetes.pod_generator.make_safe_label_value for more details. + See airflow.providers.cncf.kubernetes.pod_generator.make_safe_label_value for more details. """ - # airflow.kubernetes is an expensive import, locally import it here to + # airflow.providers.cncf.kubernetes is an expensive import, locally import it here to # speed up load times of the kubernetes_executor module. - from airflow.kubernetes import pod_generator + from airflow.providers.cncf.kubernetes import pod_generator if isinstance(input_value, datetime): return pod_generator.datetime_to_label_safe_datestring(input_value) @@ -188,8 +198,10 @@ def start(self) -> None: self.log.info("Start Kubernetes executor") self.scheduler_job_id = str(self.job_id) self.log.debug("Start with scheduler_job_id: %s", self.scheduler_job_id) - from airflow.executors.kubernetes_executor_utils import AirflowKubernetesScheduler - from airflow.kubernetes.kube_client import get_kube_client + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ( + AirflowKubernetesScheduler, + ) + from airflow.providers.cncf.kubernetes.kube_client import get_kube_client self.kube_client = get_kube_client() self.kube_scheduler = AirflowKubernetesScheduler( @@ -224,7 +236,7 @@ def execute_async( else: self.log.info("Add task %s with command %s", key, command) - from airflow.kubernetes.pod_generator import PodGenerator + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator try: kube_executor_config = PodGenerator.from_obj(executor_config) @@ -282,7 +294,7 @@ def sync(self) -> None: except Empty: break - from airflow.executors.kubernetes_executor_utils import ResourceVersion + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ResourceVersion resource_instance = ResourceVersion() for ns in resource_instance.resource_version.keys(): @@ -386,8 +398,8 @@ def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], li log = [] try: - from airflow.kubernetes.kube_client import get_kube_client - from airflow.kubernetes.pod_generator import PodGenerator + from airflow.providers.cncf.kubernetes.kube_client import get_kube_client + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator client = get_kube_client() @@ -463,7 +475,7 @@ def cleanup_stuck_queued_tasks(self, tis: list[TaskInstance]) -> list[str]: :param tis: List of Task Instances to clean up :return: List of readable task instances for a warning message """ - from airflow.kubernetes.pod_generator import PodGenerator + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator if TYPE_CHECKING: assert self.kube_client diff --git a/airflow/executors/kubernetes_executor_types.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py similarity index 100% rename from airflow/executors/kubernetes_executor_types.py rename to airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py diff --git a/airflow/executors/kubernetes_executor_utils.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py similarity index 93% rename from airflow/executors/kubernetes_executor_utils.py rename to airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py index b1ee49cb695e..c1ee9d1ebe31 100644 --- a/airflow/executors/kubernetes_executor_utils.py +++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py @@ -20,7 +20,7 @@ import multiprocessing import time from queue import Empty, Queue -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Generic, TypeVar from kubernetes import client, watch from kubernetes.client import Configuration, models as k8s @@ -28,26 +28,50 @@ from urllib3.exceptions import ReadTimeoutError from airflow.exceptions import AirflowException -from airflow.kubernetes.kube_client import get_kube_client -from airflow.kubernetes.kubernetes_helper_functions import ( +from airflow.providers.cncf.kubernetes.kube_client import get_kube_client +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import ( annotations_for_logging_task_metadata, annotations_to_key, create_pod_id, ) -from airflow.kubernetes.pod_generator import PodGenerator +from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.utils.log.logging_mixin import LoggingMixin -from airflow.utils.singleton import Singleton from airflow.utils.state import State +try: + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import ( + ALL_NAMESPACES, + POD_EXECUTOR_DONE_KEY, + ) +except ImportError: + # avoid failing import when Airflow pre 2.7 is installed + from airflow.kubernetes.kubernetes_executor_types import ( # type: ignore[no-redef] + ALL_NAMESPACES, + POD_EXECUTOR_DONE_KEY, + ) + if TYPE_CHECKING: - from airflow.executors.kubernetes_executor_types import ( + from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import ( KubernetesJobType, KubernetesResultsType, KubernetesWatchType, ) +# Singleton here is duplicated version of airflow.utils.singleton.Singleton until +# min-airflow version is 2.7.0 for the provider. then it can be imported from airflow.utils.singleton. + +T = TypeVar("T") + + +class Singleton(type, Generic[T]): + """Metaclass that allows to implement singleton pattern.""" + + _instances: dict[Singleton[T], T] = {} -from airflow.executors.kubernetes_executor_types import ALL_NAMESPACES, POD_EXECUTOR_DONE_KEY + def __call__(cls: Singleton[T], *args, **kwargs) -> T: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] class ResourceVersion(metaclass=Singleton): diff --git a/airflow/executors/local_kubernetes_executor.py b/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py similarity index 99% rename from airflow/executors/local_kubernetes_executor.py rename to airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py index cc95232c9151..ee8225209680 100644 --- a/airflow/executors/local_kubernetes_executor.py +++ b/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py @@ -22,8 +22,8 @@ from airflow.callbacks.base_callback_sink import BaseCallbackSink from airflow.callbacks.callback_requests import CallbackRequest from airflow.configuration import conf -from airflow.executors.kubernetes_executor import KubernetesExecutor from airflow.executors.local_executor import LocalExecutor +from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubernetesExecutor from airflow.utils.log.logging_mixin import LoggingMixin if TYPE_CHECKING: diff --git a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py index f237461155df..56852fb1a200 100644 --- a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +++ b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py @@ -30,8 +30,8 @@ from airflow.exceptions import AirflowException, AirflowNotFoundException from airflow.hooks.base import BaseHook -from airflow.kubernetes.kube_client import _disable_verify_ssl, _enable_tcp_keepalive from airflow.models import Connection +from airflow.providers.cncf.kubernetes.kube_client import _disable_verify_ssl, _enable_tcp_keepalive from airflow.providers.cncf.kubernetes.utils.pod_manager import PodOperatorHookProtocol from airflow.utils import yaml diff --git a/airflow/providers/cncf/kubernetes/k8s_model.py b/airflow/providers/cncf/kubernetes/k8s_model.py new file mode 100644 index 000000000000..8280a3265f09 --- /dev/null +++ b/airflow/providers/cncf/kubernetes/k8s_model.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Classes for interacting with Kubernetes API.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from functools import reduce + +from kubernetes.client import models as k8s + + +class K8SModel(ABC): + """ + Airflow Kubernetes models are here for backwards compatibility reasons only. + + Ideally clients should use the kubernetes API + and the process of + + client input -> Airflow k8s models -> k8s models + + can be avoided. All of these models implement the + `attach_to_pod` method so that they integrate with the kubernetes client. + """ + + @abstractmethod + def attach_to_pod(self, pod: k8s.V1Pod) -> k8s.V1Pod: + """ + Attaches to pod. + + :param pod: A pod to attach this Kubernetes object to + :return: The pod with the object attached + """ + + +def append_to_pod(pod: k8s.V1Pod, k8s_objects: list[K8SModel] | None): + """ + Attach additional specs to an existing pod object. + + :param pod: A pod to attach a list of Kubernetes objects to + :param k8s_objects: a potential None list of K8SModels + :return: pod with the objects attached if they exist + """ + if not k8s_objects: + return pod + return reduce(lambda p, o: o.attach_to_pod(p), k8s_objects, pod) diff --git a/airflow/providers/cncf/kubernetes/kube_client.py b/airflow/providers/cncf/kubernetes/kube_client.py new file mode 100644 index 000000000000..b9dec69402ef --- /dev/null +++ b/airflow/providers/cncf/kubernetes/kube_client.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Client for kubernetes communication.""" +from __future__ import annotations + +import logging + +import urllib3.util + +from airflow.configuration import conf + +log = logging.getLogger(__name__) + +try: + from kubernetes import client, config + from kubernetes.client import Configuration + from kubernetes.client.rest import ApiException + + has_kubernetes = True + + def _get_default_configuration() -> Configuration: + if hasattr(Configuration, "get_default_copy"): + return Configuration.get_default_copy() + return Configuration() + + def _disable_verify_ssl() -> None: + configuration = _get_default_configuration() + configuration.verify_ssl = False + Configuration.set_default(configuration) + +except ImportError as e: + # We need an exception class to be able to use it in ``except`` elsewhere + # in the code base + ApiException = BaseException + has_kubernetes = False + _import_err = e + + +def _enable_tcp_keepalive() -> None: + """ + This function enables TCP keepalive mechanism. + + This prevents urllib3 connection to hang indefinitely when idle connection + is time-outed on services like cloud load balancers or firewalls. + + See https://github.com/apache/airflow/pull/11406 for detailed explanation. + + Please ping @michalmisiewicz or @dimberman in the PR if you want to modify this function. + """ + import socket + + from urllib3.connection import HTTPConnection, HTTPSConnection + + tcp_keep_idle = conf.getint("kubernetes_executor", "tcp_keep_idle") + tcp_keep_intvl = conf.getint("kubernetes_executor", "tcp_keep_intvl") + tcp_keep_cnt = conf.getint("kubernetes_executor", "tcp_keep_cnt") + + socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] + + if hasattr(socket, "TCP_KEEPIDLE"): + socket_options.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, tcp_keep_idle)) + else: + log.debug("Unable to set TCP_KEEPIDLE on this platform") + + if hasattr(socket, "TCP_KEEPINTVL"): + socket_options.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, tcp_keep_intvl)) + else: + log.debug("Unable to set TCP_KEEPINTVL on this platform") + + if hasattr(socket, "TCP_KEEPCNT"): + socket_options.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, tcp_keep_cnt)) + else: + log.debug("Unable to set TCP_KEEPCNT on this platform") + + HTTPSConnection.default_socket_options = HTTPSConnection.default_socket_options + socket_options + HTTPConnection.default_socket_options = HTTPConnection.default_socket_options + socket_options + + +def get_kube_client( + in_cluster: bool | None = None, + cluster_context: str | None = None, + config_file: str | None = None, +) -> client.CoreV1Api: + """ + Retrieves Kubernetes client. + + :param in_cluster: whether we are in cluster + :param cluster_context: context of the cluster + :param config_file: configuration file + :return: kubernetes client + """ + if in_cluster is None: + in_cluster = conf.getboolean("kubernetes_executor", "in_cluster") + if not has_kubernetes: + raise _import_err + + if conf.getboolean("kubernetes_executor", "enable_tcp_keepalive"): + _enable_tcp_keepalive() + + configuration = _get_default_configuration() + api_client_retry_configuration = conf.getjson( + "kubernetes_executor", "api_client_retry_configuration", fallback={} + ) + + if not conf.getboolean("kubernetes_executor", "verify_ssl"): + _disable_verify_ssl() + + if isinstance(api_client_retry_configuration, dict): + configuration.retries = urllib3.util.Retry(**api_client_retry_configuration) + else: + raise ValueError("api_client_retry_configuration should be a dictionary") + + if in_cluster: + config.load_incluster_config(client_configuration=configuration) + else: + if cluster_context is None: + cluster_context = conf.get("kubernetes_executor", "cluster_context", fallback=None) + if config_file is None: + config_file = conf.get("kubernetes_executor", "config_file", fallback=None) + config.load_kube_config( + config_file=config_file, context=cluster_context, client_configuration=configuration + ) + + if not conf.getboolean("kubernetes_executor", "verify_ssl"): + configuration.verify_ssl = False + + ssl_ca_cert = conf.get("kubernetes_executor", "ssl_ca_cert") + if ssl_ca_cert: + configuration.ssl_ca_cert = ssl_ca_cert + + api_client = client.ApiClient(configuration=configuration) + return client.CoreV1Api(api_client) diff --git a/airflow/kubernetes/kube_config.py b/airflow/providers/cncf/kubernetes/kube_config.py similarity index 99% rename from airflow/kubernetes/kube_config.py rename to airflow/providers/cncf/kubernetes/kube_config.py index 20bddf82f38a..8db861a7f5d5 100644 --- a/airflow/kubernetes/kube_config.py +++ b/airflow/providers/cncf/kubernetes/kube_config.py @@ -43,7 +43,6 @@ def __init__(self): self.worker_pods_creation_batch_size = conf.getint( self.kubernetes_section, "worker_pods_creation_batch_size" ) - self.worker_container_repository = conf.get(self.kubernetes_section, "worker_container_repository") self.worker_container_tag = conf.get(self.kubernetes_section, "worker_container_tag") if self.worker_container_repository and self.worker_container_tag: diff --git a/tests/executors/kubernetes_executor_template_files/__init__.py b/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py similarity index 100% rename from tests/executors/kubernetes_executor_template_files/__init__.py rename to airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py diff --git a/airflow/kubernetes_executor_templates/basic_template.yaml b/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml similarity index 100% rename from airflow/kubernetes_executor_templates/basic_template.yaml rename to airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml diff --git a/airflow/kubernetes/kubernetes_helper_functions.py b/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py similarity index 94% rename from airflow/kubernetes/kubernetes_helper_functions.py rename to airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py index 27762421ca51..b54519fe2733 100644 --- a/airflow/kubernetes/kubernetes_helper_functions.py +++ b/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py @@ -43,8 +43,14 @@ def rand_str(num): return "".join(secrets.choice(alphanum_lower) for _ in range(num)) -def add_pod_suffix(*, pod_name, rand_len=8, max_len=80): - """Add random string to pod name while staying under max len.""" +def add_pod_suffix(*, pod_name: str, rand_len: int = 8, max_len: int = 80) -> str: + """Add random string to pod name while staying under max length. + + :param pod_name: name of the pod + :param rand_len: length of the random string to append + :param max_len: maximum length of the pod name + :meta private: + """ suffix = "-" + rand_str(rand_len) return pod_name[: max_len - len(suffix)].strip("-.") + suffix diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 49940144b5dd..6e6acbec4e38 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -35,10 +35,8 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException -from airflow.kubernetes import pod_generator -from airflow.kubernetes.pod_generator import PodGenerator -from airflow.kubernetes.secret import Secret from airflow.models import BaseOperator +from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.backcompat.backwards_compat_converters import ( convert_affinity, convert_configmap, @@ -51,6 +49,8 @@ convert_volume_mount, ) from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook +from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.providers.cncf.kubernetes.secret import Secret from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger from airflow.providers.cncf.kubernetes.utils import xcom_sidecar # type: ignore[attr-defined] from airflow.providers.cncf.kubernetes.utils.pod_manager import ( diff --git a/airflow/kubernetes/pod_generator.py b/airflow/providers/cncf/kubernetes/pod_generator.py similarity index 97% rename from airflow/kubernetes/pod_generator.py rename to airflow/providers/cncf/kubernetes/pod_generator.py index 844a1abfd8a8..c0f2e7e89f94 100644 --- a/airflow/kubernetes/pod_generator.py +++ b/airflow/providers/cncf/kubernetes/pod_generator.py @@ -38,14 +38,21 @@ from airflow.exceptions import ( AirflowConfigException, + RemovedInAirflow3Warning, +) +from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import ( PodMutationHookException, PodReconciliationError, - RemovedInAirflow3Warning, ) -from airflow.kubernetes.kubernetes_helper_functions import add_pod_suffix, rand_str -from airflow.kubernetes.pod_generator_deprecated import PodDefaults, PodGenerator as PodGeneratorDeprecated +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import add_pod_suffix, rand_str +from airflow.providers.cncf.kubernetes.pod_generator_deprecated import ( + PodDefaults, + PodGenerator as PodGeneratorDeprecated, +) + +# replace it with airflow.utils.hashlib_wrapper.md5 when min airflow version for k8s provider is 2.6.0 +from airflow.providers.cncf.kubernetes.utils.k8s_hashlib_wrapper import md5 from airflow.utils import yaml -from airflow.utils.hashlib_wrapper import md5 from airflow.version import version as airflow_version log = logging.getLogger(__name__) @@ -626,7 +633,10 @@ def extend_object_field(base_obj, client_obj, field_name): if (not isinstance(base_obj_field, list) and base_obj_field is not None) or ( not isinstance(client_obj_field, list) and client_obj_field is not None ): - raise ValueError("The chosen field must be a list.") + raise ValueError( + f"The chosen field must be a list. Got {type(base_obj_field)} base_object_field " + f"and {type(client_obj_field)} client_object_field." + ) if not base_obj_field: return client_obj_cp diff --git a/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py b/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py new file mode 100644 index 000000000000..8d64e96d6c40 --- /dev/null +++ b/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py @@ -0,0 +1,309 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Backwards compatibility for Pod generation. + +This module provides an interface between the previous Pod +API and outputs a kubernetes.client.models.V1Pod. +The advantage being that the full Kubernetes API +is supported and no serialization need be written. +""" +from __future__ import annotations + +import copy +import uuid + +import re2 +from kubernetes.client import models as k8s + +# replace it with airflow.utils.hashlib_wrapper.md5 when min airflow version for k8s provider is 2.6.0 +from airflow.providers.cncf.kubernetes.utils.k8s_hashlib_wrapper import md5 + +MAX_POD_ID_LEN = 253 + +MAX_LABEL_LEN = 63 + + +class PodDefaults: + """Static defaults for Pods.""" + + XCOM_MOUNT_PATH = "/airflow/xcom" + SIDECAR_CONTAINER_NAME = "airflow-xcom-sidecar" + XCOM_CMD = 'trap "exit 0" INT; while true; do sleep 30; done;' + VOLUME_MOUNT = k8s.V1VolumeMount(name="xcom", mount_path=XCOM_MOUNT_PATH) + VOLUME = k8s.V1Volume(name="xcom", empty_dir=k8s.V1EmptyDirVolumeSource()) + SIDECAR_CONTAINER = k8s.V1Container( + name=SIDECAR_CONTAINER_NAME, + command=["sh", "-c", XCOM_CMD], + image="alpine", + volume_mounts=[VOLUME_MOUNT], + resources=k8s.V1ResourceRequirements( + requests={ + "cpu": "1m", + } + ), + ) + + +def make_safe_label_value(string): + """ + Normalize a provided label to be of valid length and characters. + + Valid label values must be 63 characters or less and must be empty or begin and + end with an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_), + dots (.), and alphanumerics between. + + If the label value is greater than 63 chars once made safe, or differs in any + way from the original value sent to this function, then we need to truncate to + 53 chars, and append it with a unique hash. + """ + safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) + + if len(safe_label) > MAX_LABEL_LEN or string != safe_label: + safe_hash = md5(string.encode()).hexdigest()[:9] + safe_label = safe_label[: MAX_LABEL_LEN - len(safe_hash) - 1] + "-" + safe_hash + + return safe_label + + +class PodGenerator: + """ + Contains Kubernetes Airflow Worker configuration logic. + + Represents a kubernetes pod and manages execution of a single pod. + Any configuration that is container specific gets applied to + the first container in the list of containers. + + :param image: The docker image + :param name: name in the metadata section (not the container name) + :param namespace: pod namespace + :param volume_mounts: list of kubernetes volumes mounts + :param envs: A dict containing the environment variables + :param cmds: The command to be run on the first container + :param args: The arguments to be run on the pod + :param labels: labels for the pod metadata + :param node_selectors: node selectors for the pod + :param ports: list of ports. Applies to the first container. + :param volumes: Volumes to be attached to the first container + :param image_pull_policy: Specify a policy to cache or always pull an image + :param restart_policy: The restart policy of the pod + :param image_pull_secrets: Any image pull secrets to be given to the pod. + If more than one secret is required, provide a comma separated list: + secret_a,secret_b + :param init_containers: A list of init containers + :param service_account_name: Identity for processes that run in a Pod + :param resources: Resource requirements for the first containers + :param annotations: annotations for the pod + :param affinity: A dict containing a group of affinity scheduling rules + :param hostnetwork: If True enable host networking on the pod + :param tolerations: A list of kubernetes tolerations + :param security_context: A dict containing the security context for the pod + :param configmaps: Any configmap refs to read ``configmaps`` for environments from. + If more than one configmap is required, provide a comma separated list + configmap_a,configmap_b + :param dnspolicy: Specify a dnspolicy for the pod + :param schedulername: Specify a schedulername for the pod + :param pod: The fully specified pod. Mutually exclusive with `path_or_string` + :param extract_xcom: Whether to bring up a container for xcom + :param priority_class_name: priority class name for the launched Pod + """ + + def __init__( + self, + image: str | None = None, + name: str | None = None, + namespace: str | None = None, + volume_mounts: list[k8s.V1VolumeMount | dict] | None = None, + envs: dict[str, str] | None = None, + cmds: list[str] | None = None, + args: list[str] | None = None, + labels: dict[str, str] | None = None, + node_selectors: dict[str, str] | None = None, + ports: list[k8s.V1ContainerPort | dict] | None = None, + volumes: list[k8s.V1Volume | dict] | None = None, + image_pull_policy: str | None = None, + restart_policy: str | None = None, + image_pull_secrets: str | None = None, + init_containers: list[k8s.V1Container] | None = None, + service_account_name: str | None = None, + resources: k8s.V1ResourceRequirements | dict | None = None, + annotations: dict[str, str] | None = None, + affinity: dict | None = None, + hostnetwork: bool = False, + tolerations: list | None = None, + security_context: k8s.V1PodSecurityContext | dict | None = None, + configmaps: list[str] | None = None, + dnspolicy: str | None = None, + schedulername: str | None = None, + extract_xcom: bool = False, + priority_class_name: str | None = None, + ): + self.pod = k8s.V1Pod() + self.pod.api_version = "v1" + self.pod.kind = "Pod" + + # Pod Metadata + self.metadata = k8s.V1ObjectMeta() + self.metadata.labels = labels + self.metadata.name = name + self.metadata.namespace = namespace + self.metadata.annotations = annotations + + # Pod Container + self.container = k8s.V1Container(name="base") + self.container.image = image + self.container.env = [] + + if envs: + if isinstance(envs, dict): + for key, val in envs.items(): + self.container.env.append(k8s.V1EnvVar(name=key, value=val)) + elif isinstance(envs, list): + self.container.env.extend(envs) + + configmaps = configmaps or [] + self.container.env_from = [] + for configmap in configmaps: + self.container.env_from.append( + k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name=configmap)) + ) + + self.container.command = cmds or [] + self.container.args = args or [] + if image_pull_policy: + self.container.image_pull_policy = image_pull_policy + self.container.ports = ports or [] + self.container.resources = resources + self.container.volume_mounts = volume_mounts or [] + + # Pod Spec + self.spec = k8s.V1PodSpec(containers=[]) + self.spec.security_context = security_context + self.spec.tolerations = tolerations + if dnspolicy: + self.spec.dns_policy = dnspolicy + self.spec.scheduler_name = schedulername + self.spec.host_network = hostnetwork + self.spec.affinity = affinity + self.spec.service_account_name = service_account_name + self.spec.init_containers = init_containers + self.spec.volumes = volumes or [] + self.spec.node_selector = node_selectors + if restart_policy: + self.spec.restart_policy = restart_policy + self.spec.priority_class_name = priority_class_name + + self.spec.image_pull_secrets = [] + + if image_pull_secrets: + for image_pull_secret in image_pull_secrets.split(","): + self.spec.image_pull_secrets.append(k8s.V1LocalObjectReference(name=image_pull_secret)) + + # Attach sidecar + self.extract_xcom = extract_xcom + + def gen_pod(self) -> k8s.V1Pod: + """Generates pod.""" + result = None + + if result is None: + result = self.pod + result.spec = self.spec + result.metadata = self.metadata + result.spec.containers = [self.container] + + result.metadata.name = self.make_unique_pod_id(result.metadata.name) + + if self.extract_xcom: + result = self.add_sidecar(result) + + return result + + @staticmethod + def add_sidecar(pod: k8s.V1Pod) -> k8s.V1Pod: + """Adds sidecar.""" + pod_cp = copy.deepcopy(pod) + pod_cp.spec.volumes = pod.spec.volumes or [] + pod_cp.spec.volumes.insert(0, PodDefaults.VOLUME) + pod_cp.spec.containers[0].volume_mounts = pod_cp.spec.containers[0].volume_mounts or [] + pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaults.VOLUME_MOUNT) + pod_cp.spec.containers.append(PodDefaults.SIDECAR_CONTAINER) + + return pod_cp + + @staticmethod + def from_obj(obj) -> k8s.V1Pod | None: + """Converts to pod from obj.""" + if obj is None: + return None + + if isinstance(obj, PodGenerator): + return obj.gen_pod() + + if not isinstance(obj, dict): + raise TypeError( + "Cannot convert a non-dictionary or non-PodGenerator " + "object into a KubernetesExecutorConfig" + ) + + # We do not want to extract constant here from ExecutorLoader because it is just + # A name in dictionary rather than executor selection mechanism and it causes cyclic import + namespaced = obj.get("KubernetesExecutor", {}) + + if not namespaced: + return None + + resources = namespaced.get("resources") + + if resources is None: + requests = { + "cpu": namespaced.get("request_cpu"), + "memory": namespaced.get("request_memory"), + "ephemeral-storage": namespaced.get("ephemeral-storage"), + } + limits = { + "cpu": namespaced.get("limit_cpu"), + "memory": namespaced.get("limit_memory"), + "ephemeral-storage": namespaced.get("ephemeral-storage"), + } + all_resources = list(requests.values()) + list(limits.values()) + if all(r is None for r in all_resources): + resources = None + else: + resources = k8s.V1ResourceRequirements(requests=requests, limits=limits) + namespaced["resources"] = resources + return PodGenerator(**namespaced).gen_pod() + + @staticmethod + def make_unique_pod_id(dag_id): + r""" + Generate a unique Pod name. + + Kubernetes pod names must be <= 253 chars and must pass the following regex for + validation + ``^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`` + + :param dag_id: a dag_id with only alphanumeric characters + :return: ``str`` valid Pod name of appropriate length + """ + if not dag_id: + return None + + safe_uuid = uuid.uuid4().hex + safe_pod_id = dag_id[: MAX_POD_ID_LEN - len(safe_uuid) - 1] + "-" + safe_uuid + + return safe_pod_id diff --git a/airflow/kubernetes/pod_launcher_deprecated.py b/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py similarity index 98% rename from airflow/kubernetes/pod_launcher_deprecated.py rename to airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py index fa5c884a5176..9fc964cdb6e0 100644 --- a/airflow/kubernetes/pod_launcher_deprecated.py +++ b/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py @@ -32,8 +32,8 @@ from requests.exceptions import HTTPError from airflow.exceptions import AirflowException, RemovedInAirflow3Warning -from airflow.kubernetes.kube_client import get_kube_client -from airflow.kubernetes.pod_generator import PodDefaults +from airflow.providers.cncf.kubernetes.kube_client import get_kube_client +from airflow.providers.cncf.kubernetes.pod_generator import PodDefaults from airflow.settings import pod_mutation_hook from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.state import State diff --git a/airflow/kubernetes/pod_runtime_info_env.py b/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py similarity index 58% rename from airflow/kubernetes/pod_runtime_info_env.py rename to airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py index 32e178263b12..13a83393a912 100644 --- a/airflow/kubernetes/pod_runtime_info_env.py +++ b/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,19 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This module is deprecated. Please use :mod:`kubernetes.client.models.V1EnvVar`.""" -from __future__ import annotations - -import warnings - -from airflow.exceptions import RemovedInAirflow3Warning - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", RemovedInAirflow3Warning) - from airflow.providers.cncf.kubernetes.backcompat.pod_runtime_info_env import PodRuntimeInfoEnv # noqa - -warnings.warn( - "This module is deprecated. Please use `kubernetes.client.models.V1EnvVar`.", - RemovedInAirflow3Warning, - stacklevel=2, -) diff --git a/airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml b/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml similarity index 100% rename from airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml rename to airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml diff --git a/airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml b/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml similarity index 100% rename from airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml rename to airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml diff --git a/airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml b/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml similarity index 100% rename from airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml rename to airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 86ab5f70160b..d9b2966bba30 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 7.4.0 - 7.3.0 - 7.2.0 - 7.1.0 @@ -124,3 +125,212 @@ connection-types: task-decorators: - class-name: airflow.providers.cncf.kubernetes.decorators.kubernetes.kubernetes_task name: kubernetes + +config: + local_kubernetes_executor: + description: | + This section only applies if you are using the ``LocalKubernetesExecutor`` in + ``[core]`` section above + options: + kubernetes_queue: + description: | + Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``. + When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), + the task is executed via ``KubernetesExecutor``, + otherwise via ``LocalExecutor`` + version_added: ~ + type: string + example: ~ + default: "kubernetes" + kubernetes_executor: + description: ~ + options: + api_client_retry_configuration: + description: | + Kwargs to override the default urllib3 Retry used in the kubernetes API client + version_added: ~ + type: string + example: '{ "total": 3, "backoff_factor": 0.5 }' + default: "" + logs_task_metadata: + description: | + Flag to control the information added to kubernetes executor logs for better traceability + version_added: ~ + type: boolean + example: ~ + default: "False" + pod_template_file: + description: | + Path to the YAML pod file that forms the basis for KubernetesExecutor workers. + version_added: ~ + type: string + example: ~ + default: "" + see_also: ":ref:`concepts:pod_template_file`" + worker_container_repository: + description: | + The repository of the Kubernetes Image for the Worker to Run + version_added: ~ + type: string + example: ~ + default: "" + worker_container_tag: + description: | + The tag of the Kubernetes Image for the Worker to Run + version_added: ~ + type: string + example: ~ + default: "" + namespace: + description: | + The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` + version_added: ~ + type: string + example: ~ + default: "default" + delete_worker_pods: + description: | + If True, all worker pods will be deleted upon termination + version_added: ~ + type: string + example: ~ + default: "True" + delete_worker_pods_on_failure: + description: | + If False (and delete_worker_pods is True), + failed worker pods will not be deleted so users can investigate them. + This only prevents removal of worker pods where the worker itself failed, + not when the task it ran failed. + version_added: ~ + type: string + example: ~ + default: "False" + worker_pods_creation_batch_size: + description: | + Number of Kubernetes Worker Pod creation calls per scheduler loop. + Note that the current default of "1" will only launch a single pod + per-heartbeat. It is HIGHLY recommended that users increase this + number to match the tolerance of their kubernetes cluster for + better performance. + version_added: ~ + type: string + example: ~ + default: "1" + multi_namespace_mode: + description: | + Allows users to launch pods in multiple namespaces. + Will require creating a cluster-role for the scheduler, + or use multi_namespace_mode_namespace_list configuration. + version_added: ~ + type: boolean + example: ~ + default: "False" + multi_namespace_mode_namespace_list: + description: | + If multi_namespace_mode is True while scheduler does not have a cluster-role, + give the list of namespaces where the scheduler will schedule jobs + Scheduler needs to have the necessary permissions in these namespaces. + version_added: ~ + type: string + example: ~ + default: "" + in_cluster: + description: | + Use the service account kubernetes gives to pods to connect to kubernetes cluster. + It's intended for clients that expect to be running inside a pod running on kubernetes. + It will raise an exception if called from a process not running in a kubernetes environment. + version_added: ~ + type: string + example: ~ + default: "True" + cluster_context: + description: | + When running with in_cluster=False change the default cluster_context or config_file + options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. + version_added: ~ + type: string + example: ~ + default: ~ + config_file: + description: | + Path to the kubernetes configfile to be used when ``in_cluster`` is set to False + version_added: ~ + type: string + example: ~ + default: ~ + kube_client_request_args: + description: | + Keyword parameters to pass while calling a kubernetes client core_v1_api methods + from Kubernetes Executor provided as a single line formatted JSON dictionary string. + List of supported params are similar for all core_v1_apis, hence a single config + variable for all apis. See: + https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py + version_added: ~ + type: string + example: ~ + default: "" + delete_option_kwargs: + description: | + Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client + ``core_v1_api`` method when using the Kubernetes Executor. + This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` + class defined here: + https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 + version_added: ~ + type: string + example: '{"grace_period_seconds": 10}' + default: "" + enable_tcp_keepalive: + description: | + Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely + when idle connection is time-outed on services like cloud load balancers or firewalls. + version_added: ~ + type: boolean + example: ~ + default: "True" + tcp_keep_idle: + description: | + When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has + been idle for `tcp_keep_idle` seconds. + version_added: ~ + type: integer + example: ~ + default: "120" + tcp_keep_intvl: + description: | + When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond + to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. + version_added: ~ + type: integer + example: ~ + default: "30" + tcp_keep_cnt: + description: | + When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond + to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before + a connection is considered to be broken. + version_added: ~ + type: integer + example: ~ + default: "6" + verify_ssl: + description: | + Set this to false to skip verifying SSL certificate of Kubernetes python client. + version_added: ~ + type: boolean + example: ~ + default: "True" + worker_pods_queued_check_interval: + description: | + How often in seconds to check for task instances stuck in "queued" status without a pod + version_added: ~ + type: integer + example: ~ + default: "60" + ssl_ca_cert: + description: | + Path to a CA certificate to be used by the Kubernetes client to verify the server's SSL certificate. + version_added: ~ + type: string + example: ~ + default: "" diff --git a/airflow/kubernetes/secret.py b/airflow/providers/cncf/kubernetes/secret.py similarity index 98% rename from airflow/kubernetes/secret.py rename to airflow/providers/cncf/kubernetes/secret.py index 32ce92e2de01..d4fba36f16b5 100644 --- a/airflow/kubernetes/secret.py +++ b/airflow/providers/cncf/kubernetes/secret.py @@ -23,7 +23,7 @@ from kubernetes.client import models as k8s from airflow.exceptions import AirflowConfigException -from airflow.kubernetes.k8s_model import K8SModel +from airflow.providers.cncf.kubernetes.k8s_model import K8SModel class Secret(K8SModel): diff --git a/airflow/providers/cncf/kubernetes/template_rendering.py b/airflow/providers/cncf/kubernetes/template_rendering.py new file mode 100644 index 000000000000..5461284e4625 --- /dev/null +++ b/airflow/providers/cncf/kubernetes/template_rendering.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from jinja2 import TemplateAssertionError, UndefinedError +from kubernetes.client.api_client import ApiClient + +from airflow import AirflowException +from airflow.models.taskinstance import TaskInstance +from airflow.providers.cncf.kubernetes.kube_config import KubeConfig +from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import ( + create_pod_id, +) +from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.utils.session import NEW_SESSION, provide_session + + +def render_k8s_pod_yaml(task_instance: TaskInstance) -> dict | None: + """Render k8s pod yaml.""" + kube_config = KubeConfig() + pod = PodGenerator.construct_pod( + dag_id=task_instance.dag_id, + run_id=task_instance.run_id, + task_id=task_instance.task_id, + map_index=task_instance.map_index, + date=None, + pod_id=create_pod_id(task_instance.dag_id, task_instance.task_id), + try_number=task_instance.try_number, + kube_image=kube_config.kube_image, + args=task_instance.command_as_list(), + pod_override_object=PodGenerator.from_obj(task_instance.executor_config), + scheduler_job_id="0", + namespace=kube_config.executor_namespace, + base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file), + with_mutation_hook=True, + ) + sanitized_pod = ApiClient().sanitize_for_serialization(pod) + return sanitized_pod + + +@provide_session +def get_rendered_k8s_spec(task_instance: TaskInstance, session=NEW_SESSION) -> dict | None: + """Fetch rendered template fields from DB.""" + from airflow.models.renderedtifields import RenderedTaskInstanceFields + + rendered_k8s_spec = RenderedTaskInstanceFields.get_k8s_pod_yaml(task_instance, session=session) + if not rendered_k8s_spec: + try: + rendered_k8s_spec = render_k8s_pod_yaml(task_instance) + except (TemplateAssertionError, UndefinedError) as e: + raise AirflowException(f"Unable to render a k8s spec for this taskinstance: {e}") from e + return rendered_k8s_spec diff --git a/airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py b/airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py new file mode 100644 index 000000000000..72a1f8dac6b5 --- /dev/null +++ b/airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# this is a backcompat hashlib wrapper for kubernetes provider. It should be removed (with corresponding +# imports in the provider when min airflow version is 2.6.0 for the provider + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from _typeshed import ReadableBuffer + +from airflow import PY39 + + +def md5(__string: ReadableBuffer = b"") -> hashlib._Hash: + """ + Safely allows calling the ``hashlib.md5`` function when ``usedforsecurity`` is disabled in + the configuration. + + :param __string: The data to hash. Default to empty str byte. + :return: The hashed value. + """ + if PY39: + return hashlib.md5(__string, usedforsecurity=False) # type: ignore + return hashlib.md5(__string) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index acd40d4b1755..c8ac74382d6d 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -43,7 +43,7 @@ from urllib3.response import HTTPResponse from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning -from airflow.kubernetes.pod_generator import PodDefaults +from airflow.providers.cncf.kubernetes.pod_generator import PodDefaults from airflow.typing_compat import Literal, Protocol from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.timezone import utcnow @@ -89,7 +89,7 @@ class PodOperatorHookProtocol(Protocol): @property def core_v1_client(self) -> client.CoreV1Api: - """Get authenticated CoreV1Api object.""" + """Get authenticated client object.""" @property def is_in_cluster(self) -> bool: diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 60e2d4b02d7d..5efa3b3da574 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -73,7 +73,7 @@ try: from kubernetes.client import models as k8s - from airflow.kubernetes.pod_generator import PodGenerator + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator except ImportError: pass @@ -1482,7 +1482,12 @@ def _has_kubernetes() -> bool: try: from kubernetes.client import models as k8s - from airflow.kubernetes.pod_generator import PodGenerator + try: + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator + except ImportError: + from airflow.kubernetes.pre_7_4_0_compatibility.pod_generator import ( # type: ignore[assignment] + PodGenerator, + ) globals()["k8s"] = k8s globals()["PodGenerator"] = PodGenerator diff --git a/airflow/serialization/serializers/kubernetes.py b/airflow/serialization/serializers/kubernetes.py index d38836affc4a..f095400ee926 100644 --- a/airflow/serialization/serializers/kubernetes.py +++ b/airflow/serialization/serializers/kubernetes.py @@ -44,7 +44,12 @@ def serialize(o: object) -> tuple[U, str, int, bool]: return "", "", 0, False if isinstance(o, (k8s.V1Pod, k8s.V1ResourceRequirements)): - from airflow.kubernetes.pod_generator import PodGenerator + try: + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator + except ImportError: + from airflow.kubernetes.pre_7_4_0_compatibility.pod_generator import ( # type: ignore[assignment] + PodGenerator, + ) # We're running this in an except block, so we don't want it to fail # under any circumstances, e.g. accessing a non-existing attribute. diff --git a/airflow/timetables/_cron.py b/airflow/timetables/_cron.py index 49f5771966ad..89cae4bdcbed 100644 --- a/airflow/timetables/_cron.py +++ b/airflow/timetables/_cron.py @@ -65,10 +65,10 @@ def __init__(self, cron: str, timezone: str | Timezone) -> None: # as Croniter has inconsistent evaluation with other libraries if len(croniter(self._expression).expanded) > 5: raise FormatException() - interval_description = descriptor.get_description() + interval_description: str = descriptor.get_description() except (CroniterBadCronError, FormatException, MissingFieldException): interval_description = "" - self.description = interval_description + self.description: str = interval_description def __eq__(self, other: Any) -> bool: """Both expression and timezone should match. diff --git a/airflow/utils/deprecation_tools.py b/airflow/utils/deprecation_tools.py index a189fe5479ee..6be6650f093e 100644 --- a/airflow/utils/deprecation_tools.py +++ b/airflow/utils/deprecation_tools.py @@ -23,25 +23,75 @@ from types import ModuleType -def getattr_with_deprecation(imports: dict[str, str], module: str, name: str): +def getattr_with_deprecation( + imports: dict[str, str], + module: str, + override_deprecated_classes: dict[str, str], + extra_message: str, + name: str, +): + """ + Retrieve the imported attribute from the redirected module and raises a deprecation warning. + + :param imports: dict of imports and their redirection for the module + :param module: name of the module in the package to get the attribute from + :param override_deprecated_classes: override target classes with deprecated ones. If target class is + found in the dictionary, it will be displayed in the warning message. + :param extra_message: extra message to display in the warning or import error message + :param name: attribute name + :return: + """ target_class_full_name = imports.get(name) if not target_class_full_name: raise AttributeError(f"The module `{module!r}` has no attribute `{name!r}`") - warnings.warn( - f"The `{module}.{name}` class is deprecated. Please use `{target_class_full_name!r}`.", - DeprecationWarning, - stacklevel=2, - ) + warning_class_name = target_class_full_name + if override_deprecated_classes and name in override_deprecated_classes: + warning_class_name = override_deprecated_classes[name] + message = f"The `{module}.{name}` class is deprecated. Please use `{warning_class_name!r}`." + if extra_message: + message += f" {extra_message}." + warnings.warn(message, DeprecationWarning, stacklevel=2) new_module, new_class_name = target_class_full_name.rsplit(".", 1) - return getattr(importlib.import_module(new_module), new_class_name) + try: + return getattr(importlib.import_module(new_module), new_class_name) + except ImportError as e: + error_message = ( + f"Could not import `{new_module}.{new_class_name}` while trying to import `{module}.{name}`." + ) + if extra_message: + error_message += f" {extra_message}." + raise ImportError(error_message) from e -def add_deprecated_classes(module_imports: dict[str, dict[str, str]], package: str): +def add_deprecated_classes( + module_imports: dict[str, dict[str, str]], + package: str, + override_deprecated_classes: dict[str, dict[str, str]] | None = None, + extra_message: str | None = None, +): + """ + Add deprecated class PEP-563 imports and warnings modules to the package. + + :param module_imports: imports to use + :param package: package name + :param override_deprecated_classes: override target classes with deprecated ones. If module + + target class is found in the dictionary, it will be displayed in the warning message. + :param extra_message: extra message to display in the warning or import error message + """ for module_name, imports in module_imports.items(): full_module_name = f"{package}.{module_name}" module_type = ModuleType(full_module_name) + if override_deprecated_classes and module_name in override_deprecated_classes: + override_deprecated_classes_for_module = override_deprecated_classes[module_name] + else: + override_deprecated_classes_for_module = {} + # Mypy is not able to derive the right function signature https://github.com/python/mypy/issues/2427 module_type.__getattr__ = functools.partial( # type: ignore[assignment] - getattr_with_deprecation, imports, full_module_name + getattr_with_deprecation, + imports, + full_module_name, + override_deprecated_classes_for_module, + extra_message if extra_message else "", ) sys.modules.setdefault(full_module_name, module_type) diff --git a/airflow/kubernetes/volume.py b/airflow/utils/empty_set.py similarity index 62% rename from airflow/kubernetes/volume.py rename to airflow/utils/empty_set.py index ecb39e457fd4..0871c1ec5a12 100644 --- a/airflow/kubernetes/volume.py +++ b/airflow/utils/empty_set.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,19 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This module is deprecated. Please use :mod:`kubernetes.client.models.V1Volume`.""" + from __future__ import annotations -import warnings -from airflow.exceptions import RemovedInAirflow3Warning +def _get_empty_set_for_configuration() -> set[tuple[str, str]]: + """ + Retrieves an empty_set_for_configuration. -with warnings.catch_warnings(): - warnings.simplefilter("ignore", RemovedInAirflow3Warning) - from airflow.providers.cncf.kubernetes.backcompat.volume import Volume # noqa: autoflake + This method is only needed because configuration module has a deprecated method called set, and it + confuses mypy. This method will be removed when we remove the deprecated method. -warnings.warn( - "This module is deprecated. Please use `kubernetes.client.models.V1Volume`.", - RemovedInAirflow3Warning, - stacklevel=2, -) + :meta private: + :return: empty set + """ + return set() diff --git a/airflow/utils/hashlib_wrapper.py b/airflow/utils/hashlib_wrapper.py index 09850c565cb0..65a18566b775 100644 --- a/airflow/utils/hashlib_wrapper.py +++ b/airflow/utils/hashlib_wrapper.py @@ -28,9 +28,10 @@ def md5(__string: ReadableBuffer = b"") -> hashlib._Hash: """ - Safely allows calling the hashlib.md5 function when "usedforsecurity" is disabled in the configuration. + Safely allows calling the ``hashlib.md5`` function when ``usedforsecurity`` is disabled in + the configuration. - :param string: The data to hash. Default to empty str byte. + :param __string: The data to hash. Default to empty str byte. :return: The hashed value. """ if PY39: diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 499aa31c3497..5690799ce468 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -234,8 +234,12 @@ def ensure_pod_is_valid_after_unpickling(pod: V1Pod) -> V1Pod | None: if not isinstance(pod, V1Pod): return None try: - from airflow.kubernetes.pod_generator import PodGenerator - + try: + from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator + except ImportError: + from airflow.kubernetes.pre_7_4_0_compatibility.pod_generator import ( # type: ignore[assignment] + PodGenerator, + ) # now we actually reserialize / deserialize the pod pod_dict = sanitize_for_serialization(pod) return PodGenerator.deserialize_model_dict(pod_dict) diff --git a/airflow/www/views.py b/airflow/www/views.py index 54f46575be35..ae8811c05cc3 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -1541,6 +1541,10 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION): """Get rendered k8s yaml.""" if not settings.IS_K8S_OR_K8SCELERY_EXECUTOR: abort(404) + # This part is only used for k8s executor so providers.cncf.kubernetes must be installed + # with the get_rendered_k8s_spec method + from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec + dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") if task_id is None: @@ -1560,7 +1564,7 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION): pod_spec = None try: - pod_spec = ti.get_rendered_k8s_spec(session=session) + pod_spec = get_rendered_k8s_spec(ti, session=session) except AirflowException as e: if not e.__cause__: flash(f"Error rendering Kubernetes POD Spec: {e}", "error") diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index fae38c0a53b5..b96829539aa3 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -577,7 +577,7 @@ def _rebuild_k8s_image( COPY airflow/example_dags/ /opt/airflow/dags/ -COPY airflow/kubernetes_executor_templates/ /opt/airflow/pod_templates/ +COPY airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ ENV GUNICORN_CMD_ARGS='--preload' AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=0 """ diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 0ef8ef65b908..019058eeff82 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -27,6 +27,7 @@ "black", "blacken-docs", "check-aiobotocore-optional", + "check-airflow-k8s-not-used", "check-airflow-provider-compatibility", "check-apache-license-rat", "check-base-operator-partial-arguments", @@ -35,6 +36,7 @@ "check-breeze-top-dependencies-limited", "check-builtin-literals", "check-changelog-has-no-duplicates", + "check-cncf-k8s-only-for-executors", "check-core-deprecation-classes", "check-daysago-import-from-utils", "check-decorated-operator-implements-custom-name", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 7437642e4e9b..799a8a5ae5cc 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1049,7 +1049,8 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: ("airflow/providers/celery/file.py",), { "docs-filter-list-as-string": "--package-filter apache-airflow " - "--package-filter apache-airflow-providers-celery", + "--package-filter apache-airflow-providers-celery " + "--package-filter apache-airflow-providers-cncf-kubernetes" }, id="Celery python files changed", ), diff --git a/docs/apache-airflow-providers-apache-spark/changelog.rst b/docs/apache-airflow-providers-apache-spark/changelog.rst index 1a8c7b1c3ea1..36a0eb511894 100644 --- a/docs/apache-airflow-providers-apache-spark/changelog.rst +++ b/docs/apache-airflow-providers-apache-spark/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/apache/spark/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-spark/commits.rst b/docs/apache-airflow-providers-apache-spark/commits.rst index ba6baeae5a04..6b2f87851096 100644 --- a/docs/apache-airflow-providers-apache-spark/commits.rst +++ b/docs/apache-airflow-providers-apache-spark/commits.rst @@ -28,6 +28,21 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` +`3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== =================================================================== + 4.1.1 ..... @@ -36,6 +51,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ================================================================= Commit Committed Subject ================================================================================================= =========== ================================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` `6becb70316 `_ 2023-06-16 ``SparkSubmitOperator: rename spark_conn_id to conn_id (#31952)`` `13890788ae `_ 2023-06-07 ``Apache provider docstring improvements (#31730)`` diff --git a/docs/apache-airflow-providers-apache-spark/index.rst b/docs/apache-airflow-providers-apache-spark/index.rst index 1bd8a1def93c..300a89e540ee 100644 --- a/docs/apache-airflow-providers-apache-spark/index.rst +++ b/docs/apache-airflow-providers-apache-spark/index.rst @@ -76,7 +76,7 @@ Package apache-airflow-providers-apache-spark `Apache Spark `__ -Release: 4.1.1 +Release: 4.1.2 Provider package ---------------- @@ -102,3 +102,31 @@ PIP package Version required ``apache-airflow`` ``>=2.4.0`` ``pyspark`` ================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-spark[cncf.kubernetes] + + +====================================================================================================================== =================== +Dependent package Extra +====================================================================================================================== =================== +`apache-airflow-providers-cncf-kubernetes `_ ``cncf.kubernetes`` +====================================================================================================================== =================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-apache-spark 4.1.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-spark 4.1.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-celery/changelog.rst b/docs/apache-airflow-providers-celery/changelog.rst index 301ee64f5426..70dcac56ec81 100644 --- a/docs/apache-airflow-providers-celery/changelog.rst +++ b/docs/apache-airflow-providers-celery/changelog.rst @@ -16,6 +16,4 @@ specific language governing permissions and limitations under the License. - - .. include:: ../../airflow/providers/celery/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-celery/commits.rst b/docs/apache-airflow-providers-celery/commits.rst index 760e1b64aed7..66647f17f951 100644 --- a/docs/apache-airflow-providers-celery/commits.rst +++ b/docs/apache-airflow-providers-celery/commits.rst @@ -28,6 +28,26 @@ For high-level changelog, see :doc:`package information including changelog `_ 2023-07-22 ``Introduce decorator to load providers configuration (#32765)`` +`73b90c48b1 `_ 2023-07-21 ``Allow configuration to be contributed by providers (#32604)`` +`ea0deaa993 `_ 2023-07-16 ``Move default_celery.py to inside the provider (#32628)`` +`624211f33f `_ 2023-07-12 ``Add Executors discovery and documentation (#32532)`` +`40d54eac1a `_ 2023-07-12 ``Move CeleryExecutor to the celery provider (#32526)`` +`225e3041d2 `_ 2023-07-06 ``Prepare docs for July 2023 wave of Providers (RC2) (#32381)`` +`3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` +`cb4927a018 `_ 2023-07-05 ``Prepare docs for July 2023 wave of Providers (#32298)`` +`8c37b74a20 `_ 2023-06-28 ``D205 Support - Providers: Apache to Common (inclusive) (#32226)`` +`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` +================================================================================================= =========== =================================================================== + 3.2.1 ..... @@ -36,6 +56,7 @@ Latest change: 2023-06-20 ================================================================================================= =========== ============================================================= Commit Committed Subject ================================================================================================= =========== ============================================================= +`79bcc2e668 `_ 2023-06-20 ``Prepare RC1 docs for June 2023 wave of Providers (#32001)`` `8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` ================================================================================================= =========== ============================================================= diff --git a/docs/apache-airflow-providers-celery/index.rst b/docs/apache-airflow-providers-celery/index.rst index 464f12cd92e4..88a2a0ddd306 100644 --- a/docs/apache-airflow-providers-celery/index.rst +++ b/docs/apache-airflow-providers-celery/index.rst @@ -56,7 +56,7 @@ Package apache-airflow-providers-celery `Celery `__ -Release: 3.2.1 +Release: 3.3.0 Provider package ---------------- @@ -83,3 +83,31 @@ PIP package Version required ``celery`` ``>=5.2.3,<6`` ``flower`` ``>=1.0.0`` ================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider packages in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-celery[cncf.kubernetes] + + +====================================================================================================================== =================== +Dependent package Extra +====================================================================================================================== =================== +`apache-airflow-providers-cncf-kubernetes `_ ``cncf.kubernetes`` +====================================================================================================================== =================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-celery 3.3.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-celery 3.3.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst b/docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst new file mode 100644 index 000000000000..5885c9d91b6e --- /dev/null +++ b/docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst index 95ce846ab13e..9a19797d8f80 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/cncf/kubernetes/index> .. toctree:: diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst index 2c020a53e087..7b7a77b4f773 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst @@ -97,7 +97,7 @@ like this: With this API object, you can have access to all Kubernetes API objects in the form of python classes. Using this method will ensure correctness and type safety. While we have removed almost all Kubernetes convenience classes, we have kept the -:class:`~airflow.kubernetes.secret.Secret` class to simplify the process of generating secret volumes/env variables. +:class:`~airflow.providers.cncf.kubernetes.secret.Secret` class to simplify the process of generating secret volumes/env variables. .. exampleinclude:: /../../tests/system/providers/cncf/kubernetes/example_kubernetes.py :language: python diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index 7a32f1790dea..c4882a8b903c 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -37,6 +37,7 @@ in the provider's documentation. The pre-installed providers that you may want t * :doc:`Configuration Reference for Celery Provider ` * :doc:`Configuration Reference for Apache Hive Provider ` +* :doc:`Configuration Reference for CNCF Kubernetes Provider ` .. note:: For more information see :doc:`/howto/set-config`. diff --git a/docs/apache-airflow/core-concepts/executor/celery.rst b/docs/apache-airflow/core-concepts/executor/celery.rst index f69da1570fc3..1113bda53068 100644 --- a/docs/apache-airflow/core-concepts/executor/celery.rst +++ b/docs/apache-airflow/core-concepts/executor/celery.rst @@ -21,6 +21,13 @@ Celery Executor =============== +.. note:: + + As of Airflow 2.7.0, you need to install the ``celery`` provider package to use this executor. + This can be done by installing ``apache-airflow-providers-celery>=3.3.0`` or by installing Airflow + with the ``celery`` extra: ``pip install 'apache-airflow[celery]'``. + + ``CeleryExecutor`` is one of the ways you can scale out the number of workers. For this to work, you need to setup a Celery backend (**RabbitMQ**, **Redis**, **Redis Sentinel** ...) and change your ``airflow.cfg`` to point the executor parameter to diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst index 1eaf9bda5843..6416f5d2fa58 100644 --- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst +++ b/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst @@ -21,6 +21,14 @@ CeleryKubernetes Executor ========================= +.. note:: + + As of Airflow 2.7.0, you need to install both the ``celery`` and ``cncf.kubernetes`` provider package to use + this executor. This can be done by installing ``apache-airflow-providers-celery>=3.3.0`` and + ``apache-airflow-providers-cncf-kubernetes>=7.4.0`` or by installing Airflow + with the ``celery`` and ``cncf.kubernetes`` extras: ``pip install 'apache-airflow[celery,cncf.kubernetes]'``. + + The :class:`~airflow.providers.celery.executors.celery_kubernetes_executor.CeleryKubernetesExecutor` allows users to run simultaneously a ``CeleryExecutor`` and a ``KubernetesExecutor``. An executor is chosen to run a task based on the task's queue. diff --git a/docs/apache-airflow/core-concepts/executor/kubernetes.rst b/docs/apache-airflow/core-concepts/executor/kubernetes.rst index c5361e341737..e19264c96e5e 100644 --- a/docs/apache-airflow/core-concepts/executor/kubernetes.rst +++ b/docs/apache-airflow/core-concepts/executor/kubernetes.rst @@ -21,6 +21,14 @@ Kubernetes Executor =================== +.. note:: + + As of Airflow 2.7.0, you need to install the ``cncf.kubernetes`` provider package to use + this executor. This can done by installing ``apache-airflow-providers-cncf-kubernetes>=7.4.0`` + or by installing Airflow with the ``cncf.kubernetes`` extras: + ``pip install 'apache-airflow[cncf.kubernetes]'``. + + The Kubernetes executor runs each task instance in its own pod on a Kubernetes cluster. KubernetesExecutor runs as a process in the Airflow Scheduler. The scheduler itself does @@ -100,21 +108,21 @@ With these requirements in mind, here are some examples of basic ``pod_template_ Storing DAGs in the image: -.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml :language: yaml :start-after: [START template_with_dags_in_image] :end-before: [END template_with_dags_in_image] Storing DAGs in a ``persistentVolume``: -.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml :language: yaml :start-after: [START template_with_dags_in_volume] :end-before: [END template_with_dags_in_volume] Pulling DAGs from ``git``: -.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml +.. literalinclude:: /../../airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml :language: yaml :start-after: [START git_sync_template] :end-before: [END git_sync_template] diff --git a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst b/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst index ecb0dc536596..d50c7fc8780b 100644 --- a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst +++ b/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst @@ -21,7 +21,14 @@ LocalKubernetes Executor ========================= -The :class:`~airflow.executors.local_kubernetes_executor.LocalKubernetesExecutor` allows users +.. note:: + + As of Airflow 2.7.0, you need to install the ``cncf.kubernetes`` provider package to use + this executor. This can be done by installing ``apache-airflow-providers-cncf-kubernetes>=7.4.0`` + or by installing Airflow with the ``cncf.kubernetes`` extras: + ``pip install 'apache-airflow[cncf.kubernetes]'``. + +The :class:`~airflow.providers.cncf.kubernetes.executors.local_kubernetes_executor.LocalKubernetesExecutor` allows users to simultaneously run a ``LocalExecutor`` and a ``KubernetesExecutor``. An executor is chosen to run a task based on the task's queue. diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index ef6c4bf1b212..a17bfa073636 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -37,47 +37,43 @@ These are core airflow extras that extend capabilities of core Airflow. They usu packages (with the exception of ``celery`` and ``cncf.kubernetes`` extras), they just install necessary python dependencies for the provided package. -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| extra | install command | enables | Preinstalled | -+=====================+=====================================================+============================================================================+==============+ -| aiobotocore | ``pip install 'apache-airflow[aiobotocore]'`` | Support for asynchronous (deferrable) operators for Amazon integration | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor (also installs the celery provider package!) | * | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| cgroups | ``pip install 'apache-airflow[cgroups]'`` | Needed To use CgroupTaskRunner | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| cncf.kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor (also installs the Kubernetes provider package) | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| daskexecutor | ``pip install 'apache-airflow[daskexecutor]'`` | DaskExecutor ((also installs the Daskexecutor provider package) | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| deprecated_api | ``pip install 'apache-airflow[deprecated_api]'`` | Deprecated, experimental API that is replaced with the new REST API | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized services (Hadoop, Presto, Trino) | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| otel | ``pip install 'apache-airflow[otel]'`` | Required for OpenTelemetry metrics | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| pandas | ``pip install 'apache-airflow[pandas]'`` | Install Pandas library compatible with Airflow | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| statsd | ``pip install 'apache-airflow[statsd]'`` | Needed by StatsD metrics | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ -| virtualenv | ``pip install 'apache-airflow[virtualenv]'`` | Running python tasks in local virtualenv | | -+---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+--------------+ ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| extra | install command | enables | ++=====================+=====================================================+============================================================================+ +| aiobotocore | ``pip install 'apache-airflow[aiobotocore]'`` | Support for asynchronous (deferrable) operators for Amazon integration | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| cgroups | ``pip install 'apache-airflow[cgroups]'`` | Needed To use CgroupTaskRunner | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| daskexecutor | ``pip install 'apache-airflow[daskexecutor]'`` | DaskExecutor ((also installs the Daskexecutor provider package) | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| deprecated_api | ``pip install 'apache-airflow[deprecated_api]'`` | Deprecated, experimental API that is replaced with the new REST API | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized services (Hadoop, Presto, Trino) | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| otel | ``pip install 'apache-airflow[otel]'`` | Required for OpenTelemetry metrics | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| pandas | ``pip install 'apache-airflow[pandas]'`` | Install Pandas library compatible with Airflow | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| statsd | ``pip install 'apache-airflow[statsd]'`` | Needed by StatsD metrics | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| virtualenv | ``pip install 'apache-airflow[virtualenv]'`` | Running python tasks in local virtualenv | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ Providers extras @@ -188,6 +184,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | facebook | ``pip install 'apache-airflow[facebook]'`` | Facebook Social | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| github | ``pip install 'apache-airflow[github]'`` | GitHub operators and hook | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | google | ``pip install 'apache-airflow[google]'`` | Google Cloud | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | hashicorp | ``pip install 'apache-airflow[hashicorp]'`` | Hashicorp Services (Vault) | @@ -232,6 +230,10 @@ These are extras that add dependencies needed for integration with other softwar +=====================+=====================================================+===========================================+ | arangodb | ``pip install 'apache-airflow[arangodb]'`` | ArangoDB operators, sensors and hook | +---------------------+-----------------------------------------------------+-------------------------------------------+ +| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor | ++---------------------+-----------------------------------------------------+-------------------------------------------+ +| cncf.kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` | Kubernetes Executor | ++---------------------+-----------------------------------------------------+-------------------------------------------+ | docker | ``pip install 'apache-airflow[docker]'`` | Docker hooks and operators | +---------------------+-----------------------------------------------------+-------------------------------------------+ | elasticsearch | ``pip install 'apache-airflow[elasticsearch]'`` | Elasticsearch hooks and Log Handler | @@ -349,10 +351,10 @@ Those are the extras that are needed to generated documentation for Airflow. Thi +---------------------+-----------------------------------------------------+----------------------------------------------------------------------+ -Deprecated extras ------------------ +Deprecated 1.10 extras +---------------------- -These are the extras that have been used before and deprecated in 2.0 and will be removed in Airflow 3.0.0. They were +These are the extras that have been deprecated in 2.0 and will be removed in Airflow 3.0.0. They were all replaced by new extras, which have naming consistent with the names of provider packages. The ``crypto`` extra is not needed any more, because all crypto dependencies are part of airflow package, diff --git a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst index 782043f23406..f1956d89b95f 100644 --- a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst +++ b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst @@ -171,7 +171,7 @@ by using the ``| default`` Jinja filter as shown below. Much like the ``KubernetesExecutor``, the ``KubernetesPodOperator`` will no longer take Airflow custom classes and will instead expect either a pod_template yaml file, or ``kubernetes.client.models`` objects. -The one notable exception is that we will continue to support the ``airflow.kubernetes.secret.Secret`` class. +The one notable exception is that we will continue to support the ``airflow.providers.cncf.kubernetes.secret.Secret`` class. Whereas previously a user would import each individual class to build the pod as so: @@ -218,7 +218,7 @@ Now the user can use the ``kubernetes.client.models`` class as a single point of .. code-block:: python from kubernetes.client import models as k8s - from airflow.kubernetes.secret import Secret + from airflow.providers.cncf.kubernetes.secret import Secret configmaps = ["test-configmap-1", "test-configmap-2"] diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 6bfca3ef37d5..fb721ada25a5 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -35,6 +35,7 @@ alibaba allAuthenticatedUsers allowinsert allUsers +alphanumerics Alphasort amazonaws amqp @@ -296,6 +297,7 @@ ContainerPort contentUrl contextmgr contrib +CoreV coroutine coverals cp @@ -678,6 +680,7 @@ Harenslak Hashable Hashicorp hashicorp +hashlib hasn HCatalog hcatalog @@ -1105,7 +1108,9 @@ pluggable pluggy plyvel png +PodManager podName +PodSpec podSpec podspec poller diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index f4eeb8a27341..646b296444f3 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -197,7 +197,9 @@ "apache-airflow>=2.4.0", "pyspark" ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "cncf.kubernetes" + ], "excluded-python-versions": [] }, "apache.sqoop": { @@ -245,7 +247,9 @@ "celery>=5.2.3,<6", "flower>=1.0.0" ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "cncf.kubernetes" + ], "excluded-python-versions": [] }, "cloudant": { diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 540a5938a00e..c0ccd98d5a1a 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,7 +62,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:cdf58a9c27af2877dc5a658ed0a1d99c shell:87e7bdcebe1180395adfec86b2a065f5 start-airflow:f7216a8126ecf14b033e2ea677e1a105 -static-checks:f9ec0d7edaba84180403d95469d94ea0 +static-checks:a6b76083afc49c0c80270facd5236b42 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index 6220e1db62e0..083d5f668118 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - + From 583f407c54a6683247a6b8a3aed75a184f8a92c6 Mon Sep 17 00:00:00 2001 From: max <42827971+moiseenkov@users.noreply.github.com> Date: Wed, 26 Jul 2023 17:28:48 +0200 Subject: [PATCH 319/533] Fixup docstring for deprecated DataprocSubmitSparkJobOperator and refactoring system tests (#32743) --- .../google/cloud/operators/dataproc.py | 4 ++++ .../cloud/dataproc/example_dataproc_spark.py | 17 ++++++++++------- .../dataproc/example_dataproc_spark_async.py | 17 +++++++++++------ .../example_dataproc_spark_deferrable.py | 17 ++++++++++------- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 69b82e34b9f8..423ebf988f93 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1384,6 +1384,10 @@ def execute(self, context: Context): class DataprocSubmitSparkJobOperator(DataprocJobBaseOperator): """Start a Spark Job on a Cloud DataProc cluster. + .. seealso:: + This operator is deprecated, please use + :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`: + :param main_jar: The HCFS URI of the jar file that contains the main class (use this or the main_class, not both together). :param main_class: Name of the job class. (use this or the main_jar, not both diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py index 761d01816623..170230abbf6a 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py @@ -35,10 +35,8 @@ DAG_ID = "dataproc_spark" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") -CLUSTER_NAME = f"cluster-dataproc-spark-{ENV_ID}" +CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") REGION = "europe-west1" -ZONE = "europe-west1-b" - # Cluster definition CLUSTER_CONFIG = { @@ -54,8 +52,6 @@ }, } -TIMEOUT = {"seconds": 1 * 24 * 60 * 60} - # Jobs definitions # [START how_to_cloud_dataproc_spark_config] SPARK_JOB = { @@ -74,7 +70,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "spark"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", @@ -96,7 +92,14 @@ trigger_rule=TriggerRule.ALL_DONE, ) - create_cluster >> spark_task >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> spark_task + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py index edb704f6bcc9..a51f7c2e1949 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py @@ -36,9 +36,8 @@ DAG_ID = "dataproc_spark_async" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") -CLUSTER_NAME = f"dataproc-spark-async-{ENV_ID}" +CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") REGION = "europe-west1" -ZONE = "europe-west1-b" # Cluster definition CLUSTER_CONFIG = { @@ -54,8 +53,6 @@ }, } -TIMEOUT = {"seconds": 1 * 24 * 60 * 60} - # Jobs definitions SPARK_JOB = { "reference": {"project_id": PROJECT_ID}, @@ -72,7 +69,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "spark", "async"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", @@ -104,7 +101,15 @@ trigger_rule=TriggerRule.ALL_DONE, ) - create_cluster >> spark_task_async >> spark_task_async_sensor >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> spark_task_async + >> spark_task_async_sensor + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py index 36e333c2ebb4..738f5269f8f6 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py @@ -36,10 +36,8 @@ DAG_ID = "dataproc_spark_deferrable" PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") -CLUSTER_NAME = f"cluster-dataproc-spark-{ENV_ID}" +CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "-") REGION = "europe-west1" -ZONE = "europe-west1-b" - # Cluster definition CLUSTER_CONFIG = { @@ -55,8 +53,6 @@ }, } -TIMEOUT = {"seconds": 1 * 24 * 60 * 60} - # Jobs definitions # [START how_to_cloud_dataproc_spark_deferrable_config] SPARK_JOB = { @@ -75,7 +71,7 @@ schedule_interval="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "dataproc", "spark", "deferrable"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", @@ -97,7 +93,14 @@ trigger_rule=TriggerRule.ALL_DONE, ) - create_cluster >> spark_task >> delete_cluster + ( + # TEST SETUP + create_cluster + # TEST BODY + >> spark_task + # TEST TEARDOWN + >> delete_cluster + ) from tests.system.utils.watcher import watcher From 9e3104b5effc56f77833772d86e7201bcc50b8c9 Mon Sep 17 00:00:00 2001 From: VladaZakharova <80038284+VladaZakharova@users.noreply.github.com> Date: Wed, 26 Jul 2023 17:29:21 +0200 Subject: [PATCH 320/533] Fix DataflowStartFlexTemplateOperator system test (#32584) --- .../dataflow/example_dataflow_template.py | 23 ++++++++++--------- .../resources/{text.csv => input.csv} | 0 2 files changed, 12 insertions(+), 11 deletions(-) rename tests/system/providers/google/cloud/dataflow/resources/{text.csv => input.csv} (100%) diff --git a/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py b/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py index fcf919cc0a29..94af20ac666e 100644 --- a/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py +++ b/tests/system/providers/google/cloud/dataflow/example_dataflow_template.py @@ -39,15 +39,16 @@ PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") DAG_ID = "dataflow_template" -BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("_", "-") -FILE_NAME = "text.txt" -SCHEMA = "schema.json" +CSV_FILE_NAME = "input.csv" +AVRO_FILE_NAME = "output.avro" +AVRO_SCHEMA = "schema.json" GCS_TMP = f"gs://{BUCKET_NAME}/temp/" GCS_STAGING = f"gs://{BUCKET_NAME}/staging/" GCS_OUTPUT = f"gs://{BUCKET_NAME}/output" -PYTHON_FILE_LOCAL_PATH = str(Path(__file__).parent / "resources" / FILE_NAME) -SCHEMA_LOCAL_PATH = str(Path(__file__).parent / "resources" / SCHEMA) +PYTHON_FILE_LOCAL_PATH = str(Path(__file__).parent / "resources" / CSV_FILE_NAME) +SCHEMA_LOCAL_PATH = str(Path(__file__).parent / "resources" / AVRO_SCHEMA) LOCATION = "europe-west3" default_args = { @@ -60,11 +61,11 @@ "launchParameter": { "jobName": "test-flex-template", "parameters": { - "inputFileSpec": f"gs://{BUCKET_NAME}/{FILE_NAME}", - "outputBucket": f"gs://{BUCKET_NAME}/output/file.avro", + "inputFileSpec": f"gs://{BUCKET_NAME}/{CSV_FILE_NAME}", + "outputBucket": f"gs://{BUCKET_NAME}/output/{AVRO_FILE_NAME}", "outputFileFormat": "avro", "inputFileFormat": "csv", - "schema": f"gs://{BUCKET_NAME}/{SCHEMA}", + "schema": f"gs://{BUCKET_NAME}/{AVRO_SCHEMA}", }, "environment": {}, "containerSpecGcsPath": "gs://dataflow-templates/latest/flex/File_Format_Conversion", @@ -84,14 +85,14 @@ upload_file = LocalFilesystemToGCSOperator( task_id="upload_file_to_bucket", src=PYTHON_FILE_LOCAL_PATH, - dst=FILE_NAME, + dst=CSV_FILE_NAME, bucket=BUCKET_NAME, ) upload_schema = LocalFilesystemToGCSOperator( task_id="upload_schema_to_bucket", src=SCHEMA_LOCAL_PATH, - dst=SCHEMA, + dst=AVRO_SCHEMA, bucket=BUCKET_NAME, ) @@ -100,7 +101,7 @@ task_id="start_template_job", project_id=PROJECT_ID, template="gs://dataflow-templates/latest/Word_Count", - parameters={"inputFile": f"gs://{BUCKET_NAME}/{FILE_NAME}", "output": GCS_OUTPUT}, + parameters={"inputFile": f"gs://{BUCKET_NAME}/{CSV_FILE_NAME}", "output": GCS_OUTPUT}, location=LOCATION, ) # [END howto_operator_start_template_job] diff --git a/tests/system/providers/google/cloud/dataflow/resources/text.csv b/tests/system/providers/google/cloud/dataflow/resources/input.csv similarity index 100% rename from tests/system/providers/google/cloud/dataflow/resources/text.csv rename to tests/system/providers/google/cloud/dataflow/resources/input.csv From 579ce065c5b0865b4aea580ee2d05c89680b7a8b Mon Sep 17 00:00:00 2001 From: Owen Leung Date: Thu, 27 Jul 2023 01:46:45 +0800 Subject: [PATCH 321/533] Override base log folder by using task handler's base_log_folder (#32781) * Override base log folder by using task handler's base_log_folder --------- Co-authored-by: Tzu-ping Chung --- airflow/utils/serve_logs.py | 25 ++++++++++++++++++++- tests/utils/test_serve_logs.py | 40 +++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py index 8c1d62722a65..bae8a97da93a 100644 --- a/airflow/utils/serve_logs.py +++ b/airflow/utils/serve_logs.py @@ -36,6 +36,7 @@ from airflow.configuration import conf from airflow.utils.docs import get_docs_url from airflow.utils.jwt_signer import JWTSigner +from airflow.utils.module_loading import import_string logger = logging.getLogger(__name__) @@ -44,7 +45,29 @@ def create_app(): flask_app = Flask(__name__, static_folder=None) expiration_time_in_seconds = conf.getint("webserver", "log_request_clock_grace", fallback=30) log_directory = os.path.expanduser(conf.get("logging", "BASE_LOG_FOLDER")) - + log_config_class = conf.get("logging", "logging_config_class") + if log_config_class: + logger.info("Detected user-defined logging config. Attempting to load %s", log_config_class) + try: + logging_config = import_string(log_config_class) + try: + base_log_folder = logging_config["handlers"]["task"]["base_log_folder"] + except KeyError: + base_log_folder = None + if base_log_folder is not None: + log_directory = base_log_folder + logger.info( + "Successfully imported user-defined logging config. Flask App will serve log from %s", + log_directory, + ) + else: + logger.warning( + "User-defined logging config does not specify 'base_log_folder'. " + "Flask App will use default log directory %s", + base_log_folder, + ) + except Exception as e: + raise ImportError(f"Unable to load {log_config_class} due to error: {e}") signer = JWTSigner( secret_key=conf.get("webserver", "secret_key"), expiration_time_in_seconds=expiration_time_in_seconds, diff --git a/tests/utils/test_serve_logs.py b/tests/utils/test_serve_logs.py index 5288e646edbf..5b751312d36a 100644 --- a/tests/utils/test_serve_logs.py +++ b/tests/utils/test_serve_logs.py @@ -17,12 +17,14 @@ from __future__ import annotations import datetime +from pathlib import Path from typing import TYPE_CHECKING import jwt import pytest import time_machine +from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG from airflow.utils.jwt_signer import JWTSigner from airflow.utils.serve_logs import create_app from tests.test_utils.config import conf_vars @@ -34,18 +36,46 @@ @pytest.fixture -def client(tmpdir): - with conf_vars({("logging", "base_log_folder"): str(tmpdir)}): +def client_without_config(tmp_path): + with conf_vars({("logging", "base_log_folder"): tmp_path.as_posix()}): app = create_app() yield app.test_client() @pytest.fixture -def sample_log(tmpdir): - f = tmpdir / "sample.log" - f.write(LOG_DATA.encode()) +def client_with_config(): + with conf_vars( + { + ( + "logging", + "logging_config_class", + ): "airflow.config_templates.airflow_local_settings.DEFAULT_LOGGING_CONFIG" + } + ): + app = create_app() + + yield app.test_client() + +@pytest.fixture(params=["client_without_config", "client_with_config"]) +def client(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_log(request, tmp_path): + client = request.getfixturevalue("client") + + if client == request.getfixturevalue("client_without_config"): + base_log_dir = tmp_path + elif client == request.getfixturevalue("client_with_config"): + base_log_dir = Path(DEFAULT_LOGGING_CONFIG["handlers"]["task"]["base_log_folder"]) + else: + raise ValueError(f"Unknown client fixture: {client}") + + f = base_log_dir.joinpath("sample.log") + f.write_bytes(LOG_DATA.encode()) return f From 3237cb3484f0e7ec611ffba13360185e2f6c33f8 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 27 Jul 2023 06:50:40 +0800 Subject: [PATCH 322/533] Adjust graph zoom based on selected task (#32792) * Center to node on task selection * Use new fitView options from reactflow * Maintain zoom, remove focusNode fn --- airflow/www/package.json | 2 +- .../www/static/js/dag/details/graph/index.tsx | 99 ++++++---------- .../www/static/js/dag/details/graph/utils.ts | 6 +- airflow/www/yarn.lock | 108 +++++++++--------- 4 files changed, 97 insertions(+), 118 deletions(-) diff --git a/airflow/www/package.json b/airflow/www/package.json index cad6a6320a15..aec3e3509f6d 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -137,7 +137,7 @@ "react-syntax-highlighter": "^15.5.0", "react-table": "^7.8.0", "react-textarea-autosize": "^8.3.4", - "reactflow": "^11.4.0", + "reactflow": "^11.7.4", "redoc": "^2.0.0-rc.72", "remark-gfm": "^3.0.1", "swagger-ui-dist": "4.1.3", diff --git a/airflow/www/static/js/dag/details/graph/index.tsx b/airflow/www/static/js/dag/details/graph/index.tsx index 4dae9477bdc5..1bdfe5ece7a6 100644 --- a/airflow/www/static/js/dag/details/graph/index.tsx +++ b/airflow/www/static/js/dag/details/graph/index.tsx @@ -17,7 +17,7 @@ * under the License. */ -import React, { useRef, useState, useEffect } from "react"; +import React, { useRef, useState, useEffect, useMemo } from "react"; import { Box, useTheme, Select, Text } from "@chakra-ui/react"; import ReactFlow, { ReactFlowProvider, @@ -26,18 +26,13 @@ import ReactFlow, { MiniMap, Node as ReactFlowNode, useReactFlow, - ControlButton, Panel, } from "reactflow"; -import { RiFocus3Line } from "react-icons/ri"; import { useGraphData, useGridData } from "src/api"; import useSelection from "src/dag/useSelection"; import { useOffsetTop } from "src/utils"; import { useGraphLayout } from "src/utils/graph"; -import Tooltip from "src/components/Tooltip"; -import { useContainerRef } from "src/context/containerRef"; -import useFilters from "src/dag/useFilters"; import Edge from "src/components/Graph/Edge"; import Node, { CustomNodeProps } from "./Node"; @@ -54,13 +49,9 @@ interface Props { const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { const graphRef = useRef(null); - const containerRef = useContainerRef(); const { data } = useGraphData(); const [arrange, setArrange] = useState(data?.arrange || "LR"); - - const { - filters: { root, filterDownstream, filterUpstream }, - } = useFilters(); + const [hasRendered, setHasRendered] = useState(false); useEffect(() => { setArrange(data?.arrange || "LR"); @@ -77,45 +68,47 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { data: { dagRuns, groups }, } = useGridData(); const { colors } = useTheme(); - const { setCenter, setViewport } = useReactFlow(); + const { getZoom, fitView } = useReactFlow(); const latestDagRunId = dagRuns[dagRuns.length - 1]?.runId; - - // Reset viewport when tasks are filtered - useEffect(() => { - setViewport({ x: 0, y: 0, zoom: 1 }); - }, [root, filterDownstream, filterUpstream, setViewport]); - const offsetTop = useOffsetTop(graphRef); - let nodes: ReactFlowNode[] = []; - - if (graphData?.children) { - nodes = flattenNodes({ - children: graphData.children, + const nodes: ReactFlowNode[] = useMemo( + () => + graphData?.children + ? flattenNodes({ + children: graphData.children, + selected, + openGroupIds, + onToggleGroups, + latestDagRunId, + groups, + hoveredTaskState, + }) + : [], + [ + graphData?.children, selected, openGroupIds, onToggleGroups, latestDagRunId, groups, hoveredTaskState, - }); - } + ] + ); - const focusNode = () => { - if (selected.taskId) { - const node = nodes.find((n) => n.id === selected.taskId); - const x = node?.positionAbsolute?.x || node?.position.x; - const y = node?.positionAbsolute?.y || node?.position.y; - if (!x || !y) return; - setCenter( - x + (node.data.width || 0) / 2, - y + (node.data.height || 0) / 2, - { - duration: 1000, - } - ); + // Zoom to/from nodes when changing selection, maintain zoom level when changing task selection + useEffect(() => { + if (hasRendered) { + const zoom = getZoom(); + fitView({ + duration: 750, + nodes: selected.taskId ? [{ id: selected.taskId }] : undefined, + minZoom: selected.taskId ? zoom : undefined, + maxZoom: selected.taskId ? zoom : undefined, + }); } - }; + setHasRendered(true); + }, [fitView, hasRendered, selected.taskId, getZoom]); const edges = buildEdges({ edges: graphData?.edges, @@ -141,6 +134,11 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { maxZoom={1} onlyRenderVisibleElements defaultEdgeOptions={{ zIndex: 1 }} + // Fit view to selected task or the whole graph on render + fitView + fitViewOptions={{ + nodes: selected.taskId ? [{ id: selected.taskId }] : undefined, + }} > @@ -157,28 +155,7 @@ const Graph = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { - - - - - - - - - + nodeStrokeColor(props, colors)} diff --git a/airflow/www/static/js/dag/details/graph/utils.ts b/airflow/www/static/js/dag/details/graph/utils.ts index b1e27fb0eda3..aef01e5d53bd 100644 --- a/airflow/www/static/js/dag/details/graph/utils.ts +++ b/airflow/www/static/js/dag/details/graph/utils.ts @@ -125,14 +125,14 @@ export const flattenNodes = ({ }; export const nodeColor = ({ - data: { height, width, instance, childCount, isActive }, + data: { height, width, instance, isActive, isOpen }, }: ReactFlowNode) => { let opacity = "90"; let color = "#cccccc"; if (!height || !width) return ""; - if (instance?.state && !childCount) + if (instance?.state && !isOpen) color = Color(stateColors[instance.state]).hex(); - if (childCount) opacity = "50"; + if (isOpen) opacity = "50"; if (!isActive) opacity = "21"; return `${color}${opacity}`; diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index a3e6e864b7bd..a8b449990440 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -1354,13 +1354,6 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/runtime@^7.18.9": - version "7.20.6" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.6.tgz#facf4879bfed9b5326326273a64220f099b0fce3" - integrity sha512-Q+8MqP7TiHMWzSfwiJwXCjyf4GYA4Dgw3emg/7xmwsdLJOZUp+nMqcOwOzzYheuM1rhDu8FSj2l0aoMygEuXuA== - dependencies: - regenerator-runtime "^0.13.11" - "@babel/runtime@^7.3.1": version "7.21.5" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.5.tgz#8492dddda9644ae3bda3b45eabe87382caee7200" @@ -2762,29 +2755,28 @@ resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.2.tgz#830beaec4b4091a9e9398ac50f865ddea52186b9" integrity sha512-92FRmppjjqz29VMJ2dn+xdyXZBrMlE42AV6Kq6BwjWV7CNUW1hs2FtxSNLQE+gJhaZ6AAmYuO9y8dshhcBl7vA== -"@reactflow/background@11.1.0": - version "11.1.0" - resolved "https://registry.yarnpkg.com/@reactflow/background/-/background-11.1.0.tgz#943c799d9f251340e9867ad8f4c6ac291163e401" - integrity sha512-EgDn3rhK+l8jKmE6KGUZvesRjdh7fOqsz5Hj7STUU5/uGsvgN9KFuudY/Ka8m+yCQxqNK8MAJcRMOZd0mvNFMQ== +"@reactflow/background@11.2.4": + version "11.2.4" + resolved "https://registry.yarnpkg.com/@reactflow/background/-/background-11.2.4.tgz#06cd4c9f222dbeb2d3ffb2a530b6d88bf130dd9c" + integrity sha512-SYQbCRCU0GuxT/40Tm7ZK+l5wByGnNJSLtZhbL9C/Hl7JhsJXV3UGXr0vrlhVZUBEtkWA7XhZM/5S9XEA5XSFA== dependencies: - "@babel/runtime" "^7.18.9" - "@reactflow/core" "11.4.0" + "@reactflow/core" "11.7.4" classcat "^5.0.3" - zustand "^4.1.1" + zustand "^4.3.1" -"@reactflow/controls@11.1.0": - version "11.1.0" - resolved "https://registry.yarnpkg.com/@reactflow/controls/-/controls-11.1.0.tgz#6d6f6dd6e53557579c6cfcea3c7376d2d00c2953" - integrity sha512-5nH1TQ9mkveUOnq7QgohzeAdGR4WwKQJMrWjb5u3Dnm5D5+oRxTE3eGBoaw6B6nYaK1rDrPCcMAuGmEPdEC+Mg== +"@reactflow/controls@11.1.15": + version "11.1.15" + resolved "https://registry.yarnpkg.com/@reactflow/controls/-/controls-11.1.15.tgz#6dc823eb67f38a50907fffcc21b6a20e4fc00e7c" + integrity sha512-//33XfBYu8vQ6brfmlZwKrDoh+8hh93xO2d88XiqfIbrPEEb32SYjsb9mS9VuHKNlSIW+eB27fBA1Gt00mEj5w== dependencies: - "@babel/runtime" "^7.18.9" - "@reactflow/core" "11.4.0" + "@reactflow/core" "11.7.4" classcat "^5.0.3" + zustand "^4.3.1" -"@reactflow/core@11.4.0": - version "11.4.0" - resolved "https://registry.yarnpkg.com/@reactflow/core/-/core-11.4.0.tgz#9af0c812eb9968b75cf55427c6be4a9205d0db48" - integrity sha512-AfFp685kmxWs2Iiq35TatG9Q8u5W+eftXECQ0ea55Oi37nrMe5jfWhjnGnnl3bSFcHqAe6avqNiFDwqugU6kzQ== +"@reactflow/core@11.7.4", "@reactflow/core@^11.6.0": + version "11.7.4" + resolved "https://registry.yarnpkg.com/@reactflow/core/-/core-11.7.4.tgz#1a7e4d6cabbd2ea888547133d507f1ab24896520" + integrity sha512-nt0T8ERp8TE7YCDQViaoEY9lb0StDPrWHVx3zBjhStFYET3wc88t8QRasZdf99xRTmyNtI3U3M40M5EBLNUpMw== dependencies: "@types/d3" "^7.4.0" "@types/d3-drag" "^3.0.1" @@ -2794,31 +2786,40 @@ d3-drag "^3.0.0" d3-selection "^3.0.0" d3-zoom "^3.0.0" - zustand "^4.1.1" + zustand "^4.3.1" -"@reactflow/minimap@11.3.0": - version "11.3.0" - resolved "https://registry.yarnpkg.com/@reactflow/minimap/-/minimap-11.3.0.tgz#2f89dbab4c10b754c452f70857172d959cca60aa" - integrity sha512-nvb4qmbsogjhrn7GWXpvLMtmAyE7mjs0BXvtbpcFVpKqQ3Lbf76zCa8c2krUMnBBqu+9yF0Ftkn7mMCTV2gPLQ== +"@reactflow/minimap@11.5.4": + version "11.5.4" + resolved "https://registry.yarnpkg.com/@reactflow/minimap/-/minimap-11.5.4.tgz#b072094f7d827660f0205796d5f22fbbf6e31cc3" + integrity sha512-1tDBj2zX2gxu2oHU6qvH5RGNrOWRfRxu8369KhDotuuBN5yJrGXJzWIKikwhzjsNsQJYOB+B0cS44yWAfwSwzw== dependencies: - "@babel/runtime" "^7.18.9" - "@reactflow/core" "11.4.0" + "@reactflow/core" "11.7.4" "@types/d3-selection" "^3.0.3" "@types/d3-zoom" "^3.0.1" classcat "^5.0.3" d3-selection "^3.0.0" d3-zoom "^3.0.0" - zustand "^4.1.1" + zustand "^4.3.1" -"@reactflow/node-toolbar@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@reactflow/node-toolbar/-/node-toolbar-1.1.0.tgz#c7de2fb5c7aef02a1e575ce12a35d23cd45c18bd" - integrity sha512-kibrTGGvwhFGndVSgwr9E6l9Uddr44csr06X+PJ7FJ0SXgeOHbSw4MaM/9dSFxkFoCi77fPXSdMONNTReSBnIg== +"@reactflow/node-resizer@2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@reactflow/node-resizer/-/node-resizer-2.1.1.tgz#8f9b4e362e572dcddb54d43a67b5c5919b25937f" + integrity sha512-5Q+IBmZfpp/bYsw3+KRVJB1nUbj6W3XAp5ycx4uNWH+K98vbssymyQsW0vvKkIhxEPg6tkiMzO4UWRWvwBwt1g== dependencies: - "@babel/runtime" "^7.18.9" - "@reactflow/core" "11.4.0" + "@reactflow/core" "^11.6.0" + classcat "^5.0.4" + d3-drag "^3.0.0" + d3-selection "^3.0.0" + zustand "^4.3.1" + +"@reactflow/node-toolbar@1.2.3": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@reactflow/node-toolbar/-/node-toolbar-1.2.3.tgz#8ff8408dffee7920752479cd19e7ab7c9c47b4d2" + integrity sha512-uFQy9xpog92s0G1wsPLniwV9nyH4i/MmL7QoMsWdnKaOi7XMhd8SJcCzUdHC3imR21HltsuQITff/XQ51ApMbg== + dependencies: + "@reactflow/core" "11.7.4" classcat "^5.0.3" - zustand "^4.1.1" + zustand "^4.3.1" "@redocly/ajv@^8.6.4": version "8.6.4" @@ -4518,7 +4519,7 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz#9f84ba3244a512f3a54e5277e8eef4c489864e40" integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== -classcat@^5.0.3: +classcat@^5.0.3, classcat@^5.0.4: version "5.0.4" resolved "https://registry.yarnpkg.com/classcat/-/classcat-5.0.4.tgz#e12d1dfe6df6427f260f03b80dc63571a5107ba6" integrity sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g== @@ -9975,16 +9976,17 @@ react@^18.0.0: dependencies: loose-envify "^1.1.0" -reactflow@^11.4.0: - version "11.4.0" - resolved "https://registry.yarnpkg.com/reactflow/-/reactflow-11.4.0.tgz#aeb4b030ba93e8e656094f59226e55ec538f55b4" - integrity sha512-Y+LZ3XZX7UejW4vukeyLwDDfqNT0RxyNNSHD1FJOIu2IvyVMkj+wKTcbp3ehm7brBkMOOaPyugcEWlLwFXcrjg== +reactflow@^11.7.4: + version "11.7.4" + resolved "https://registry.yarnpkg.com/reactflow/-/reactflow-11.7.4.tgz#b00159c3471d007bc4865b23005c636b1f08ab26" + integrity sha512-QI6+oc1Ft6oFeLSdHlp+SmgymbI5Tm49wj5JyE84O4A54yN/ImfYaBhLit9Cmfzxn9Tz6tDqmGMGbk4bdtB8/w== dependencies: - "@reactflow/background" "11.1.0" - "@reactflow/controls" "11.1.0" - "@reactflow/core" "11.4.0" - "@reactflow/minimap" "11.3.0" - "@reactflow/node-toolbar" "1.1.0" + "@reactflow/background" "11.2.4" + "@reactflow/controls" "11.1.15" + "@reactflow/core" "11.7.4" + "@reactflow/minimap" "11.5.4" + "@reactflow/node-resizer" "2.1.1" + "@reactflow/node-toolbar" "1.2.3" read-pkg-up@^8.0.0: version "8.0.0" @@ -11863,10 +11865,10 @@ zrender@5.4.3: dependencies: tslib "2.3.0" -zustand@^4.1.1: - version "4.1.5" - resolved "https://registry.yarnpkg.com/zustand/-/zustand-4.1.5.tgz#7402b511f5b23ccb0f9ba6d20ae01ec817e16eb6" - integrity sha512-PsdRT8Bvq22Yyh1tvpgdHNE7OAeFKqJXUxtJvj1Ixw2B9O2YZ1M34ImQ+xyZah4wZrR4lENMoDUutKPpyXCQ/Q== +zustand@^4.3.1: + version "4.3.9" + resolved "https://registry.yarnpkg.com/zustand/-/zustand-4.3.9.tgz#a7d4332bbd75dfd25c6848180b3df1407217f2ad" + integrity sha512-Tat5r8jOMG1Vcsj8uldMyqYKC5IZvQif8zetmLHs9WoZlntTHmIoNM8TpLRY31ExncuUvUOXehd0kvahkuHjDw== dependencies: use-sync-external-store "1.2.0" From 9194144dab01d1898877215379e1c019fe6f10cd Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 27 Jul 2023 01:32:43 -0400 Subject: [PATCH 323/533] Replace Ruff setting known-third-party with namespace-packages (#32873) --- airflow/__init__.py | 7 ++--- airflow/providers/apprise/hooks/apprise.py | 3 +- .../apprise/notifications/apprise.py | 3 +- .../providers/openlineage/extractors/base.py | 4 +-- .../providers/openlineage/extractors/bash.py | 3 +- .../openlineage/extractors/manager.py | 3 +- .../openlineage/extractors/python.py | 3 +- .../providers/openlineage/plugins/adapter.py | 12 +++---- .../providers/openlineage/plugins/facets.py | 1 - airflow/providers/openlineage/sqlparser.py | 6 ++-- airflow/providers/openlineage/utils/sql.py | 3 +- airflow/providers/openlineage/utils/utils.py | 6 ++-- dev/breeze/pyproject.toml | 4 +++ pyproject.toml | 31 +++---------------- 14 files changed, 36 insertions(+), 53 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index 2e8e088ebc25..ebdd7d87771c 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -49,8 +49,7 @@ # very easily cause import cycles in the conf init/validate code (since downstream code from # those functions likely import settings). # configuration is therefore initted early here, simply by importing it. -from airflow import configuration -from airflow import settings +from airflow import configuration, settings __all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"] @@ -125,7 +124,7 @@ def __getattr__(name: str): STATICA_HACK = True globals()["kcah_acitats"[::-1].upper()] = False if STATICA_HACK: # pragma: no cover - from airflow.models.dag import DAG - from airflow.models.xcom_arg import XComArg from airflow.exceptions import AirflowException + from airflow.models.dag import DAG from airflow.models.dataset import Dataset + from airflow.models.xcom_arg import XComArg diff --git a/airflow/providers/apprise/hooks/apprise.py b/airflow/providers/apprise/hooks/apprise.py index 61548be76ab2..5ab9e16da92d 100644 --- a/airflow/providers/apprise/hooks/apprise.py +++ b/airflow/providers/apprise/hooks/apprise.py @@ -21,9 +21,10 @@ from typing import Any, Iterable import apprise -from airflow.hooks.base import BaseHook from apprise import AppriseConfig, NotifyFormat, NotifyType +from airflow.hooks.base import BaseHook + class AppriseHook(BaseHook): """ diff --git a/airflow/providers/apprise/notifications/apprise.py b/airflow/providers/apprise/notifications/apprise.py index cabae9bdca6e..cf78d68f375e 100644 --- a/airflow/providers/apprise/notifications/apprise.py +++ b/airflow/providers/apprise/notifications/apprise.py @@ -29,9 +29,10 @@ "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0" ) -from airflow.providers.apprise.hooks.apprise import AppriseHook from apprise import AppriseConfig, NotifyFormat, NotifyType +from airflow.providers.apprise.hooks.apprise import AppriseHook + class AppriseNotifier(BaseNotifier): """ diff --git a/airflow/providers/openlineage/extractors/base.py b/airflow/providers/openlineage/extractors/base.py index de84f0f63f25..51c9281e5621 100644 --- a/airflow/providers/openlineage/extractors/base.py +++ b/airflow/providers/openlineage/extractors/base.py @@ -20,11 +20,11 @@ from abc import ABC, abstractmethod from attrs import Factory, define +from openlineage.client.facet import BaseFacet +from openlineage.client.run import Dataset from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.state import TaskInstanceState -from openlineage.client.facet import BaseFacet -from openlineage.client.run import Dataset @define diff --git a/airflow/providers/openlineage/extractors/bash.py b/airflow/providers/openlineage/extractors/bash.py index 9d7c40b114c1..cfb022032ea3 100644 --- a/airflow/providers/openlineage/extractors/bash.py +++ b/airflow/providers/openlineage/extractors/bash.py @@ -17,13 +17,14 @@ from __future__ import annotations +from openlineage.client.facet import SourceCodeJobFacet + from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage from airflow.providers.openlineage.plugins.facets import ( UnknownOperatorAttributeRunFacet, UnknownOperatorInstance, ) from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled -from openlineage.client.facet import SourceCodeJobFacet """ :meta private: diff --git a/airflow/providers/openlineage/extractors/manager.py b/airflow/providers/openlineage/extractors/manager.py index b22afb5fdc35..02a4124840b5 100644 --- a/airflow/providers/openlineage/extractors/manager.py +++ b/airflow/providers/openlineage/extractors/manager.py @@ -173,9 +173,10 @@ def extract_inlets_and_outlets( @staticmethod def convert_to_ol_dataset(obj): - from airflow.lineage.entities import Table from openlineage.client.run import Dataset + from airflow.lineage.entities import Table + if isinstance(obj, Dataset): return obj elif isinstance(obj, Table): diff --git a/airflow/providers/openlineage/extractors/python.py b/airflow/providers/openlineage/extractors/python.py index 50d84014fdee..017e6488cc68 100644 --- a/airflow/providers/openlineage/extractors/python.py +++ b/airflow/providers/openlineage/extractors/python.py @@ -20,13 +20,14 @@ import inspect from typing import Callable +from openlineage.client.facet import SourceCodeJobFacet + from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage from airflow.providers.openlineage.plugins.facets import ( UnknownOperatorAttributeRunFacet, UnknownOperatorInstance, ) from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled -from openlineage.client.facet import SourceCodeJobFacet """ :meta private: diff --git a/airflow/providers/openlineage/plugins/adapter.py b/airflow/providers/openlineage/plugins/adapter.py index 0e530e5c53fe..0f19aa0f14e4 100644 --- a/airflow/providers/openlineage/plugins/adapter.py +++ b/airflow/providers/openlineage/plugins/adapter.py @@ -22,12 +22,6 @@ import requests.exceptions import yaml - -from airflow.configuration import conf -from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION -from airflow.providers.openlineage.extractors import OperatorLineage -from airflow.providers.openlineage.utils.utils import OpenLineageRedactor -from airflow.utils.log.logging_mixin import LoggingMixin from openlineage.client import OpenLineageClient, set_producer from openlineage.client.facet import ( BaseFacet, @@ -42,6 +36,12 @@ ) from openlineage.client.run import Job, Run, RunEvent, RunState +from airflow.configuration import conf +from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION +from airflow.providers.openlineage.extractors import OperatorLineage +from airflow.providers.openlineage.utils.utils import OpenLineageRedactor +from airflow.utils.log.logging_mixin import LoggingMixin + if TYPE_CHECKING: from airflow.models.dagrun import DagRun from airflow.utils.log.secrets_masker import SecretsMasker diff --git a/airflow/providers/openlineage/plugins/facets.py b/airflow/providers/openlineage/plugins/facets.py index f73ff1a98beb..2c301856f7c0 100644 --- a/airflow/providers/openlineage/plugins/facets.py +++ b/airflow/providers/openlineage/plugins/facets.py @@ -17,7 +17,6 @@ from __future__ import annotations from attrs import define - from openlineage.client.facet import BaseFacet from openlineage.client.utils import RedactMixin diff --git a/airflow/providers/openlineage/sqlparser.py b/airflow/providers/openlineage/sqlparser.py index fdf36c94d37f..2b18e28d7d85 100644 --- a/airflow/providers/openlineage/sqlparser.py +++ b/airflow/providers/openlineage/sqlparser.py @@ -20,6 +20,9 @@ import sqlparse from attrs import define +from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet +from openlineage.client.run import Dataset +from openlineage.common.sql import DbTableMeta, SqlMeta, parse from airflow.providers.openlineage.extractors.base import OperatorLineage from airflow.providers.openlineage.utils.sql import ( @@ -28,9 +31,6 @@ get_table_schemas, ) from airflow.typing_compat import TypedDict -from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet -from openlineage.client.run import Dataset -from openlineage.common.sql import DbTableMeta, SqlMeta, parse if TYPE_CHECKING: from sqlalchemy.engine import Engine diff --git a/airflow/providers/openlineage/utils/sql.py b/airflow/providers/openlineage/utils/sql.py index fe43a25bae67..5d2fa80cd68c 100644 --- a/airflow/providers/openlineage/utils/sql.py +++ b/airflow/providers/openlineage/utils/sql.py @@ -23,10 +23,9 @@ from typing import TYPE_CHECKING, Dict, List, Optional from attrs import define -from sqlalchemy import Column, MetaData, Table, and_, union_all - from openlineage.client.facet import SchemaDatasetFacet, SchemaField from openlineage.client.run import Dataset +from sqlalchemy import Column, MetaData, Table, and_, union_all if TYPE_CHECKING: from sqlalchemy.engine import Engine diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py index 9d1cab8eca87..ca8b559e3a49 100644 --- a/airflow/providers/openlineage/utils/utils.py +++ b/airflow/providers/openlineage/utils/utils.py @@ -29,6 +29,9 @@ import attrs from attrs import asdict +# TODO: move this maybe to Airflow's logic? +from openlineage.client.utils import RedactMixin + from airflow.compat.functools import cache from airflow.configuration import conf from airflow.providers.openlineage.plugins.facets import ( @@ -37,9 +40,6 @@ ) from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key -# TODO: move this maybe to Airflow's logic? -from openlineage.client.utils import RedactMixin - if TYPE_CHECKING: from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index ca71f72a7113..b18b68f8033c 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -33,3 +33,7 @@ python_files = [ testpaths = [ "tests", ] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] diff --git a/pyproject.toml b/pyproject.toml index bad61144d76e..de7dc5a905e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,8 @@ extend-exclude = [ # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can # ignore them in ruff "airflow/providers/common/sql/*/*.pyi", - "airflow/migrations/versions/*.py" + "airflow/migrations/versions/*.py", + "tests/dags/test_imports.py", ] extend-select = [ @@ -73,6 +74,8 @@ extend-ignore = [ "E731", ] +namespace-packages = ["airflow/providers"] + [tool.pytest.ini_options] # * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker. # * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8 @@ -106,39 +109,13 @@ testpaths = [ ] [tool.ruff.isort] -known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"] required-imports = ["from __future__ import annotations"] combine-as-imports = true -# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817 -known-third-party = [ - "asana", - "atlassian", - "celery", - "cloudant", - "databricks", - "datadog", - "docker", - "elasticsearch", - "github", - "google", - "grpc", - "jenkins", - "mysql", - "neo4j", - "papermill", - "redis", - "sendgrid", - "snowflake", - "telegram", - "trino", -] - [tool.ruff.per-file-ignores] "airflow/models/__init__.py" = ["F401"] "airflow/models/sqla_models.py" = ["F401"] - # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are # needed for the test to work "tests/decorators/test_python.py" = ["I002"] From 5aa62de16a70ba427e085ce7ca2525dcd112fbf2 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Thu, 27 Jul 2023 08:45:20 +0300 Subject: [PATCH 324/533] Set openlineage provider as ready to be released (#32854) * Set openlineage provider as ready to be released --- dev/provider_packages/prepare_provider_packages.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 82f3de998880..5aacd7cc6837 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -1676,11 +1676,12 @@ def verify_changelog_exists(package: str) -> str: def list_providers_packages(): """List all provider packages.""" providers = get_all_providers() - # For now we should exclude open-lineage from being consider for releasing until it is ready to - # be released - if "openlineage" in providers: - providers.remove("openlineage") + # if provider needs to be not considered in release add it here + # this is useful for cases where provider is WIP for a long period thus we don't want to release it yet. + providers_to_remove_from_release = [] for provider in providers: + if provider in providers_to_remove_from_release: + continue console.print(provider) From b73366799d98195a5ccc49a2008932186c4763b5 Mon Sep 17 00:00:00 2001 From: Maciej Obuchowski Date: Thu, 27 Jul 2023 07:53:43 +0200 Subject: [PATCH 325/533] openlineage, gcs: add openlineage methods for GcsToGcsOperator (#31350) Signed-off-by: Maciej Obuchowski --- .../google/cloud/transfers/gcs_to_gcs.py | 29 ++++++++ .../providers/openlineage/extractors/base.py | 18 +++-- dev/breeze/tests/test_selective_checks.py | 12 +-- generated/provider_dependencies.json | 1 + .../google/cloud/transfers/test_gcs_to_gcs.py | 73 +++++++++++++++++++ 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py index 17d163855986..cf4cf3c0ce65 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py @@ -233,6 +233,8 @@ def __init__( self.source_object_required = source_object_required self.exact_match = exact_match self.match_glob = match_glob + self.resolved_source_objects: set[str] = set() + self.resolved_target_objects: set[str] = set() def execute(self, context: Context): @@ -540,7 +542,34 @@ def _copy_single_object(self, hook, source_object, destination_object): destination_object, ) + self.resolved_source_objects.add(source_object) + if not destination_object: + self.resolved_target_objects.add(source_object) + else: + self.resolved_target_objects.add(destination_object) + hook.rewrite(self.source_bucket, source_object, self.destination_bucket, destination_object) if self.move_object: hook.delete(self.source_bucket, source_object) + + def get_openlineage_events_on_complete(self, task_instance): + """ + Implementing _on_complete because execute method does preprocessing on internals. + This means we won't have to normalize self.source_object and self.source_objects, + destination bucket and so on. + """ + from openlineage.client.run import Dataset + + from airflow.providers.openlineage.extractors import OperatorLineage + + return OperatorLineage( + inputs=[ + Dataset(namespace=f"gs://{self.source_bucket}", name=source) + for source in sorted(self.resolved_source_objects) + ], + outputs=[ + Dataset(namespace=f"gs://{self.destination_bucket}", name=target) + for target in sorted(self.resolved_target_objects) + ], + ) diff --git a/airflow/providers/openlineage/extractors/base.py b/airflow/providers/openlineage/extractors/base.py index 51c9281e5621..95d8fa6f2821 100644 --- a/airflow/providers/openlineage/extractors/base.py +++ b/airflow/providers/openlineage/extractors/base.py @@ -83,6 +83,7 @@ def get_operator_classnames(cls) -> list[str]: return [] def extract(self) -> OperatorLineage | None: + # OpenLineage methods are optional - if there's no method, return None try: return self._get_openlineage_facets(self.operator.get_openlineage_facets_on_start) # type: ignore except AttributeError: @@ -100,7 +101,15 @@ def extract_on_complete(self, task_instance) -> OperatorLineage | None: def _get_openlineage_facets(self, get_facets_method, *args) -> OperatorLineage | None: try: - facets = get_facets_method(*args) + facets: OperatorLineage = get_facets_method(*args) + # "rewrite" OperatorLineage to safeguard against different version of the same class + # that was existing in openlineage-airflow package outside of Airflow repo + return OperatorLineage( + inputs=facets.inputs, + outputs=facets.outputs, + run_facets=facets.run_facets, + job_facets=facets.job_facets, + ) except ImportError: self.log.exception( "OpenLineage provider method failed to import OpenLineage integration. " @@ -108,11 +117,4 @@ def _get_openlineage_facets(self, get_facets_method, *args) -> OperatorLineage | ) except Exception: self.log.exception("OpenLineage provider method failed to extract data from provider. ") - else: - return OperatorLineage( - inputs=facets.inputs, - outputs=facets.outputs, - run_facets=facets.run_facets, - job_facets=facets.job_facets, - ) return None diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ff051b57ad11..60ad9015c288 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -539,7 +539,7 @@ def test_expected_output_full_tests_needed( { "affected-providers-list-as-string": "amazon apache.beam apache.cassandra cncf.kubernetes " "common.sql facebook google hashicorp microsoft.azure microsoft.mssql " - "mysql oracle postgres presto salesforce sftp ssh trino", + "mysql openlineage oracle postgres presto salesforce sftp ssh trino", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", "needs-helm-tests": "false", @@ -564,8 +564,8 @@ def test_expected_output_full_tests_needed( { "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " "cncf.kubernetes common.sql facebook google " - "hashicorp microsoft.azure microsoft.mssql mysql oracle postgres presto " - "salesforce sftp ssh trino", + "hashicorp microsoft.azure microsoft.mssql mysql openlineage oracle postgres " + "presto salesforce sftp ssh trino", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", "image-build": "true", @@ -666,7 +666,7 @@ def test_expected_output_pull_request_v2_3( "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " "cncf.kubernetes common.sql " "facebook google hashicorp microsoft.azure microsoft.mssql mysql " - "oracle postgres presto salesforce sftp ssh trino", + "openlineage oracle postgres presto salesforce sftp ssh trino", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", "image-build": "true", @@ -685,6 +685,7 @@ def test_expected_output_pull_request_v2_3( "--package-filter apache-airflow-providers-microsoft-azure " "--package-filter apache-airflow-providers-microsoft-mssql " "--package-filter apache-airflow-providers-mysql " + "--package-filter apache-airflow-providers-openlineage " "--package-filter apache-airflow-providers-oracle " "--package-filter apache-airflow-providers-postgres " "--package-filter apache-airflow-providers-presto " @@ -697,7 +698,7 @@ def test_expected_output_pull_request_v2_3( "skip-provider-tests": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always CLI " "Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.sql,facebook," - "hashicorp,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto," + "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," "salesforce,sftp,ssh,trino] Providers[google]", }, id="CLI tests and Google-related provider tests should run if cli/chart files changed", @@ -965,6 +966,7 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: "--package-filter apache-airflow-providers-microsoft-azure " "--package-filter apache-airflow-providers-microsoft-mssql " "--package-filter apache-airflow-providers-mysql " + "--package-filter apache-airflow-providers-openlineage " "--package-filter apache-airflow-providers-oracle " "--package-filter apache-airflow-providers-postgres " "--package-filter apache-airflow-providers-presto " diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 94fc01793ac2..f9b7722f9bd5 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -467,6 +467,7 @@ "microsoft.azure", "microsoft.mssql", "mysql", + "openlineage", "oracle", "postgres", "presto", diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py index d29a505ba329..cf525235b14a 100644 --- a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py @@ -21,6 +21,7 @@ from unittest import mock import pytest +from openlineage.client.run import Dataset from airflow.exceptions import AirflowException from airflow.providers.google.cloud.transfers.gcs_to_gcs import WILDCARD, GCSToGCSOperator @@ -827,3 +828,75 @@ def test_copy_files_into_a_folder( for src, dst in zip(expected_source_objects, expected_destination_objects) ] mock_hook.return_value.rewrite.assert_has_calls(mock_calls) + + @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") + def test_execute_simple_reports_openlineage(self, mock_hook): + operator = GCSToGCSOperator( + task_id=TASK_ID, + source_bucket=TEST_BUCKET, + source_object=SOURCE_OBJECTS_SINGLE_FILE[0], + destination_bucket=DESTINATION_BUCKET, + ) + + operator.execute(None) + + lineage = operator.get_openlineage_events_on_complete(None) + assert len(lineage.inputs) == 1 + assert len(lineage.outputs) == 1 + assert lineage.inputs[0] == Dataset( + namespace=f"gs://{TEST_BUCKET}", name=SOURCE_OBJECTS_SINGLE_FILE[0] + ) + assert lineage.outputs[0] == Dataset( + namespace=f"gs://{DESTINATION_BUCKET}", name=SOURCE_OBJECTS_SINGLE_FILE[0] + ) + + @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") + def test_execute_multiple_reports_openlineage(self, mock_hook): + operator = GCSToGCSOperator( + task_id=TASK_ID, + source_bucket=TEST_BUCKET, + source_objects=SOURCE_OBJECTS_LIST, + destination_bucket=DESTINATION_BUCKET, + destination_object=DESTINATION_OBJECT, + ) + + operator.execute(None) + + lineage = operator.get_openlineage_events_on_complete(None) + assert len(lineage.inputs) == 3 + assert len(lineage.outputs) == 1 + assert lineage.inputs == [ + Dataset(namespace=f"gs://{TEST_BUCKET}", name=SOURCE_OBJECTS_LIST[0]), + Dataset(namespace=f"gs://{TEST_BUCKET}", name=SOURCE_OBJECTS_LIST[1]), + Dataset(namespace=f"gs://{TEST_BUCKET}", name=SOURCE_OBJECTS_LIST[2]), + ] + assert lineage.outputs[0] == Dataset(namespace=f"gs://{DESTINATION_BUCKET}", name=DESTINATION_OBJECT) + + @mock.patch("airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSHook") + def test_execute_wildcard_reports_openlineage(self, mock_hook): + mock_hook.return_value.list.return_value = [ + "test_object1.txt", + "test_object2.txt", + ] + + operator = GCSToGCSOperator( + task_id=TASK_ID, + source_bucket=TEST_BUCKET, + source_object=SOURCE_OBJECT_WILDCARD_SUFFIX, + destination_bucket=DESTINATION_BUCKET, + destination_object=DESTINATION_OBJECT, + ) + + operator.execute(None) + + lineage = operator.get_openlineage_events_on_complete(None) + assert len(lineage.inputs) == 2 + assert len(lineage.outputs) == 2 + assert lineage.inputs == [ + Dataset(namespace=f"gs://{TEST_BUCKET}", name="test_object1.txt"), + Dataset(namespace=f"gs://{TEST_BUCKET}", name="test_object2.txt"), + ] + assert lineage.outputs == [ + Dataset(namespace=f"gs://{DESTINATION_BUCKET}", name="foo/bar/1.txt"), + Dataset(namespace=f"gs://{DESTINATION_BUCKET}", name="foo/bar/2.txt"), + ] From 9d68492f875464f505afef2ecd81a28d8e4922b8 Mon Sep 17 00:00:00 2001 From: VladaZakharova <80038284+VladaZakharova@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:17:16 +0200 Subject: [PATCH 326/533] misc: update MLEngine system tests (#32881) --- .../google/cloud/operators/mlengine.py | 3 +++ .../cloud/ml_engine/example_mlengine.py | 18 ++++++++--------- .../cloud/ml_engine/example_mlengine_async.py | 20 +++++++++---------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/airflow/providers/google/cloud/operators/mlengine.py b/airflow/providers/google/cloud/operators/mlengine.py index 554167662e17..b39104137227 100644 --- a/airflow/providers/google/cloud/operators/mlengine.py +++ b/airflow/providers/google/cloud/operators/mlengine.py @@ -1002,6 +1002,9 @@ class MLEngineStartTrainingJobOperator(GoogleCloudBaseOperator): For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:MLEngineStartTrainingJobOperator` + For more information about used parameters, check: + https://cloud.google.com/sdk/gcloud/reference/ml-engine/jobs/submit/training + :param job_id: A unique templated id for the submitted Google MLEngine training job. (templated) :param region: The Google Compute Engine region to run the MLEngine training diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py index 6edd147bbf3a..51bf14fb5e88 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py +++ b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py @@ -49,14 +49,14 @@ DAG_ID = "example_gcp_mlengine" PREDICT_FILE_NAME = "predict.json" -MODEL_NAME = f"example_ml_model_{ENV_ID}" -BUCKET_NAME = f"example_ml_bucket_{ENV_ID}" +MODEL_NAME = f"model_{DAG_ID}_{ENV_ID}".replace("_", "-") +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("_", "-") BUCKET_PATH = f"gs://{BUCKET_NAME}" JOB_DIR = f"{BUCKET_PATH}/job-dir" SAVED_MODEL_PATH = f"{JOB_DIR}/" PREDICTION_INPUT = f"{BUCKET_PATH}/{PREDICT_FILE_NAME}" PREDICTION_OUTPUT = f"{BUCKET_PATH}/prediction_output/" -TRAINER_URI = "gs://system-tests-resources/example_gcp_mlengine/trainer-0.2.tar.gz" +TRAINER_URI = "gs://airflow-system-tests-resources/ml-engine/trainer-0.2.tar.gz" TRAINER_PY_MODULE = "trainer.task" SUMMARY_TMP = f"{BUCKET_PATH}/tmp/" SUMMARY_STAGING = f"{BUCKET_PATH}/staging/" @@ -104,8 +104,8 @@ def write_predict_file(path_to_file: str): project_id=PROJECT_ID, region="us-central1", job_id="training-job-{{ ts_nodash }}-{{ params.model_name }}", - runtime_version="2.1", - python_version="3.8", + runtime_version="1.15", + python_version="3.7", job_dir=JOB_DIR, package_uris=[TRAINER_URI], training_python_module=TRAINER_PY_MODULE, @@ -148,10 +148,10 @@ def write_predict_file(path_to_file: str): "name": "v1", "description": "First-version", "deployment_uri": JOB_DIR, - "runtime_version": "2.1", + "runtime_version": "1.15", "machineType": "mls1-c1-m2", "framework": "TENSORFLOW", - "pythonVersion": "3.8", + "pythonVersion": "3.7", }, ) # [END howto_operator_gcp_mlengine_create_version1] @@ -165,10 +165,10 @@ def write_predict_file(path_to_file: str): "name": "v2", "description": "Second version", "deployment_uri": JOB_DIR, - "runtime_version": "2.1", + "runtime_version": "1.15", "machineType": "mls1-c1-m2", "framework": "TENSORFLOW", - "pythonVersion": "3.8", + "pythonVersion": "3.7", }, ) # [END howto_operator_gcp_mlengine_create_version2] diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine_async.py b/tests/system/providers/google/cloud/ml_engine/example_mlengine_async.py index 873b57401601..7c2caef846cf 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine_async.py +++ b/tests/system/providers/google/cloud/ml_engine/example_mlengine_async.py @@ -49,14 +49,14 @@ DAG_ID = "async_example_gcp_mlengine" PREDICT_FILE_NAME = "async_predict.json" -MODEL_NAME = f"example_async_ml_model_{ENV_ID}" -BUCKET_NAME = f"example_async_ml_bucket_{ENV_ID}" +MODEL_NAME = f"model_{DAG_ID}_{ENV_ID}".replace("-", "_") +BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("_", "-") BUCKET_PATH = f"gs://{BUCKET_NAME}" JOB_DIR = f"{BUCKET_PATH}/job-dir" SAVED_MODEL_PATH = f"{JOB_DIR}/" PREDICTION_INPUT = f"{BUCKET_PATH}/{PREDICT_FILE_NAME}" PREDICTION_OUTPUT = f"{BUCKET_PATH}/prediction_output/" -TRAINER_URI = "gs://system-tests-resources/example_gcp_mlengine/async-trainer-0.2.tar.gz" +TRAINER_URI = "gs://airflow-system-tests-resources/ml-engine/async-trainer-0.2.tar.gz" TRAINER_PY_MODULE = "trainer.task" SUMMARY_TMP = f"{BUCKET_PATH}/tmp/" SUMMARY_STAGING = f"{BUCKET_PATH}/staging/" @@ -74,7 +74,7 @@ def generate_model_predict_input_data() -> list[int]: schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "ml_engine"], + tags=["example", "ml_engine", "deferrable"], params={"model_name": MODEL_NAME}, ) as dag: create_bucket = GCSCreateBucketOperator( @@ -104,8 +104,8 @@ def write_predict_file(path_to_file: str): project_id=PROJECT_ID, region="us-central1", job_id="async_training-job-{{ ts_nodash }}-{{ params.model_name }}", - runtime_version="2.1", - python_version="3.8", + runtime_version="1.15", + python_version="3.7", job_dir=JOB_DIR, package_uris=[TRAINER_URI], training_python_module=TRAINER_PY_MODULE, @@ -149,10 +149,10 @@ def write_predict_file(path_to_file: str): "name": "v1", "description": "First-version", "deployment_uri": JOB_DIR, - "runtime_version": "2.1", + "runtime_version": "1.15", "machineType": "mls1-c1-m2", "framework": "TENSORFLOW", - "pythonVersion": "3.8", + "pythonVersion": "3.7", }, ) # [END howto_operator_gcp_mlengine_create_version1] @@ -166,10 +166,10 @@ def write_predict_file(path_to_file: str): "name": "v2", "description": "Second version", "deployment_uri": JOB_DIR, - "runtime_version": "2.1", + "runtime_version": "1.15", "machineType": "mls1-c1-m2", "framework": "TENSORFLOW", - "pythonVersion": "3.8", + "pythonVersion": "3.7", }, ) # [END howto_operator_gcp_mlengine_create_version2] From ed000264178056ec3c0f8f96ffe14eba97e2ddd2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 27 Jul 2023 22:56:59 +0200 Subject: [PATCH 327/533] Fix hash for breeze dependencies (#32895) The #32873 has been merged without merging and commiting breeze's hash accounting for changed dependencies, breaking unit tests of breeze. This PR brings hash of breeze back adter running `breeze setup version` manually and commiting the changed hash. --- .github/workflows/ci.yml | 5 ++++- dev/breeze/README.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cda3148c59a1..daec76e48c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -383,7 +383,10 @@ jobs: cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: python -m pip install --editable ./dev/breeze/ - - run: breeze setup version + - run: > + breeze setup version || + (echo "\033[0;31mERROR: Breeze version check failed\033[0m" && + echo "Please run 'breeze setup version' and commit the changes." && false) - run: python -m pytest -n auto --color=yes working-directory: ./dev/breeze/ - run: breeze setup check-all-params-in-groups diff --git a/dev/breeze/README.md b/dev/breeze/README.md index a91598a0628b..775150252451 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: c2bade421865f94fb91f652317d76f3686ad017067349265d8a1d465bbfc2f0796d570bf4777acfb31a7703f1ccb6b0a63c655b97cec9eb9fdeeb1676c804c02 +Package config hash: 9d095d522c9f6fcf0c5834fcdc050bc98231d17fad07ec054c4e437580129d547b693b66b61442757f81fc1a505483da5267cc973dbf86babba7cd2c11697708 --------------------------------------------------------------------------------------------------------- From 42d3377b60c23c8cda2b15a25f70c06f14b1faca Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 27 Jul 2023 23:16:29 +0200 Subject: [PATCH 328/533] Apply pypi suffix to provider dependencies (including apache-airflow) (#32896) So far we only applied PyPi suffix for cross-provider dependencies - i.e. when one airflow package depended on another one released at the same time we added "dev0" suffix to their cross-dependencies, when building them on CI so that they can be installed together without conflicts (this is because of decisions made with PEP-440 where version suffixes are in different namespaces than the final versions, so it is impossible to specify "final" dependency as minimal and have the ".dev0" satisfy it only for that single package. With openlineage provider, we need to release it before Airflow 2.7.0 gets released, and openlineage provider depends on Airflow 2.7.0 so we need to also handle the situation, where OpenLineage has >= 2.7.0 for Airflow, but the dependency that is used to resolve dependencies in CI uses 2.7.0dev0. This is done by dynamically manipulating the dependencies in setup.py based on VERSION_SUFFIX_FOR_PYPI variable. This variable in CI is set to "dev0" thus all packages built have "dev0" added as version, with this change if any package has >= specified, it will also be extended with the same "dev0" suffix. --- .github/workflows/build-images.yml | 1 + .github/workflows/ci.yml | 1 + Dockerfile | 5 +- Dockerfile.ci | 5 +- .../commands/ci_image_commands.py | 2 + .../commands/ci_image_commands_config.py | 1 + .../commands/production_image_commands.py | 2 + .../production_image_commands_config.py | 1 + .../airflow_breeze/params/build_ci_params.py | 1 + .../params/build_prod_params.py | 1 + .../params/common_build_params.py | 1 + images/breeze/output-commands-hash.txt | 8 +- images/breeze/output_ci-image.svg | 24 +-- images/breeze/output_ci-image_build.svg | 156 +++++++++--------- images/breeze/output_prod-image_build.svg | 68 ++++---- setup.py | 42 ++++- 16 files changed, 187 insertions(+), 132 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 25d086f0ff41..d613e059d19b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -175,6 +175,7 @@ jobs: DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} RUNS_ON: ${{ needs.build-info.outputs.runs-on }} BACKEND: sqlite + VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daec76e48c9a..b152bff75973 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -305,6 +305,7 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" # Force more parallelism for build even on public images PARALLELISM: 6 + VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" diff --git a/Dockerfile b/Dockerfile index 16446c0b03e7..21c17afa71c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1290,9 +1290,12 @@ ARG ADDITIONAL_PYTHON_DEPS="" # !!! MAKE SURE YOU SYNCHRONIZE THE LIST BETWEEN: Dockerfile, Dockerfile.ci ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="" +ARG VERSION_SUFFIX_FOR_PYPI="" + ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ - EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ + VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} WORKDIR ${AIRFLOW_HOME} diff --git a/Dockerfile.ci b/Dockerfile.ci index 5aa9d72ec0d9..9ae0091c6b0f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1376,8 +1376,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # !!! MAKE SURE YOU SYNCHRONIZE THE LIST BETWEEN: Dockerfile, Dockerfile.ci ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" +ARG VERSION_SUFFIX_FOR_PYPI="" + ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} + UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} \ + VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 5b6b9ee3b8d6..141908864eda 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -70,6 +70,7 @@ option_upgrade_to_newer_dependencies, option_verbose, option_verify, + option_version_suffix_for_pypi, option_wait_for_image, ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm @@ -195,6 +196,7 @@ def prepare_for_building_ci_image(params: BuildCiParams): @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository +@option_version_suffix_for_pypi @option_verbose @option_dry_run @option_answer diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index b408bb9f7d36..c011fde22ade 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -66,6 +66,7 @@ "--additional-dev-apt-command", "--dev-apt-deps", "--dev-apt-command", + "--version-suffix-for-pypi", ], }, { diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index cb2d3d419be0..a1c6f2df1536 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -72,6 +72,7 @@ option_upgrade_to_newer_dependencies, option_verbose, option_verify, + option_version_suffix_for_pypi, option_wait_for_image, ) from airflow_breeze.utils.console import Output, get_console @@ -223,6 +224,7 @@ def prod_image(): @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository +@option_version_suffix_for_pypi @option_verbose @option_dry_run def build( diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 42c58c6e847c..40609bffa899 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -72,6 +72,7 @@ "--runtime-apt-command", "--dev-apt-deps", "--dev-apt-command", + "--version-suffix-for-pypi", ], }, { diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index 04d4b147f9e2..97324a27abf7 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -98,6 +98,7 @@ def optional_image_args(self) -> list[str]: "additional_airflow_extras", "additional_pip_install_flags", "additional_python_deps", + "version_suffix_for_pypi", ] def __post_init__(self): diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 8dfbdc0935e4..721cac8da703 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -240,4 +240,5 @@ def optional_image_args(self) -> list[str]: "dev_apt_deps", "runtime_apt_command", "runtime_apt_deps", + "version_suffix_for_pypi", ] diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index bbc0da9d9a61..914a2b02e2e1 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -65,6 +65,7 @@ class CommonBuildParams: upgrade_to_newer_dependencies: bool = False upgrade_on_failure: bool = False dry_run: bool = False + version_suffix_for_pypi: str = "" verbose: bool = False @property diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index c0ccd98d5a1a..94c4c589d89f 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -9,10 +9,10 @@ ci:get-workflow-info:8246038093359b9c3c110043419473e2 ci:resource-check:bfcca92f18a403ca630955074eb5e9ad ci:selective-check:6657ed5d42affb7264b5efcc86f17a2a ci:65d145d7746bde5f8dc8f6f484cfc134 -ci-image:build:a06f3ac7e0110213db25d06e6f5702da +ci-image:build:517f74610049c6c1d42a11693d40f89f ci-image:pull:7f14482a588f018f76df84719e77723f ci-image:verify:c90dc7e20fce2351eb89d8d1ebbd35e7 -ci-image:8b6a9a00849da08e89e8af2b6bd3672c +ci-image:8a6c5e10fa1901522e128d5a0bdcc033 cleanup:8d92d453a6700f6d8cb11fb6a8b50461 compile-www-assets:0963f1409f0aa1e3b137cddd4cc52e87 down:4580f5b3b178ea00182694f134a751f3 @@ -31,10 +31,10 @@ k8s:status:1b1b6fb6ccc7a211a3eb44568da71659 k8s:tests:2319c6f5ba0681ff7627e3166a1479ef k8s:upload-k8s-image:a9ac79e2f5e3d6b01fa45e764885913f k8s:db08df55a8200bff5858a3b7549bff47 -prod-image:build:1a321a1243555ab4e2cc95f44e85c92e +prod-image:build:d58b9222d141fd3620459ffaa9ee0235 prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 -prod-image:ad55d9dce93cbfcc1c5ad8640a007340 +prod-image:a7018fc4de9d087d69b25cd1cba71216 release-management:add-back-references:8dab6a30076a55f2d31c6d22a94e0ccb release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485 diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg index bf1df9347a16..3025e6e470c3 100644 --- a/images/breeze/output_ci-image.svg +++ b/images/breeze/output_ci-image.svg @@ -35,8 +35,8 @@ .breeze-ci-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-ci-image-r2 { fill: #c5c8c6 } .breeze-ci-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-ci-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-ci-image-r5 { fill: #868887 } +.breeze-ci-image-r4 { fill: #868887 } +.breeze-ci-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-ci-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage CI images +Tools that developers can use to manually manage CI images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build   Build CI image. Include building multiple images for all python versions.                                 │ -│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ -│verify  Verify CI image.                                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions.                                 │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 875f8212fe69..ef457feb9dca 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -264,9 +264,12 @@ + + + - Command: ci-image build + Command: ci-image build @@ -277,79 +280,80 @@ -Usage: breeze ci-image build [OPTIONS] +Usage: breeze ci-image build [OPTIONS] -Build CI image. Include building multiple images for all python versions. +Build CI image. Include building multiple images for all python versions. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ -│[default: 3.8]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ -│file. It could be full remote URL to the location file, or local file placed in │ -│`docker-context-files` (in this case it has to start with                       │ -│/opt/airflow/docker-context-files).                                             │ -│(TEXT)                                                                          │ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│[default: 3.8]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11]                                                  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index 882393e5d0df..2b4931a8dac2 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -1,4 +1,4 @@ - +
    + {instance?.queuedDttm && ( + + {isMappedOrGroupSummary && "Total "}Queued Duration:{" "} + {formatDuration(queuedDuration)} + + )} + + {isMappedOrGroupSummary && "Total "}Run Duration:{" "} + {formatDuration(taskDuration)} + +
    + {instance?.queuedDttm && ( + + {isMappedOrGroupSummary && "Earliest "}Queued At:{" "} + + )} + {instance?.startDate && ( + + {isMappedOrGroupSummary && "Earliest "}Start:{" "} + + )} + {instance?.endDate && ( + + {isMappedOrGroupSummary && "Latest "}End:{" "} + + )} + + ); +}; + +export default GanttTooltip; diff --git a/airflow/www/static/js/dag/details/gantt/Row.tsx b/airflow/www/static/js/dag/details/gantt/Row.tsx new file mode 100644 index 000000000000..96f079154e1e --- /dev/null +++ b/airflow/www/static/js/dag/details/gantt/Row.tsx @@ -0,0 +1,139 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from "react"; +import { Box, Tooltip, Flex } from "@chakra-ui/react"; +import useSelection from "src/dag/useSelection"; +import { getDuration } from "src/datetime_utils"; +import { SimpleStatus } from "src/dag/StatusBox"; +import { useContainerRef } from "src/context/containerRef"; +import { hoverDelay } from "src/utils"; +import type { DagRun, Task } from "src/types"; +import GanttTooltip from "./GanttTooltip"; + +interface Props { + ganttWidth?: number; + openGroupIds: string[]; + dagRun: DagRun; + task: Task; +} + +const Row = ({ ganttWidth = 500, openGroupIds, task, dagRun }: Props) => { + const { + selected: { runId, taskId }, + onSelect, + } = useSelection(); + const containerRef = useContainerRef(); + + const runDuration = getDuration(dagRun?.startDate, dagRun?.endDate); + + const instance = task.instances.find((ti) => ti.runId === runId); + const isSelected = taskId === instance?.taskId; + const hasQueuedDttm = !!instance?.queuedDttm; + const isOpen = openGroupIds.includes(task.label || ""); + + // Calculate durations in ms + const taskDuration = getDuration(instance?.startDate, instance?.endDate); + const queuedDuration = hasQueuedDttm + ? getDuration(instance?.queuedDttm, instance?.startDate) + : 0; + const taskStartOffset = getDuration( + dagRun.startDate, + instance?.queuedDttm || instance?.startDate + ); + + // Percent of each duration vs the overall dag run + const taskDurationPercent = taskDuration / runDuration; + const taskStartOffsetPercent = taskStartOffset / runDuration; + const queuedDurationPercent = queuedDuration / runDuration; + + // Calculate the pixel width of the queued and task bars and the position in the graph + // Min width should be 5px + let width = ganttWidth * taskDurationPercent; + if (width < 5) width = 5; + let queuedWidth = hasQueuedDttm ? ganttWidth * queuedDurationPercent : 0; + if (hasQueuedDttm && queuedWidth < 5) queuedWidth = 5; + const offsetMargin = taskStartOffsetPercent * ganttWidth; + + return ( +
    + + {instance ? ( + } + hasArrow + portalProps={{ containerRef }} + placement="top" + openDelay={hoverDelay} + > + { + onSelect({ + runId: instance.runId, + taskId: instance.taskId, + }); + }} + > + {instance.state !== "queued" && hasQueuedDttm && ( + + )} + + + + ) : ( + + )} + + {isOpen && + !!task.children && + task.children.map((c) => ( + + ))} +
    + ); +}; + +export default Row; diff --git a/airflow/www/static/js/dag/details/gantt/index.tsx b/airflow/www/static/js/dag/details/gantt/index.tsx new file mode 100644 index 000000000000..931c48b5662e --- /dev/null +++ b/airflow/www/static/js/dag/details/gantt/index.tsx @@ -0,0 +1,173 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { Box, Divider, Text } from "@chakra-ui/react"; + +import useSelection from "src/dag/useSelection"; +import { useGridData } from "src/api"; +import Time from "src/components/Time"; +import { getDuration } from "src/datetime_utils"; + +import Row from "./Row"; + +interface Props { + openGroupIds: string[]; + gridScrollRef: React.RefObject; + ganttScrollRef: React.RefObject; +} + +const Gantt = ({ openGroupIds, gridScrollRef, ganttScrollRef }: Props) => { + const ganttRef = useRef(null); + const [top, setTop] = useState(0); + const [width, setWidth] = useState(500); + const [height, setHeight] = useState("100%"); + const { + selected: { runId, taskId }, + } = useSelection(); + const { + data: { dagRuns, groups }, + } = useGridData(); + + const calculateGanttDimensions = useCallback(() => { + if (ganttRef?.current) { + const tbody = gridScrollRef.current?.getElementsByTagName("tbody")[0]; + const tableTop = + (tbody?.getBoundingClientRect().top || 0) + + (gridScrollRef?.current?.scrollTop || 0); + const ganttRect = ganttRef?.current?.getBoundingClientRect(); + const offsetTop = ganttRect?.top; + setTop(tableTop && offsetTop ? tableTop - offsetTop : 0); + if (ganttRect?.width) setWidth(ganttRect.width); + const gridHeight = gridScrollRef.current?.getBoundingClientRect().height; + if (gridHeight) setHeight(`${gridHeight - 155}px`); + } + }, [ganttRef, gridScrollRef]); + + // Calculate top, height and width when changing selections + useEffect(() => { + calculateGanttDimensions(); + }, [runId, taskId, openGroupIds, calculateGanttDimensions]); + + const onGridScroll = (e: Event) => { + const { scrollTop } = e.currentTarget as HTMLDivElement; + if (scrollTop && ganttScrollRef?.current) { + ganttScrollRef.current.scrollTo(0, scrollTop); + + // Double check scroll position after 100ms + setTimeout(() => { + const gridScrollTop = gridScrollRef.current?.scrollTop; + const ganttScrollTop = ganttScrollRef.current?.scrollTop; + if (ganttScrollTop !== gridScrollTop && ganttScrollRef.current) { + ganttScrollRef.current.scrollTo(0, gridScrollTop || 0); + } + }, 100); + } + }; + + // Sync grid and gantt scroll + useEffect(() => { + const grid = gridScrollRef.current; + grid?.addEventListener("scroll", onGridScroll); + return () => { + grid?.removeEventListener("scroll", onGridScroll); + }; + }); + + // Calculate top, height and width on resize + useEffect(() => { + const ganttChart = ganttRef.current; + const ganttObserver = new ResizeObserver(calculateGanttDimensions); + + if (ganttChart) { + ganttObserver.observe(ganttChart); + return () => { + ganttObserver.unobserve(ganttChart); + }; + } + return () => {}; + }, [ganttRef, calculateGanttDimensions]); + + const dagRun = dagRuns.find((dr) => dr.runId === runId); + + const startDate = dagRun?.startDate; + + const numBars = Math.round(width / 100); + const runDuration = getDuration(dagRun?.startDate, dagRun?.endDate); + const intervals = runDuration / numBars; + + return ( + + + {Array.from(Array(numBars)).map((_, i) => ( + + + + + + ))} + + + + + +
    + {!!runId && + !!dagRun && + !!groups.children && + groups.children.map((c) => ( + + ))} +
    +     +     + ); +}; + +export default Gantt; diff --git a/airflow/www/static/js/dag/details/index.tsx b/airflow/www/static/js/dag/details/index.tsx index a4caa7236a0a..e505fd406ef1 100644 --- a/airflow/www/static/js/dag/details/index.tsx +++ b/airflow/www/static/js/dag/details/index.tsx @@ -17,7 +17,7 @@ * under the License. */ -import React, { useCallback } from "react"; +import React, { useCallback, useEffect } from "react"; import { Flex, Divider, @@ -33,7 +33,13 @@ import { useSearchParams } from "react-router-dom"; import useSelection from "src/dag/useSelection"; import { getTask, getMetaValue } from "src/utils"; import { useGridData, useTaskInstance } from "src/api"; -import { MdDetails, MdAccountTree, MdReorder, MdCode } from "react-icons/md"; +import { + MdDetails, + MdAccountTree, + MdReorder, + MdCode, + MdOutlineViewTimeline, +} from "react-icons/md"; import { BiBracket } from "react-icons/bi"; import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper"; @@ -42,6 +48,7 @@ import TaskInstanceContent from "./taskInstance"; import DagRunContent from "./dagRun"; import DagContent from "./Dag"; import Graph from "./graph"; +import Gantt from "./gantt"; import DagCode from "./dagCode"; import MappedInstances from "./taskInstance/MappedInstances"; import Logs from "./taskInstance/Logs"; @@ -58,16 +65,20 @@ interface Props { openGroupIds: string[]; onToggleGroups: (groupIds: string[]) => void; hoveredTaskState?: string | null; + gridScrollRef: React.RefObject; + ganttScrollRef: React.RefObject; } const tabToIndex = (tab?: string) => { switch (tab) { case "graph": return 1; + case "gantt": + return 2; case "code": case "logs": case "mapped_tasks": - return 2; + return 3; case "details": default: return 0; @@ -84,6 +95,8 @@ const indexToTab = ( case 1: return "graph"; case 2: + return "gantt"; + case 3: if (!taskId) return "code"; if (showMappedTasks) return "mapped_tasks"; if (showLogs) return "logs"; @@ -96,7 +109,13 @@ const indexToTab = ( const TAB_PARAM = "tab"; -const Details = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { +const Details = ({ + openGroupIds, + onToggleGroups, + hoveredTaskState, + gridScrollRef, + ganttScrollRef, +}: Props) => { const { selected: { runId, taskId, mapIndex }, onSelect, @@ -134,6 +153,15 @@ const Details = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { [setSearchParams, searchParams, showLogs, showMappedTasks, taskId] ); + useEffect(() => { + // We only have 3 tabs for when nothing or a task group are selected + const tabCount = + (runId && !taskId) || (runId && taskId && !isGroup) ? 4 : 3; + if (tabCount === 3 && tabIndex > 2) { + onChangeTab(1); + } + }, [taskId, runId, tabIndex, isGroup, onChangeTab]); + const run = dagRuns.find((r) => r.runId === runId); const { data: mappedTaskInstance } = useTaskInstance({ dagId, @@ -212,6 +240,14 @@ const Details = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { Graph + {run && ( + + + + Gantt + + + )} {showDagCode && ( @@ -262,6 +298,15 @@ const Details = ({ openGroupIds, onToggleGroups, hoveredTaskState }: Props) => { hoveredTaskState={hoveredTaskState} /> + {run && ( + + + + )} {showDagCode && ( diff --git a/airflow/www/static/js/dag/grid/index.tsx b/airflow/www/static/js/dag/grid/index.tsx index b598a7fb5895..a67305994a26 100644 --- a/airflow/www/static/js/dag/grid/index.tsx +++ b/airflow/www/static/js/dag/grid/index.tsx @@ -39,6 +39,8 @@ interface Props { onToggleGroups: (groupIds: string[]) => void; isGridCollapsed?: boolean; setIsGridCollapsed?: (collapsed: boolean) => void; + gridScrollRef?: React.RefObject; + ganttScrollRef?: React.RefObject; } const Grid = ({ @@ -49,8 +51,9 @@ const Grid = ({ onToggleGroups, isGridCollapsed, setIsGridCollapsed, + gridScrollRef, + ganttScrollRef, }: Props) => { - const scrollRef = useRef(null); const tableRef = useRef(null); const offsetTop = useOffsetTop(tableRef); const { selected } = useSelection(); @@ -68,9 +71,34 @@ const Grid = ({ return true; }); + const onGanttScroll = (e: Event) => { + const { scrollTop } = e.currentTarget as HTMLDivElement; + if (scrollTop && gridScrollRef?.current) { + gridScrollRef.current.scrollTo(0, scrollTop); + + // Double check the scroll position after 100ms + setTimeout(() => { + const gridScrollTop = gridScrollRef?.current?.scrollTop; + const ganttScrollTop = ganttScrollRef?.current?.scrollTop; + if (ganttScrollTop !== gridScrollTop && gridScrollRef?.current) { + gridScrollRef.current.scrollTo(0, ganttScrollTop || 0); + } + }, 100); + } + }; + + // Sync grid and gantt scroll + useEffect(() => { + const gantt = ganttScrollRef?.current; + gantt?.addEventListener("scroll", onGanttScroll); + return () => { + gantt?.removeEventListener("scroll", onGanttScroll); + }; + }); + useEffect(() => { const scrollOnResize = new ResizeObserver(() => { - const runsContainer = scrollRef.current; + const runsContainer = gridScrollRef?.current; // Set scroll to top right if it is scrollable if ( tableRef?.current && @@ -90,7 +118,7 @@ const Grid = ({ }; } return () => {}; - }, [tableRef, isGridCollapsed]); + }, [tableRef, isGridCollapsed, gridScrollRef]); return ( @@ -134,11 +162,12 @@ const Grid = ({ )} diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts index 09a7ca4f9bf7..4fbb1b0ccc4e 100644 --- a/airflow/www/static/js/types/index.ts +++ b/airflow/www/static/js/types/index.ts @@ -71,6 +71,7 @@ interface TaskInstance { mappedStates?: { [key: string]: number; }; + queuedDttm?: string | null; mapIndex?: number; tryNumber?: number; triggererJob?: Job; diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 8d5ee60d0d38..6d5f4d74205f 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -125,6 +125,10 @@ def get_mapped_summary(parent_instance, task_instances): group_state = state break + group_queued_dttm = datetime_to_string( + min((ti.queued_dttm for ti in task_instances if ti.queued_dttm), default=None) + ) + group_start_date = datetime_to_string( min((ti.start_date for ti in task_instances if ti.start_date), default=None) ) @@ -136,6 +140,7 @@ def get_mapped_summary(parent_instance, task_instances): "task_id": parent_instance.task_id, "run_id": parent_instance.run_id, "state": group_state, + "queued_dttm": group_queued_dttm, "start_date": group_start_date, "end_date": group_end_date, "mapped_states": mapped_states, diff --git a/airflow/www/views.py b/airflow/www/views.py index ae8811c05cc3..636d7c0f877b 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -304,6 +304,7 @@ def dag_to_grid(dag: DagModel, dag_runs: Sequence[DagRun], session: Session): TaskInstance._try_number, func.min(TaskInstanceNote.content).label("note"), func.count(func.coalesce(TaskInstance.state, sqla.literal("no_status"))).label("state_count"), + func.min(TaskInstance.queued_dttm).label("queued_dttm"), func.min(TaskInstance.start_date).label("start_date"), func.max(TaskInstance.end_date).label("end_date"), ) @@ -334,6 +335,7 @@ def _get_summary(task_instance): "task_id": task_instance.task_id, "run_id": task_instance.run_id, "state": task_instance.state, + "queued_dttm": task_instance.queued_dttm, "start_date": task_instance.start_date, "end_date": task_instance.end_date, "try_number": wwwutils.get_try_count(task_instance._try_number, task_instance.state), @@ -363,15 +365,26 @@ def set_overall_state(record): record = { "task_id": ti_summary.task_id, "run_id": run_id, + "queued_dttm": ti_summary.queued_dttm, "start_date": ti_summary.start_date, "end_date": ti_summary.end_date, "mapped_states": {ti_summary.state: ti_summary.state_count}, "state": None, # We change this before yielding } continue + record["queued_dttm"] = min( + filter(None, [record["queued_dttm"], ti_summary.queued_dttm]), default=None + ) record["start_date"] = min( filter(None, [record["start_date"], ti_summary.start_date]), default=None ) + # Sometimes the start date of a group might be before the queued date of the group + if ( + record["queued_dttm"] + and record["start_date"] + and record["queued_dttm"] > record["start_date"] + ): + record["queued_dttm"] = None record["end_date"] = max( filter(None, [record["end_date"], ti_summary.end_date]), default=None ) @@ -421,18 +434,24 @@ def get_summary(dag_run: DagRun): if item ] + children_queued_dttms = (item["queued_dttm"] for item in child_instances) children_start_dates = (item["start_date"] for item in child_instances) children_end_dates = (item["end_date"] for item in child_instances) children_states = {item["state"] for item in child_instances} group_state = next((state for state in wwwutils.priority if state in children_states), None) + group_queued_dttm = min(filter(None, children_queued_dttms), default=None) group_start_date = min(filter(None, children_start_dates), default=None) group_end_date = max(filter(None, children_end_dates), default=None) + # Sometimes the start date of a group might be before the queued date of the group + if group_queued_dttm and group_start_date and group_queued_dttm > group_start_date: + group_queued_dttm = None return { "task_id": task_group.group_id, "run_id": dag_run.run_id, "state": group_state, + "queued_dttm": group_queued_dttm, "start_date": group_start_date, "end_date": group_end_date, } @@ -462,6 +481,7 @@ def get_mapped_group_summary(run_id: str, mapped_instances: Mapping[int, list[Ta if item and item["run_id"] == run_id ] + children_queued_dttms = (item["queued_dttm"] for item in child_instances) children_start_dates = (item["start_date"] for item in child_instances) children_end_dates = (item["end_date"] for item in child_instances) children_states = {item["state"] for item in child_instances} @@ -477,6 +497,7 @@ def get_mapped_group_summary(run_id: str, mapped_instances: Mapping[int, list[Ta mapped_states[value] += 1 group_state = next((state for state in wwwutils.priority if state in children_states), None) + group_queued_dttm = min(filter(None, children_queued_dttms), default=None) group_start_date = min(filter(None, children_start_dates), default=None) group_end_date = max(filter(None, children_end_dates), default=None) @@ -484,6 +505,7 @@ def get_mapped_group_summary(run_id: str, mapped_instances: Mapping[int, list[Ta "task_id": task_group.group_id, "run_id": run_id, "state": group_state, + "queued_dttm": group_queued_dttm, "start_date": group_start_date, "end_date": group_end_date, "mapped_states": mapped_states, diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index a8b449990440..26f41798faad 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -9805,10 +9805,10 @@ react-focus-lock@^2.9.1: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-icons@^4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.3.1.tgz#2fa92aebbbc71f43d2db2ed1aed07361124e91ca" - integrity sha512-cB10MXLTs3gVuXimblAdI71jrJx8njrJZmNMEMC+sQu5B/BIOmlsAjskdqpn81y8UBVEGuHODd7/ci5DvoSzTQ== +react-icons@^4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.9.0.tgz#ba44f436a053393adb1bdcafbc5c158b7b70d2a3" + integrity sha512-ijUnFr//ycebOqujtqtV9PFS7JjhWg0QU6ykURVHuL4cbofvRCf3f6GMn9+fBktEFQOIVZnuAYLZdiyadRQRFg== react-is@^16.13.1, react-is@^16.7.0, react-is@^16.8.1: version "16.13.1" diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index 171b48211f54..57e7e5686558 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -238,6 +238,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): "instances": [ { "run_id": "run_1", + "queued_dttm": None, "start_date": None, "end_date": None, "note": None, @@ -247,6 +248,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): }, { "run_id": "run_2", + "queued_dttm": None, "start_date": None, "end_date": None, "note": None, @@ -270,6 +272,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): { "run_id": "run_1", "mapped_states": {"success": 3}, + "queued_dttm": None, "start_date": None, "end_date": None, "state": "success", @@ -278,6 +281,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): { "run_id": "run_2", "mapped_states": {"no_status": 3}, + "queued_dttm": None, "start_date": None, "end_date": None, "state": None, @@ -297,12 +301,14 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): "end_date": None, "run_id": "run_1", "mapped_states": {"success": 3}, + "queued_dttm": None, "start_date": None, "state": "success", "task_id": "mapped_task_group", }, { "run_id": "run_2", + "queued_dttm": None, "start_date": None, "end_date": None, "state": None, @@ -323,6 +329,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): { "run_id": "run_1", "mapped_states": {"success": 4}, + "queued_dttm": None, "start_date": None, "end_date": None, "state": "success", @@ -331,6 +338,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): { "run_id": "run_2", "mapped_states": {"no_status": 2, "running": 1, "success": 1}, + "queued_dttm": None, "start_date": "2021-07-01T01:00:00+00:00", "end_date": "2021-07-01T01:02:03+00:00", "state": "running", @@ -348,12 +356,14 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): { "end_date": None, "run_id": "run_1", + "queued_dttm": None, "start_date": None, "state": "success", "task_id": "group", }, { "run_id": "run_2", + "queued_dttm": None, "start_date": "2021-07-01T01:00:00+00:00", "end_date": "2021-07-01T01:02:03+00:00", "state": "running", From db8d737ad690b721270d0c2fd3a83f08d7ce5c3f Mon Sep 17 00:00:00 2001 From: raphaelauv Date: Fri, 28 Jul 2023 06:47:40 +0200 Subject: [PATCH 332/533] Fix typo in Dockerfile (#32898) --- Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 9ae0091c6b0f..9a0e30090167 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1281,7 +1281,7 @@ ENV HOME=${HOME} \ # We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an # unexpected result - the cache for Dockerfiles might get invalidated in case the host system # had different umask set and group x bit was not set. In Azure the bit might be not set at all. -# That also protects against AUFS Docker backen dproblem where changing the executable bit required sync +# That also protects against AUFS Docker backend problem where changing the executable bit required sync RUN bash /scripts/docker/install_mysql.sh prod \ && bash /scripts/docker/install_mysql.sh dev \ && bash /scripts/docker/install_mssql.sh dev \ From 7ba7fb1173e55c24c94fe01f0742fd00cd9c0d82 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 28 Jul 2023 10:23:00 +0530 Subject: [PATCH 333/533] Order the release commands properly (#32888) --- dev/README_RELEASE_AIRFLOW.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 0147f418a0ad..fbcf2dbb7636 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -952,8 +952,8 @@ This includes: - For major/minor release, update version in `airflow/__main__.py`, `docs/docker-stack/` and `airflow/api_connexion/openapi/v1.yaml` to the next likely minor version release. - Update the `REVISION_HEADS_MAP` at airflow/utils/db.py to include the revision head of the release even if there are no migrations. - Sync `RELEASE_NOTES.rst` (including deleting relevant `newsfragments`) and `README.md` changes. -- Updating `airflow_bug_report.yml` issue template in `.github/ISSUE_TEMPLATE/` with the new version. - Updating `Dockerfile` with the new version. +- Updating `airflow_bug_report.yml` issue template in `.github/ISSUE_TEMPLATE/` with the new version. ## Update default Airflow version in the helm chart From 980065aaf1b25f1d8d6504446589da33ca699689 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:34:46 -0400 Subject: [PATCH 334/533] Make `get_url_user_profile()` in auth manager optional (#32886) --- airflow/auth/managers/base_auth_manager.py | 2 +- airflow/auth/managers/fab/fab_auth_manager.py | 4 ++-- airflow/www/templates/appbuilder/navbar_right.html | 7 +++++-- tests/auth/managers/fab/test_fab_auth_manager.py | 13 +++++++++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index 645616069d3c..17b7b61aebf3 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -53,7 +53,7 @@ def get_url_login(self, **kwargs) -> str: ... @abstractmethod - def get_url_user_profile(self) -> str: + def get_url_user_profile(self) -> str | None: """Return the url to a page displaying info about the current user.""" ... diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index c009ff6247f2..ac5ae45c43c5 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -60,8 +60,8 @@ def get_url_login(self, **kwargs) -> str: else: return url_for(f"{self.security_manager.auth_view.endpoint}.login") - def get_url_user_profile(self) -> str: + def get_url_user_profile(self) -> str | None: """Return the url to a page displaying info about the current user.""" if not self.security_manager.user_view: - raise AirflowException("`user_view` not defined in the security manager.") + return None return url_for(f"{self.security_manager.user_view.endpoint}.userinfo") diff --git a/airflow/www/templates/appbuilder/navbar_right.html b/airflow/www/templates/appbuilder/navbar_right.html index 7252b9647a4a..2c16e09d96b3 100644 --- a/airflow/www/templates/appbuilder/navbar_right.html +++ b/airflow/www/templates/appbuilder/navbar_right.html @@ -73,8 +73,11 @@ diff --git a/tests/auth/managers/fab/test_fab_auth_manager.py b/tests/auth/managers/fab/test_fab_auth_manager.py index ef8baba45546..313165b56b91 100644 --- a/tests/auth/managers/fab/test_fab_auth_manager.py +++ b/tests/auth/managers/fab/test_fab_auth_manager.py @@ -81,3 +81,16 @@ def test_get_url_login_with_next(self, mock_url_for, auth_manager): auth_manager.security_manager.auth_view.endpoint = "test_endpoint" auth_manager.get_url_login(next_url="next_url") mock_url_for.assert_called_once_with("test_endpoint.login", next="next_url") + + def test_get_url_user_profile_when_auth_view_not_defined(self, auth_manager): + assert auth_manager.get_url_user_profile() is None + + @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + def test_get_url_user_profile(self, mock_url_for, auth_manager): + expected_url = "test_url" + mock_url_for.return_value = expected_url + auth_manager.security_manager.user_view = Mock() + auth_manager.security_manager.user_view.endpoint = "test_endpoint" + actual_url = auth_manager.get_url_user_profile() + mock_url_for.assert_called_once_with("test_endpoint.userinfo") + assert actual_url == expected_url From 4b005147c18b5ec8d23c6206ad25090557219575 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 28 Jul 2023 13:57:25 +0800 Subject: [PATCH 335/533] Encourage PR description to leave an empty line (#32905) --- .github/PULL_REQUEST_TEMPLATE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 011b4caa5e71..64a21ff29024 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -33,6 +33,9 @@ How to write a good git commit message: http://chris.beams.io/posts/git-commit/ --> + + + --- **^ Add meaningful description above** From 319045492d2559bd856a43a1fa810adf59358d7d Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Fri, 28 Jul 2023 02:16:13 -0400 Subject: [PATCH 336/533] Add get_user method to auth manager (#32838) --- airflow/api/auth/backend/basic_auth.py | 2 +- .../endpoints/dag_run_endpoint.py | 6 +-- .../endpoints/role_and_permission_endpoint.py | 2 +- .../endpoints/task_instance_endpoint.py | 4 +- .../api_connexion/endpoints/user_endpoint.py | 2 +- .../schemas/role_and_permission_schema.py | 2 +- airflow/api_connexion/schemas/user_schema.py | 2 +- airflow/auth/managers/base_auth_manager.py | 11 +++++ .../auth/managers/fab/auth/anonymous_user.py | 4 +- airflow/auth/managers/fab/fab_auth_manager.py | 16 +++++-- .../sqla => auth/managers/fab}/models.py | 5 ++- airflow/auth/managers/model/__init__.py | 17 ++++++++ airflow/auth/managers/model/base_user.py | 42 +++++++++++++++++++ airflow/cli/commands/role_command.py | 2 +- .../0073_2_0_0_prefix_dag_permissions.py | 2 +- airflow/models/__init__.py | 2 - airflow/utils/db.py | 6 +-- airflow/www/decorators.py | 4 +- airflow/www/fab_security/manager.py | 5 +-- airflow/www/fab_security/sqla/manager.py | 4 +- airflow/www/security.py | 2 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- .../test_role_and_permission_endpoint.py | 2 +- .../endpoints/test_user_endpoint.py | 2 +- .../api_connexion/schemas/test_user_schema.py | 2 +- .../managers/fab/test_fab_auth_manager.py | 27 +++++++++--- tests/cli/commands/test_role_command.py | 2 +- tests/test_utils/db.py | 2 +- tests/www/test_security.py | 2 +- 29 files changed, 140 insertions(+), 43 deletions(-) rename airflow/{www/fab_security/sqla => auth/managers/fab}/models.py (98%) create mode 100644 airflow/auth/managers/model/__init__.py create mode 100644 airflow/auth/managers/model/base_user.py diff --git a/airflow/api/auth/backend/basic_auth.py b/airflow/api/auth/backend/basic_auth.py index 3f802fde636a..dbfeaa5f806d 100644 --- a/airflow/api/auth/backend/basic_auth.py +++ b/airflow/api/auth/backend/basic_auth.py @@ -24,8 +24,8 @@ from flask_appbuilder.const import AUTH_LDAP from flask_login import login_user +from airflow.auth.managers.fab.models import User from airflow.utils.airflow_flask_app import get_airflow_app -from airflow.www.fab_security.sqla.models import User CLIENT_AUTH: tuple[str, str] | Any | None = None diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index 490923c6c381..377b79aa113b 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -21,7 +21,6 @@ import pendulum from connexion import NoContent from flask import g -from flask_login import current_user from marshmallow import ValidationError from sqlalchemy import delete, or_, select from sqlalchemy.orm import Session @@ -68,6 +67,7 @@ from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType from airflow.www.decorators import action_logging +from airflow.www.extensions.init_auth_manager import get_auth_manager RESOURCE_EVENT_PREFIX = "dag_run" @@ -357,7 +357,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse: ) dag_run_note = post_body.get("note") if dag_run_note: - current_user_id = getattr(current_user, "id", None) + current_user_id = get_auth_manager().get_user_id() dag_run.note = (dag_run_note, current_user_id) return dagrun_schema.dump(dag_run) except ValueError as ve: @@ -478,7 +478,7 @@ def set_dag_run_note(*, dag_id: str, dag_run_id: str, session: Session = NEW_SES except ValidationError as err: raise BadRequest(detail=str(err)) - current_user_id = getattr(current_user, "id", None) + current_user_id = get_auth_manager().get_user_id() if dag_run.dag_run_note is None: dag_run.note = (new_note, current_user_id) else: diff --git a/airflow/api_connexion/endpoints/role_and_permission_endpoint.py b/airflow/api_connexion/endpoints/role_and_permission_endpoint.py index 609c45893fc3..34b0f5478a57 100644 --- a/airflow/api_connexion/endpoints/role_and_permission_endpoint.py +++ b/airflow/api_connexion/endpoints/role_and_permission_endpoint.py @@ -34,9 +34,9 @@ role_schema, ) from airflow.api_connexion.types import APIResponse, UpdateMask +from airflow.auth.managers.fab.models import Action, Role from airflow.security import permissions from airflow.utils.airflow_flask_app import get_airflow_app -from airflow.www.fab_security.sqla.models import Action, Role from airflow.www.security import AirflowSecurityManager diff --git a/airflow/api_connexion/endpoints/task_instance_endpoint.py b/airflow/api_connexion/endpoints/task_instance_endpoint.py index 55db7ef8b937..c0f1f7079ec0 100644 --- a/airflow/api_connexion/endpoints/task_instance_endpoint.py +++ b/airflow/api_connexion/endpoints/task_instance_endpoint.py @@ -51,6 +51,7 @@ from airflow.utils.db import get_query_count from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState, TaskInstanceState +from airflow.www.extensions.init_auth_manager import get_auth_manager T = TypeVar("T") @@ -692,9 +693,8 @@ def set_task_instance_note( raise NotFound(error_message) ti, sla_miss = result - from flask_login import current_user - current_user_id = getattr(current_user, "id", None) + current_user_id = get_auth_manager().get_user_id() if ti.task_instance_note is None: ti.note = (new_note, current_user_id) else: diff --git a/airflow/api_connexion/endpoints/user_endpoint.py b/airflow/api_connexion/endpoints/user_endpoint.py index 2a88fb1b24ca..c8ba1a7cc751 100644 --- a/airflow/api_connexion/endpoints/user_endpoint.py +++ b/airflow/api_connexion/endpoints/user_endpoint.py @@ -34,9 +34,9 @@ user_schema, ) from airflow.api_connexion.types import APIResponse, UpdateMask +from airflow.auth.managers.fab.models import Role, User from airflow.security import permissions from airflow.utils.airflow_flask_app import get_airflow_app -from airflow.www.fab_security.sqla.models import Role, User @security.requires_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_USER)]) diff --git a/airflow/api_connexion/schemas/role_and_permission_schema.py b/airflow/api_connexion/schemas/role_and_permission_schema.py index 324336c28866..30f23e2f3585 100644 --- a/airflow/api_connexion/schemas/role_and_permission_schema.py +++ b/airflow/api_connexion/schemas/role_and_permission_schema.py @@ -21,7 +21,7 @@ from marshmallow import Schema, fields from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field -from airflow.www.fab_security.sqla.models import Action, Permission, Resource, Role +from airflow.auth.managers.fab.models import Action, Permission, Resource, Role class ActionSchema(SQLAlchemySchema): diff --git a/airflow/api_connexion/schemas/user_schema.py b/airflow/api_connexion/schemas/user_schema.py index 843ad32f0245..fb30da9a02c0 100644 --- a/airflow/api_connexion/schemas/user_schema.py +++ b/airflow/api_connexion/schemas/user_schema.py @@ -23,7 +23,7 @@ from airflow.api_connexion.parameters import validate_istimezone from airflow.api_connexion.schemas.role_and_permission_schema import RoleSchema -from airflow.www.fab_security.sqla.models import User +from airflow.auth.managers.fab.models import User class UserCollectionItemSchema(SQLAlchemySchema): diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index 17b7b61aebf3..a234efcaca6c 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -20,6 +20,7 @@ from abc import abstractmethod from typing import TYPE_CHECKING +from airflow.auth.managers.model.base_user import BaseUser from airflow.exceptions import AirflowException from airflow.utils.log.logging_mixin import LoggingMixin @@ -42,6 +43,16 @@ def get_user_name(self) -> str: """Return the username associated to the user in session.""" ... + @abstractmethod + def get_user(self) -> BaseUser: + """Return the user associated to the user in session.""" + ... + + @abstractmethod + def get_user_id(self) -> str: + """Return the user ID associated to the user in session.""" + ... + @abstractmethod def is_logged_in(self) -> bool: """Return whether the user is logged in.""" diff --git a/airflow/auth/managers/fab/auth/anonymous_user.py b/airflow/auth/managers/fab/auth/anonymous_user.py index b97ec69e3ef0..c11669f0c6c3 100644 --- a/airflow/auth/managers/fab/auth/anonymous_user.py +++ b/airflow/auth/managers/fab/auth/anonymous_user.py @@ -20,8 +20,10 @@ from flask import current_app from flask_login import AnonymousUserMixin +from airflow.auth.managers.model.base_user import BaseUser -class AnonymousUser(AnonymousUserMixin): + +class AnonymousUser(AnonymousUserMixin, BaseUser): """User object used when no active user is logged in.""" _roles: set[tuple[str, str]] = set() diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index ac5ae45c43c5..085b6e997b03 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -22,6 +22,7 @@ from airflow import AirflowException from airflow.auth.managers.base_auth_manager import BaseAuthManager +from airflow.auth.managers.fab.models import User from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride @@ -39,13 +40,22 @@ def get_user_name(self) -> str: For backward compatibility reasons, the username in FAB auth manager is the concatenation of the first name and the last name. """ - first_name = current_user.first_name or "" - last_name = current_user.last_name or "" + user = self.get_user() + first_name = user.first_name or "" + last_name = user.last_name or "" return f"{first_name} {last_name}".strip() + def get_user(self) -> User: + """Return the user associated to the user in session.""" + return current_user + + def get_user_id(self) -> str: + """Return the user ID associated to the user in session.""" + return str(self.get_user().get_id()) + def is_logged_in(self) -> bool: """Return whether the user is logged in.""" - return current_user and not current_user.is_anonymous + return not self.get_user().is_anonymous def get_security_manager_override_class(self) -> type: """Return the security manager override.""" diff --git a/airflow/www/fab_security/sqla/models.py b/airflow/auth/managers/fab/models.py similarity index 98% rename from airflow/www/fab_security/sqla/models.py rename to airflow/auth/managers/fab/models.py index 5c835888010f..475499707eaf 100644 --- a/airflow/www/fab_security/sqla/models.py +++ b/airflow/auth/managers/fab/models.py @@ -40,6 +40,7 @@ ) from sqlalchemy.orm import backref, declared_attr, relationship +from airflow.auth.managers.model.base_user import BaseUser from airflow.models.base import Base """ @@ -139,7 +140,7 @@ def __repr__(self): ) -class User(Model): +class User(Model, BaseUser): """Represents an Airflow user which has roles assigned to it.""" __tablename__ = "ab_user" @@ -185,7 +186,7 @@ def changed_by_fk(self): @classmethod def get_user_id(cls): try: - return g.user.id + return g.user.get_id() except Exception: return None diff --git a/airflow/auth/managers/model/__init__.py b/airflow/auth/managers/model/__init__.py new file mode 100644 index 000000000000..217e5db96078 --- /dev/null +++ b/airflow/auth/managers/model/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/model/base_user.py b/airflow/auth/managers/model/base_user.py new file mode 100644 index 000000000000..329846881a74 --- /dev/null +++ b/airflow/auth/managers/model/base_user.py @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from abc import abstractmethod + + +class BaseUser: + """User model interface.""" + + @property + def is_authenticated(self) -> bool: + return not self.is_anonymous + + @property + @abstractmethod + def is_active(self) -> bool: + ... + + @property + @abstractmethod + def is_anonymous(self) -> bool: + ... + + @abstractmethod + def get_id(self) -> str: + ... diff --git a/airflow/cli/commands/role_command.py b/airflow/cli/commands/role_command.py index db11d69dd2f8..4e439fa2fc9c 100644 --- a/airflow/cli/commands/role_command.py +++ b/airflow/cli/commands/role_command.py @@ -23,11 +23,11 @@ import json import os +from airflow.auth.managers.fab.models import Action, Permission, Resource, Role from airflow.cli.simple_table import AirflowConsole from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning from airflow.utils.providers_configuration_loader import providers_configuration_loaded -from airflow.www.fab_security.sqla.models import Action, Permission, Resource, Role from airflow.www.security import EXISTING_ROLES diff --git a/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py b/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py index 660da4ac2c4f..44f930a04ba7 100644 --- a/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py +++ b/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py @@ -28,7 +28,7 @@ from airflow import settings from airflow.security import permissions -from airflow.www.fab_security.sqla.models import Action, Permission, Resource +from airflow.auth.managers.fab.models import Action, Permission, Resource # revision identifiers, used by Alembic. revision = "849da589634d" diff --git a/airflow/models/__init__.py b/airflow/models/__init__.py index 45690b5ae0cb..4d15dfe5c787 100644 --- a/airflow/models/__init__.py +++ b/airflow/models/__init__.py @@ -59,12 +59,10 @@ def import_all_models(): for name in __lazy_imports: __getattr__(name) - import airflow.jobs.job import airflow.models.dagwarning import airflow.models.dataset import airflow.models.serialized_dag import airflow.models.tasklog - import airflow.www.fab_security.sqla.models def __getattr__(name): diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 9ef504e73946..f0f60d1dc002 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -720,8 +720,8 @@ def _get_flask_db(sql_database_uri): def _create_db_from_orm(session): from alembic import command + from airflow.auth.managers.fab.models import Model from airflow.models.base import Base - from airflow.www.fab_security.sqla.models import Model def _create_flask_session_tbl(sql_database_uri): db = _get_flask_db(sql_database_uri) @@ -998,7 +998,7 @@ def check_username_duplicates(session: Session) -> Iterable[str]: :param session: session of the sqlalchemy :rtype: str """ - from airflow.www.fab_security.sqla.models import RegisterUser, User + from airflow.auth.managers.fab.models import RegisterUser, User for model in [User, RegisterUser]: dups = [] @@ -1730,8 +1730,8 @@ def drop_airflow_models(connection): :param connection: SQLAlchemy Connection :return: None """ + from airflow.auth.managers.fab.models import Model from airflow.models.base import Base - from airflow.www.fab_security.sqla.models import Model Base.metadata.drop_all(connection) Model.metadata.drop_all(connection) diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index af316e3ed0fd..c74be2635e1f 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -26,7 +26,7 @@ from typing import Callable, TypeVar, cast import pendulum -from flask import after_this_request, g, request +from flask import after_this_request, request from pendulum.parsing.exceptions import ParserError from airflow.models import Log @@ -88,7 +88,7 @@ def wrapper(*args, **kwargs): if not get_auth_manager().is_logged_in(): user = "anonymous" else: - user = f"{g.user.username} ({g.user.get_full_name()})" + user = get_auth_manager().get_user_name() fields_skip_logging = {"csrf_token", "_csrf_token"} extra_fields = [ diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index dc31a3552221..fca6a3083ceb 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -67,12 +67,11 @@ from flask_jwt_extended import current_user as current_user_jwt from flask_limiter import Limiter from flask_limiter.util import get_remote_address -from flask_login import current_user from werkzeug.security import check_password_hash +from airflow.auth.managers.fab.models import Action, Permission, RegisterUser, Resource, Role, User from airflow.configuration import conf from airflow.www.extensions.init_auth_manager import get_auth_manager -from airflow.www.fab_security.sqla.models import Action, Permission, RegisterUser, Resource, Role, User # This product contains a modified portion of 'Flask App Builder' developed by Daniel Vaz Gaspar. # (https://github.com/dpgaspar/Flask-AppBuilder). @@ -1539,4 +1538,4 @@ def remove_permission_from_role(self, role, permission) -> None: @staticmethod def before_request(): """Hook runs before request.""" - g.user = current_user + g.user = get_auth_manager().get_user() diff --git a/airflow/www/fab_security/sqla/manager.py b/airflow/www/fab_security/sqla/manager.py index c90ca52654a0..6ce9580c2927 100644 --- a/airflow/www/fab_security/sqla/manager.py +++ b/airflow/www/fab_security/sqla/manager.py @@ -26,8 +26,7 @@ from sqlalchemy.orm.exc import MultipleResultsFound from werkzeug.security import generate_password_hash -from airflow.www.fab_security.manager import BaseSecurityManager -from airflow.www.fab_security.sqla.models import ( +from airflow.auth.managers.fab.models import ( Action, Permission, RegisterUser, @@ -36,6 +35,7 @@ User, assoc_permission_role, ) +from airflow.www.fab_security.manager import BaseSecurityManager log = logging.getLogger(__name__) diff --git a/airflow/www/security.py b/airflow/www/security.py index 7f534e78b8f9..98384839aba8 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -24,6 +24,7 @@ from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload +from airflow.auth.managers.fab.models import Permission, Resource, Role, User from airflow.auth.managers.fab.views.user_details import CustomUserDBModelView from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.models import DagBag, DagModel @@ -32,7 +33,6 @@ from airflow.utils.session import NEW_SESSION, provide_session from airflow.www.extensions.init_auth_manager import get_auth_manager from airflow.www.fab_security.sqla.manager import SecurityManager -from airflow.www.fab_security.sqla.models import Permission, Resource, Role, User from airflow.www.fab_security.views import ( ActionModelView, CustomResetMyPasswordView, diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index dc75b8aee675..f10af90f5bb5 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -902b20991528af2fceec9c967f4a7a59b921e0fc2e39b6bb2c4727b3f61a56b7 \ No newline at end of file +7ff18b1eafa528dbdfb62d75151a526de280f73e1edf7fea18da7c64644f0da9 \ No newline at end of file diff --git a/tests/api_connexion/endpoints/test_role_and_permission_endpoint.py b/tests/api_connexion/endpoints/test_role_and_permission_endpoint.py index 4ba36597e699..0a62de1043fb 100644 --- a/tests/api_connexion/endpoints/test_role_and_permission_endpoint.py +++ b/tests/api_connexion/endpoints/test_role_and_permission_endpoint.py @@ -19,8 +19,8 @@ import pytest from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP +from airflow.auth.managers.fab.models import Role from airflow.security import permissions -from airflow.www.fab_security.sqla.models import Role from airflow.www.security import EXISTING_ROLES from tests.test_utils.api_connexion_utils import ( assert_401, diff --git a/tests/api_connexion/endpoints/test_user_endpoint.py b/tests/api_connexion/endpoints/test_user_endpoint.py index a84b7f1087af..f29f998a8491 100644 --- a/tests/api_connexion/endpoints/test_user_endpoint.py +++ b/tests/api_connexion/endpoints/test_user_endpoint.py @@ -22,10 +22,10 @@ from sqlalchemy.sql.functions import count from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP +from airflow.auth.managers.fab.models import User from airflow.security import permissions from airflow.utils import timezone from airflow.utils.session import create_session -from airflow.www.fab_security.sqla.models import User from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user from tests.test_utils.config import conf_vars diff --git a/tests/api_connexion/schemas/test_user_schema.py b/tests/api_connexion/schemas/test_user_schema.py index 042948429a33..cf52e61b8525 100644 --- a/tests/api_connexion/schemas/test_user_schema.py +++ b/tests/api_connexion/schemas/test_user_schema.py @@ -19,8 +19,8 @@ import pytest from airflow.api_connexion.schemas.user_schema import user_collection_item_schema, user_schema +from airflow.auth.managers.fab.models import User from airflow.utils import timezone -from airflow.www.fab_security.sqla.models import User from tests.test_utils.api_connexion_utils import create_role, delete_role TEST_EMAIL = "test@example.org" diff --git a/tests/auth/managers/fab/test_fab_auth_manager.py b/tests/auth/managers/fab/test_fab_auth_manager.py index 313165b56b91..96ddbc7f3e61 100644 --- a/tests/auth/managers/fab/test_fab_auth_manager.py +++ b/tests/auth/managers/fab/test_fab_auth_manager.py @@ -23,8 +23,8 @@ from airflow import AirflowException from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager +from airflow.auth.managers.fab.models import User from airflow.auth.managers.fab.security_manager_override import FabAirflowSecurityManagerOverride -from airflow.www.fab_security.sqla.models import User from airflow.www.security import ApplessAirflowSecurityManager @@ -44,21 +44,38 @@ class TestFabAuthManager: (None, "Last", "Last"), ], ) - @mock.patch("flask_login.utils._get_user") - def test_get_user_name(self, mock_current_user, first_name, last_name, expected, auth_manager): + @mock.patch.object(FabAuthManager, "get_user") + def test_get_user_name(self, mock_get_user, first_name, last_name, expected, auth_manager): user = User() user.first_name = first_name user.last_name = last_name - mock_current_user.return_value = user + mock_get_user.return_value = user assert auth_manager.get_user_name() == expected @mock.patch("flask_login.utils._get_user") - def test_is_logged_in(self, mock_current_user, auth_manager): + def test_get_user(self, mock_current_user, auth_manager): user = Mock() user.is_anonymous.return_value = True mock_current_user.return_value = user + assert auth_manager.get_user() == user + + @mock.patch.object(FabAuthManager, "get_user") + def test_get_user_id(self, mock_get_user, auth_manager): + user_id = "test" + user = Mock() + user.get_id.return_value = user_id + mock_get_user.return_value = user + + assert auth_manager.get_user_id() == user_id + + @mock.patch.object(FabAuthManager, "get_user") + def test_is_logged_in(self, mock_get_user, auth_manager): + user = Mock() + user.is_anonymous.return_value = True + mock_get_user.return_value = user + assert auth_manager.is_logged_in() is False def test_get_security_manager_override_class_return_fab_security_manager_override(self, auth_manager): diff --git a/tests/cli/commands/test_role_command.py b/tests/cli/commands/test_role_command.py index 8d8772dcf95a..544d8e956062 100644 --- a/tests/cli/commands/test_role_command.py +++ b/tests/cli/commands/test_role_command.py @@ -23,10 +23,10 @@ import pytest +from airflow.auth.managers.fab.models import Role from airflow.cli.commands import role_command from airflow.security import permissions from airflow.utils.cli_app_builder import get_application_builder -from airflow.www.fab_security.sqla.models import Role TEST_USER1_EMAIL = "test-user1@example.com" TEST_USER2_EMAIL = "test-user2@example.com" diff --git a/tests/test_utils/db.py b/tests/test_utils/db.py index 79074a1ddb80..43b6185eea7c 100644 --- a/tests/test_utils/db.py +++ b/tests/test_utils/db.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +from airflow.auth.managers.fab.models import Permission, Resource, assoc_permission_role from airflow.jobs.job import Job from airflow.models import ( Connection, @@ -50,7 +51,6 @@ from airflow.security.permissions import RESOURCE_DAG_PREFIX from airflow.utils.db import add_default_pool_if_not_exists, create_default_connections, reflect_tables from airflow.utils.session import create_session -from airflow.www.fab_security.sqla.models import Permission, Resource, assoc_permission_role def clear_db_runs(): diff --git a/tests/www/test_security.py b/tests/www/test_security.py index 58ccc778e14d..99659de2c795 100644 --- a/tests/www/test_security.py +++ b/tests/www/test_security.py @@ -31,13 +31,13 @@ from airflow.auth.managers.fab.auth.anonymous_user import AnonymousUser from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager +from airflow.auth.managers.fab.models import User, assoc_permission_role from airflow.exceptions import AirflowException from airflow.models import DagModel from airflow.models.base import Base from airflow.models.dag import DAG from airflow.security import permissions from airflow.www import app as application -from airflow.www.fab_security.sqla.models import User, assoc_permission_role from airflow.www.utils import CustomSQLAInterface from tests.test_utils.api_connexion_utils import ( create_user, From 12228d16be13afb2918139ea3c5a285a23242bd0 Mon Sep 17 00:00:00 2001 From: "YAMAMORI, Akihiro" <6166778+gecko655@users.noreply.github.com> Date: Fri, 28 Jul 2023 15:17:51 +0900 Subject: [PATCH 337/533] Unwrap Proxy before checking __iter__ in is_container() (#32850) * unwrap Proxy before checking __iter__ in is_container() Fixes: https://github.com/apache/airflow/issues/32804 --------- Co-authored-by: Tzu-ping Chung --- airflow/utils/helpers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 24e2bc9e66be..d40728620469 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -26,6 +26,8 @@ from itertools import filterfalse, tee from typing import TYPE_CHECKING, Any, Callable, Generator, Iterable, Mapping, MutableMapping, TypeVar, cast +from lazy_object_proxy import Proxy + from airflow.configuration import conf from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.utils.context import Context @@ -116,6 +118,11 @@ def handler(signum, frame): def is_container(obj: Any) -> bool: """Test if an object is a container (iterable) but not a string.""" + if isinstance(obj, Proxy): + # Proxy of any object is considered a container because it implements __iter__ + # to forward the call to the lazily initialized object + # Unwrap Proxy before checking __iter__ to evaluate the proxied object + obj = obj.__wrapped__ return hasattr(obj, "__iter__") and not isinstance(obj, str) From f8aae7946e2c940e2d3680eb81a1887a910197af Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 28 Jul 2023 08:23:31 +0200 Subject: [PATCH 338/533] Also add PyPI suffix to Arm images (#32902) Follow up after #32896 - also apply the suffix to ARM images --- .github/workflows/build-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index d613e059d19b..82a857bdabc6 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -253,6 +253,7 @@ jobs: RUNS_ON: ${{ needs.build-info.outputs.runs-on }} BACKEND: sqlite outputs: ${{toJSON(needs.build-info.outputs) }} + VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" From 8a0f410010cc39ce8d31ee7b64a352fbd2ad19ef Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Fri, 28 Jul 2023 09:24:23 +0300 Subject: [PATCH 339/533] Update openlineage provider to min version of airflow 2.7.0 (#32882) --- airflow/providers/openlineage/provider.yaml | 2 +- generated/provider_dependencies.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index e79e0691dba8..bbaa6a5fc9ff 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -26,7 +26,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.6.0 + - apache-airflow>=2.7.0 - apache-airflow-providers-common-sql>=1.3.1 - attrs>=22.2 - openlineage-integration-common>=0.28.0 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index b783da6b8c64..23c9c7f14b1c 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -653,7 +653,7 @@ "openlineage": { "deps": [ "apache-airflow-providers-common-sql>=1.3.1", - "apache-airflow>=2.6.0", + "apache-airflow>=2.7.0", "attrs>=22.2", "openlineage-integration-common>=0.28.0", "openlineage-python>=0.28.0" From 5c8223c33598f06820aa215f2cd07760ccbb063e Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:54:31 +0300 Subject: [PATCH 340/533] Bump common-sql version in provider (#32907) The support for openlineage was added in apache-airflow-providers-common-sql==1.6.0 --- airflow/providers/openlineage/provider.yaml | 2 +- generated/provider_dependencies.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index bbaa6a5fc9ff..2a2c1a66695c 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -27,7 +27,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-common-sql>=1.6.0 - attrs>=22.2 - openlineage-integration-common>=0.28.0 - openlineage-python>=0.28.0 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 23c9c7f14b1c..5a0ddbc68a36 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -652,7 +652,7 @@ }, "openlineage": { "deps": [ - "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-common-sql>=1.6.0", "apache-airflow>=2.7.0", "attrs>=22.2", "openlineage-integration-common>=0.28.0", From d800c1bc3967265280116a05d1855a4da0e1ba10 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Fri, 28 Jul 2023 01:20:46 -0700 Subject: [PATCH 341/533] Support task mapping with setup teardown (#32820) Co-authored-by: Tzu-ping Chung --- airflow/decorators/base.py | 17 +- ...128_2_7_0_add_is_setup_to_task_instance.py | 51 ++ airflow/models/abstractoperator.py | 44 +- airflow/models/baseoperator.py | 42 ++ airflow/models/dag.py | 14 +- airflow/models/dagrun.py | 2 +- airflow/models/mappedoperator.py | 49 +- airflow/models/taskinstance.py | 5 + airflow/models/xcom_arg.py | 2 - airflow/ti_deps/deps/trigger_rule_dep.py | 18 +- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/migrations-ref.rst | 4 +- tests/models/test_mappedoperator.py | 436 ++++++++++++++++++ tests/models/test_taskinstance.py | 1 + tests/models/test_taskmixin.py | 55 --- tests/serialization/test_dag_serialization.py | 49 +- tests/www/views/test_views_tasks.py | 7 + 17 files changed, 649 insertions(+), 149 deletions(-) create mode 100644 airflow/migrations/versions/0128_2_7_0_add_is_setup_to_task_instance.py diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py index b2059eb6b422..af37f191be2d 100644 --- a/airflow/decorators/base.py +++ b/airflow/decorators/base.py @@ -302,9 +302,9 @@ class _TaskDecorator(ExpandableFactory, Generic[FParams, FReturn, OperatorSubcla decorator_name: str = attr.ib(repr=False, default="task") _airflow_is_task_decorator: ClassVar[bool] = True - is_setup: ClassVar[bool] = False - is_teardown: ClassVar[bool] = False - on_failure_fail_dagrun: ClassVar[bool] = False + is_setup: bool = False + is_teardown: bool = False + on_failure_fail_dagrun: bool = False @multiple_outputs.default def _infer_multiple_outputs(self): @@ -382,6 +382,10 @@ def expand(self, **map_kwargs: OperatorExpandArgument) -> XComArg: prevent_duplicates(self.kwargs, map_kwargs, fail_reason="mapping already partial") # Since the input is already checked at parse time, we can set strict # to False to skip the checks on execution. + if self.is_teardown: + if "trigger_rule" in self.kwargs: + raise ValueError("Trigger rule not configurable for teardown tasks.") + self.kwargs.update(trigger_rule=TriggerRule.ALL_DONE_SETUP_SUCCESS) return self._expand(DictOfListsExpandInput(map_kwargs), strict=False) def expand_kwargs(self, kwargs: OperatorExpandKwargsArgument, *, strict: bool = True) -> XComArg: @@ -406,7 +410,12 @@ def _expand(self, expand_input: ExpandInput, *, strict: bool) -> XComArg: task_params=task_kwargs.pop("params", None), task_default_args=task_kwargs.pop("default_args", None), ) - partial_kwargs.update(task_kwargs) + partial_kwargs.update( + task_kwargs, + is_setup=self.is_setup, + is_teardown=self.is_teardown, + on_failure_fail_dagrun=self.on_failure_fail_dagrun, + ) task_id = get_unique_task_id(partial_kwargs.pop("task_id"), dag, task_group) if task_group: diff --git a/airflow/migrations/versions/0128_2_7_0_add_is_setup_to_task_instance.py b/airflow/migrations/versions/0128_2_7_0_add_is_setup_to_task_instance.py new file mode 100644 index 000000000000..07e4fb4482a8 --- /dev/null +++ b/airflow/migrations/versions/0128_2_7_0_add_is_setup_to_task_instance.py @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add is_setup to task_instance + +Revision ID: 0646b768db47 +Revises: 788397e78828 +Create Date: 2023-07-24 10:12:07.630608 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "0646b768db47" +down_revision = "788397e78828" +branch_labels = None +depends_on = None +airflow_version = "2.7.0" + + +TABLE_NAME = "task_instance" + + +def upgrade(): + """Apply is_setup column to task_instance""" + with op.batch_alter_table(TABLE_NAME) as batch_op: + batch_op.add_column(sa.Column("is_setup", sa.Boolean(), nullable=False, server_default="0")) + + +def downgrade(): + """Remove is_setup column from task_instance""" + with op.batch_alter_table(TABLE_NAME) as batch_op: + batch_op.drop_column("is_setup", mssql_drop_default=True) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 908cb3258074..7cba88f28dfd 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -109,8 +109,6 @@ class AbstractOperator(Templater, DAGNode): inlets: list trigger_rule: TriggerRule - _is_setup = False - _is_teardown = False _on_failure_fail_dagrun = False HIDE_ATTRS_FROM_UI: ClassVar[frozenset[str]] = frozenset( @@ -160,44 +158,20 @@ def node_id(self) -> str: return self.task_id @property - def is_setup(self): - """ - Whether the operator is a setup task. - - :meta private: - """ - return self._is_setup + def is_setup(self) -> bool: + raise NotImplementedError() @is_setup.setter - def is_setup(self, value): - """ - Setter for is_setup property. - - :meta private: - """ - if self.is_teardown is True and value is True: - raise ValueError(f"Cannot mark task '{self.task_id}' as setup; task is already a teardown.") - self._is_setup = value + def is_setup(self, value: bool) -> None: + raise NotImplementedError() @property - def is_teardown(self): - """ - Whether the operator is a teardown task. - - :meta private: - """ - return self._is_teardown + def is_teardown(self) -> bool: + raise NotImplementedError() @is_teardown.setter - def is_teardown(self, value): - """ - Setter for is_teardown property. - - :meta private: - """ - if self.is_setup is True and value is True: - raise ValueError(f"Cannot mark task '{self.task_id}' as teardown; task is already a setup.") - self._is_teardown = value + def is_teardown(self, value: bool) -> None: + raise NotImplementedError() @property def on_failure_fail_dagrun(self): @@ -233,8 +207,6 @@ def as_teardown( on_failure_fail_dagrun=NOTSET, ): self.is_teardown = True - if TYPE_CHECKING: - assert isinstance(self, BaseOperator) # is_teardown not supported for MappedOperator self.trigger_rule = TriggerRule.ALL_DONE_SETUP_SUCCESS if on_failure_fail_dagrun is not NOTSET: self.on_failure_fail_dagrun = on_failure_fail_dagrun diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index e389a8df8ab4..e72fcc2940a6 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -957,6 +957,8 @@ def __init__( ) self.template_fields = [self.template_fields] + self._is_setup = False + self._is_teardown = False if SetupTeardownContext.active: SetupTeardownContext.update_context_map(self) @@ -1415,6 +1417,43 @@ def output(self) -> XComArg: return XComArg(operator=self) + @property + def is_setup(self) -> bool: + """Whether the operator is a setup task. + + :meta private: + """ + return self._is_setup + + @is_setup.setter + def is_setup(self, value: bool) -> None: + """Setter for is_setup property. + + :meta private: + """ + if self.is_teardown and value: + raise ValueError(f"Cannot mark task '{self.task_id}' as setup; task is already a teardown.") + self._is_setup = value + + @property + def is_teardown(self) -> bool: + """Whether the operator is a teardown task. + + :meta private: + """ + return self._is_teardown + + @is_teardown.setter + def is_teardown(self, value: bool) -> None: + """ + Setter for is_teardown property. + + :meta private: + """ + if self.is_setup and value: + raise ValueError(f"Cannot mark task '{self.task_id}' as teardown; task is already a setup.") + self._is_teardown = value + @staticmethod def xcom_push( context: Any, @@ -1501,6 +1540,9 @@ def get_serialized_fields(cls): "_BaseOperator__instantiated", "_BaseOperator__init_kwargs", "_BaseOperator__from_mapped", + "_is_setup", + "_is_teardown", + "_on_failure_fail_dagrun", } | { # Class level defaults need to be added to this list "start_date", diff --git a/airflow/models/dag.py b/airflow/models/dag.py index ae8ed2b6b615..5d2172c56d1f 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2674,7 +2674,7 @@ def test( conn_file_path: str | None = None, variable_file_path: str | None = None, session: Session = NEW_SESSION, - ) -> None: + ) -> DagRun: """ Execute one single DagRun for a given DAG and execution date. @@ -2738,19 +2738,17 @@ def add_logger_if_needed(ti: TaskInstance): # than creating a BackfillJob and allows us to surface logs to the user while dr.state == DagRunState.RUNNING: schedulable_tis, _ = dr.update_state(session=session) - try: - for ti in schedulable_tis: + for ti in schedulable_tis: + try: add_logger_if_needed(ti) ti.task = tasks[ti.task_id] _run_task(ti, session=session) - except Exception: - self.log.info( - "Task failed. DAG will continue to run until finished and be marked as failed.", - exc_info=True, - ) + except Exception: + self.log.exception("Task failed; ti=%s", ti) if conn_file_path or variable_file_path: # Remove the local variables we have added to the secrets_backend_list secrets_backend_list.pop(0) + return dr @provide_session def create_dagrun( diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index baca09602473..a6e1da6a0ec1 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -696,7 +696,7 @@ def recalculate(self) -> _UnfinishedStates: msg="all_tasks_deadlocked", ) - # finally, if the roots aren't done, the dag is still running + # finally, if the leaves aren't done, the dag is still running else: self.set_state(DagRunState.RUNNING) diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py index dd8b49fb98e0..0cf8852ea2f3 100644 --- a/airflow/models/mappedoperator.py +++ b/airflow/models/mappedoperator.py @@ -324,24 +324,6 @@ def __attrs_post_init__(self): f"{self.task_id!r}." ) - @AbstractOperator.is_setup.setter # type: ignore[attr-defined] - def is_setup(self, value): - """ - Setter for is_setup property. Disabled for MappedOperator. - - :meta private: - """ - raise ValueError("Cannot set is_setup for mapped operator.") - - @AbstractOperator.is_teardown.setter # type: ignore[attr-defined] - def is_teardown(self, value): - """ - Setter for is_teardown property. Disabled for MappedOperator. - - :meta private: - """ - raise ValueError("Cannot set is_teardown for mapped operator.") - @classmethod @cache def get_serialized_fields(cls): @@ -354,9 +336,9 @@ def get_serialized_fields(cls): "task_group", "upstream_task_ids", "supports_lineage", - "is_setup", - "is_teardown", - "on_failure_fail_dagrun", + "_is_setup", + "_is_teardown", + "_on_failure_fail_dagrun", } @staticmethod @@ -408,8 +390,23 @@ def trigger_rule(self) -> TriggerRule: @trigger_rule.setter def trigger_rule(self, value): - # required for mypy which complains about overriding writeable attr with read-only property - raise ValueError("Cannot set trigger_rule for mapped operator.") + self.partial_kwargs["trigger_rule"] = value + + @property + def is_setup(self) -> bool: + return bool(self.partial_kwargs.get("is_setup")) + + @is_setup.setter + def is_setup(self, value: bool) -> None: + self.partial_kwargs["is_setup"] = value + + @property + def is_teardown(self) -> bool: + return bool(self.partial_kwargs.get("is_teardown")) + + @is_teardown.setter + def is_teardown(self, value: bool) -> None: + self.partial_kwargs["is_teardown"] = value @property def depends_on_past(self) -> bool: @@ -640,12 +637,18 @@ def unmap(self, resolve: None | Mapping[str, Any] | tuple[Context, Session]) -> else: raise RuntimeError("cannot unmap a non-serialized operator without context") kwargs = self._get_unmap_kwargs(kwargs, strict=self._disallow_kwargs_override) + is_setup = kwargs.pop("is_setup", False) + is_teardown = kwargs.pop("is_teardown", False) + on_failure_fail_dagrun = kwargs.pop("on_failure_fail_dagrun", False) op = self.operator_class(**kwargs, _airflow_from_mapped=True) # We need to overwrite task_id here because BaseOperator further # mangles the task_id based on the task hierarchy (namely, group_id # is prepended, and '__N' appended to deduplicate). This is hacky, # but better than duplicating the whole mangling logic. op.task_id = self.task_id + op.is_setup = is_setup + op.is_teardown = is_teardown + op.on_failure_fail_dagrun = on_failure_fail_dagrun return op # After a mapped operator is serialized, there's no real way to actually diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 6eafbab4a42c..c31e2d8c15ec 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -41,6 +41,7 @@ import pendulum from jinja2 import TemplateAssertionError, UndefinedError from sqlalchemy import ( + Boolean, Column, DateTime, Float, @@ -419,6 +420,7 @@ class TaskInstance(Base, LoggingMixin): # Usually used when resuming from DEFERRED. next_method = Column(String(1000)) next_kwargs = Column(MutableDict.as_mutable(ExtendedJSON)) + is_setup = Column(Boolean, nullable=False, default=False, server_default="0") # If adding new fields here then remember to add them to # refresh_from_db() or they won't display in the UI correctly @@ -577,6 +579,7 @@ def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any "operator": task.task_type, "custom_operator_name": getattr(task, "custom_operator_name", None), "map_index": map_index, + "is_setup": task.is_setup, } @reconstructor @@ -875,6 +878,7 @@ def refresh_from_db(self, session: Session = NEW_SESSION, lock_for_update: bool self.trigger_id = ti.trigger_id self.next_method = ti.next_method self.next_kwargs = ti.next_kwargs + self.is_setup = ti.is_setup else: self.state = None @@ -896,6 +900,7 @@ def refresh_from_task(self, task: Operator, pool_override: str | None = None) -> self.executor_config = task.executor_config self.operator = task.task_type self.custom_operator_name = getattr(task, "custom_operator_name", None) + self.is_setup = task.is_setup @provide_session def clear_xcom_data(self, session: Session = NEW_SESSION) -> None: diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py index 2383861483ab..e8cde16db8cf 100644 --- a/airflow/models/xcom_arg.py +++ b/airflow/models/xcom_arg.py @@ -346,8 +346,6 @@ def as_teardown( ): for operator, _ in self.iter_references(): operator.is_teardown = True - if TYPE_CHECKING: - assert isinstance(operator, BaseOperator) # Can't set MappedOperator as teardown operator.trigger_rule = TriggerRule.ALL_DONE_SETUP_SUCCESS if on_failure_fail_dagrun is not NOTSET: operator.on_failure_fail_dagrun = on_failure_fail_dagrun diff --git a/airflow/ti_deps/deps/trigger_rule_dep.py b/airflow/ti_deps/deps/trigger_rule_dep.py index b1c2d09b969b..3d2cc21d0667 100644 --- a/airflow/ti_deps/deps/trigger_rule_dep.py +++ b/airflow/ti_deps/deps/trigger_rule_dep.py @@ -22,9 +22,8 @@ import functools from typing import TYPE_CHECKING, Iterator, NamedTuple -from sqlalchemy import and_, func, or_ +from sqlalchemy import and_, case, func, or_, true -from airflow.models import MappedOperator from airflow.models.taskinstance import PAST_DEPENDS_MET from airflow.ti_deps.dep_context import DepContext from airflow.ti_deps.deps.base_ti_dep import BaseTIDep, TIDepStatus @@ -69,7 +68,7 @@ def calculate(cls, finished_upstreams: Iterator[TaskInstance]) -> _UpstreamTISta curr_state = {ti.state: 1} counter.update(curr_state) # setup task cannot be mapped - if not isinstance(ti.task, MappedOperator) and ti.task.is_setup: + if ti.task.is_setup: setup_counter.update(curr_state) return _UpstreamTIStates( success=counter.get(TaskInstanceState.SUCCESS, 0), @@ -227,18 +226,15 @@ def _iter_upstream_conditions() -> Iterator[ColumnOperators]: # "simple" tasks (no task or task group mapping involved). if not any(needs_expansion(t) for t in upstream_tasks.values()): upstream = len(upstream_tasks) - upstream_setup = len( - [x for x in upstream_tasks.values() if not isinstance(x, MappedOperator) and x.is_setup] - ) + upstream_setup = len([x for x in upstream_tasks.values() if x.is_setup]) else: - upstream = ( - session.query(func.count()) + upstream, upstream_setup = ( + session.query(func.count(), func.sum(case((TaskInstance.is_setup == true(), 1), else_=0))) .filter(TaskInstance.dag_id == ti.dag_id, TaskInstance.run_id == ti.run_id) .filter(or_(*_iter_upstream_conditions())) - .scalar() + .one() ) - # todo: add support for mapped setup? - upstream_setup = None + upstream_done = done >= upstream changed = False diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index f10af90f5bb5..7e68174d5b8c 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -7ff18b1eafa528dbdfb62d75151a526de280f73e1edf7fea18da7c64644f0da9 \ No newline at end of file +aed14092a8168884ceda6b63c269779ccec09b6302da07a7914cc36aaf9a8d3b \ No newline at end of file diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 4f5abde251f9..c0f0443a9dab 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``788397e78828`` (head) | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column | +| ``0646b768db47`` (head) | ``788397e78828`` | ``2.7.0`` | Add is_setup to task_instance | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``788397e78828`` | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index d6aef854284a..1cfb53628a61 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -18,12 +18,15 @@ from __future__ import annotations import logging +from collections import defaultdict from datetime import timedelta from unittest.mock import patch import pendulum import pytest +from airflow.decorators import setup, task, task_group, teardown +from airflow.exceptions import AirflowSkipException from airflow.models import DAG from airflow.models.baseoperator import BaseOperator from airflow.models.mappedoperator import MappedOperator @@ -651,3 +654,436 @@ def test_task_mapping_with_explicit_task_group(): assert finish.upstream_list == [mapped] assert mapped.downstream_list == [finish] + + +class TestMappedSetupTeardown: + @staticmethod + def get_states(dr): + ti_dict = defaultdict(dict) + for ti in dr.get_task_instances(): + if ti.map_index == -1: + ti_dict[ti.task_id] = ti.state + else: + ti_dict[ti.task_id][ti.map_index] = ti.state + return ti_dict + + def test_one_to_many_work_failed(self, session, dag_maker): + """ + Work task failed. Setup maps to teardown. Should have 3 teardowns all successful even + though the work task has failed. + """ + with dag_maker(dag_id="one_to_many") as dag: + + @setup + def my_setup(): + print("setting up multiple things") + return [1, 2, 3] + + @task + def my_work(val): + print(f"doing work with multiple things: {val}") + raise ValueError("fail!") + return val + + @teardown + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup() + t = my_teardown.expand(val=s) + with t: + my_work(s) + + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": "success", + "my_work": "failed", + "my_teardown": {0: "success", 1: "success", 2: "success"}, + } + assert states == expected + + def test_many_one_explicit_odd_setup_mapped_setups_fail(self, dag_maker): + """ + one unmapped setup goes to two different teardowns + one mapped setup goes to same teardown + mapped setups fail + teardowns should still run + """ + with dag_maker( + dag_id="many_one_explicit_odd_setup_mapped_setups_fail", + ) as dag: + + @task + def other_setup(): + print("other setup") + return "other setup" + + @task + def other_work(): + print("other work") + return "other work" + + @task + def other_teardown(): + print("other teardown") + return "other teardown" + + @task + def my_setup(val): + print(f"setup: {val}") + raise ValueError("fail") + return val + + @task + def my_work(val): + print(f"work: {val}") + + @task + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup.expand(val=["data1.json", "data2.json", "data3.json"]) + o_setup = other_setup() + o_teardown = other_teardown() + with o_teardown.as_teardown(setups=o_setup): + other_work() + t = my_teardown(s).as_teardown(setups=s) + with t: + my_work(s) + o_setup >> t + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": {0: "failed", 1: "failed", 2: "failed"}, + "other_setup": "success", + "other_teardown": "success", + "other_work": "success", + "my_teardown": "success", + "my_work": "upstream_failed", + } + assert states == expected + + def test_many_one_explicit_odd_setup_all_setups_fail(self, dag_maker): + """ + one unmapped setup goes to two different teardowns + one mapped setup goes to same teardown + all setups fail + teardowns should not run + """ + with dag_maker( + dag_id="many_one_explicit_odd_setup_all_setups_fail", + ) as dag: + + @task + def other_setup(): + print("other setup") + raise ValueError("fail") + return "other setup" + + @task + def other_work(): + print("other work") + return "other work" + + @task + def other_teardown(): + print("other teardown") + return "other teardown" + + @task + def my_setup(val): + print(f"setup: {val}") + raise ValueError("fail") + return val + + @task + def my_work(val): + print(f"work: {val}") + + @task + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup.expand(val=["data1.json", "data2.json", "data3.json"]) + o_setup = other_setup() + o_teardown = other_teardown() + with o_teardown.as_teardown(setups=o_setup): + other_work() + t = my_teardown(s).as_teardown(setups=s) + with t: + my_work(s) + o_setup >> t + + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_teardown": "upstream_failed", + "other_setup": "failed", + "other_work": "upstream_failed", + "other_teardown": "upstream_failed", + "my_setup": {0: "failed", 1: "failed", 2: "failed"}, + "my_work": "upstream_failed", + } + assert states == expected + + def test_many_one_explicit_odd_setup_one_mapped_fails(self, dag_maker): + """ + one unmapped setup goes to two different teardowns + one mapped setup goes to same teardown + one of the mapped setup instances fails + teardowns should all run + """ + with dag_maker(dag_id="many_one_explicit_odd_setup_one_mapped_fails") as dag: + + @task + def other_setup(): + print("other setup") + return "other setup" + + @task + def other_work(): + print("other work") + return "other work" + + @task + def other_teardown(): + print("other teardown") + return "other teardown" + + @task + def my_setup(val): + if val == "data2.json": + raise ValueError("fail!") + elif val == "data3.json": + raise AirflowSkipException("skip!") + print(f"setup: {val}") + return val + + @task + def my_work(val): + print(f"work: {val}") + + @task + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup.expand(val=["data1.json", "data2.json", "data3.json"]) + o_setup = other_setup() + o_teardown = other_teardown() + with o_teardown.as_teardown(setups=o_setup): + other_work() + t = my_teardown(s).as_teardown(setups=s) + with t: + my_work(s) + o_setup >> t + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": {0: "success", 1: "failed", 2: "skipped"}, + "other_setup": "success", + "other_teardown": "success", + "other_work": "success", + "my_teardown": "success", + "my_work": "upstream_failed", + } + assert states == expected + + def test_one_to_many_as_teardown(self, dag_maker, session): + """ + 1 setup mapping to 3 teardowns + 1 work task + work fails + teardowns succeed + dagrun should be failure + """ + with dag_maker(dag_id="one_to_many_as_teardown") as dag: + + @task + def my_setup(): + print("setting up multiple things") + return [1, 2, 3] + + @task + def my_work(val): + print(f"doing work with multiple things: {val}") + raise ValueError("this fails") + return val + + @task + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup() + t = my_teardown.expand(val=s).as_teardown(setups=s) + with t: + my_work(s) + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": "success", + "my_teardown": {0: "success", 1: "success", 2: "success"}, + "my_work": "failed", + } + assert states == expected + + def test_one_to_many_as_teardown_offd(self, dag_maker, session): + """ + 1 setup mapping to 3 teardowns + 1 work task + work succeeds + all but one teardown succeed + offd=True + dagrun should be success + """ + with dag_maker(dag_id="one_to_many_as_teardown_offd") as dag: + + @task + def my_setup(): + print("setting up multiple things") + return [1, 2, 3] + + @task + def my_work(val): + print(f"doing work with multiple things: {val}") + return val + + @task + def my_teardown(val): + print(f"teardown: {val}") + if val == 2: + raise ValueError("failure") + + s = my_setup() + t = my_teardown.expand(val=s).as_teardown(setups=s, on_failure_fail_dagrun=True) + with t: + my_work(s) + # todo: if on_failure_fail_dagrun=True, should we still regard the WORK task as a leaf? + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": "success", + "my_teardown": {0: "success", 1: "failed", 2: "success"}, + "my_work": "success", + } + assert states == expected + + def test_mapped_task_group_simple(self, dag_maker, session): + """ + Mapped task group wherein there's a simple s >> w >> t pipeline. + When s is skipped, all should be skipped + When s is failed, all should be upstream failed + """ + with dag_maker(dag_id="mapped_task_group_simple") as dag: + + @setup + def my_setup(val): + if val == "data2.json": + raise ValueError("fail!") + elif val == "data3.json": + raise AirflowSkipException("skip!") + print(f"setup: {val}") + + @task + def my_work(val): + print(f"work: {val}") + + @teardown + def my_teardown(val): + print(f"teardown: {val}") + + @task_group + def file_transforms(filename): + s = my_setup(filename) + t = my_teardown(filename).as_teardown(setups=s) + with t: + my_work(filename) + + file_transforms.expand(filename=["data1.json", "data2.json", "data3.json"]) + dr = dag.test() + states = self.get_states(dr) + expected = { + "file_transforms.my_setup": {0: "success", 1: "failed", 2: "skipped"}, + "file_transforms.my_work": {0: "success", 1: "upstream_failed", 2: "skipped"}, + "file_transforms.my_teardown": {0: "success", 1: "upstream_failed", 2: "skipped"}, + } + + assert states == expected + + def test_mapped_task_group_work_fail_or_skip(self, dag_maker, session): + """ + Mapped task group wherein there's a simple s >> w >> t pipeline. + When w is skipped, teardown should still run + When w is failed, teardown should still run + """ + with dag_maker(dag_id="mapped_task_group_work_fail_or_skip") as dag: + + @setup + def my_setup(val): + print(f"setup: {val}") + + @task + def my_work(val): + if val == "data2.json": + raise ValueError("fail!") + elif val == "data3.json": + raise AirflowSkipException("skip!") + print(f"work: {val}") + + @teardown + def my_teardown(val): + print(f"teardown: {val}") + + @task_group + def file_transforms(filename): + s = my_setup(filename) + t = my_teardown(filename).as_teardown(setups=s) + with t: + my_work(filename) + + file_transforms.expand(filename=["data1.json", "data2.json", "data3.json"]) + dr = dag.test() + states = self.get_states(dr) + expected = { + "file_transforms.my_setup": {0: "success", 1: "success", 2: "success"}, + "file_transforms.my_teardown": {0: "success", 1: "success", 2: "success"}, + "file_transforms.my_work": {0: "success", 1: "failed", 2: "skipped"}, + } + assert states == expected + + def test_teardown_many_one_explicit(self, dag_maker, session): + """-- passing + one mapped setup going to one unmapped work + 3 diff states for setup: success / failed / skipped + teardown still runs, and receives the xcom from the single successful setup + """ + with dag_maker(dag_id="teardown_many_one_explicit") as dag: + + @task + def my_setup(val): + if val == "data2.json": + raise ValueError("fail!") + elif val == "data3.json": + raise AirflowSkipException("skip!") + print(f"setup: {val}") + return val + + @task + def my_work(val): + print(f"work: {val}") + + @task + def my_teardown(val): + print(f"teardown: {val}") + + s = my_setup.expand(val=["data1.json", "data2.json", "data3.json"]) + with my_teardown(s).as_teardown(setups=s): + my_work(s) + dr = dag.test() + states = self.get_states(dr) + expected = { + "my_setup": {0: "success", 1: "failed", 2: "skipped"}, + "my_teardown": "success", + "my_work": "upstream_failed", + } + assert states == expected diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 9e7f5eb48a17..78f7fb9690bc 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -3026,6 +3026,7 @@ def test_refresh_from_db(self, create_task_instance): "_try_number": 1, "max_tries": 1, "hostname": "some_unique_hostname", + "is_setup": False, "unixname": "some_unique_unixname", "job_id": 1234, "pool": "some_fake_pool_id", diff --git a/tests/models/test_taskmixin.py b/tests/models/test_taskmixin.py index 2d55b30cb369..c1795f22a84a 100644 --- a/tests/models/test_taskmixin.py +++ b/tests/models/test_taskmixin.py @@ -211,61 +211,6 @@ def my_ok_task(): m.operator.is_setup = True -def test_no_setup_or_teardown_for_mapped_operator(dag_maker): - @task - def add_one(x): - return x + 1 - - @task - def print_task(values): - print(sum(values)) - - # vanilla mapped task - with dag_maker(): - added_vals = add_one.expand(x=[1, 2, 3]) - print_task(added_vals) - - # combining setup and teardown with vanilla mapped task is fine - with dag_maker(): - s1 = BaseOperator(task_id="s1").as_setup() - t1 = BaseOperator(task_id="t1").as_teardown(setups=s1) - added_vals = add_one.expand(x=[1, 2, 3]) - print_task_task = print_task(added_vals) - s1 >> added_vals - print_task_task >> t1 - # confirm structure - assert s1.downstream_task_ids == {"add_one", "t1"} - assert t1.upstream_task_ids == {"print_task", "s1"} - assert added_vals.operator.upstream_task_ids == {"s1"} - assert added_vals.operator.downstream_task_ids == {"print_task"} - assert print_task_task.operator.upstream_task_ids == {"add_one"} - assert print_task_task.operator.downstream_task_ids == {"t1"} - - # but you can't use a mapped task as setup or teardown - with dag_maker(): - added_vals = add_one.expand(x=[1, 2, 3]) - with pytest.raises(ValueError, match="Cannot set is_teardown for mapped operator"): - added_vals.as_teardown() - - # ... no matter how hard you try - with dag_maker(): - added_vals = add_one.expand(x=[1, 2, 3]) - with pytest.raises(ValueError, match="Cannot set is_teardown for mapped operator"): - added_vals.is_teardown = True - - # same with setup - with dag_maker(): - added_vals = add_one.expand(x=[1, 2, 3]) - with pytest.raises(ValueError, match="Cannot set is_setup for mapped operator"): - added_vals.as_setup() - - # and again, trying harder... - with dag_maker(): - added_vals = add_one.expand(x=[1, 2, 3]) - with pytest.raises(ValueError, match="Cannot set is_setup for mapped operator"): - added_vals.is_setup = True - - def test_set_setup_teardown_ctx_dependencies_using_decorated_tasks(dag_maker): with dag_maker(): diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index c89122879e21..f59794509ecf 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -53,6 +53,7 @@ from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.security import permissions from airflow.sensors.bash import BashSensor +from airflow.serialization.enums import Encoding from airflow.serialization.json_schema import load_dag_schema_dict from airflow.serialization.serialized_objects import ( DagDependency, @@ -112,7 +113,8 @@ def detect_task_dependencies(task: Operator) -> DagDependency | None: # type: i ] ), ) - +TYPE = Encoding.TYPE +VAR = Encoding.VAR serialized_simple_dag_ground_truth = { "__version": 1, "dag": { @@ -383,7 +385,11 @@ def test_serialization(self): serialized_dags[v.dag_id] = dag # Compares with the ground truth of JSON string. - self.validate_serialized_dag(serialized_dags["simple_dag"], serialized_simple_dag_ground_truth) + actual, expected = self.prepare_ser_dags_for_comparison( + actual=serialized_dags["simple_dag"], + expected=serialized_simple_dag_ground_truth, + ) + assert actual == expected @pytest.mark.parametrize( "timetable, serialized_timetable", @@ -412,7 +418,14 @@ def test_dag_serialization_to_timetable(self, timetable, serialized_timetable): del expected["dag"]["schedule_interval"] expected["dag"]["timetable"] = serialized_timetable - self.validate_serialized_dag(serialized_dag, expected) + actual, expected = self.prepare_ser_dags_for_comparison( + actual=serialized_dag, + expected=expected, + ) + for task in actual["dag"]["tasks"]: + for k, v in task.items(): + print(task["task_id"], k, v) + assert actual == expected def test_dag_serialization_unregistered_custom_timetable(self): """Verify serialization fails without timetable registration.""" @@ -429,10 +442,10 @@ def test_dag_serialization_unregistered_custom_timetable(self): ) assert str(ctx.value) == message - def validate_serialized_dag(self, json_dag, ground_truth_dag): + def prepare_ser_dags_for_comparison(self, actual, expected): """Verify serialized DAGs match the ground truth.""" - assert json_dag["dag"]["fileloc"].split("/")[-1] == "test_dag_serialization.py" - json_dag["dag"]["fileloc"] = None + assert actual["dag"]["fileloc"].split("/")[-1] == "test_dag_serialization.py" + actual["dag"]["fileloc"] = None def sorted_serialized_dag(dag_dict: dict): """ @@ -447,7 +460,11 @@ def sorted_serialized_dag(dag_dict: dict): ) return dag_dict - assert sorted_serialized_dag(ground_truth_dag) == sorted_serialized_dag(json_dag) + # by roundtripping to json we get a cleaner diff + # if not doing this, we get false alarms such as "__var" != VAR + actual = json.loads(json.dumps(sorted_serialized_dag(actual))) + expected = json.loads(json.dumps(sorted_serialized_dag(expected))) + return actual, expected def test_deserialization_across_process(self): """A serialized DAG can be deserialized in another process.""" @@ -2290,6 +2307,9 @@ def x(arg1, arg2, arg3): "_operator_name": "@task", "downstream_task_ids": [], "partial_kwargs": { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": { "__type": "dict", @@ -2329,6 +2349,9 @@ def x(arg1, arg2, arg3): value={"arg2": {"a": 1, "b": 2}, "arg3": _XComRef({"task_id": "op1", "key": XCOM_RETURN_KEY})}, ) assert deserialized.partial_kwargs == { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": {"arg1": [1, 2, {"a": "b"}]}, "retry_delay": timedelta(seconds=30), @@ -2344,6 +2367,9 @@ def x(arg1, arg2, arg3): value={"arg2": {"a": 1, "b": 2}, "arg3": _XComRef({"task_id": "op1", "key": XCOM_RETURN_KEY})}, ) assert pickled.partial_kwargs == { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": {"arg1": [1, 2, {"a": "b"}]}, "retry_delay": timedelta(seconds=30), @@ -2376,6 +2402,9 @@ def x(arg1, arg2, arg3): "_operator_name": "@task", "downstream_task_ids": [], "partial_kwargs": { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": { "__type": "dict", @@ -2413,6 +2442,9 @@ def x(arg1, arg2, arg3): value=_XComRef({"task_id": "op1", "key": XCOM_RETURN_KEY}), ) assert deserialized.partial_kwargs == { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": {"arg1": [1, 2, {"a": "b"}]}, "retry_delay": timedelta(seconds=30), @@ -2428,6 +2460,9 @@ def x(arg1, arg2, arg3): _XComRef({"task_id": "op1", "key": XCOM_RETURN_KEY}), ) assert pickled.partial_kwargs == { + "is_setup": False, + "is_teardown": False, + "on_failure_fail_dagrun": False, "op_args": [], "op_kwargs": {"arg1": [1, 2, {"a": "b"}]}, "retry_delay": timedelta(seconds=30), diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 32df99c0476f..93bc52884c61 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -1017,6 +1017,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1049,6 +1050,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1081,6 +1083,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1113,6 +1116,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1145,6 +1149,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1177,6 +1182,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, @@ -1209,6 +1215,7 @@ def test_task_instances(admin_client): "executor_config": {}, "external_executor_id": None, "hostname": "", + "is_setup": False, "job_id": None, "map_index": -1, "max_tries": 0, From 0924389a877c5461733ef8a048e860b951d81a56 Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Fri, 28 Jul 2023 18:05:56 +0300 Subject: [PATCH 342/533] Fix MIN_AIRFLOW_VERSION_EXCEPTIONS for openlineage (#32909) --- airflow/providers/openlineage/__init__.py | 6 ++++-- dev/provider_packages/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 | 4 +++- dev/provider_packages/prepare_provider_packages.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/airflow/providers/openlineage/__init__.py b/airflow/providers/openlineage/__init__.py index 229077d2475f..b4606cee7ee6 100644 --- a/airflow/providers/openlineage/__init__.py +++ b/airflow/providers/openlineage/__init__.py @@ -35,7 +35,9 @@ except ImportError: from airflow.version import version as airflow_version -if packaging.version.parse(airflow_version) < packaging.version.parse("2.6.0"): +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.7.0" +): raise RuntimeError( - f"The package `apache-airflow-providers-openlineage:{__version__}` requires Apache Airflow 2.6.0+" + f"The package `apache-airflow-providers-openlineage:{__version__}` requires Apache Airflow 2.7.0+" ) diff --git a/dev/provider_packages/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 b/dev/provider_packages/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 index c2abb903e08e..8d659164166d 100644 --- a/dev/provider_packages/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 +++ b/dev/provider_packages/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 @@ -53,7 +53,9 @@ try: except ImportError: from airflow.version import version as airflow_version -if packaging.version.parse(airflow_version) < packaging.version.parse("{{ MIN_AIRFLOW_VERSION }}"): +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "{{ MIN_AIRFLOW_VERSION }}" +): raise RuntimeError( f"The package `{{ PACKAGE_PIP_NAME }}:{__version__}` requires Apache Airflow {{ MIN_AIRFLOW_VERSION }}+" # NOQA: E501 ) diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 5aacd7cc6837..f9cff84e1945 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -59,7 +59,7 @@ # In case you have some providers that you want to have different min-airflow version for, # Add them as exceptions here. Make sure to remove it once the min-airflow version is bumped # to the same version that is required by the exceptional provider -MIN_AIRFLOW_VERSION_EXCEPTIONS = {"openlineage": "2.6.0"} +MIN_AIRFLOW_VERSION_EXCEPTIONS = {"openlineage": "2.7.0"} INITIAL_CHANGELOG_CONTENT = """ .. Licensed to the Apache Software Foundation (ASF) under one From f91c003092e0e2c0e5b9a5073352b023a51a887f Mon Sep 17 00:00:00 2001 From: raphaelauv Date: Fri, 28 Jul 2023 18:14:35 +0200 Subject: [PATCH 343/533] fix: remove K8S 1.23 support (#32899) --- README.md | 18 +-- chart/README.md | 2 +- .../src/airflow_breeze/global_constants.py | 2 +- images/breeze/output-commands-hash.txt | 24 ++-- images/breeze/output-commands.svg | 108 +++++++++--------- .../breeze/output_k8s_configure-cluster.svg | 6 +- images/breeze/output_k8s_create-cluster.svg | 6 +- images/breeze/output_k8s_delete-cluster.svg | 4 +- images/breeze/output_k8s_deploy-airflow.svg | 6 +- images/breeze/output_k8s_k9s.svg | 4 +- images/breeze/output_k8s_logs.svg | 4 +- .../breeze/output_k8s_run-complete-tests.svg | 6 +- images/breeze/output_k8s_shell.svg | 4 +- images/breeze/output_k8s_status.svg | 4 +- images/breeze/output_k8s_tests.svg | 6 +- images/breeze/output_k8s_upload-k8s-image.svg | 6 +- 16 files changed, 105 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 5a3ae51aec6a..f69be701d26a 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,15 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.6.3) | -|-------------|------------------------------|---------------------------| -| Python | 3.8, 3.9, 3.10, 3.11 | 3.7, 3.8, 3.9, 3.10, 3.11 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.23, 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | -| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | -| MySQL | 5.7, 8 | 5.7, 8 | -| SQLite | 3.15.0+ | 3.15.0+ | -| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | +| | Main version (dev) | Stable version (2.6.3) | +|-------------|------------------------|---------------------------| +| Python | 3.8, 3.9, 3.10, 3.11 | 3.7, 3.8, 3.9, 3.10, 3.11 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | +| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | +| MySQL | 5.7, 8 | 5.7, 8 | +| SQLite | 3.15.0+ | 3.15.0+ | +| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | \* Experimental diff --git a/chart/README.md b/chart/README.md index 674e5a6722ef..af613c8734fb 100644 --- a/chart/README.md +++ b/chart/README.md @@ -30,7 +30,7 @@ cluster using the [Helm](https://helm.sh) package manager. ## Requirements -- Kubernetes 1.23+ cluster +- Kubernetes 1.24+ cluster - Helm 3.0+ - PV provisioner support in the underlying infrastructure (optionally) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 444bd1e01048..e16b6874bf52 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -66,7 +66,7 @@ # - https://endoflife.date/amazon-eks # - https://endoflife.date/azure-kubernetes-service # - https://endoflife.date/google-kubernetes-engine -ALLOWED_KUBERNETES_VERSIONS = ["v1.23.17", "v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3"] +ALLOWED_KUBERNETES_VERSIONS = ["v1.24.15", "v1.25.11", "v1.26.6", "v1.27.3"] ALLOWED_EXECUTORS = ["KubernetesExecutor", "CeleryExecutor", "LocalExecutor", "CeleryKubernetesExecutor"] START_AIRFLOW_ALLOWED_EXECUTORS = ["CeleryExecutor", "LocalExecutor"] START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS = START_AIRFLOW_ALLOWED_EXECUTORS[1] diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 94c4c589d89f..54868e36dc9c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -18,19 +18,19 @@ compile-www-assets:0963f1409f0aa1e3b137cddd4cc52e87 down:4580f5b3b178ea00182694f134a751f3 exec:9d0fb86607526afb6b161115ae7bf9cc k8s:build-k8s-image:4c299179e91066626e5b4a61b06fd6c6 -k8s:configure-cluster:450f6aaa0623bbfd6ce05dfa0bc13156 -k8s:create-cluster:dfeb9f49a113fc28e8d4029fae140b96 -k8s:delete-cluster:4b6ee64171fd556277fa9546e14da2b3 -k8s:deploy-airflow:a9646df64d666d9447fbe6d10f34a1fc -k8s:k9s:6a4602852d4d159db826876e794478bc -k8s:logs:9c952975d5c11b67a337c308c3f52624 -k8s:run-complete-tests:e5675291696c35561dad8ea5228726b1 +k8s:configure-cluster:3bd5d4614ffcd02952fadd0f61bc0cdf +k8s:create-cluster:cef6b3203035af1741d6acc87a26fe12 +k8s:delete-cluster:76a40b5a9001d8a933672ae3175ad59f +k8s:deploy-airflow:bb5f9212ad00de2f4131df2bb2beebf3 +k8s:k9s:f25f0df6b0fdb2604822e0c8d5b64d06 +k8s:logs:547c07e859eafcf67591952b785d381d +k8s:run-complete-tests:0cc62d0fbea8ae35849dc84fa786c960 k8s:setup-env:a34e94744ca4e0592371fe55478c3d54 -k8s:shell:9ee5244d1dc9c625dbd8c87504a6c4ce -k8s:status:1b1b6fb6ccc7a211a3eb44568da71659 -k8s:tests:2319c6f5ba0681ff7627e3166a1479ef -k8s:upload-k8s-image:a9ac79e2f5e3d6b01fa45e764885913f -k8s:db08df55a8200bff5858a3b7549bff47 +k8s:shell:ced753917db3d1bd89e7ea3314ac0d69 +k8s:status:1529ccd444b41c4b0b5f943289957100 +k8s:tests:2a1e2928faea2eddafaff94176a46690 +k8s:upload-k8s-image:6b3a20cdeb692f3c3d727f6b9e68c901 +k8s:8c1e4287deb0533a74f3b302f9c574be prod-image:build:d58b9222d141fd3620459ffaa9ee0235 prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 11b7038cecc5..ea72db06740e 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #868887 } -.breeze-help-r5 { fill: #68a0b3;font-weight: bold } +.breeze-help-r4 { fill: #68a0b3;font-weight: bold } +.breeze-help-r5 { fill: #868887 } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -217,59 +217,59 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_configure-cluster.svg b/images/breeze/output_k8s_configure-cluster.svg index 172cf94c67bb..6a7dc8ec78da 100644 --- a/images/breeze/output_k8s_configure-cluster.svg +++ b/images/breeze/output_k8s_configure-cluster.svg @@ -160,8 +160,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ @@ -172,7 +172,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_k8s_create-cluster.svg b/images/breeze/output_k8s_create-cluster.svg index f0d275384972..6dc10384bc4f 100644 --- a/images/breeze/output_k8s_create-cluster.svg +++ b/images/breeze/output_k8s_create-cluster.svg @@ -163,8 +163,8 @@ │(>3.8< | 3.9 | 3.10 | 3.11)                                 │ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--force-recreate-clusterForce recreation of the cluster even if it is already created.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ @@ -176,7 +176,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_k8s_delete-cluster.svg b/images/breeze/output_k8s_delete-cluster.svg index 14e197d0de69..8767a7c44aba 100644 --- a/images/breeze/output_k8s_delete-cluster.svg +++ b/images/breeze/output_k8s_delete-cluster.svg @@ -117,8 +117,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--allApply it to all created clusters│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ diff --git a/images/breeze/output_k8s_deploy-airflow.svg b/images/breeze/output_k8s_deploy-airflow.svg index b37f697bd195..694ab621a786 100644 --- a/images/breeze/output_k8s_deploy-airflow.svg +++ b/images/breeze/output_k8s_deploy-airflow.svg @@ -171,8 +171,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--executorExecutor to use for a kubernetes cluster.                                         │ │(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ │[default: KubernetesExecutor]                                                     │ @@ -188,7 +188,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_k8s_k9s.svg b/images/breeze/output_k8s_k9s.svg index 7d90c3c11f57..a888d2ddb468 100644 --- a/images/breeze/output_k8s_k9s.svg +++ b/images/breeze/output_k8s_k9s.svg @@ -114,8 +114,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/images/breeze/output_k8s_logs.svg b/images/breeze/output_k8s_logs.svg index be34b530c96f..a5a49aa9f3d4 100644 --- a/images/breeze/output_k8s_logs.svg +++ b/images/breeze/output_k8s_logs.svg @@ -117,8 +117,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--allApply it to all created clusters│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ diff --git a/images/breeze/output_k8s_run-complete-tests.svg b/images/breeze/output_k8s_run-complete-tests.svg index 5e8df4bf2133..82db183c2b64 100644 --- a/images/breeze/output_k8s_run-complete-tests.svg +++ b/images/breeze/output_k8s_run-complete-tests.svg @@ -216,8 +216,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--executorExecutor to use for a kubernetes cluster.                                         │ │(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ │[default: KubernetesExecutor]                                                     │ @@ -232,7 +232,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_k8s_shell.svg b/images/breeze/output_k8s_shell.svg index 994cc183ff4f..e3f0cfd2d987 100644 --- a/images/breeze/output_k8s_shell.svg +++ b/images/breeze/output_k8s_shell.svg @@ -126,8 +126,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--executorExecutor to use for a kubernetes cluster.                                         │ │(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ │[default: KubernetesExecutor]                                                     │ diff --git a/images/breeze/output_k8s_status.svg b/images/breeze/output_k8s_status.svg index 94c9e6377abf..153241b5edbc 100644 --- a/images/breeze/output_k8s_status.svg +++ b/images/breeze/output_k8s_status.svg @@ -120,8 +120,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ │--allApply it to all created clusters│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_tests.svg b/images/breeze/output_k8s_tests.svg index 58cf76488229..c61d1fc57ffe 100644 --- a/images/breeze/output_k8s_tests.svg +++ b/images/breeze/output_k8s_tests.svg @@ -168,8 +168,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ │--executorExecutor to use for a kubernetes cluster.                                         │ │(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ │[default: KubernetesExecutor]                                                     │ @@ -184,7 +184,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_k8s_upload-k8s-image.svg b/images/breeze/output_k8s_upload-k8s-image.svg index 56efc756963f..c99009e777c7 100644 --- a/images/breeze/output_k8s_upload-k8s-image.svg +++ b/images/breeze/output_k8s_upload-k8s-image.svg @@ -153,8 +153,8 @@ │--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ │[default: 3.8]                                              │ │--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.23.17< | v1.24.15 | v1.25.11 | v1.26.6 | v1.27.3)│ -│[default: v1.23.17]                                   │ +│(>v1.24.15< | v1.25.11 | v1.26.6 | v1.27.3)           │ +│[default: v1.24.15]                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ @@ -164,7 +164,7 @@ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11]                                                  │ │--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.23.17 v1.24.15 v1.25.11 v1.26.6 v1.27.3]         │ +│[default: v1.24.15 v1.25.11 v1.26.6 v1.27.3]                  │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ From 0fbb05751acf6061f9f6fa83ee53816d568055d0 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Sat, 29 Jul 2023 04:10:55 +0800 Subject: [PATCH 344/533] Remove old gantt chart and redirect to grid views gantt tab (#32908) * Remove old gantt chart and redirect to grid views gantt tab * Add ts-ignore for some reason --- LICENSE | 1 - airflow/www/static/css/gantt.css | 50 -- airflow/www/static/js/api/useDatasets.ts | 1 + .../js/cluster-activity/nav/FilterBar.tsx | 2 + .../static/js/cluster-activity/useFilters.tsx | 3 + .../www/static/js/dag/details/gantt/index.tsx | 1 + airflow/www/static/js/dag/nav/FilterBar.tsx | 1 + airflow/www/static/js/dag/useFilters.test.tsx | 1 + airflow/www/static/js/dag/useFilters.tsx | 1 + airflow/www/static/js/gantt.js | 431 ------------------ airflow/www/templates/airflow/gantt.html | 83 ---- airflow/www/views.py | 97 +--- airflow/www/webpack.config.js | 1 - docs/apache-airflow/img/gantt.png | Bin 70606 -> 352148 bytes licenses/LICENSE-moment-strftime.txt | 9 - tests/www/views/test_views_extra_links.py | 30 +- ...ews_graph_gantt.py => test_views_graph.py} | 1 - 17 files changed, 15 insertions(+), 698 deletions(-) delete mode 100644 airflow/www/static/css/gantt.css delete mode 100644 airflow/www/static/js/gantt.js delete mode 100644 airflow/www/templates/airflow/gantt.html delete mode 100644 licenses/LICENSE-moment-strftime.txt rename tests/www/views/{test_views_graph_gantt.py => test_views_graph.py} (99%) diff --git a/LICENSE b/LICENSE index 81899d0fee17..b83a80d26d24 100644 --- a/LICENSE +++ b/LICENSE @@ -237,7 +237,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/) (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock) (MIT License) MomentJS v2.24.0 (http://momentjs.com/) - (MIT License) moment-strftime v0.5.0 (https://github.com/benjaminoakes/moment-strftime) (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/) ======================================================================== diff --git a/airflow/www/static/css/gantt.css b/airflow/www/static/css/gantt.css deleted file mode 100644 index ab076e5dc58b..000000000000 --- a/airflow/www/static/css/gantt.css +++ /dev/null @@ -1,50 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.axis path, -.axis line { - fill: none; - stroke: #000; - shape-rendering: crispEdges; -} - -rect { - stroke: #000; - cursor: pointer; -} - -/* Creates a small triangle extender for the tooltip */ -.d3-tip::after { - box-sizing: border-box; - display: inline; - width: 100%; - line-height: 1; - color: rgba(0, 0, 0, 0.8); - content: "\25BC"; - pointer-events: none; - position: absolute; - text-align: center; -} - -/* Style northward tooltips differently */ -.d3-tip.n::after { - margin: -1px 0 0 0; - top: 100%; - left: 0; -} diff --git a/airflow/www/static/js/api/useDatasets.ts b/airflow/www/static/js/api/useDatasets.ts index 62d2583d905a..6150f51ce359 100644 --- a/airflow/www/static/js/api/useDatasets.ts +++ b/airflow/www/static/js/api/useDatasets.ts @@ -58,6 +58,7 @@ export default function useDatasets({ const updatedAfterParam = updatedAfter && updatedAfter.count && updatedAfter.unit ? { + // @ts-ignore updated_after: moment() .subtract(updatedAfter.count, updatedAfter.unit) .toISOString(), diff --git a/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx b/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx index 29fa456df44a..10cc6813f1b7 100644 --- a/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx +++ b/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx @@ -35,7 +35,9 @@ const FilterBar = () => { useFilters(); const { timezone } = useTimezone(); + // @ts-ignore const startDate = moment(filters.startDate); + // @ts-ignore const endDate = moment(filters.endDate); const formattedStartDate = startDate.tz(timezone).format(isoFormatWithoutTZ); const formattedEndDate = endDate.tz(timezone).format(isoFormatWithoutTZ); diff --git a/airflow/www/static/js/cluster-activity/useFilters.tsx b/airflow/www/static/js/cluster-activity/useFilters.tsx index 414745d0de14..6e30683e3a5c 100644 --- a/airflow/www/static/js/cluster-activity/useFilters.tsx +++ b/airflow/www/static/js/cluster-activity/useFilters.tsx @@ -52,6 +52,7 @@ const useFilters = (): FilterHookReturn => { const endDate = searchParams.get(END_DATE_PARAM) || now; const startDate = searchParams.get(START_DATE_PARAM) || + // @ts-ignore moment(endDate).subtract(1, "d").toISOString(); const makeOnChangeFn = @@ -68,10 +69,12 @@ const useFilters = (): FilterHookReturn => { const onStartDateChange = makeOnChangeFn( START_DATE_PARAM, + // @ts-ignore (localDate: string) => moment(localDate).utc().format() ); const onEndDateChange = makeOnChangeFn(END_DATE_PARAM, (localDate: string) => + // @ts-ignore moment(localDate).utc().format() ); diff --git a/airflow/www/static/js/dag/details/gantt/index.tsx b/airflow/www/static/js/dag/details/gantt/index.tsx index 931c48b5662e..d881543a0dd9 100644 --- a/airflow/www/static/js/dag/details/gantt/index.tsx +++ b/airflow/www/static/js/dag/details/gantt/index.tsx @@ -131,6 +131,7 @@ const Gantt = ({ openGroupIds, gridScrollRef, ganttScrollRef }: Props) => { ml={-9} >