From e54a2820db70a4f248e8e3f2329aeee43eec2eb2 Mon Sep 17 00:00:00 2001 From: GPK Date: Wed, 2 Oct 2024 12:48:12 +0100 Subject: [PATCH] =?UTF-8?q?Revert=20"Move=20FSHook/PackageIndexHook/Subpro?= =?UTF-8?q?cessHook=20to=20standard=20provider=20(#42=E2=80=A6"=20(#42659)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 61d1dbbc7feb9728da125dc00ad05314758036eb. --- .../standard => }/hooks/filesystem.py | 0 .../standard => }/hooks/package_index.py | 0 .../standard => }/hooks/subprocess.py | 4 +- airflow/operators/bash.py | 2 +- airflow/providers/standard/hooks/__init__.py | 16 -------- airflow/providers/standard/provider.yaml | 7 ---- airflow/providers_manager.py | 4 +- airflow/sensors/filesystem.py | 2 +- .../logging-monitoring/errors.rst | 2 +- .../operators-and-hooks-ref.rst | 4 +- .../standard => }/hooks/test_package_index.py | 6 +-- .../standard => }/hooks/test_subprocess.py | 6 +-- tests/providers/standard/hooks/__init__.py | 16 -------- .../standard/hooks/test_filesystem.py | 39 ------------------- tests/sensors/test_filesystem.py | 2 +- 15 files changed, 16 insertions(+), 94 deletions(-) rename airflow/{providers/standard => }/hooks/filesystem.py (100%) rename airflow/{providers/standard => }/hooks/package_index.py (100%) rename airflow/{providers/standard => }/hooks/subprocess.py (96%) delete mode 100644 airflow/providers/standard/hooks/__init__.py rename tests/{providers/standard => }/hooks/test_package_index.py (93%) rename tests/{providers/standard => }/hooks/test_subprocess.py (95%) delete mode 100644 tests/providers/standard/hooks/__init__.py delete mode 100644 tests/providers/standard/hooks/test_filesystem.py diff --git a/airflow/providers/standard/hooks/filesystem.py b/airflow/hooks/filesystem.py similarity index 100% rename from airflow/providers/standard/hooks/filesystem.py rename to airflow/hooks/filesystem.py diff --git a/airflow/providers/standard/hooks/package_index.py b/airflow/hooks/package_index.py similarity index 100% rename from airflow/providers/standard/hooks/package_index.py rename to airflow/hooks/package_index.py diff --git a/airflow/providers/standard/hooks/subprocess.py b/airflow/hooks/subprocess.py similarity index 96% rename from airflow/providers/standard/hooks/subprocess.py rename to airflow/hooks/subprocess.py index 9e578a7d8034b..bc20b5c20b4c5 100644 --- a/airflow/providers/standard/hooks/subprocess.py +++ b/airflow/hooks/subprocess.py @@ -52,8 +52,8 @@ def run_command( :param env: Optional dict containing environment variables to be made available to the shell environment in which ``command`` will be executed. If omitted, ``os.environ`` will be used. Note, that in case you have Sentry configured, original variables from the environment - will also be passed to the subprocess with ``SUBPROCESS_`` prefix. See: - https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/errors.html for details. + will also be passed to the subprocess with ``SUBPROCESS_`` prefix. See + :doc:`/administration-and-deployment/logging-monitoring/errors` for details. :param output_encoding: encoding to use for decoding stdout :param cwd: Working directory to run the command in. If None (default), the command is run in a temporary directory. diff --git a/airflow/operators/bash.py b/airflow/operators/bash.py index bf4a943df6e08..2ec0341a0d1e2 100644 --- a/airflow/operators/bash.py +++ b/airflow/operators/bash.py @@ -24,8 +24,8 @@ from typing import TYPE_CHECKING, Any, Callable, Container, Sequence, cast from airflow.exceptions import AirflowException, AirflowSkipException +from airflow.hooks.subprocess import SubprocessHook from airflow.models.baseoperator import BaseOperator -from airflow.providers.standard.hooks.subprocess import SubprocessHook from airflow.utils.operator_helpers import context_to_airflow_vars from airflow.utils.types import ArgNotSet diff --git a/airflow/providers/standard/hooks/__init__.py b/airflow/providers/standard/hooks/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/airflow/providers/standard/hooks/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index 068fde1fe3761..83d8acf0a68b3 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -50,10 +50,3 @@ sensors: - airflow.providers.standard.sensors.time_delta - airflow.providers.standard.sensors.time - airflow.providers.standard.sensors.weekday - -hooks: - - integration-name: Standard - python-modules: - - airflow.providers.standard.hooks.filesystem - - airflow.providers.standard.hooks.package_index - - airflow.providers.standard.hooks.subprocess diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index e276c465ef689..2c673063cb23e 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -36,8 +36,8 @@ from packaging.utils import canonicalize_name from airflow.exceptions import AirflowOptionalProviderFeatureException -from airflow.providers.standard.hooks.filesystem import FSHook -from airflow.providers.standard.hooks.package_index import PackageIndexHook +from airflow.hooks.filesystem import FSHook +from airflow.hooks.package_index import PackageIndexHook from airflow.typing_compat import ParamSpec from airflow.utils import yaml from airflow.utils.entry_points import entry_points_with_dist diff --git a/airflow/sensors/filesystem.py b/airflow/sensors/filesystem.py index 4496f5d6abfa4..5d32ab07ad4e7 100644 --- a/airflow/sensors/filesystem.py +++ b/airflow/sensors/filesystem.py @@ -25,7 +25,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException -from airflow.providers.standard.hooks.filesystem import FSHook +from airflow.hooks.filesystem import FSHook from airflow.sensors.base import BaseSensorOperator from airflow.triggers.base import StartTriggerArgs from airflow.triggers.file import FileTrigger diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst index 0ad3fa8c5127a..cb09843422321 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/errors.rst @@ -96,7 +96,7 @@ Impact of Sentry on Environment variables passed to Subprocess Hook When Sentry is enabled, by default it changes the standard library to pass all environment variables to subprocesses opened by Airflow. This changes the default behaviour of -:class:`airflow.providers.standard.hooks.subprocess.SubprocessHook` - always all environment variables are passed to the +:class:`airflow.hooks.subprocess.SubprocessHook` - always all environment variables are passed to the subprocess executed with specific set of environment variables. In this case not only the specified environment variables are passed but also all existing environment variables are passed with ``SUBPROCESS_`` prefix added. This happens also for all other subprocesses. diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index d4ac6bda74c34..16b74305a958b 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -106,8 +106,8 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Hooks - Guides - * - :mod:`airflow.providers.standard.hooks.filesystem` + * - :mod:`airflow.hooks.filesystem` - - * - :mod:`airflow.providers.standard.hooks.subprocess` + * - :mod:`airflow.hooks.subprocess` - diff --git a/tests/providers/standard/hooks/test_package_index.py b/tests/hooks/test_package_index.py similarity index 93% rename from tests/providers/standard/hooks/test_package_index.py rename to tests/hooks/test_package_index.py index 6a90db0715d81..9da429c5a09cf 100644 --- a/tests/providers/standard/hooks/test_package_index.py +++ b/tests/hooks/test_package_index.py @@ -21,8 +21,8 @@ import pytest +from airflow.hooks.package_index import PackageIndexHook from airflow.models.connection import Connection -from airflow.providers.standard.hooks.package_index import PackageIndexHook class MockConnection(Connection): @@ -73,7 +73,7 @@ def mock_get_connection(monkeypatch: pytest.MonkeyPatch, request: pytest.Fixture password: str | None = testdata.get("password", None) expected_result: str | None = testdata.get("expected_result", None) monkeypatch.setattr( - "airflow.providers.standard.hooks.package_index.PackageIndexHook.get_connection", + "airflow.hooks.package_index.PackageIndexHook.get_connection", lambda *_: MockConnection(host, login, password), ) return expected_result @@ -104,7 +104,7 @@ class MockProc: return MockProc() - monkeypatch.setattr("airflow.providers.standard.hooks.package_index.subprocess.run", mock_run) + monkeypatch.setattr("airflow.hooks.package_index.subprocess.run", mock_run) hook_instance = PackageIndexHook() if mock_get_connection: diff --git a/tests/providers/standard/hooks/test_subprocess.py b/tests/hooks/test_subprocess.py similarity index 95% rename from tests/providers/standard/hooks/test_subprocess.py rename to tests/hooks/test_subprocess.py index 2b2e9473359e5..0f625be816887 100644 --- a/tests/providers/standard/hooks/test_subprocess.py +++ b/tests/hooks/test_subprocess.py @@ -26,7 +26,7 @@ import pytest -from airflow.providers.standard.hooks.subprocess import SubprocessHook +from airflow.hooks.subprocess import SubprocessHook OS_ENV_KEY = "SUBPROCESS_ENV_TEST" OS_ENV_VAL = "this-is-from-os-environ" @@ -81,11 +81,11 @@ def test_return_value(self, val, expected): @mock.patch.dict("os.environ", clear=True) @mock.patch( - "airflow.providers.standard.hooks.subprocess.TemporaryDirectory", + "airflow.hooks.subprocess.TemporaryDirectory", return_value=MagicMock(__enter__=MagicMock(return_value="/tmp/airflowtmpcatcat")), ) @mock.patch( - "airflow.providers.standard.hooks.subprocess.Popen", + "airflow.hooks.subprocess.Popen", return_value=MagicMock(stdout=MagicMock(readline=MagicMock(side_effect=StopIteration), returncode=0)), ) def test_should_exec_subprocess(self, mock_popen, mock_temporary_directory): diff --git a/tests/providers/standard/hooks/__init__.py b/tests/providers/standard/hooks/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/standard/hooks/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/standard/hooks/test_filesystem.py b/tests/providers/standard/hooks/test_filesystem.py deleted file mode 100644 index bbcd22dc94219..0000000000000 --- a/tests/providers/standard/hooks/test_filesystem.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import pytest - -from airflow.providers.standard.hooks.filesystem import FSHook - -pytestmark = pytest.mark.db_test - - -class TestFSHook: - def test_get_ui_field_behaviour(self): - fs_hook = FSHook() - assert fs_hook.get_ui_field_behaviour() == { - "hidden_fields": ["host", "schema", "port", "login", "password", "extra"], - "relabeling": {}, - "placeholders": {}, - } - - def test_get_path(self): - fs_hook = FSHook(fs_conn_id="fs_default") - - assert fs_hook.get_path() == "/" diff --git a/tests/sensors/test_filesystem.py b/tests/sensors/test_filesystem.py index 641f2f218f2db..1fb123cfe7248 100644 --- a/tests/sensors/test_filesystem.py +++ b/tests/sensors/test_filesystem.py @@ -40,7 +40,7 @@ @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode class TestFileSensor: def setup_method(self): - from airflow.providers.standard.hooks.filesystem import FSHook + from airflow.hooks.filesystem import FSHook hook = FSHook() args = {"owner": "airflow", "start_date": DEFAULT_DATE}