From 2c4bf76ebfe95276ea558da43edae43a3c492e9d Mon Sep 17 00:00:00 2001 From: eladkal <45845474+eladkal@users.noreply.github.com> Date: Sun, 1 May 2022 09:47:35 +0300 Subject: [PATCH] `DataprocHook`: Remove deprecated function `submit` --- airflow/providers/google/CHANGELOG.rst | 2 ++ .../providers/google/cloud/hooks/dataproc.py | 24 +------------------ .../google/cloud/hooks/test_dataproc.py | 9 ------- 3 files changed, 3 insertions(+), 32 deletions(-) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 0630283c7ca53..8329bdbe7a53c 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -91,6 +91,8 @@ Breaking changes * ``DataprocHook``: order of parameters in ``wait_for_job`` function has changed. +* ``DataprocHook``: Remove function ``submit``. Please use ``submit_job`` + * ``DataprocSubmitJobOperator``: order of parameters has changed. * ``CloudDatastoreImportEntitiesOperator`` : Remove ``xcom_push``. Please use ``BaseOperator.do_xcom_push`` diff --git a/airflow/providers/google/cloud/hooks/dataproc.py b/airflow/providers/google/cloud/hooks/dataproc.py index ae5c330d922ff..784b4dd16977e 100644 --- a/airflow/providers/google/cloud/hooks/dataproc.py +++ b/airflow/providers/google/cloud/hooks/dataproc.py @@ -20,8 +20,7 @@ import time import uuid -import warnings -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union from google.api_core.client_options import ClientOptions from google.api_core.exceptions import ServerError @@ -767,27 +766,6 @@ def submit_job( metadata=metadata, ) - def submit( - self, - project_id: str, - job: dict, - region: str = 'global', - job_error_states: Optional[Iterable[str]] = None, - ) -> None: - """ - Submits Google Cloud Dataproc job. - - :param project_id: The id of Google Cloud Dataproc project. - :param job: The job to be submitted - :param region: The region of Google Dataproc cluster. - :param job_error_states: Job states that should be considered error states. - """ - # TODO: Remover one day - warnings.warn("This method is deprecated. Please use `submit_job`", DeprecationWarning, stacklevel=2) - job_object = self.submit_job(region=region, project_id=project_id, job=job) - job_id = job_object.reference.job_id - self.wait_for_job(job_id=job_id, region=region, project_id=project_id) - @GoogleBaseHook.fallback_to_default_project_id def cancel_job( self, diff --git a/tests/providers/google/cloud/hooks/test_dataproc.py b/tests/providers/google/cloud/hooks/test_dataproc.py index b52039dd2ddd2..2356f3cc00038 100644 --- a/tests/providers/google/cloud/hooks/test_dataproc.py +++ b/tests/providers/google/cloud/hooks/test_dataproc.py @@ -372,15 +372,6 @@ def test_submit_job_missing_region(self, mock_client): with pytest.raises(TypeError): self.hook.submit_job(job=JOB, project_id=GCP_PROJECT) - @mock.patch(DATAPROC_STRING.format("DataprocHook.wait_for_job")) - @mock.patch(DATAPROC_STRING.format("DataprocHook.submit_job")) - def test_submit(self, mock_submit_job, mock_wait_for_job): - mock_submit_job.return_value.reference.job_id = JOB_ID - with pytest.warns(DeprecationWarning): - self.hook.submit(project_id=GCP_PROJECT, job=JOB, region=GCP_LOCATION) - mock_submit_job.assert_called_once_with(region=GCP_LOCATION, project_id=GCP_PROJECT, job=JOB) - mock_wait_for_job.assert_called_once_with(region=GCP_LOCATION, project_id=GCP_PROJECT, job_id=JOB_ID) - @mock.patch(DATAPROC_STRING.format("DataprocHook.get_job_client")) def test_cancel_job(self, mock_client): self.hook.cancel_job(region=GCP_LOCATION, job_id=JOB_ID, project_id=GCP_PROJECT)