From f3538c1a2f2248f05386241d0416480f8c75a099 Mon Sep 17 00:00:00 2001 From: Julian LaNeve Date: Mon, 30 Oct 2023 21:39:39 -0400 Subject: [PATCH 1/9] Switch from Black to Ruff --- .pre-commit-config.yaml | 18 +- airflow/api/common/delete_dag.py | 2 +- airflow/contrib/operators/__init__.py | 189 +++++++----------- airflow/contrib/secrets/__init__.py | 2 +- airflow/contrib/sensors/__init__.py | 10 +- airflow/contrib/utils/__init__.py | 4 +- airflow/lineage/__init__.py | 8 +- airflow/models/dag.py | 2 +- .../providers/apache/cassandra/__init__.py | 2 +- .../apache/kafka/operators/consume.py | 4 +- .../cncf/kubernetes/utils/pod_manager.py | 2 +- airflow/providers/ftp/operators/ftp.py | 2 +- .../providers/google/cloud/hooks/bigquery.py | 4 +- .../providers/google/cloud/hooks/mlengine.py | 4 +- .../google/cloud/hooks/secret_manager.py | 4 +- .../google/cloud/operators/bigquery.py | 4 +- .../google/cloud/operators/dataproc.py | 2 +- .../cloud/transfers/bigquery_to_mssql.py | 3 +- .../providers/microsoft/azure/hooks/asb.py | 2 +- .../providers/microsoft/azure/hooks/wasb.py | 4 +- .../providers/openlineage/plugins/listener.py | 5 +- airflow/providers/openlineage/utils/utils.py | 3 +- airflow/providers/qubole/hooks/qubole.py | 12 +- airflow/providers/qubole/operators/qubole.py | 3 +- airflow/providers/sftp/operators/sftp.py | 2 +- .../snowflake/operators/snowflake.py | 4 +- airflow/providers/trino/hooks/trino.py | 4 +- airflow/settings.py | 4 +- airflow/ti_deps/deps/trigger_rule_dep.py | 24 ++- airflow/timetables/events.py | 3 +- airflow/www/views.py | 4 +- tests/models/test_taskinstance.py | 16 +- .../amazon/aws/hooks/test_base_aws.py | 6 +- tests/providers/amazon/aws/hooks/test_eks.py | 3 +- .../amazon/aws/log/test_s3_task_handler.py | 2 +- .../amazon/aws/utils/eks_test_constants.py | 12 +- .../flink/sensors/test_flink_kubernetes.py | 8 +- .../hive/transfers/test_mysql_to_hive.py | 4 +- .../celery/executors/test_celery_executor.py | 4 +- .../common/sql/operators/test_sql.py | 4 +- .../google/cloud/hooks/test_datacatalog.py | 6 +- .../google/cloud/hooks/test_dataflow.py | 8 +- .../google/cloud/hooks/test_datapipeline.py | 4 +- .../google/cloud/hooks/test_functions.py | 20 +- .../providers/google/cloud/hooks/test_gcs.py | 8 +- .../google/cloud/log/test_gcs_task_handler.py | 2 +- .../cloud/operators/test_datacatalog.py | 12 +- .../google/cloud/operators/test_dataproc.py | 2 +- .../cloud/transfers/test_postgres_to_gcs.py | 2 +- .../hooks/test_display_video.py | 4 +- tests/providers/http/hooks/test_http.py | 8 +- .../providers/amazon/aws/example_emr_eks.py | 4 +- .../amazon/aws/example_sagemaker_pipeline.py | 2 +- .../system/providers/docker/example_docker.py | 4 +- .../docker/example_docker_copy_data.py | 4 +- tests/system/providers/ftp/__init__.py | 32 +-- .../cloud/dataproc/example_dataproc_gke.py | 2 +- .../microsoft/azure/example_azure_synapse.py | 4 +- tests/utils/test_log_handlers.py | 34 ++-- tests/utils/test_logging_mixin.py | 4 +- tests/www/views/test_views_base.py | 4 +- 61 files changed, 257 insertions(+), 312 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7c10d53ff0f0f..83006f4010febb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,13 +155,6 @@ repos: - --fuzzy-match-generates-todo files: > \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ - - repo: https://github.com/psf/black - rev: 23.10.0 - hooks: - - id: black - name: Run black (Python formatter) - args: [--config=./pyproject.toml] - exclude: ^.*/.*_vendor/|^airflow/contrib/ - repo: local hooks: - id: update-common-sql-api-stubs @@ -181,13 +174,16 @@ repos: pass_filenames: false require_serial: true - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.3 hooks: # Since ruff makes use of multiple cores we _purposefully_ don't run this in docker so it can use the # host CPU to it's fullest + - id: ruff-format + name: ruff-format + exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py - id: ruff - name: ruff - args: [ --fix ] + name: ruff-lint + args: [--fix] exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 @@ -200,7 +196,7 @@ repos: - --target-version=py38 - --target-version=py39 - --target-version=py310 - alias: black + alias: blacken-docs additional_dependencies: [black==23.10.0] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 diff --git a/airflow/api/common/delete_dag.py b/airflow/api/common/delete_dag.py index 1a3346775544d3..8af05056a72af7 100644 --- a/airflow/api/common/delete_dag.py +++ b/airflow/api/common/delete_dag.py @@ -72,7 +72,7 @@ def delete_dag(dag_id: str, keep_records_in_log: bool = True, session: Session = ) ) - dags_to_delete = [dag_id for dag_id, in dags_to_delete_query] + dags_to_delete = [dag_id for (dag_id,) in dags_to_delete_query] # Scheduler removes DAGs without files from serialized_dag table every dag_dir_list_interval. # There may be a lag, so explicitly removes serialized DAG here. diff --git a/airflow/contrib/operators/__init__.py b/airflow/contrib/operators/__init__.py index fe090c8b0d7fd3..f4ec7dcb2b1e06 100644 --- a/airflow/contrib/operators/__init__.py +++ b/airflow/contrib/operators/__init__.py @@ -180,9 +180,8 @@ "DataprocDeleteClusterOperator": ( "airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator" ), - "DataprocInstantiateInlineWorkflowTemplateOperator": - "airflow.providers.google.cloud.operators.dataproc." - "DataprocInstantiateInlineWorkflowTemplateOperator", + "DataprocInstantiateInlineWorkflowTemplateOperator": "airflow.providers.google.cloud.operators.dataproc." + "DataprocInstantiateInlineWorkflowTemplateOperator", "DataprocInstantiateWorkflowTemplateOperator": ( "airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateWorkflowTemplateOperator" ), @@ -240,9 +239,8 @@ "DataProcSparkSqlOperator": ( "airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkSqlJobOperator" ), - "DataprocWorkflowTemplateInstantiateInlineOperator": - "airflow.providers.google.cloud.operators.dataproc." - "DataprocInstantiateInlineWorkflowTemplateOperator", + "DataprocWorkflowTemplateInstantiateInlineOperator": "airflow.providers.google.cloud.operators.dataproc." + "DataprocInstantiateInlineWorkflowTemplateOperator", "DataprocWorkflowTemplateInstantiateOperator": ( "airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateWorkflowTemplateOperator" ), @@ -351,9 +349,8 @@ "ComputeEngineCopyInstanceTemplateOperator": ( "airflow.providers.google.cloud.operators.compute.ComputeEngineCopyInstanceTemplateOperator" ), - "ComputeEngineInstanceGroupUpdateManagerTemplateOperator": - "airflow.providers.google.cloud.operators.compute." - "ComputeEngineInstanceGroupUpdateManagerTemplateOperator", + "ComputeEngineInstanceGroupUpdateManagerTemplateOperator": "airflow.providers.google.cloud.operators.compute." + "ComputeEngineInstanceGroupUpdateManagerTemplateOperator", "ComputeEngineSetMachineTypeOperator": ( "airflow.providers.google.cloud.operators.compute.ComputeEngineSetMachineTypeOperator" ), @@ -364,9 +361,8 @@ "airflow.providers.google.cloud.operators.compute.ComputeEngineStopInstanceOperator" ), "GceBaseOperator": "airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator", - "GceInstanceGroupManagerUpdateTemplateOperator": - "airflow.providers.google.cloud.operators.compute." - "ComputeEngineInstanceGroupUpdateManagerTemplateOperator", + "GceInstanceGroupManagerUpdateTemplateOperator": "airflow.providers.google.cloud.operators.compute." + "ComputeEngineInstanceGroupUpdateManagerTemplateOperator", "GceInstanceStartOperator": ( "airflow.providers.google.cloud.operators.compute.ComputeEngineStartInstanceOperator" ), @@ -513,30 +509,22 @@ ), }, "gcp_natural_language_operator": { - "CloudNaturalLanguageAnalyzeEntitiesOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeEntitiesOperator", - "CloudNaturalLanguageAnalyzeEntitySentimentOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeEntitySentimentOperator", - "CloudNaturalLanguageAnalyzeSentimentOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeSentimentOperator", - "CloudNaturalLanguageClassifyTextOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageClassifyTextOperator", - "CloudLanguageAnalyzeEntitiesOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeEntitiesOperator", - "CloudLanguageAnalyzeEntitySentimentOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeEntitySentimentOperator", - "CloudLanguageAnalyzeSentimentOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageAnalyzeSentimentOperator", - "CloudLanguageClassifyTextOperator": - "airflow.providers.google.cloud.operators.natural_language." - "CloudNaturalLanguageClassifyTextOperator", + "CloudNaturalLanguageAnalyzeEntitiesOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeEntitiesOperator", + "CloudNaturalLanguageAnalyzeEntitySentimentOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeEntitySentimentOperator", + "CloudNaturalLanguageAnalyzeSentimentOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeSentimentOperator", + "CloudNaturalLanguageClassifyTextOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageClassifyTextOperator", + "CloudLanguageAnalyzeEntitiesOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeEntitiesOperator", + "CloudLanguageAnalyzeEntitySentimentOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeEntitySentimentOperator", + "CloudLanguageAnalyzeSentimentOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageAnalyzeSentimentOperator", + "CloudLanguageClassifyTextOperator": "airflow.providers.google.cloud.operators.natural_language." + "CloudNaturalLanguageClassifyTextOperator", }, "gcp_spanner_operator": { "SpannerDeleteDatabaseInstanceOperator": ( @@ -692,66 +680,46 @@ ), }, "gcp_transfer_operator": { - "CloudDataTransferServiceCancelOperationOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceCancelOperationOperator", - "CloudDataTransferServiceCreateJobOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceCreateJobOperator", - "CloudDataTransferServiceDeleteJobOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceDeleteJobOperator", - "CloudDataTransferServiceGCSToGCSOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceGCSToGCSOperator", - "CloudDataTransferServiceGetOperationOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceGetOperationOperator", - "CloudDataTransferServiceListOperationsOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceListOperationsOperator", - "CloudDataTransferServicePauseOperationOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServicePauseOperationOperator", - "CloudDataTransferServiceResumeOperationOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceResumeOperationOperator", - "CloudDataTransferServiceS3ToGCSOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceS3ToGCSOperator", - "CloudDataTransferServiceUpdateJobOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceUpdateJobOperator", - "GcpTransferServiceJobCreateOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceCreateJobOperator", - "GcpTransferServiceJobDeleteOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceDeleteJobOperator", - "GcpTransferServiceJobUpdateOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceUpdateJobOperator", - "GcpTransferServiceOperationCancelOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceCancelOperationOperator", - "GcpTransferServiceOperationGetOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceGetOperationOperator", - "GcpTransferServiceOperationPauseOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServicePauseOperationOperator", - "GcpTransferServiceOperationResumeOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceResumeOperationOperator", - "GcpTransferServiceOperationsListOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceListOperationsOperator", - "GoogleCloudStorageToGoogleCloudStorageTransferOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceGCSToGCSOperator", - "S3ToGoogleCloudStorageTransferOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceS3ToGCSOperator", + "CloudDataTransferServiceCancelOperationOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceCancelOperationOperator", + "CloudDataTransferServiceCreateJobOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceCreateJobOperator", + "CloudDataTransferServiceDeleteJobOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceDeleteJobOperator", + "CloudDataTransferServiceGCSToGCSOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceGCSToGCSOperator", + "CloudDataTransferServiceGetOperationOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceGetOperationOperator", + "CloudDataTransferServiceListOperationsOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceListOperationsOperator", + "CloudDataTransferServicePauseOperationOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServicePauseOperationOperator", + "CloudDataTransferServiceResumeOperationOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceResumeOperationOperator", + "CloudDataTransferServiceS3ToGCSOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceS3ToGCSOperator", + "CloudDataTransferServiceUpdateJobOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceUpdateJobOperator", + "GcpTransferServiceJobCreateOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceCreateJobOperator", + "GcpTransferServiceJobDeleteOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceDeleteJobOperator", + "GcpTransferServiceJobUpdateOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceUpdateJobOperator", + "GcpTransferServiceOperationCancelOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceCancelOperationOperator", + "GcpTransferServiceOperationGetOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceGetOperationOperator", + "GcpTransferServiceOperationPauseOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServicePauseOperationOperator", + "GcpTransferServiceOperationResumeOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceResumeOperationOperator", + "GcpTransferServiceOperationsListOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceListOperationsOperator", + "GoogleCloudStorageToGoogleCloudStorageTransferOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceGCSToGCSOperator", + "S3ToGoogleCloudStorageTransferOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceS3ToGCSOperator", }, "gcp_translate_operator": { "CloudTranslateTextOperator": ( @@ -767,15 +735,12 @@ ), }, "gcp_video_intelligence_operator": { - "CloudVideoIntelligenceDetectVideoExplicitContentOperator": - "airflow.providers.google.cloud.operators.video_intelligence." - "CloudVideoIntelligenceDetectVideoExplicitContentOperator", - "CloudVideoIntelligenceDetectVideoLabelsOperator": - "airflow.providers.google.cloud.operators.video_intelligence." - "CloudVideoIntelligenceDetectVideoLabelsOperator", - "CloudVideoIntelligenceDetectVideoShotsOperator": - "airflow.providers.google.cloud.operators.video_intelligence." - "CloudVideoIntelligenceDetectVideoShotsOperator", + "CloudVideoIntelligenceDetectVideoExplicitContentOperator": "airflow.providers.google.cloud.operators.video_intelligence." + "CloudVideoIntelligenceDetectVideoExplicitContentOperator", + "CloudVideoIntelligenceDetectVideoLabelsOperator": "airflow.providers.google.cloud.operators.video_intelligence." + "CloudVideoIntelligenceDetectVideoLabelsOperator", + "CloudVideoIntelligenceDetectVideoShotsOperator": "airflow.providers.google.cloud.operators.video_intelligence." + "CloudVideoIntelligenceDetectVideoShotsOperator", }, "gcp_vision_operator": { "CloudVisionAddProductToProductSetOperator": ( @@ -945,9 +910,7 @@ "JiraOperator": "airflow.providers.atlassian.jira.operators.jira.JiraOperator", }, "kubernetes_pod_operator": { - "KubernetesPodOperator": ( - "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator" - ), + "KubernetesPodOperator": ("airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator"), }, "mlengine_operator": { "MLEngineManageModelOperator": ( @@ -997,9 +960,8 @@ "OpsgenieAlertOperator": "airflow.providers.opsgenie.operators.opsgenie.OpsgenieCreateAlertOperator", }, "oracle_to_azure_data_lake_transfer": { - "OracleToAzureDataLakeOperator": - "airflow.providers.microsoft.azure.transfers." - "oracle_to_azure_data_lake.OracleToAzureDataLakeOperator", + "OracleToAzureDataLakeOperator": "airflow.providers.microsoft.azure.transfers." + "oracle_to_azure_data_lake.OracleToAzureDataLakeOperator", }, "oracle_to_oracle_transfer": { "OracleToOracleOperator": ( @@ -1065,9 +1027,8 @@ "S3ToGCSOperator": "airflow.providers.google.cloud.transfers.s3_to_gcs.S3ToGCSOperator", }, "s3_to_gcs_transfer_operator": { - "CloudDataTransferServiceS3ToGCSOperator": - "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." - "CloudDataTransferServiceS3ToGCSOperator", + "CloudDataTransferServiceS3ToGCSOperator": "airflow.providers.google.cloud.operators.cloud_storage_transfer_service." + "CloudDataTransferServiceS3ToGCSOperator", }, "s3_to_sftp_operator": { "S3ToSFTPOperator": "airflow.providers.amazon.aws.transfers.s3_to_sftp.S3ToSFTPOperator", diff --git a/airflow/contrib/secrets/__init__.py b/airflow/contrib/secrets/__init__.py index 7498cf0fc540a6..d97107bfed13b3 100644 --- a/airflow/contrib/secrets/__init__.py +++ b/airflow/contrib/secrets/__init__.py @@ -26,7 +26,7 @@ warnings.warn( "This module is deprecated. Please use airflow.providers.*.secrets.", RemovedInAirflow3Warning, - stacklevel=2 + stacklevel=2, ) __deprecated_classes = { diff --git a/airflow/contrib/sensors/__init__.py b/airflow/contrib/sensors/__init__.py index 9da2c3684578d4..3c313626e7217b 100644 --- a/airflow/contrib/sensors/__init__.py +++ b/airflow/contrib/sensors/__init__.py @@ -85,12 +85,10 @@ "FTPSSensor": "airflow.providers.ftp.sensors.ftp.FTPSSensor", }, "gcp_transfer_sensor": { - "CloudDataTransferServiceJobStatusSensor": - "airflow.providers.google.cloud.sensors.cloud_storage_transfer_service." - "CloudDataTransferServiceJobStatusSensor", - "GCPTransferServiceWaitForJobStatusSensor": - "airflow.providers.google.cloud.sensors.cloud_storage_transfer_service." - "CloudDataTransferServiceJobStatusSensor", + "CloudDataTransferServiceJobStatusSensor": "airflow.providers.google.cloud.sensors.cloud_storage_transfer_service." + "CloudDataTransferServiceJobStatusSensor", + "GCPTransferServiceWaitForJobStatusSensor": "airflow.providers.google.cloud.sensors.cloud_storage_transfer_service." + "CloudDataTransferServiceJobStatusSensor", }, "gcs_sensor": { "GCSObjectExistenceSensor": "airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor", diff --git a/airflow/contrib/utils/__init__.py b/airflow/contrib/utils/__init__.py index c24b7a24e98465..0202a1c7c13d27 100644 --- a/airflow/contrib/utils/__init__.py +++ b/airflow/contrib/utils/__init__.py @@ -24,9 +24,7 @@ from airflow.utils.deprecation_tools import add_deprecated_classes warnings.warn( - "This module is deprecated. Please use `airflow.utils`.", - RemovedInAirflow3Warning, - stacklevel=2 + "This module is deprecated. Please use `airflow.utils`.", RemovedInAirflow3Warning, stacklevel=2 ) __deprecated_classes = { diff --git a/airflow/lineage/__init__.py b/airflow/lineage/__init__.py index 76b9cc1e0c7a5a..79f1f664638fad 100644 --- a/airflow/lineage/__init__.py +++ b/airflow/lineage/__init__.py @@ -119,11 +119,9 @@ def wrapper(self, context, *args, **kwargs): if self.inlets and isinstance(self.inlets, list): # get task_ids that are specified as parameter and make sure they are upstream - task_ids = ( - {o for o in self.inlets if isinstance(o, str)} - .union(op.task_id for op in self.inlets if isinstance(op, AbstractOperator)) - .intersection(self.get_flat_relative_ids(upstream=True)) - ) + task_ids = {o for o in self.inlets if isinstance(o, str)}.union( + op.task_id for op in self.inlets if isinstance(op, AbstractOperator) + ).intersection(self.get_flat_relative_ids(upstream=True)) # pick up unique direct upstream task_ids if AUTO is specified if AUTO.upper() in self.inlets or AUTO.lower() in self.inlets: diff --git a/airflow/models/dag.py b/airflow/models/dag.py index bef0bde9a71640..897c0aa44ce44f 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -3618,7 +3618,7 @@ def get_paused_dag_ids(dag_ids: list[str], session: Session = NEW_SESSION) -> se .where(DagModel.dag_id.in_(dag_ids)) ) - paused_dag_ids = {paused_dag_id for paused_dag_id, in paused_dag_ids} + paused_dag_ids = {paused_dag_id for (paused_dag_id,) in paused_dag_ids} return paused_dag_ids def get_default_view(self) -> str: diff --git a/airflow/providers/apache/cassandra/__init__.py b/airflow/providers/apache/cassandra/__init__.py index 2be119519a7ce6..295a102df93313 100644 --- a/airflow/providers/apache/cassandra/__init__.py +++ b/airflow/providers/apache/cassandra/__init__.py @@ -39,5 +39,5 @@ "2.5.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-cassandra:{__version__}` requires Apache Airflow 2.5.0+" # NOQA: E501 + f"The package `apache-airflow-providers-apache-cassandra:{__version__}` requires Apache Airflow 2.5.0+" ) diff --git a/airflow/providers/apache/kafka/operators/consume.py b/airflow/providers/apache/kafka/operators/consume.py index 312ba238c09f59..8e5fda8f5cd7ab 100644 --- a/airflow/providers/apache/kafka/operators/consume.py +++ b/airflow/providers/apache/kafka/operators/consume.py @@ -137,7 +137,9 @@ def execute(self, context) -> Any: if self.apply_function: apply_callable = partial( - self.apply_function, *self.apply_function_args, **self.apply_function_kwargs # type: ignore + self.apply_function, + *self.apply_function_args, + **self.apply_function_kwargs, # type: ignore ) if self.apply_function_batch: diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 71ba391e53c422..edbc910980d232 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -719,7 +719,7 @@ def extract_xcom_json(self, pod: V1Pod) -> str: ) as resp: result = self._exec_pod_command( resp, - f"if [ -s {PodDefaults.XCOM_MOUNT_PATH}/return.json ]; then cat {PodDefaults.XCOM_MOUNT_PATH}/return.json; else echo __airflow_xcom_result_empty__; fi", # noqa + f"if [ -s {PodDefaults.XCOM_MOUNT_PATH}/return.json ]; then cat {PodDefaults.XCOM_MOUNT_PATH}/return.json; else echo __airflow_xcom_result_empty__; fi", ) if result and result.rstrip() != "__airflow_xcom_result_empty__": # Note: result string is parsed to check if its valid json. diff --git a/airflow/providers/ftp/operators/ftp.py b/airflow/providers/ftp/operators/ftp.py index 5118aa667c61c3..89ea9f8f73ad8b 100644 --- a/airflow/providers/ftp/operators/ftp.py +++ b/airflow/providers/ftp/operators/ftp.py @@ -156,7 +156,7 @@ def get_openlineage_facets_on_start(self): local_host = socket.gethostbyname(local_host) except Exception as e: self.log.warning( - f"Failed to resolve local hostname. Using the hostname got by socket.gethostbyname() without resolution. {e}", # noqa: E501 + f"Failed to resolve local hostname. Using the hostname got by socket.gethostbyname() without resolution. {e}", exc_info=True, ) diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index 7d51bc113e6f43..e140f4b5d36111 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -1725,7 +1725,7 @@ def run_load( # we check to make sure the passed source format is valid # if it's not, we raise a ValueError # Refer to this link for more details: - # https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat # noqa + # https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat if schema_fields is None and not autodetect: raise ValueError("You must either pass a schema or autodetect=True.") @@ -2137,7 +2137,7 @@ def run_query( # BigQuery also allows you to define how you want a table's schema to change # as a side effect of a query job # for more details: - # https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions # noqa + # https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions allowed_schema_update_options = ["ALLOW_FIELD_ADDITION", "ALLOW_FIELD_RELAXATION"] diff --git a/airflow/providers/google/cloud/hooks/mlengine.py b/airflow/providers/google/cloud/hooks/mlengine.py index 5b3fc480ed1ffc..234ad2329a5d39 100644 --- a/airflow/providers/google/cloud/hooks/mlengine.py +++ b/airflow/providers/google/cloud/hooks/mlengine.py @@ -590,7 +590,9 @@ async def get_job_status( async with ClientSession() as session: try: job = await self.get_job( - project_id=project_id, job_id=job_id, session=session # type: ignore + project_id=project_id, + job_id=job_id, + session=session, # type: ignore ) job = await job.json(content_type=None) self.log.info("Retrieving json_response: %s", job) diff --git a/airflow/providers/google/cloud/hooks/secret_manager.py b/airflow/providers/google/cloud/hooks/secret_manager.py index 484b147a8f7d90..396d292d3656a8 100644 --- a/airflow/providers/google/cloud/hooks/secret_manager.py +++ b/airflow/providers/google/cloud/hooks/secret_manager.py @@ -81,5 +81,7 @@ def get_secret( :param project_id: Project id (if you want to override the project_id from credentials) """ return self.get_conn().get_secret( - secret_id=secret_id, secret_version=secret_version, project_id=project_id # type: ignore + secret_id=secret_id, + secret_version=secret_version, + project_id=project_id, # type: ignore ) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index a0e0d884a8b585..3bbe3b0de83632 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -2850,7 +2850,9 @@ def execute(self, context: Any): project_id = self.project_id or self.hook.project_id if project_id: job_id_path = convert_job_id( - job_id=self.job_id, project_id=project_id, location=self.location # type: ignore[arg-type] + job_id=self.job_id, + project_id=project_id, + location=self.location, # type: ignore[arg-type] ) context["ti"].xcom_push(key="job_id_path", value=job_id_path) # Wait for the job to complete diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 86d7d9fd6278d8..bb6e069e95506a 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -152,7 +152,7 @@ class ClusterGenerator: ``projects/[PROJECT_STORING_KEYS]/locations/[LOCATION]/keyRings/[KEY_RING_NAME]/cryptoKeys/[KEY_NAME]`` # noqa :param enable_component_gateway: Provides access to the web interfaces of default and selected optional components on the cluster. - """ # noqa: E501 + """ def __init__( self, diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py index c1e3e9f550ce4d..5052e795113392 100644 --- a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +++ b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py @@ -63,8 +63,7 @@ def __init__( if mssql_table is not None: warnings.warn( # fmt: off - "The `mssql_table` parameter has been deprecated. " - "Use `target_table_name` instead.", + "The `mssql_table` parameter has been deprecated. Use `target_table_name` instead.", # fmt: on AirflowProviderDeprecationWarning, ) diff --git a/airflow/providers/microsoft/azure/hooks/asb.py b/airflow/providers/microsoft/azure/hooks/asb.py index 80273d6f960507..34b21b4f23b65b 100644 --- a/airflow/providers/microsoft/azure/hooks/asb.py +++ b/airflow/providers/microsoft/azure/hooks/asb.py @@ -64,7 +64,7 @@ def get_ui_field_behaviour() -> dict[str, Any]: ".servicebus.windows.net (for Azure AD authenticaltion)" ), "credential": "credential", - "schema": "Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=", # noqa + "schema": "Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=", }, } diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 2a1f853cad542c..f76ac7439aacb7 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -600,7 +600,9 @@ async def get_async_conn(self) -> AsyncBlobServiceClient: tenant, app_id, app_secret, **client_secret_auth_config ) self.blob_service_client = AsyncBlobServiceClient( - account_url=account_url, credential=token_credential, **extra # type:ignore[arg-type] + account_url=account_url, + credential=token_credential, + **extra, # type:ignore[arg-type] ) return self.blob_service_client diff --git a/airflow/providers/openlineage/plugins/listener.py b/airflow/providers/openlineage/plugins/listener.py index 7810fcf2578015..2f4b60008d867c 100644 --- a/airflow/providers/openlineage/plugins/listener.py +++ b/airflow/providers/openlineage/plugins/listener.py @@ -51,7 +51,10 @@ def __init__(self): @hookimpl def on_task_instance_running( - self, previous_state, task_instance: TaskInstance, session: Session # This will always be QUEUED + self, + previous_state, + task_instance: TaskInstance, + session: Session, # This will always be QUEUED ): if not hasattr(task_instance, "task"): self.log.warning( diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py index 7c0ea6506ae048..73b95f5d6205b1 100644 --- a/airflow/providers/openlineage/utils/utils.py +++ b/airflow/providers/openlineage/utils/utils.py @@ -356,7 +356,8 @@ def _redact(self, item: Redactable, name: str | None, depth: int, max_depth: int if attrs.has(type(item)): # TODO: fixme when mypy gets compatible with new attrs for dict_key, subval in attrs.asdict( - item, recurse=False # type: ignore[arg-type] + item, + recurse=False, # type: ignore[arg-type] ).items(): if _is_name_redactable(dict_key, item): setattr( diff --git a/airflow/providers/qubole/hooks/qubole.py b/airflow/providers/qubole/hooks/qubole.py index 3cbc827aa3604c..9e1c535540ee90 100644 --- a/airflow/providers/qubole/hooks/qubole.py +++ b/airflow/providers/qubole/hooks/qubole.py @@ -175,12 +175,16 @@ def execute(self, context: Context) -> None: time.sleep(Qubole.poll_interval) self.cmd = self.cls.find(self.cmd.id) # type: ignore[attr-defined] self.log.info( - "Command Id: %s and Status: %s", self.cmd.id, self.cmd.status # type: ignore[attr-defined] + "Command Id: %s and Status: %s", + self.cmd.id, + self.cmd.status, # type: ignore[attr-defined] ) if "fetch_logs" in self.kwargs and self.kwargs["fetch_logs"] is True: self.log.info( - "Logs for Command Id: %s \n%s", self.cmd.id, self.cmd.get_log() # type: ignore[attr-defined] + "Logs for Command Id: %s \n%s", + self.cmd.id, + self.cmd.get_log(), # type: ignore[attr-defined] ) if self.cmd.status != "done": # type: ignore[attr-defined] @@ -236,9 +240,7 @@ def get_results( self.cmd = self.cls.find(cmd_id) include_headers_str = "true" if include_headers else "false" - self.cmd.get_results( - fp, inline, delim, fetch, arguments=[include_headers_str] - ) # type: ignore[attr-defined] + self.cmd.get_results(fp, inline, delim, fetch, arguments=[include_headers_str]) # type: ignore[attr-defined] fp.flush() fp.close() return fp.name diff --git a/airflow/providers/qubole/operators/qubole.py b/airflow/providers/qubole/operators/qubole.py index 3f5b8c987e4dad..6869273f566133 100644 --- a/airflow/providers/qubole/operators/qubole.py +++ b/airflow/providers/qubole/operators/qubole.py @@ -54,8 +54,7 @@ def get_link( :return: url link """ conn = BaseHook.get_connection( - getattr(operator, "qubole_conn_id", None) - or operator.kwargs["qubole_conn_id"] # type: ignore[attr-defined] + getattr(operator, "qubole_conn_id", None) or operator.kwargs["qubole_conn_id"] # type: ignore[attr-defined] ) if conn and conn.host: host = re.sub(r"api$", "v2/analyze?command_id=", conn.host) diff --git a/airflow/providers/sftp/operators/sftp.py b/airflow/providers/sftp/operators/sftp.py index 32a29b3fdd7634..8e04b0d0b9c89f 100644 --- a/airflow/providers/sftp/operators/sftp.py +++ b/airflow/providers/sftp/operators/sftp.py @@ -210,7 +210,7 @@ def get_openlineage_facets_on_start(self): local_host = socket.gethostbyname(local_host) except Exception as e: self.log.warning( - f"Failed to resolve local hostname. Using the hostname got by socket.gethostbyname() without resolution. {e}", # noqa: E501 + f"Failed to resolve local hostname. Using the hostname got by socket.gethostbyname() without resolution. {e}", exc_info=True, ) diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 8cde2c197a60c2..5c136f1bae76e8 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -505,7 +505,9 @@ def execute(self, context: Context) -> None: deferrable=self.deferrable, ) self.query_ids = self._hook.execute_query( - self.sql, statement_count=self.statement_count, bindings=self.bindings # type: ignore[arg-type] + self.sql, + statement_count=self.statement_count, + bindings=self.bindings, # type: ignore[arg-type] ) self.log.info("List of query ids %s", self.query_ids) diff --git a/airflow/providers/trino/hooks/trino.py b/airflow/providers/trino/hooks/trino.py index aaa77a45bc3455..1d1bbd1cd2729e 100644 --- a/airflow/providers/trino/hooks/trino.py +++ b/airflow/providers/trino/hooks/trino.py @@ -177,9 +177,7 @@ def get_first( except DatabaseError as e: raise TrinoException(e) - def get_pandas_df( - self, sql: str = "", parameters: Iterable | Mapping[str, Any] | None = None, **kwargs - ): # type: ignore[override] + def get_pandas_df(self, sql: str = "", parameters: Iterable | Mapping[str, Any] | None = None, **kwargs): # type: ignore[override] import pandas as pd cursor = self.get_cursor() diff --git a/airflow/settings.py b/airflow/settings.py index 23bf1cbfcbc8c2..e0e2bce6a2a1f3 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -273,9 +273,7 @@ def prepare_engine_args(disable_connection_pool=False, pool_class=None): default_args = default.copy() break - engine_args: dict = conf.getjson( - "database", "sql_alchemy_engine_args", fallback=default_args - ) # type: ignore + engine_args: dict = conf.getjson("database", "sql_alchemy_engine_args", fallback=default_args) # type: ignore if pool_class: # Don't use separate settings for size etc, only those from sql_alchemy_engine_args diff --git a/airflow/ti_deps/deps/trigger_rule_dep.py b/airflow/ti_deps/deps/trigger_rule_dep.py index 9cc4582bbb8d92..ca2a6100a27846 100644 --- a/airflow/ti_deps/deps/trigger_rule_dep.py +++ b/airflow/ti_deps/deps/trigger_rule_dep.py @@ -275,9 +275,12 @@ def _evaluate_setup_constraint(*, relevant_setups) -> Iterator[tuple[TIDepStatus task_ids=ti.task_id, key=PAST_DEPENDS_MET, session=session, default=False ) if not past_depends_met: - yield self._failing_status( - reason="Task should be skipped but the past depends are not met" - ), changed + yield ( + self._failing_status( + reason="Task should be skipped but the past depends are not met" + ), + changed, + ) return changed = ti.set_state(new_state, session) @@ -288,13 +291,16 @@ def _evaluate_setup_constraint(*, relevant_setups) -> Iterator[tuple[TIDepStatus if ti.map_index > -1: non_successes -= removed if non_successes > 0: - yield self._failing_status( - reason=( - f"All setup tasks must complete successfully. Relevant setups: {relevant_setups}: " - f"upstream_states={upstream_states}, " - f"upstream_task_ids={task.upstream_task_ids}" + yield ( + self._failing_status( + reason=( + f"All setup tasks must complete successfully. Relevant setups: {relevant_setups}: " + f"upstream_states={upstream_states}, " + f"upstream_task_ids={task.upstream_task_ids}" + ), ), - ), changed + changed, + ) def _evaluate_direct_relatives() -> Iterator[TIDepStatus]: """Evaluate whether ``ti``'s trigger rule was met. diff --git a/airflow/timetables/events.py b/airflow/timetables/events.py index 4a062cc5201402..2d55966a0520f6 100644 --- a/airflow/timetables/events.py +++ b/airflow/timetables/events.py @@ -83,7 +83,8 @@ def next_dagrun_info( next_event = self.event_dates[0] else: future_dates = itertools.dropwhile( - lambda when: when <= last_automated_data_interval.end, self.event_dates # type: ignore + lambda when: when <= last_automated_data_interval.end, + self.event_dates, # type: ignore ) next_event = next(future_dates, None) # type: ignore if next_event is None: diff --git a/airflow/www/views.py b/airflow/www/views.py index 7530b6e193bd0d..bf8fa858734432 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -929,7 +929,7 @@ def index(self): dagtags = session.execute(select(func.distinct(DagTag.name)).order_by(DagTag.name)).all() tags = [ {"name": name, "selected": bool(arg_tags_filter and name in arg_tags_filter)} - for name, in dagtags + for (name,) in dagtags ] owner_links_dict = DagOwnerAttributes.get_all(session) @@ -2447,7 +2447,7 @@ def _mark_dagrun_state_as_queued( ) ) - completed_tis_ids = [task_id for task_id, in existing_tis] + completed_tis_ids = [task_id for (task_id,) in existing_tis] tasks_with_no_state = list(set(all_task_ids) - set(completed_tis_ids)) details = [str(t) for t in tasks_with_no_state] diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 39b6a147d93838..6193680e44b089 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -2940,11 +2940,13 @@ def user_defined_macro(): def foo(arg): print(arg) - PythonOperator( - task_id="context_inside_template", - python_callable=foo, - op_kwargs={"arg": "{{ user_defined_macro() }}"}, - ), + ( + PythonOperator( + task_id="context_inside_template", + python_callable=foo, + op_kwargs={"arg": "{{ user_defined_macro() }}"}, + ), + ) dagrun = dag_maker.create_dagrun() tis = dagrun.get_task_instances() ti: TaskInstance = next(x for x in tis if x.task_id == "context_inside_template") @@ -3607,11 +3609,11 @@ def pull(arg1, arg2): ("taskflow", 1, None), ] - ti = tis[((downstream, 0, None))] + ti = tis[(downstream, 0, None)] ti.run() ti.xcom_pull(task_ids=downstream, map_indexes=0, session=session) == ["a", "d"] - ti = tis[((downstream, 1, None))] + ti = tis[(downstream, 1, None)] if strict: with pytest.raises(TypeError) as ctx: ti.run() diff --git a/tests/providers/amazon/aws/hooks/test_base_aws.py b/tests/providers/amazon/aws/hooks/test_base_aws.py index 13e21f0ad93ccd..f1e1389c51587e 100644 --- a/tests/providers/amazon/aws/hooks/test_base_aws.py +++ b/tests/providers/amazon/aws/hooks/test_base_aws.py @@ -114,14 +114,12 @@ -""".format( # noqa: E501 +""".format( account_id=DEFAULT_ACCOUNT_ID, role_name="test-role", provider_name="TestProvFed", username="testuser", -).replace( - "\n", "" -) +).replace("\n", "") class CustomSessionFactory(BaseSessionFactory): diff --git a/tests/providers/amazon/aws/hooks/test_eks.py b/tests/providers/amazon/aws/hooks/test_eks.py index 8d8aa9f6f84517..d3e04f4a1e2402 100644 --- a/tests/providers/amazon/aws/hooks/test_eks.py +++ b/tests/providers/amazon/aws/hooks/test_eks.py @@ -274,7 +274,8 @@ def test_create_cluster_throws_exception_when_cluster_exists( with pytest.raises(ClientError) as raised_exception: eks_hook.create_cluster( - name=generated_test_data.existing_cluster_name, **dict(ClusterInputs.REQUIRED) # type: ignore + name=generated_test_data.existing_cluster_name, + **dict(ClusterInputs.REQUIRED), # type: ignore ) assert_client_error_exception_thrown( diff --git a/tests/providers/amazon/aws/log/test_s3_task_handler.py b/tests/providers/amazon/aws/log/test_s3_task_handler.py index 1e7e32d93117f4..758bf1244a5471 100644 --- a/tests/providers/amazon/aws/log/test_s3_task_handler.py +++ b/tests/providers/amazon/aws/log/test_s3_task_handler.py @@ -144,7 +144,7 @@ def test_read_when_s3_log_missing(self): assert 1 == len(log) assert len(log) == len(metadata) actual = log[0][0][-1] - expected = "*** No logs found on s3 for ti=\n" # noqa: E501 + expected = "*** No logs found on s3 for ti=\n" assert actual == expected assert {"end_of_log": True, "log_pos": 0} == metadata[0] diff --git a/tests/providers/amazon/aws/utils/eks_test_constants.py b/tests/providers/amazon/aws/utils/eks_test_constants.py index 63ff4cca5d72ec..3d284a40cd5193 100644 --- a/tests/providers/amazon/aws/utils/eks_test_constants.py +++ b/tests/providers/amazon/aws/utils/eks_test_constants.py @@ -189,12 +189,12 @@ class PageCount: LARGE: int = 10 -FARGATE_PROFILE_UUID_PATTERN: str = ( - r"(?P[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})" -) -NODEGROUP_UUID_PATTERN: str = ( - r"(?P[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})" -) +FARGATE_PROFILE_UUID_PATTERN: ( + str +) = r"(?P[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})" +NODEGROUP_UUID_PATTERN: ( + str +) = r"(?P[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})" class RegExTemplates: diff --git a/tests/providers/apache/flink/sensors/test_flink_kubernetes.py b/tests/providers/apache/flink/sensors/test_flink_kubernetes.py index 6907665712f97e..2e5b635fdcdc6c 100644 --- a/tests/providers/apache/flink/sensors/test_flink_kubernetes.py +++ b/tests/providers/apache/flink/sensors/test_flink_kubernetes.py @@ -238,7 +238,7 @@ "state": "RECONCILING", }, "reconciliationStatus": { - "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', # noqa: E501 + "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', "reconciliationTimestamp": 1664124436834, "state": "DEPLOYED", }, @@ -382,7 +382,7 @@ "state": "RECONCILING", }, "reconciliationStatus": { - "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', # noqa: E501 + "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', "reconciliationTimestamp": 1664124436834, "state": "DEPLOYED", }, @@ -693,7 +693,7 @@ "updateTime": "1664124454261", }, "reconciliationStatus": { - "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', # noqa: E501 + "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', "reconciliationTimestamp": 1664124436834, "state": "DEPLOYED", }, @@ -835,7 +835,7 @@ } }, "reconciliationStatus": { - "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/TopSpeedWindowing.jar","parallelism":4,"entryClass":null,"args":[],"state":"suspended","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG\\n"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', # noqa: E501 + "lastReconciledSpec": '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/TopSpeedWindowing.jar","parallelism":4,"entryClass":null,"args":[],"state":"suspended","savepointTriggerNonce":null,"initialSavepointPath":null,"upgradeMode":"stateless","allowNonRestoredState":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.15","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_15","ingress":{"template":"{{name}}.{{namespace}}.flink.k8s.io","className":null,"annotations":null},"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m"},"replicas":null,"podTemplate":null},"logConfiguration":{"log4j-console.properties":"rootLogger.level = DEBUG\\n"}},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2}}}', "reconciliationTimestamp": 1664131800071, "state": "UPGRADING", }, diff --git a/tests/providers/apache/hive/transfers/test_mysql_to_hive.py b/tests/providers/apache/hive/transfers/test_mysql_to_hive.py index 87d67f306da8e5..59ffa9d743750e 100644 --- a/tests/providers/apache/hive/transfers/test_mysql_to_hive.py +++ b/tests/providers/apache/hive/transfers/test_mysql_to_hive.py @@ -270,9 +270,7 @@ def test_mysql_to_hive_verify_csv_special_char(self, spy_on_hive): INSERT INTO {} VALUES ( '{}', '{}' ) - """.format( - mysql_table, *db_record - ) + """.format(mysql_table, *db_record) ) conn.commit() diff --git a/tests/providers/celery/executors/test_celery_executor.py b/tests/providers/celery/executors/test_celery_executor.py index 9cdb0d7cbff871..0130bccc4c4c0f 100644 --- a/tests/providers/celery/executors/test_celery_executor.py +++ b/tests/providers/celery/executors/test_celery_executor.py @@ -168,9 +168,7 @@ def test_command_validation(self, command, raise_exception): "airflow.providers.celery.executors.celery_executor_utils._execute_in_subprocess" ) as mock_subproc, mock.patch( "airflow.providers.celery.executors.celery_executor_utils._execute_in_fork" - ) as mock_fork, mock.patch( - "celery.app.task.Task.request" - ) as mock_task: + ) as mock_fork, mock.patch("celery.app.task.Task.request") as mock_task: mock_task.id = "abcdef-124215-abcdef" with expected_context: celery_executor_utils.execute_command(command) diff --git a/tests/providers/common/sql/operators/test_sql.py b/tests/providers/common/sql/operators/test_sql.py index 2344e1c2a3a030..764b22c0436dce 100644 --- a/tests/providers/common/sql/operators/test_sql.py +++ b/tests/providers/common/sql/operators/test_sql.py @@ -131,7 +131,7 @@ class TestColumnCheckOperator: UNION ALL SELECT 'X' AS col_name, 'distinct_check' AS check_type, X_distinct_check AS check_result FROM (SELECT COUNT(DISTINCT(X)) AS X_distinct_check FROM test_table WHERE Y > 1) AS sq - """ # noqa 501 + """ # 501 correct_generate_sql_query_with_partition_and_where = """ SELECT 'X' AS col_name, 'null_check' AS check_type, X_null_check AS check_result @@ -139,7 +139,7 @@ class TestColumnCheckOperator: UNION ALL SELECT 'X' AS col_name, 'distinct_check' AS check_type, X_distinct_check AS check_result FROM (SELECT COUNT(DISTINCT(X)) AS X_distinct_check FROM test_table WHERE Y > 1) AS sq - """ # noqa 501 + """ # 501 correct_generate_sql_query_with_where = """ SELECT 'X' AS col_name, 'null_check' AS check_type, X_null_check AS check_result diff --git a/tests/providers/google/cloud/hooks/test_datacatalog.py b/tests/providers/google/cloud/hooks/test_datacatalog.py index c97cef7c4ae1b6..5dc65aba50fa76 100644 --- a/tests/providers/google/cloud/hooks/test_datacatalog.py +++ b/tests/providers/google/cloud/hooks/test_datacatalog.py @@ -65,9 +65,9 @@ TEST_NAME: str = "test-name" TEST_TAG_ID: str = "test-tag-id" TEST_LOCATION_PATH: str = f"projects/{{}}/locations/{TEST_LOCATION}" -TEST_ENTRY_PATH: str = ( - f"projects/{{}}/locations/{TEST_LOCATION}/entryGroups/{TEST_ENTRY_GROUP_ID}/entries/{TEST_ENTRY_ID}" -) +TEST_ENTRY_PATH: ( + str +) = f"projects/{{}}/locations/{TEST_LOCATION}/entryGroups/{TEST_ENTRY_GROUP_ID}/entries/{TEST_ENTRY_ID}" TEST_ENTRY_GROUP_PATH: str = f"projects/{{}}/locations/{TEST_LOCATION}/entryGroups/{TEST_ENTRY_GROUP_ID}" TEST_TAG_TEMPLATE_PATH: str = f"projects/{{}}/locations/{TEST_LOCATION}/tagTemplates/{TEST_TAG_TEMPLATE_ID}" TEST_TAG_TEMPLATE_FIELD_PATH: str = ( diff --git a/tests/providers/google/cloud/hooks/test_dataflow.py b/tests/providers/google/cloud/hooks/test_dataflow.py index aeaaabccb3b6c0..3544826bdc0dd4 100644 --- a/tests/providers/google/cloud/hooks/test_dataflow.py +++ b/tests/providers/google/cloud/hooks/test_dataflow.py @@ -1717,12 +1717,8 @@ def test_dataflow_job_cancel_job_no_running_jobs(self): mock_jobs.return_value.update.assert_not_called() def test_fetch_list_job_messages_responses(self): - mock_list = ( - self.mock_dataflow.projects.return_value.locations.return_value.jobs.return_value.messages.return_value.list - ) - mock_list_next = ( - self.mock_dataflow.projects.return_value.locations.return_value.jobs.return_value.messages.return_value.list_next - ) + mock_list = self.mock_dataflow.projects.return_value.locations.return_value.jobs.return_value.messages.return_value.list + mock_list_next = self.mock_dataflow.projects.return_value.locations.return_value.jobs.return_value.messages.return_value.list_next mock_list.return_value.execute.return_value = "response_1" mock_list_next.return_value = None diff --git a/tests/providers/google/cloud/hooks/test_datapipeline.py b/tests/providers/google/cloud/hooks/test_datapipeline.py index 728ba575e850f0..4bac7e7847d52f 100644 --- a/tests/providers/google/cloud/hooks/test_datapipeline.py +++ b/tests/providers/google/cloud/hooks/test_datapipeline.py @@ -120,9 +120,7 @@ def test_run_data_pipeline(self, mock_connection): Test that run_data_pipeline is called with correct parameters and calls Google Data Pipelines API """ - mock_request = ( - mock_connection.return_value.projects.return_value.locations.return_value.pipelines.return_value.run - ) + mock_request = mock_connection.return_value.projects.return_value.locations.return_value.pipelines.return_value.run mock_request.return_value.execute.return_value = {"job": {"id": TEST_JOB_ID}} result = self.datapipeline_hook.run_data_pipeline( diff --git a/tests/providers/google/cloud/hooks/test_functions.py b/tests/providers/google/cloud/hooks/test_functions.py index fbe79a9db8c186..f7bb02b1e2c9e5 100644 --- a/tests/providers/google/cloud/hooks/test_functions.py +++ b/tests/providers/google/cloud/hooks/test_functions.py @@ -81,9 +81,7 @@ def test_create_new_function_overridden_project_id(self, wait_for_operation_to_c def test_upload_function_zip_overridden_project_id(self, get_conn, requests_put): mck, open_module = get_open_mock() with mock.patch(f"{open_module}.open", mck): - generate_upload_url_method = ( - get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl - ) + generate_upload_url_method = get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl execute_method = generate_upload_url_method.return_value.execute execute_method.return_value = {"uploadUrl": "http://uploadHere"} @@ -222,9 +220,7 @@ def test_update_function(self, wait_for_operation_to_complete, get_conn): def test_upload_function_zip(self, get_conn, requests_put, mock_project_id): mck, open_module = get_open_mock() with mock.patch(f"{open_module}.open", mck): - generate_upload_url_method = ( - get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl - ) + generate_upload_url_method = get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl execute_method = generate_upload_url_method.return_value.execute execute_method.return_value = {"uploadUrl": "http://uploadHere"} @@ -250,9 +246,7 @@ def test_upload_function_zip(self, get_conn, requests_put, mock_project_id): def test_upload_function_zip_overridden_project_id(self, get_conn, requests_put): mck, open_module = get_open_mock() with mock.patch(f"{open_module}.open", mck): - generate_upload_url_method = ( - get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl - ) + generate_upload_url_method = get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.generateUploadUrl execute_method = generate_upload_url_method.return_value.execute execute_method.return_value = {"uploadUrl": "http://uploadHere"} @@ -275,9 +269,7 @@ def test_upload_function_zip_overridden_project_id(self, get_conn, requests_put) def test_call_function(self, mock_get_conn): payload = {"executionId": "wh41ppcyoa6l", "result": "Hello World!"} - call = ( - mock_get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.call - ) + call = mock_get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.call call.return_value.execute.return_value = payload @@ -299,9 +291,7 @@ def test_call_function(self, mock_get_conn): def test_call_function_error(self, mock_get_conn): payload = {"error": "Something very bad"} - call = ( - mock_get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.call - ) + call = mock_get_conn.return_value.projects.return_value.locations.return_value.functions.return_value.call call.return_value.execute.return_value = payload diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/tests/providers/google/cloud/hooks/test_gcs.py index ed767b05229df6..33df98e37b0023 100644 --- a/tests/providers/google/cloud/hooks/test_gcs.py +++ b/tests/providers/google/cloud/hooks/test_gcs.py @@ -299,7 +299,9 @@ def test_is_older_than_with_true_cond(self, mock_service): # When response = self.gcs_hook.is_older_than( - bucket_name=test_bucket, object_name=test_object, seconds=86400 # 24hr + bucket_name=test_bucket, + object_name=test_object, + seconds=86400, # 24hr ) # Then @@ -318,7 +320,9 @@ def test_is_older_than_with_false_cond(self, mock_service): # When response = self.gcs_hook.is_older_than( - bucket_name=test_bucket, object_name=test_object, seconds=86400 # 24hr + bucket_name=test_bucket, + object_name=test_object, + seconds=86400, # 24hr ) # Then assert not response diff --git a/tests/providers/google/cloud/log/test_gcs_task_handler.py b/tests/providers/google/cloud/log/test_gcs_task_handler.py index 68b45e9b9fa206..2d4dd7340d8059 100644 --- a/tests/providers/google/cloud/log/test_gcs_task_handler.py +++ b/tests/providers/google/cloud/log/test_gcs_task_handler.py @@ -240,7 +240,7 @@ def test_write_to_remote_on_close_failed_read_old_logs(self, mock_blob, mock_cli mock.call.from_string().download_as_bytes(), mock.call.from_string("gs://bucket/remote/log/location/1.log", mock_client.return_value), mock.call.from_string().upload_from_string( - "MESSAGE\nError checking for previous log; if exists, may be overwritten: Fail to download\n", # noqa: E501 + "MESSAGE\nError checking for previous log; if exists, may be overwritten: Fail to download\n", content_type="text/plain", ), ], diff --git a/tests/providers/google/cloud/operators/test_datacatalog.py b/tests/providers/google/cloud/operators/test_datacatalog.py index 4247831933337f..71b6420b1c9e7b 100644 --- a/tests/providers/google/cloud/operators/test_datacatalog.py +++ b/tests/providers/google/cloud/operators/test_datacatalog.py @@ -81,12 +81,12 @@ f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}" f"/entryGroups/{TEST_ENTRY_GROUP_ID}/entries/{TEST_ENTRY_ID}" ) -TEST_ENTRY_GROUP_PATH: str = ( - f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/entryGroups/{TEST_ENTRY_GROUP_ID}" -) -TEST_TAG_TEMPLATE_PATH: str = ( - f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/tagTemplates/{TEST_TAG_TEMPLATE_ID}" -) +TEST_ENTRY_GROUP_PATH: ( + str +) = f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/entryGroups/{TEST_ENTRY_GROUP_ID}" +TEST_TAG_TEMPLATE_PATH: ( + str +) = f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/tagTemplates/{TEST_TAG_TEMPLATE_ID}" TEST_TAG_PATH: str = ( f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/entryGroups/" f"{TEST_ENTRY_GROUP_ID}/entries/{TEST_ENTRY_ID}/tags/{TEST_TAG_ID}" diff --git a/tests/providers/google/cloud/operators/test_dataproc.py b/tests/providers/google/cloud/operators/test_dataproc.py index 0f731210c622ad..39361c5a9840cd 100644 --- a/tests/providers/google/cloud/operators/test_dataproc.py +++ b/tests/providers/google/cloud/operators/test_dataproc.py @@ -139,7 +139,7 @@ "gke_cluster_target": "projects/project_id/locations/region/clusters/gke_cluster_name", "node_pool_target": [ { - "node_pool": "projects/project_id/locations/region/clusters/gke_cluster_name/nodePools/dp", # noqa + "node_pool": "projects/project_id/locations/region/clusters/gke_cluster_name/nodePools/dp", "roles": ["DEFAULT"], } ], diff --git a/tests/providers/google/cloud/transfers/test_postgres_to_gcs.py b/tests/providers/google/cloud/transfers/test_postgres_to_gcs.py index 8d0660b8faa48c..141e10f24dc11d 100644 --- a/tests/providers/google/cloud/transfers/test_postgres_to_gcs.py +++ b/tests/providers/google/cloud/transfers/test_postgres_to_gcs.py @@ -34,7 +34,7 @@ FILENAME = "test_{}.ndjson" NDJSON_LINES = [ - b'{"some_json": {"firtname": "John", "lastname": "Smith", "nested_dict": {"a": null, "b": "something"}}, "some_num": 42, "some_str": "mock_row_content_1"}\n', # noqa + b'{"some_json": {"firtname": "John", "lastname": "Smith", "nested_dict": {"a": null, "b": "something"}}, "some_num": 42, "some_str": "mock_row_content_1"}\n', b'{"some_json": {}, "some_num": 43, "some_str": "mock_row_content_2"}\n', b'{"some_json": {}, "some_num": 44, "some_str": "mock_row_content_3"}\n', ] diff --git a/tests/providers/google/marketing_platform/hooks/test_display_video.py b/tests/providers/google/marketing_platform/hooks/test_display_video.py index d46becb3dbd829..c6eae2aa8f9b17 100644 --- a/tests/providers/google/marketing_platform/hooks/test_display_video.py +++ b/tests/providers/google/marketing_platform/hooks/test_display_video.py @@ -496,9 +496,7 @@ def test_get_report(self, get_conn_mock): report_id = "REPORT_ID" return_value = "TEST" - get_conn_mock.return_value.queries.return_value.reports.return_value.get.return_value.execute.return_value = ( # noqa - return_value - ) + get_conn_mock.return_value.queries.return_value.reports.return_value.get.return_value.execute.return_value = return_value self.hook.get_report(query_id=query_id, report_id=report_id) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 8b03b9ab1334fa..617009d5750d63 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -408,9 +408,7 @@ def test_keep_alive_enabled(self): "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ), mock.patch( "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" - ) as tcp_keep_alive_send, mock.patch( - "requests.adapters.HTTPAdapter.send" - ) as http_send: + ) as tcp_keep_alive_send, mock.patch("requests.adapters.HTTPAdapter.send") as http_send: hook = HttpHook(method="GET") response = Response() response.status_code = HTTPStatus.OK @@ -425,9 +423,7 @@ def test_keep_alive_disabled(self): "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ), mock.patch( "requests_toolbelt.adapters.socket_options.TCPKeepAliveAdapter.send" - ) as tcp_keep_alive_send, mock.patch( - "requests.adapters.HTTPAdapter.send" - ) as http_send: + ) as tcp_keep_alive_send, mock.patch("requests.adapters.HTTPAdapter.send") as http_send: hook = HttpHook(method="GET", tcp_keep_alive=False) response = Response() response.status_code = HTTPStatus.OK diff --git a/tests/system/providers/amazon/aws/example_emr_eks.py b/tests/system/providers/amazon/aws/example_emr_eks.py index 384b85f883770b..28dc7ac3c27e15 100644 --- a/tests/system/providers/amazon/aws/example_emr_eks.py +++ b/tests/system/providers/amazon/aws/example_emr_eks.py @@ -133,9 +133,7 @@ def create_iam_oidc_identity_provider(cluster_name): def delete_iam_oidc_identity_provider(cluster_name): oidc_provider_issuer_url = boto3.client("eks").describe_cluster( name=cluster_name, - )["cluster"][ - "identity" - ]["oidc"]["issuer"] + )["cluster"]["identity"]["oidc"]["issuer"] oidc_provider_issuer_endpoint = oidc_provider_issuer_url.replace("https://", "") account_id = boto3.client("sts").get_caller_identity()["Account"] diff --git a/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py b/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py index acad53f2ae45b5..07eac5bae59c1d 100644 --- a/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py +++ b/tests/system/providers/amazon/aws/example_sagemaker_pipeline.py @@ -47,7 +47,7 @@ def create_pipeline(name: str, role_arn: str): # Each step takes roughly 1 second to execute, so the pipeline runtimes is ~30 seconds, which should be # enough to test stopping and awaiting without race conditions. # Built using sagemaker sdk, and using json.loads(pipeline.definition()) - pipeline_json_definition = """{"Version": "2020-12-01", "Metadata": {}, "Parameters": [], "PipelineExperimentConfig": {"ExperimentName": {"Get": "Execution.PipelineName"}, "TrialName": {"Get": "Execution.PipelineExecutionId"}}, "Steps": [{"Name": "DummyCond29", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond28", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond27", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond26", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond25", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond24", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond23", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond22", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond21", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond20", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond19", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond18", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond17", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond16", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond15", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond14", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond13", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond12", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond11", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond10", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond9", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond8", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond7", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond6", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond5", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond4", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond3", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond2", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond1", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond0", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}]}""" # noqa: E501 + pipeline_json_definition = """{"Version": "2020-12-01", "Metadata": {}, "Parameters": [], "PipelineExperimentConfig": {"ExperimentName": {"Get": "Execution.PipelineName"}, "TrialName": {"Get": "Execution.PipelineExecutionId"}}, "Steps": [{"Name": "DummyCond29", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond28", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond27", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond26", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond25", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond24", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond23", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond22", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond21", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond20", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond19", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond18", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond17", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond16", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond15", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond14", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond13", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond12", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond11", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond10", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond9", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond8", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond7", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond6", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond5", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond4", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond3", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond2", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond1", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond0", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [{"Name": "DummyCond", "Type": "Condition", "Arguments": {"Conditions": [{"Type": "LessThanOrEqualTo", "LeftValue": 3.0, "RightValue": 6.0}], "IfSteps": [], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}], "ElseSteps": []}}]}""" sgmk_client = boto3.client("sagemaker") sgmk_client.create_pipeline( PipelineName=name, PipelineDefinition=pipeline_json_definition, RoleArn=role_arn diff --git a/tests/system/providers/docker/example_docker.py b/tests/system/providers/docker/example_docker.py index 1bc77338cc6473..069f4794de6322 100644 --- a/tests/system/providers/docker/example_docker.py +++ b/tests/system/providers/docker/example_docker.py @@ -54,9 +54,7 @@ ( # TEST BODY - t1 - >> [t2, t3] - >> t4 + t1 >> [t2, t3] >> t4 ) from tests.system.utils import get_test_run # noqa: E402 diff --git a/tests/system/providers/docker/example_docker_copy_data.py b/tests/system/providers/docker/example_docker_copy_data.py index 02cfb100e8e6de..1154790bdcbc94 100644 --- a/tests/system/providers/docker/example_docker_copy_data.py +++ b/tests/system/providers/docker/example_docker_copy_data.py @@ -99,9 +99,7 @@ ( # TEST BODY - t_is_data_available - >> t_move - >> t_print + t_is_data_available >> t_move >> t_print ) from tests.system.utils import get_test_run # noqa: E402 diff --git a/tests/system/providers/ftp/__init__.py b/tests/system/providers/ftp/__init__.py index e0533d99236c22..13a83393a9124b 100644 --- a/tests/system/providers/ftp/__init__.py +++ b/tests/system/providers/ftp/__init__.py @@ -1,16 +1,16 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py index c839136528863a..564b33180a7fd6 100644 --- a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py +++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py @@ -65,7 +65,7 @@ "gke_cluster_target": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{GKE_CLUSTER_NAME}", "node_pool_target": [ { - "node_pool": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{GKE_CLUSTER_NAME}/nodePools/dp", # noqa + "node_pool": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{GKE_CLUSTER_NAME}/nodePools/dp", "roles": ["DEFAULT"], "node_pool_config": { "config": { diff --git a/tests/system/providers/microsoft/azure/example_azure_synapse.py b/tests/system/providers/microsoft/azure/example_azure_synapse.py index 5d8f65fac927a3..7465b71677deba 100644 --- a/tests/system/providers/microsoft/azure/example_azure_synapse.py +++ b/tests/system/providers/microsoft/azure/example_azure_synapse.py @@ -63,7 +63,9 @@ ) as dag: # [START howto_operator_azure_synapse] run_spark_job = AzureSynapseRunSparkBatchOperator( - task_id="run_spark_job", spark_pool="provsparkpool", payload=SPARK_JOB_PAYLOAD # type: ignore + task_id="run_spark_job", + spark_pool="provsparkpool", + payload=SPARK_JOB_PAYLOAD, # type: ignore ) # [END howto_operator_azure_synapse] diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index d2c8aa90ec917e..4b47c4ac49279d 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -565,7 +565,7 @@ def test_log_retrieval_valid_trigger(self, create_task_instance): AIRFLOW_CTX_TRY_NUMBER=1 AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00 [2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554 -""" # noqa: E501 +""" def test_parse_timestamps(): @@ -604,41 +604,41 @@ def test_interleave_interleaves(): ) log_sample2 = "\n".join( [ - "[2022-11-16T00:05:54.295-0800] {taskinstance.py:1278} INFO - Executing on 2022-11-16 08:05:52.324532+00:00", # noqa: E501 - "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", # noqa: E501 - "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", # noqa: E501 - "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", # noqa: E501 - "[2022-11-16T00:05:54.306-0800] {standard_task_runner.py:82} INFO - Running: ['airflow', 'tasks', 'run', 'simple_async_timedelta', 'wait', 'manual__2022-11-16T08:05:52.324532+00:00', '--job-id', '33648', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmp725r305n']", # noqa: E501 + "[2022-11-16T00:05:54.295-0800] {taskinstance.py:1278} INFO - Executing on 2022-11-16 08:05:52.324532+00:00", + "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", + "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", + "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", + "[2022-11-16T00:05:54.306-0800] {standard_task_runner.py:82} INFO - Running: ['airflow', 'tasks', 'run', 'simple_async_timedelta', 'wait', 'manual__2022-11-16T08:05:52.324532+00:00', '--job-id', '33648', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmp725r305n']", "[2022-11-16T00:05:54.309-0800] {standard_task_runner.py:83} INFO - Job 33648: Subtask wait", ] ) log_sample3 = "\n".join( [ - "[2022-11-16T00:05:54.457-0800] {task_command.py:376} INFO - Running on host daniels-mbp-2.lan", # noqa: E501 - "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", # noqa: E501 + "[2022-11-16T00:05:54.457-0800] {task_command.py:376} INFO - Running on host daniels-mbp-2.lan", + "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", "AIRFLOW_CTX_DAG_ID=simple_async_timedelta", "AIRFLOW_CTX_TASK_ID=wait", "AIRFLOW_CTX_EXECUTION_DATE=2022-11-16T08:05:52.324532+00:00", "AIRFLOW_CTX_TRY_NUMBER=1", "AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00", - "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", # noqa: E501 + "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", ] ) expected = "\n".join( [ "[2022-11-16T00:05:54.278-0800] {taskinstance.py:1258} INFO - Starting attempt 1 of 1", - "[2022-11-16T00:05:54.295-0800] {taskinstance.py:1278} INFO - Executing on 2022-11-16 08:05:52.324532+00:00", # noqa: E501 - "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", # noqa: E501 - "[2022-11-16T00:05:54.306-0800] {standard_task_runner.py:82} INFO - Running: ['airflow', 'tasks', 'run', 'simple_async_timedelta', 'wait', 'manual__2022-11-16T08:05:52.324532+00:00', '--job-id', '33648', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmp725r305n']", # noqa: E501 + "[2022-11-16T00:05:54.295-0800] {taskinstance.py:1278} INFO - Executing on 2022-11-16 08:05:52.324532+00:00", + "[2022-11-16T00:05:54.300-0800] {standard_task_runner.py:55} INFO - Started process 52536 to run task", + "[2022-11-16T00:05:54.306-0800] {standard_task_runner.py:82} INFO - Running: ['airflow', 'tasks', 'run', 'simple_async_timedelta', 'wait', 'manual__2022-11-16T08:05:52.324532+00:00', '--job-id', '33648', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmp725r305n']", "[2022-11-16T00:05:54.309-0800] {standard_task_runner.py:83} INFO - Job 33648: Subtask wait", - "[2022-11-16T00:05:54.457-0800] {task_command.py:376} INFO - Running on host daniels-mbp-2.lan", # noqa: E501 - "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", # noqa: E501 + "[2022-11-16T00:05:54.457-0800] {task_command.py:376} INFO - Running on host daniels-mbp-2.lan", + "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", "AIRFLOW_CTX_DAG_ID=simple_async_timedelta", "AIRFLOW_CTX_TASK_ID=wait", "AIRFLOW_CTX_EXECUTION_DATE=2022-11-16T08:05:52.324532+00:00", "AIRFLOW_CTX_TRY_NUMBER=1", "AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00", - "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", # noqa: E501 + "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", ] ) assert "\n".join(_interleave_logs(log_sample2, log_sample1, log_sample3)) == expected @@ -702,7 +702,7 @@ def test_interleave_interleaves(): [2023-01-15T22:37:12.359-0800] {temporal.py:71} INFO - sleeping 1 second... [2023-01-15T22:37:13.360-0800] {temporal.py:74} INFO - yielding event with payload DateTime(2023, 1, 16, 6, 37, 13, 44492, tzinfo=Timezone('UTC')) [2023-01-15T22:37:13.361-0800] {triggerer_job.py:540} INFO - Trigger (ID 106) fired: TriggerEvent -""" # noqa: E501 +""" def test_interleave_logs_correct_ordering(): @@ -716,6 +716,6 @@ def test_interleave_logs_correct_ordering(): [2023-01-17T12:47:10.882-0800] {temporal.py:71} INFO - sleeping 1 second... [2023-01-17T12:47:11.883-0800] {temporal.py:74} INFO - yielding event with payload DateTime(2023, 1, 17, 20, 47, 11, 254388, tzinfo=Timezone('UTC')) [2023-01-17T12:47:11.883-0800] {triggerer_job.py:540} INFO - Trigger (ID 1) fired: TriggerEvent - """ # noqa: E501 + """ assert sample_with_dupe == "\n".join(_interleave_logs(sample_with_dupe, "", sample_with_dupe)) diff --git a/tests/utils/test_logging_mixin.py b/tests/utils/test_logging_mixin.py index 522b83fc2d891b..c1b06ef0073845 100644 --- a/tests/utils/test_logging_mixin.py +++ b/tests/utils/test_logging_mixin.py @@ -52,7 +52,7 @@ def parent_child_handlers(child_logger): logger = child_logger.parent logger.addHandler(parent_handler) - child_logger.addHandler(child_handler), + (child_logger.addHandler(child_handler),) child_logger.propagate = True yield parent_handler, child_handler @@ -74,7 +74,7 @@ def test_set_context(self, child_logger, parent_child_handlers): parent.propagate = False parent.addHandler(handler1) log = parent.getChild("child") - log.addHandler(handler2), + (log.addHandler(handler2),) log.propagate = True value = "test" diff --git a/tests/www/views/test_views_base.py b/tests/www/views/test_views_base.py index 45ceb69e74ed77..ea903eca156981 100644 --- a/tests/www/views/test_views_base.py +++ b/tests/www/views/test_views_base.py @@ -68,7 +68,7 @@ def heartbeat_healthy(): state="running", latest_heartbeat=last_heartbeat, ) - SchedulerJobRunner(job=job), + (SchedulerJobRunner(job=job),) with create_session() as session: session.add(job) yield "healthy", last_heartbeat.isoformat() @@ -88,7 +88,7 @@ def heartbeat_too_slow(): state="running", latest_heartbeat=last_heartbeat, ) - SchedulerJobRunner(job=job), + (SchedulerJobRunner(job=job),) with create_session() as session: session.query(Job).filter( Job.job_type == "SchedulerJob", From 20cc5120a6ea3af7f6c2aff6fb564d3866c939d7 Mon Sep 17 00:00:00 2001 From: Julian LaNeve Date: Mon, 30 Oct 2023 22:32:20 -0400 Subject: [PATCH 2/9] swap order to recommended --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83006f4010febb..deec43f1ddd687 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -178,13 +178,13 @@ repos: hooks: # Since ruff makes use of multiple cores we _purposefully_ don't run this in docker so it can use the # host CPU to it's fullest - - id: ruff-format - name: ruff-format - exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py - id: ruff name: ruff-lint args: [--fix] exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py + - id: ruff-format + name: ruff-format + exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 hooks: From 430f80a70098d1113d7bab55c9b15b7e51598c1a Mon Sep 17 00:00:00 2001 From: Julian LaNeve Date: Mon, 30 Oct 2023 22:38:24 -0400 Subject: [PATCH 3/9] update pre_commit_ids --- dev/breeze/src/airflow_breeze/pre_commit_ids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index a083ee15ea5be4..d63651e1a043fd 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -24,7 +24,6 @@ PRE_COMMIT_LIST = [ "all", - "black", "blacken-docs", "check-aiobotocore-optional", "check-airflow-k8s-not-used", @@ -106,6 +105,7 @@ "replace-bad-characters", "rst-backticks", "ruff", + "ruff-format", "shellcheck", "trailing-whitespace", "ts-compile-format-lint-www", From 1fde80dc3596c715e77e6f4fd4b06047030ee2e6 Mon Sep 17 00:00:00 2001 From: Julian LaNeve Date: Mon, 30 Oct 2023 23:06:16 -0400 Subject: [PATCH 4/9] update STATIC_CODE_CHECKS.rst --- STATIC_CODE_CHECKS.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index be446735cbd13e..f496b56f1604a3 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -140,8 +140,6 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | ID | Description | Image | +===========================================================+==============================================================+=========+ -| black | Run black (Python formatter) | | -+-----------------------------------------------------------+--------------------------------------------------------------+---------+ | blacken-docs | Run black on Python code blocks in documentation files | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-aiobotocore-optional | Check if aiobotocore is an optional dependency only | | @@ -319,7 +317,9 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | rst-backticks | Check if RST files use double backticks for code | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ -| ruff | ruff | | +| ruff | ruff-lint | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| ruff-format | ruff-format | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | shellcheck | Check Shell scripts syntax correctness | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ From d325d230b6bad33ab046a6e3f273bf12659a82fe Mon Sep 17 00:00:00 2001 From: Julian LaNeve Date: Mon, 30 Oct 2023 23:14:06 -0400 Subject: [PATCH 5/9] fix issues from formatting --- .../apache/kafka/operators/consume.py | 4 +- .../google/cloud/operators/bigquery.py | 4 +- airflow/providers/openlineage/utils/utils.py | 4 +- .../snowflake/operators/snowflake.py | 4 +- airflow/timetables/events.py | 4 +- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_static-checks.svg | 148 +++++++++--------- 7 files changed, 83 insertions(+), 87 deletions(-) diff --git a/airflow/providers/apache/kafka/operators/consume.py b/airflow/providers/apache/kafka/operators/consume.py index 8e5fda8f5cd7ab..8f835edc86b1ec 100644 --- a/airflow/providers/apache/kafka/operators/consume.py +++ b/airflow/providers/apache/kafka/operators/consume.py @@ -137,9 +137,9 @@ def execute(self, context) -> Any: if self.apply_function: apply_callable = partial( - self.apply_function, + self.apply_function, # type: ignore *self.apply_function_args, - **self.apply_function_kwargs, # type: ignore + **self.apply_function_kwargs, ) if self.apply_function_batch: diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 3bbe3b0de83632..33285331d6000c 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -2850,9 +2850,9 @@ def execute(self, context: Any): project_id = self.project_id or self.hook.project_id if project_id: job_id_path = convert_job_id( - job_id=self.job_id, + job_id=self.job_id, # type: ignore[arg-type] project_id=project_id, - location=self.location, # type: ignore[arg-type] + location=self.location, ) context["ti"].xcom_push(key="job_id_path", value=job_id_path) # Wait for the job to complete diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py index 73b95f5d6205b1..a52c7022cac767 100644 --- a/airflow/providers/openlineage/utils/utils.py +++ b/airflow/providers/openlineage/utils/utils.py @@ -356,8 +356,8 @@ def _redact(self, item: Redactable, name: str | None, depth: int, max_depth: int if attrs.has(type(item)): # TODO: fixme when mypy gets compatible with new attrs for dict_key, subval in attrs.asdict( - item, - recurse=False, # type: ignore[arg-type] + item, # type: ignore[arg-type] + recurse=False, ).items(): if _is_name_redactable(dict_key, item): setattr( diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index 5c136f1bae76e8..9e0bf3d1cfbb56 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -505,9 +505,9 @@ def execute(self, context: Context) -> None: deferrable=self.deferrable, ) self.query_ids = self._hook.execute_query( - self.sql, + self.sql, # type: ignore[arg-type] statement_count=self.statement_count, - bindings=self.bindings, # type: ignore[arg-type] + bindings=self.bindings, ) self.log.info("List of query ids %s", self.query_ids) diff --git a/airflow/timetables/events.py b/airflow/timetables/events.py index 2d55966a0520f6..1998b12d46172f 100644 --- a/airflow/timetables/events.py +++ b/airflow/timetables/events.py @@ -83,8 +83,8 @@ def next_dagrun_info( next_event = self.event_dates[0] else: future_dates = itertools.dropwhile( - lambda when: when <= last_automated_data_interval.end, - self.event_dates, # type: ignore + lambda when: when <= last_automated_data_interval.end, # type: ignore + self.event_dates, ) next_event = next(future_dates, None) # type: ignore if next_event is None: diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 393ba795987507..a9a4bbd88eb5e1 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -65,7 +65,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:9dff88972f7c2ba4da2ad47f77ba2d06 shell:619e9de6bb9e6eb26acab06c8aedb183 start-airflow:c238330495a57851030415e34042173b -static-checks:19926b8fcea5784b28d4a0d99865363c +static-checks:8ed3127b97b5b56dae600aa4b15d4002 testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:46d744487713d3dda7f47fca9e59c4a4 diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index 422e9edd75626b..388f02b7e8776b 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - +