From bd3c9009a41d191a6c2da1b2c66b359f0d7b923c Mon Sep 17 00:00:00 2001 From: Chenglong Yan Date: Mon, 30 May 2022 00:35:42 +0800 Subject: [PATCH] Migrate HTTP example DAGs to new design AIP-47 closes: #22448 , #22431 --- .../providers/http/example_dags/__init__.py | 17 ---------- .../operators.rst | 14 ++++---- .../http/operators/test_http_system.py | 32 ------------------- .../system/providers/http}/example_http.py | 12 ++++++- 4 files changed, 18 insertions(+), 57 deletions(-) delete mode 100644 airflow/providers/http/example_dags/__init__.py delete mode 100644 tests/providers/http/operators/test_http_system.py rename {airflow/providers/http/example_dags => tests/system/providers/http}/example_http.py (92%) diff --git a/airflow/providers/http/example_dags/__init__.py b/airflow/providers/http/example_dags/__init__.py deleted file mode 100644 index 217e5db960782..0000000000000 --- a/airflow/providers/http/example_dags/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/docs/apache-airflow-providers-http/operators.rst b/docs/apache-airflow-providers-http/operators.rst index 0deb4a021c10e..8c67a963cae8e 100644 --- a/docs/apache-airflow-providers-http/operators.rst +++ b/docs/apache-airflow-providers-http/operators.rst @@ -32,7 +32,7 @@ to ``true``. Here we are poking until httpbin gives us a response text containing ``httpbin``. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_http_sensor_check] :end-before: [END howto_operator_http_http_sensor_check] @@ -69,14 +69,14 @@ the response text back. In the first example we are calling a ``POST`` with json data and succeed when we get the same json data back otherwise the task will fail. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_post_op] :end-before: [END howto_operator_http_task_post_op] Here we are calling a ``GET`` request and pass params to it. The task will succeed regardless of the response text. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_get_op] :end-before: [END howto_operator_http_task_get_op] @@ -91,7 +91,7 @@ it on the next task downstream use ``response_filter``. This is useful if: Below is an example of retrieving data from a REST API and only returning a nested property instead of the full response body. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_get_op_response_filter] :end-before: [END howto_operator_http_task_get_op_response_filter] @@ -99,7 +99,7 @@ response body. In the third example we are performing a ``PUT`` operation to put / set data according to the data that is being provided to the request. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_put_op] :end-before: [END howto_operator_http_task_put_op] @@ -107,14 +107,14 @@ provided to the request. In this example we call a ``DELETE`` operation to the ``delete`` endpoint. This time we are passing form data to the request. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_del_op] :end-before: [END howto_operator_http_task_del_op] Here we pass form data to a ``POST`` operation which is equal to a usual form submit. -.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py +.. exampleinclude:: /../../tests/system/providers/http/example_http.py :language: python :start-after: [START howto_operator_http_task_post_op_formenc] :end-before: [END howto_operator_http_task_post_op_formenc] diff --git a/tests/providers/http/operators/test_http_system.py b/tests/providers/http/operators/test_http_system.py deleted file mode 100644 index f86a898acf79e..0000000000000 --- a/tests/providers/http/operators/test_http_system.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os - -import pytest - -from tests.test_utils import AIRFLOW_MAIN_FOLDER -from tests.test_utils.system_tests_class import SystemTest - -HTTP_DAG_FOLDER = os.path.join(AIRFLOW_MAIN_FOLDER, "airflow", "providers", "http", "example_dags") - - -@pytest.mark.backend("mysql", "postgres") -@pytest.mark.system("http") -class HttpExampleDagsSystemTest(SystemTest): - def test_run_example_dag_http(self): - self.run_dag('example_http_operator', HTTP_DAG_FOLDER) diff --git a/airflow/providers/http/example_dags/example_http.py b/tests/system/providers/http/example_http.py similarity index 92% rename from airflow/providers/http/example_dags/example_http.py rename to tests/system/providers/http/example_http.py index 59589b84d97c3..876c09eb40233 100644 --- a/airflow/providers/http/example_dags/example_http.py +++ b/tests/system/providers/http/example_http.py @@ -19,14 +19,19 @@ """Example HTTP operator and sensor""" import json +import os from datetime import datetime from airflow import DAG from airflow.providers.http.operators.http import SimpleHttpOperator from airflow.providers.http.sensors.http import HttpSensor +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = 'example_http_operator' + + dag = DAG( - 'example_http_operator', + DAG_ID, default_args={'retries': 1}, tags=['example'], start_date=datetime(2021, 1, 1), @@ -107,3 +112,8 @@ # [END howto_operator_http_http_sensor_check] task_http_sensor_check >> task_post_op >> task_get_op >> task_get_op_response_filter task_get_op_response_filter >> task_put_op >> task_del_op >> task_post_op_formenc + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag)