AIP-47 - Migrate s3 DAGs to new design apache#22438

docs/apache-airflow-providers-amazon/operators/s3.rst

@@ -38,7 +38,7 @@ Create an Amazon S3 bucket
 To create an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateBucketOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_create_bucket]
@@ -52,7 +52,7 @@ Delete an Amazon S3 bucket
 To delete an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_bucket]
@@ -66,7 +66,7 @@ Set the tags for an Amazon S3 bucket
 To set the tags for an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3PutBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_put_bucket_tagging]
@@ -80,7 +80,7 @@ Get the tag of an Amazon S3 bucket
 To get the tag set associated with an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3GetBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_get_bucket_tagging]
@@ -94,7 +94,7 @@ Delete the tags of an Amazon S3 bucket
 To delete the tags of an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteBucketTaggingOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_bucket_tagging]
@@ -108,7 +108,7 @@ Create an Amazon S3 object
 To create a new (or replace) Amazon S3 object you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CreateObjectOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_create_object]
@@ -123,7 +123,7 @@ To copy an Amazon S3 object from one bucket to another you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3CopyObjectOperator`.
 The Amazon S3 connection used here needs to have access to both source and destination bucket/key.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_copy_object]
@@ -137,7 +137,7 @@ Delete Amazon S3 objects
 To delete one or multiple Amazon S3 objects you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3DeleteObjectsOperator`.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_delete_objects]
@@ -153,7 +153,7 @@ To transform the data from one Amazon S3 object and save it to another object yo
 You can also apply an optional [Amazon S3 Select expression](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-glacier-select-sql-reference-select.html)
 to select the data you want to retrieve from ``source_s3_key`` using ``select_expression``.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_file_transform]
@@ -169,7 +169,7 @@ To list all Amazon S3 prefixes within an Amazon S3 bucket you can use
 See `here <https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html>`__
 for more information about Amazon S3 prefixes.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_list_prefixes]
@@ -184,7 +184,7 @@ To list all Amazon S3 objects within an Amazon S3 bucket you can use
 :class:`~airflow.providers.amazon.aws.operators.s3.S3ListOperator`.
 You can specify a ``prefix`` to filter the objects whose name begins with such prefix.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_s3_list]
@@ -208,15 +208,15 @@ Please keep in mind, especially when used to check a large volume of keys, that
 
 To check one file:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_single_key]
     :end-before: [END howto_sensor_s3_key_single_key]
 
 To check multiple files:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_multiple_keys]
@@ -236,13 +236,13 @@ multiple files can match one key. The list of matched S3 object attributes conta
 
     [{"Size": int}]
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_function_definition]
     :end-before: [END howto_sensor_s3_key_function_definition]
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_key_function]
@@ -259,7 +259,7 @@ the inactivity period has passed with no increase in the number of objects you c
 Note, this sensor will not behave correctly in reschedule mode,
 as the state of the listed objects in the Amazon S3 bucket will be lost between rescheduled invocations.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_s3_keys_unchanged]

docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst

@@ -48,7 +48,7 @@ To get more information visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_dynamodb_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_dynamodb_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_dynamodb_to_s3]
@@ -57,7 +57,7 @@ Example usage:
 To parallelize the replication, users can create multiple ``DynamoDBToS3Operator`` tasks using the
 ``TotalSegments`` parameter.  For instance to replicate with parallelism of 2, create two tasks:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_dynamodb_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_dynamodb_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_dynamodb_to_s3_segmented]

docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst

@@ -41,7 +41,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_ftp_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_ftp_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_ftp_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst

@@ -39,7 +39,7 @@ To copy data from a Google Cloud Storage bucket to an Amazon S3 bucket you can u
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_gcs_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_gcs_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst

@@ -38,7 +38,7 @@ Google Sheets to Amazon S3 transfer operator
 
 This example loads data from Google Sheets and save it to an Amazon S3 file.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_sheets_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_sheets_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_sheets_to_s3]
@@ -57,7 +57,7 @@ It searches for up to 50 videos (due to pagination) in a given time range
 (``YOUTUBE_VIDEO_PUBLISHED_AFTER``, ``YOUTUBE_VIDEO_PUBLISHED_BEFORE``) on a YouTube channel (``YOUTUBE_CHANNEL_ID``)
 saves the response in Amazon S3 and also pushes the data to xcom.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_youtube_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_youtube_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_youtube_search_to_s3]
@@ -66,7 +66,7 @@ saves the response in Amazon S3 and also pushes the data to xcom.
 It passes over the YouTube IDs to the next request which then gets the
 information (``YOUTUBE_VIDEO_FIELDS``) for the requested videos and saves them in Amazon S3 (``S3_BUCKET_NAME``).
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_google_api_youtube_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_google_api_youtube_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_google_api_youtube_list_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst

@@ -38,7 +38,7 @@ Imap Attachment To Amazon S3 transfer operator
 To save an email attachment via IMAP protocol from an email server to an Amazon S3 Bucket you can use
 :class:`~airflow.providers.amazon.aws.transfers.imap_attachment_to_s3.ImapAttachmentToS3Operator`
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_imap_attachment_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_imap_attachment_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst

@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_local_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_local_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_local_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst

@@ -43,7 +43,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_mongo_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_mongo_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_mongo_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst

@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_redshift_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_redshift_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_redshift_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst

@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_ftp.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_ftp.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_ftp]

docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst

@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_redshift.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_redshift]

docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst

@@ -43,7 +43,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_s3_to_sftp.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_s3_to_sftp]

docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst

@@ -38,7 +38,7 @@ Extract data from Salesforce to Amazon S3 transfer operator
 The following example demonstrates a use case of extracting account data from a Salesforce
 instance and upload to an Amazon S3 bucket.
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_salesforce_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_salesforce_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_salesforce_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst

@@ -42,7 +42,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_sftp_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_sftp_to_s3]

docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst

@@ -44,7 +44,7 @@ To get more information about this operator visit:
 
 Example usage:
 
-.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py
+.. exampleinclude:: /../../tests/system/providers/amazon/aws/s3/example_sql_to_s3.py
     :language: python
     :dedent: 4
     :start-after: [START howto_transfer_sql_to_s3]

tests/providers/amazon/aws/operators/test_s3_bucket_tagging_system.py → tests/system/providers/amazon/__init__.py

@@ -15,24 +15,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import pytest
-
-from tests.test_utils.amazon_system_helpers import AWS_DAG_FOLDER, AmazonSystemTest, provide_aws_context
-
-BUCKET_NAME = "AmazonSystemTestBucket"
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.system("amazon.aws")
-class ExampleDagsSystemTest(AmazonSystemTest):
-    @provide_aws_context()
-    def setUp(self):
-        super().setUp()
-
-    @provide_aws_context()
-    def tearDown(self):
-        super().tearDown()
-
-    @provide_aws_context()
-    def test_run_example_dag_bucket_tagging(self):
-        self.run_dag('s3_bucket_tagging_dag', AWS_DAG_FOLDER)

tests/system/providers/amazon/aws/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/system/providers/amazon/aws/s3/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/amazon/aws/example_dags/example_dynamodb_to_s3.py → tests/system/providers/amazon/aws/s3/example_dynamodb_to_s3.py

@@ -20,13 +20,16 @@
 from airflow import DAG
 from airflow.models.baseoperator import chain
 from airflow.providers.amazon.aws.transfers.dynamodb_to_s3 import DynamoDBToS3Operator
+from tests.system.providers.amazon.aws.utils import set_env_id
 
+ENV_ID = set_env_id()
+DAG_ID = 'example_dynamodb_to_s3'
 TABLE_NAME = environ.get('DYNAMO_TABLE_NAME', 'ExistingDynamoDbTableName')
 BUCKET_NAME = environ.get('S3_BUCKET_NAME', 'ExistingS3BucketName')
 
 
 with DAG(
-    dag_id='example_dynamodb_to_s3',
+    dag_id=DAG_ID,
     schedule_interval=None,
     start_date=datetime(2021, 1, 1),
     tags=['example'],
@@ -70,3 +73,8 @@
     # [END howto_transfer_dynamodb_to_s3_segmented]
 
     chain(backup_db, [backup_db_segment_1, backup_db_segment_2])
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)