diff --git a/docs/apache-airflow-providers-amazon/operators/s3/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst index 516b1712446e0..41e5c7149bb18 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3/s3.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst @@ -222,6 +222,14 @@ To check multiple files: :start-after: [START howto_sensor_s3_key_multiple_keys] :end-before: [END howto_sensor_s3_key_multiple_keys] +To check a file with regular expression: + +.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_sensor_s3_key_regex] + :end-before: [END howto_sensor_s3_key_regex] + To check with an additional custom check you can define a function which receives a list of matched S3 object attributes and returns a boolean: @@ -268,6 +276,14 @@ To check multiple files: :start-after: [START howto_sensor_s3_key_multiple_keys_deferrable] :end-before: [END howto_sensor_s3_key_multiple_keys_deferrable] +To check a file with regular expression: + +.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_sensor_s3_key_regex_deferrable] + :end-before: [END howto_sensor_s3_key_regex_deferrable] + .. _howto/sensor:S3KeysUnchangedSensor: Wait on Amazon S3 prefix changes diff --git a/tests/system/providers/amazon/aws/example_s3.py b/tests/system/providers/amazon/aws/example_s3.py index 87806a0bf0a5a..3b4a4bd38bf6f 100644 --- a/tests/system/providers/amazon/aws/example_s3.py +++ b/tests/system/providers/amazon/aws/example_s3.py @@ -71,6 +71,8 @@ key = f"{env_id}-key" key_2 = f"{env_id}-key2" + key_regex_pattern = ".*-key" + # [START howto_sensor_s3_key_function_definition] def check_fn(files: list) -> bool: """ @@ -191,7 +193,7 @@ def check_fn(files: list) -> bool: ) # [END howto_sensor_s3_key_multiple_keys_deferrable] - # [START howto_sensor_s3_key_function] + # [START howto_sensor_s3_key_function_deferrable] # Check if a file exists and match a certain pattern defined in check_fn sensor_key_with_function_deferrable = S3KeySensor( task_id="sensor_key_with_function_deferrable", @@ -200,7 +202,18 @@ def check_fn(files: list) -> bool: check_fn=check_fn, deferrable=True, ) - # [END howto_sensor_s3_key_function] + # [END howto_sensor_s3_key_function_deferrable] + + # [START howto_sensor_s3_key_regex_deferrable] + # Check if a file exists and match a certain regular expression pattern + sensor_key_with_regex_deferrable = S3KeySensor( + task_id="sensor_key_with_regex_deferrable", + bucket_name=bucket_name, + bucket_key=key_regex_pattern, + use_regex=True, + deferrable=True, + ) + # [END howto_sensor_s3_key_regex_deferrable] # [START howto_sensor_s3_key_function] # Check if a file exists and match a certain pattern defined in check_fn @@ -212,6 +225,13 @@ def check_fn(files: list) -> bool: ) # [END howto_sensor_s3_key_function] + # [START howto_sensor_s3_key_regex] + # Check if a file exists and match a certain regular expression pattern + sensor_key_with_regex = S3KeySensor( + task_id="sensor_key_with_regex", bucket_name=bucket_name, bucket_key=key_regex_pattern, use_regex=True + ) + # [END howto_sensor_s3_key_regex] + # [START howto_operator_s3_copy_object] copy_object = S3CopyObjectOperator( task_id="copy_object", @@ -286,8 +306,13 @@ def check_fn(files: list) -> bool: create_object_2, list_prefixes, list_keys, - [sensor_one_key, sensor_two_keys, sensor_key_with_function], - [sensor_one_key_deferrable, sensor_two_keys_deferrable, sensor_key_with_function_deferrable], + [sensor_one_key, sensor_two_keys, sensor_key_with_function, sensor_key_with_regex], + [ + sensor_one_key_deferrable, + sensor_two_keys_deferrable, + sensor_key_with_function_deferrable, + sensor_key_with_regex_deferrable, + ], copy_object, file_transform, branching,