Remove deprecated modules from Amazon provider package

[vincbeck]

Follow up of #25543. @potiuk

[potiuk]

Yep. Waiting for the tests to complete :)

[potiuk]

stilll a number of tests failing :( @vincbeck

[Taragolis]

@vincbeck

Selected tests not relevant anymore and could be removed

test_get_client_type_returns_a_boto3_client_of_the_requested_type

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 223 to 232 in 72a6ac5

	@unittest.skipIf(mock_emr is None, 'mock_emr package not present')
	@mock_emr
	def test_get_client_type_returns_a_boto3_client_of_the_requested_type(self):
	client = boto3.client('emr', region_name='us-east-1')
	if client.list_clusters()['Clusters']:
	raise ValueError('AWS not properly mocked')
	hook = AwsBaseHook(aws_conn_id='aws_default', client_type='emr')
	client_from_hook = hook.get_client_type('emr')
	assert client_from_hook.list_clusters()['Clusters'] == []

test_get_client_type_overwrite

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 245 to 253 in 72a6ac5

	@unittest.skipIf(mock_emr is None, 'mock_emr package not present')
	@mock_emr
	def test_get_client_type_overwrite(self):
	client = boto3.client('emr', region_name='us-east-1')
	if client.list_clusters()['Clusters']:
	raise ValueError('AWS not properly mocked')
	hook = AwsBaseHook(aws_conn_id='aws_default', client_type='dynamodb')
	client_from_hook = hook.get_client_type(client_type='emr')
	assert client_from_hook.list_clusters()['Clusters'] == []

test_get_client_type_deprecation_warning

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 255 to 262 in 72a6ac5

	@unittest.skipIf(mock_emr is None, 'mock_emr package not present')
	@mock_emr
	def test_get_client_type_deprecation_warning(self):
	hook = AwsBaseHook(aws_conn_id='aws_default', client_type='emr')
	warning_message = """client_type is deprecated. Set client_type from class attribute."""
	with pytest.warns(DeprecationWarning) as warnings:
	hook.get_client_type(client_type='emr')
	assert warning_message in [str(w.message) for w in warnings]

test_get_resource_type_returns_a_boto3_resource_of_the_requested_type

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 264 to 282 in 72a6ac5

	@unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamo2 package not present')
	@mock_dynamodb2
	def test_get_resource_type_returns_a_boto3_resource_of_the_requested_type(self):
	hook = AwsBaseHook(aws_conn_id='aws_default', resource_type='dynamodb')
	resource_from_hook = hook.get_resource_type('dynamodb')
	# this table needs to be created in production
	table = resource_from_hook.create_table(
	TableName='test_airflow',
	KeySchema=[
	{'AttributeName': 'id', 'KeyType': 'HASH'},
	],
	AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}],
	ProvisionedThroughput={'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10},
	)
	table.meta.client.get_waiter('table_exists').wait(TableName='test_airflow')
	assert table.item_count == 0

test_get_resource_type_overwrite

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 304 to 322 in 72a6ac5

	@unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamo2 package not present')
	@mock_dynamodb2
	def test_get_resource_type_overwrite(self):
	hook = AwsBaseHook(aws_conn_id='aws_default', resource_type='s3')
	resource_from_hook = hook.get_resource_type('dynamodb')
	# this table needs to be created in production
	table = resource_from_hook.create_table(
	TableName='test_airflow',
	KeySchema=[
	{'AttributeName': 'id', 'KeyType': 'HASH'},
	],
	AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}],
	ProvisionedThroughput={'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10},
	)
	table.meta.client.get_waiter('table_exists').wait(TableName='test_airflow')
	assert table.item_count == 0

test_get_resource_deprecation_warning

airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

Lines 324 to 331 in 72a6ac5

	@unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamo2 package not present')
	@mock_dynamodb2
	def test_get_resource_deprecation_warning(self):
	hook = AwsBaseHook(aws_conn_id='aws_default', resource_type='dynamodb')
	warning_message = """resource_type is deprecated. Set resource_type from class attribute."""
	with pytest.warns(DeprecationWarning) as warnings:
	hook.get_resource_type('dynamodb')
	assert warning_message in [str(w.message) for w in warnings]

---

test_refreshable_credentials uses removed attribute

Quick hotfix:

1. In hook specify client_type='sts'
   

   airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

   Lines 623 to 625 in 72a6ac5

   	mock_get_connection.return_value = mock_connection
   	hook = AwsBaseHook(aws_conn_id='aws_default', client_type='airflow_test')

2. hook.get_client_type('sts') -> hook.get_client_type()
   

   airflow/tests/providers/amazon/aws/hooks/test_base_aws.py

   Lines 662 to 664 in 72a6ac5

   	mock_refresh.side_effect = mock_refresh_credentials
   	client = hook.get_client_type('sts')
   	client.get_caller_identity()

[vincbeck]

Awesome! Thank you so much @Taragolis

[eladkal]

CI is not happy :(
some failures related to TestSsmSecrets

[potiuk]

I prepared the #25618 - but I can still include that one if we fix the issues

[potiuk]

🤞 @vincbeck - I think this is the last one I want to include in this wave of providers :) (I will make one more pass to see)

[vincbeck]

Some tests were canceled. Any idea @potiuk ?

[potiuk]

Just one - likely an intermittent error. Re-run it @vincbeck

[vincbeck]

YAY!

[potiuk]

🎉 🎉 🎉 🎉 🎉