diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a5604112664d..2128fc9834fb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ CHANGELOG Next Release (TBD) ================== +* bugfix:``aws datapipeline``: Fix issue when serializing + pipeline definitions containing list elements + (`issue 705 `__) * bugfix:``aws s3``: Fix issue when recursively removing keys containing control characters (`issue 675 `__) diff --git a/awscli/customizations/datapipeline/translator.py b/awscli/customizations/datapipeline/translator.py index d52198de75e2..48104cb7c438 100644 --- a/awscli/customizations/datapipeline/translator.py +++ b/awscli/customizations/datapipeline/translator.py @@ -44,17 +44,25 @@ def definition_to_api(definition): # with a 'key', 'stringValue'|'refValue' fields = [] for key, value in sorted(element.items()): - field = {'key': key} - if isinstance(value, dict) and list(value.keys()) == ['ref']: - field['refValue'] = value['ref'] + if isinstance(value, list): + for item in value: + fields.append(_convert_single_field(key, item)) else: - field['stringValue'] = value - fields.append(field) + fields.append(_convert_single_field(key, value)) api_object['fields'] = fields api_elements.append(api_object) return api_elements +def _convert_single_field(key, value): + field = {'key': key} + if isinstance(value, dict) and list(value.keys()) == ['ref']: + field['refValue'] = value['ref'] + else: + field['stringValue'] = value + return field + + def api_to_definition(api_response): # When we're translating from api_response -> definition # we have to be careful *not* to mutate the existing diff --git a/tests/unit/customizations/datapipeline/test_translator.py b/tests/unit/customizations/datapipeline/test_translator.py index 5eeb124ee166..9696257e3cd4 100644 --- a/tests/unit/customizations/datapipeline/test_translator.py +++ b/tests/unit/customizations/datapipeline/test_translator.py @@ -12,7 +12,6 @@ # language governing permissions and limitations under the License. from tests import unittest -import mock from botocore.compat import OrderedDict, json from awscli.customizations.datapipeline import translator @@ -114,6 +113,44 @@ def test_missing_id_field(self): with self.assertRaises(translator.PipelineDefinitionError): translator.definition_to_api(definition) + def test_list_value_with_strings(self): + definition = self.load_def("""{"objects": [ + { + "id" : "emrActivity", + "type" : "EmrActivity", + "name" : "Foo", + "step" : ["s3://foo1", "s3://foo2", "s3://foo3"] + } + ]}""") + actual = translator.definition_to_api(definition) + api = [{"name": "Foo", "id": "emrActivity", + "fields": [ + {"key": "step", "stringValue": "s3://foo1"}, + {"key": "step", "stringValue": "s3://foo2"}, + {"key": "step", "stringValue": "s3://foo3"}, + {"key": "type", "stringValue": "EmrActivity"}, + ]}] + self.assertEqual(actual, api) + + def test_value_with_refs(self): + definition = self.load_def("""{"objects": [ + { + "id" : "emrActivity", + "type" : "EmrActivity", + "name" : "Foo", + "step" : ["s3://foo1", {"ref": "otherValue"}, "s3://foo3"] + } + ]}""") + actual = translator.definition_to_api(definition) + api = [{"name": "Foo", "id": "emrActivity", + "fields": [ + {"key": "step", "stringValue": "s3://foo1"}, + {"key": "step", "refValue": "otherValue"}, + {"key": "step", "stringValue": "s3://foo3"}, + {"key": "type", "stringValue": "EmrActivity"}, + ]}] + self.assertEqual(actual, api) + # These tests check the API -> DF conversion. def test_api_to_df(self): api = [{"name": "S3ToS3Copy", "id": "S3ToS3Copy",