diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb4331b930c5..e7f3aab629ba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,23 @@ CHANGELOG ========= +1.3.23 +====== + +* feature:``aws support``: Update ``aws support`` command to + the latest version +* feature:``aws iam``: Update ``aws iam`` command to the latest + version +* feature:``aws emr``: Add ``--hive-site`` option to + ``aws emr create-cluster`` and ``aws emr install-application`` commands +* feature:``aws s3 sync``: Add an ``--exact-timestamps`` option + to the ``aws s3 sync`` command + (`issue 824 `__) +* bugfix:``aws ec2 copy-snapshot``: Fix bug when spaces in + the description caused the copy request to fail + (`issue botocore 321 `__) + + 1.3.22 ====== diff --git a/awscli/__init__.py b/awscli/__init__.py index ceb715fff443..cadbe8f64311 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -17,7 +17,7 @@ """ import os -__version__ = '1.3.22' +__version__ = '1.3.23' # # Get our data path to be added to botocore's search path diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py index ce5c87ce96d5..5a7333310f9b 100644 --- a/awscli/customizations/emr/applicationutils.py +++ b/awscli/customizations/emr/applicationutils.py @@ -33,9 +33,19 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None): if hive_version is None: hive_version = constants.LATEST step_list.append( - emrutils.build_hive_install_step( + _build_install_hive_step( region=parsed_globals.region, version=hive_version)) + args = app_config.get('Args') + if args is not None: + hive_site_path = _find_matching_arg( + key=constants.HIVE_SITE_KEY, args_list=args) + if hive_site_path is not None: + step_list.append( + _build_install_hive_site_step( + region=parsed_globals.region, + version=hive_version, + hive_site_path=hive_site_path)) elif app_name == constants.PIG: pig_version = app_config.get('Version') if pig_version is None: @@ -122,4 +132,47 @@ def build_impala_install_bootstrap_action(region, version, args=None): path=emrutils.build_s3_link( relative_path=constants.IMPALA_INSTALL_PATH, region=region), - args=args_list) \ No newline at end of file + args=args_list) + + +def _build_install_hive_step(region, version, + action_on_failure=constants.TERMINATE_CLUSTER): + step_args = [ + emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region), + constants.INSTALL_HIVE_ARG, + constants.BASE_PATH_ARG, + emrutils.build_s3_link(constants.HIVE_BASE_PATH), + constants.HIVE_VERSIONS, + version] + step = emrutils.build_step( + name=constants.INSTALL_HIVE_NAME, + action_on_failure=action_on_failure, + jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region), + args=step_args) + return step + + +def _build_install_hive_site_step(region, version, hive_site_path, + action_on_failure=constants.CANCEL_AND_WAIT): + step_args = [ + emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region), + constants.BASE_PATH_ARG, + emrutils.build_s3_link(constants.HIVE_BASE_PATH), + constants.INSTALL_HIVE_SITE_ARG, + hive_site_path, + constants.HIVE_VERSIONS, + version] + step = emrutils.build_step( + name=constants.INSTALL_HIVE_SITE_NAME, + action_on_failure=action_on_failure, + jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region), + args=step_args) + return step + + +def _find_matching_arg(key, args_list): + for arg in args_list: + if key in arg: + return arg + + return None diff --git a/awscli/customizations/emr/constants.py b/awscli/customizations/emr/constants.py index bb1709fcc8c0..ed854b435aff 100644 --- a/awscli/customizations/emr/constants.py +++ b/awscli/customizations/emr/constants.py @@ -102,6 +102,9 @@ INSTALL_PIG_NAME = 'Install Pig' INSTALL_HIVE_ARG = '--install-hive' INSTALL_HIVE_NAME = 'Install Hive' +HIVE_SITE_KEY = '--hive-site' +INSTALL_HIVE_SITE_ARG = '--install-hive-site' +INSTALL_HIVE_SITE_NAME = 'Install Hive Site Configuration' BASE_PATH_ARG = '--base-path' INSTALL_GANGLIA_NAME = 'Install Ganglia' INSTALL_HBASE_NAME = 'Install HBase' diff --git a/awscli/customizations/emr/emrutils.py b/awscli/customizations/emr/emrutils.py index 4fd931db3338..e77d03a1bc5a 100644 --- a/awscli/customizations/emr/emrutils.py +++ b/awscli/customizations/emr/emrutils.py @@ -129,14 +129,15 @@ def build_bootstrap_action( return ba_config -def build_s3_link(relative_path='', region=None): - if region and region != 'us-east-1': - return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path) - else: - return 's3://elasticmapreduce{0}'.format(relative_path) +def build_s3_link(relative_path='', region='us-east-1'): + if region is None: + region = 'us-east-1' + return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path) -def get_script_runner(region=None): +def get_script_runner(region='us-east-1'): + if region is None: + region = 'us-east-1' return build_s3_link( relative_path=constants.SCRIPT_RUNNER_PATH, region=region) @@ -164,23 +165,6 @@ def build_pig_install_step(region, version, return step -def build_hive_install_step(region, version, - action_on_failure=constants.TERMINATE_CLUSTER): - step_args = [ - build_s3_link(constants.HIVE_SCRIPT_PATH, region), - constants.INSTALL_HIVE_ARG, - constants.BASE_PATH_ARG, - build_s3_link(constants.HIVE_BASE_PATH), - constants.HIVE_VERSIONS, - version] - step = build_step( - name=constants.INSTALL_HIVE_NAME, - action_on_failure=action_on_failure, - jar=build_s3_link(constants.SCRIPT_RUNNER_PATH, region), - args=step_args) - return step - - def call(session, operation_object, parameters, region_name=None, endpoint_url=None, verify=None): # We could get an error from get_endpoint() about not having diff --git a/awscli/customizations/s3/comparator.py b/awscli/customizations/s3/comparator.py index 555bd24bf2c4..9c86eeee9e96 100644 --- a/awscli/customizations/s3/comparator.py +++ b/awscli/customizations/s3/comparator.py @@ -38,6 +38,10 @@ def __init__(self, params=None): if 'size_only' in params: self.compare_on_size_only = params['size_only'] + self.match_exact_timestamps = False + if 'exact_timestamps' in params: + self.match_exact_timestamps = params['exact_timestamps'] + def call(self, src_files, dest_files): """ This function preforms the actual comparisons. The parameters it takes @@ -198,6 +202,11 @@ def compare_time(self, src_file, dest_file): # at the source location. return False elif cmd == "download": + if self.match_exact_timestamps: + # An update is needed unless the + # timestamps match exactly. + return total_seconds(delta) == 0 + if total_seconds(delta) <= 0: return True else: diff --git a/awscli/customizations/s3/s3.py b/awscli/customizations/s3/s3.py index 1be544f053ba..ecff9a3a2077 100644 --- a/awscli/customizations/s3/s3.py +++ b/awscli/customizations/s3/s3.py @@ -813,6 +813,7 @@ def add_verify_ssl(self, parsed_globals): 'sse', 'storage-class', 'content-type', 'cache-control', 'content-disposition', 'content-encoding', 'content-language', + 'exact-timestamps', 'expires', 'size-only']}, 'ls': {'options': {'nargs': '?', 'default': 's3://'}, 'params': ['recursive'], 'default': 's3://', @@ -867,6 +868,11 @@ def add_verify_ssl(self, parsed_globals): 'size-only': {'options': {'action': 'store_true'}, 'documents': ('Makes the size of each key the only criteria used to ' 'decide whether to sync from source to destination.')}, + 'exact-timestamps': {'options': {'action': 'store_true'}, 'documents': + ('When syncing from S3 to local, same-sized items will be ' + 'ignored only when the timestamps match exactly. The ' + 'default behavior is to ignore same-sized items unless ' + 'the local version is newer than the S3 version.')}, 'index-document': {'options': {}, 'documents': ('A suffix that is appended to a request that is for a ' 'directory on the website endpoint (e.g. if the suffix ' diff --git a/awscli/examples/autoscaling/put-scheduled-update-group-action.rst b/awscli/examples/autoscaling/put-scheduled-update-group-action.rst index 81e1c0f0949c..4ef21b7a1623 100644 --- a/awscli/examples/autoscaling/put-scheduled-update-group-action.rst +++ b/awscli/examples/autoscaling/put-scheduled-update-group-action.rst @@ -8,7 +8,7 @@ The following example creates a scheduled action to scale on a recurring schedul aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4 -For more information, see `Scheduled Scaling`_ in the *Auto Scaling Developer Guide*. +For more information, see `Scheduled Scaling`__ in the *Auto Scaling Developer Guide*. -.. _`Scheduled Scaling`: http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/schedule_time.html +.. __: http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/schedule_time.html diff --git a/awscli/examples/elasticloadbalancing/apply-security-groups-to-load-balancer.rst b/awscli/examples/elb/apply-security-groups-to-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/apply-security-groups-to-load-balancer.rst rename to awscli/examples/elb/apply-security-groups-to-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/attach-load-balancer-to-subnets.rst b/awscli/examples/elb/attach-load-balancer-to-subnets.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/attach-load-balancer-to-subnets.rst rename to awscli/examples/elb/attach-load-balancer-to-subnets.rst diff --git a/awscli/examples/elasticloadbalancing/configure-health-check.rst b/awscli/examples/elb/configure-health-check.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/configure-health-check.rst rename to awscli/examples/elb/configure-health-check.rst diff --git a/awscli/examples/elasticloadbalancing/create-app-cookie-stickiness-policy.rst b/awscli/examples/elb/create-app-cookie-stickiness-policy.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/create-app-cookie-stickiness-policy.rst rename to awscli/examples/elb/create-app-cookie-stickiness-policy.rst diff --git a/awscli/examples/elasticloadbalancing/create-lb-cookie-stickiness-policy.rst b/awscli/examples/elb/create-lb-cookie-stickiness-policy.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/create-lb-cookie-stickiness-policy.rst rename to awscli/examples/elb/create-lb-cookie-stickiness-policy.rst diff --git a/awscli/examples/elasticloadbalancing/create-load-balancer-listeners.rst b/awscli/examples/elb/create-load-balancer-listeners.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/create-load-balancer-listeners.rst rename to awscli/examples/elb/create-load-balancer-listeners.rst diff --git a/awscli/examples/elasticloadbalancing/create-load-balancer-policy.rst b/awscli/examples/elb/create-load-balancer-policy.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/create-load-balancer-policy.rst rename to awscli/examples/elb/create-load-balancer-policy.rst diff --git a/awscli/examples/elasticloadbalancing/create-load-balancer.rst b/awscli/examples/elb/create-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/create-load-balancer.rst rename to awscli/examples/elb/create-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/delete-load-balancer-listeners.rst b/awscli/examples/elb/delete-load-balancer-listeners.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/delete-load-balancer-listeners.rst rename to awscli/examples/elb/delete-load-balancer-listeners.rst diff --git a/awscli/examples/elasticloadbalancing/delete-load-balancer-policy.rst b/awscli/examples/elb/delete-load-balancer-policy.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/delete-load-balancer-policy.rst rename to awscli/examples/elb/delete-load-balancer-policy.rst diff --git a/awscli/examples/elasticloadbalancing/delete-load-balancer.rst b/awscli/examples/elb/delete-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/delete-load-balancer.rst rename to awscli/examples/elb/delete-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/deregister-instances-from-load-balancer.rst b/awscli/examples/elb/deregister-instances-from-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/deregister-instances-from-load-balancer.rst rename to awscli/examples/elb/deregister-instances-from-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/describe-instance-health.rst b/awscli/examples/elb/describe-instance-health.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/describe-instance-health.rst rename to awscli/examples/elb/describe-instance-health.rst diff --git a/awscli/examples/elasticloadbalancing/describe-load-balancer-attributes.rst b/awscli/examples/elb/describe-load-balancer-attributes.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/describe-load-balancer-attributes.rst rename to awscli/examples/elb/describe-load-balancer-attributes.rst diff --git a/awscli/examples/elasticloadbalancing/describe-load-balancer-policies.rst b/awscli/examples/elb/describe-load-balancer-policies.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/describe-load-balancer-policies.rst rename to awscli/examples/elb/describe-load-balancer-policies.rst diff --git a/awscli/examples/elasticloadbalancing/describe-load-balancer-policy-types.rst b/awscli/examples/elb/describe-load-balancer-policy-types.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/describe-load-balancer-policy-types.rst rename to awscli/examples/elb/describe-load-balancer-policy-types.rst diff --git a/awscli/examples/elasticloadbalancing/describe-load-balancers.rst b/awscli/examples/elb/describe-load-balancers.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/describe-load-balancers.rst rename to awscli/examples/elb/describe-load-balancers.rst diff --git a/awscli/examples/elasticloadbalancing/detach-load-balancer-from-subnets.rst b/awscli/examples/elb/detach-load-balancer-from-subnets.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/detach-load-balancer-from-subnets.rst rename to awscli/examples/elb/detach-load-balancer-from-subnets.rst diff --git a/awscli/examples/elasticloadbalancing/disable-availaibility-zones-for-load-balancer.rst b/awscli/examples/elb/disable-availaibility-zones-for-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/disable-availaibility-zones-for-load-balancer.rst rename to awscli/examples/elb/disable-availaibility-zones-for-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/enable-availability-zones-for-load-balancer.rst b/awscli/examples/elb/enable-availability-zones-for-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/enable-availability-zones-for-load-balancer.rst rename to awscli/examples/elb/enable-availability-zones-for-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/modify-load-balancer-attributes.rst b/awscli/examples/elb/modify-load-balancer-attributes.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/modify-load-balancer-attributes.rst rename to awscli/examples/elb/modify-load-balancer-attributes.rst diff --git a/awscli/examples/elasticloadbalancing/register-instances-with-load-balancer.rst b/awscli/examples/elb/register-instances-with-load-balancer.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/register-instances-with-load-balancer.rst rename to awscli/examples/elb/register-instances-with-load-balancer.rst diff --git a/awscli/examples/elasticloadbalancing/set-load-balancer-listener-ssl-certificate.rst b/awscli/examples/elb/set-load-balancer-listener-ssl-certificate.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/set-load-balancer-listener-ssl-certificate.rst rename to awscli/examples/elb/set-load-balancer-listener-ssl-certificate.rst diff --git a/awscli/examples/elasticloadbalancing/set-load-balancer-policies-for-backend-server.rst b/awscli/examples/elb/set-load-balancer-policies-for-backend-server.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/set-load-balancer-policies-for-backend-server.rst rename to awscli/examples/elb/set-load-balancer-policies-for-backend-server.rst diff --git a/awscli/examples/elasticloadbalancing/set-load-balancer-policies-of-listener.rst b/awscli/examples/elb/set-load-balancer-policies-of-listener.rst similarity index 100% rename from awscli/examples/elasticloadbalancing/set-load-balancer-policies-of-listener.rst rename to awscli/examples/elb/set-load-balancer-policies-of-listener.rst diff --git a/awscli/examples/emr/create-cluster-examples.rst b/awscli/examples/emr/create-cluster-examples.rst index d1f5339bd1bf..36932a40add5 100644 --- a/awscli/examples/emr/create-cluster-examples.rst +++ b/awscli/examples/emr/create-cluster-examples.rst @@ -9,7 +9,7 @@ - Command:: aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - + **3. Create an Amazon EMR cluster with default roles - Command:: @@ -20,31 +20,31 @@ - Command:: - aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2 + aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2 **5. Specify whether the cluster should terminate after completing all the steps** - Create an Amazon EMR cluster that will terminate after completing all the steps:: - aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - Create an Amazon EMR cluster that will NOT terminate after completing all the steps:: - aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate + aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate **6. Specify EC2 Attributes** - Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile":: - aws emr create-cluster --ec2-attributes KeyName=myKey,InstanceProfile=myRole --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --ec2-attributes KeyName=myKey,InstanceProfile=myRole --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - Create an Amazon EMR cluster in an Amazon VPC subnet:: - aws emr create-cluster --ec2-attributes SubnetId=subnet-xxxxx --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --ec2-attributes SubnetId=subnet-xxxxx --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - Create an Amazon EMR cluster in an AvailabilityZone. For example, us-west-1b:: - aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate **7. Enable debugging and specify a Log URI** @@ -56,7 +56,7 @@ - Add a list of tags:: - aws emr create-cluster --tags name="John Doe" age=29 address="123 East NW Seattle" --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --tags name="John Doe" age=29 address="123 East NW Seattle" --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - List tags of an Amazon EMR cluster:: @@ -66,7 +66,7 @@ - Command:: - aws emr create-cluster --bootstrap-actions Path=s3://mybucket/myscript1,Name=BootstrapAction1,Args=[arg1,arg2] Path=s3://mybucket/myscript2,Name=BootstrapAction2,Args=[arg1,arg2] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --bootstrap-actions Path=s3://mybucket/myscript1,Name=BootstrapAction1,Args=[arg1,arg2] Path=s3://mybucket/myscript2,Name=BootstrapAction2,Args=[arg1,arg2] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - The following example changes the maximum number of map tasks and sets the NameNode heap size:: @@ -76,7 +76,7 @@ - Create an Amazon EMR cluster with Hive, Pig, HBase, Ganglia, and Impala installed:: - aws emr create-cluster --applications Name=Hive Name=Pig Name=HBase Name=Ganglia Name=Impala,Args=[IMPALA_BACKEND_PORT=22001,IMPALA_MEM_LIMIT=70%] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + aws emr create-cluster --applications Name=Hive Name=Pig Name=HBase Name=Ganglia Name=Impala,Args=[IMPALA_BACKEND_PORT=22001,IMPALA_MEM_LIMIT=70%] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - Create an Amazon EMR cluster with Hive and Pig installed:: @@ -109,7 +109,7 @@ **13. To add Streaming steps when creating an Amazon EMR cluster** - Command:: - + aws emr create-cluster --steps Type=STREAMING,Name='Streaming Program',ActionOnFailure=CONTINUE,Args=-mapper,mymapper,-reducer,myreducer,-input,myinput,-output,myoutput Type=STREAMING,Name='Streaming Program',ActionOnFailure=CONTINUE,Args=--files,s3://elasticmapreduce/samples/wordcount/wordSplitter.py,-mapper,wordSplitter.py,-reducer,aggregate,-input,s3://elasticmapreduce/samples/wordcount/input,-output,s3://mybucket/wordcount/output --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate - Streaming steps required parameters:: diff --git a/awscli/examples/emr/create-cluster-synopsis.rst b/awscli/examples/emr/create-cluster-synopsis.rst index b37981df62f4..ff999cfb904f 100644 --- a/awscli/examples/emr/create-cluster-synopsis.rst +++ b/awscli/examples/emr/create-cluster-synopsis.rst @@ -3,7 +3,7 @@ --instance-groups --auto-terminate | --no-auto-terminate [--use-default-role] - [--service-role ] + [--service-role ] [--name ] [--log-uri ] [--additional-info ] @@ -15,4 +15,4 @@ [--applications ] [--bootstrap-actions ] [--steps ] - [--restore-from-hbase-backup ] \ No newline at end of file + [--restore-from-hbase-backup ] diff --git a/awscli/examples/emr/create-default-roles.rst b/awscli/examples/emr/create-default-roles.rst index 77b73ea2da1e..dac9e30b76cd 100644 --- a/awscli/examples/emr/create-default-roles.rst +++ b/awscli/examples/emr/create-default-roles.rst @@ -10,102 +10,102 @@ If the role does not exist then the output will be: - [ - { - "RolePolicy": { - "Statement": [ - { - "Action": [ - "cloudwatch:*", - "dynamodb:*", - "ec2:Describe*", - "elasticmapreduce:Describe*", - "rds:Describe*", - "s3:*", - "sdb:*", - "sns:*", - "sqs:*" - ], - "Resource": [ - "*" - ], - "Effect": "Allow" - } - ] - }, - "Role": { - "AssumeRolePolicyDocument": { - "Version": "2008-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Sid": "", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ] - }, - "RoleId": "AROAJEATMSYEQGRVYQDX4", - "CreateDate": "2014-06-14T01:05:15.356Z", - "RoleName": "EMR_EC2_DefaultRole", - "Path": "/", - "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" - } - }, - { - "RolePolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CancelSpotInstanceRequests", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:Describe*", - "ec2:DeleteTags", - "ec2:ModifyImageAttribute", - "ec2:ModifyInstanceAttribute", - "ec2:RequestSpotInstances", - "ec2:RunInstances", - "ec2:TerminateInstances", - "iam:PassRole", - "iam:ListRolePolicies", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:ListInstanceProfiles", - "s3:Get*", - "s3:List*", - "s3:CreateBucket", - "sdb:BatchPutAttributes", - "sdb:Select" - ], - "Resource": "*", - "Effect": "Allow" - } - ] - }, - "Role": { - "AssumeRolePolicyDocument": { - "Version": "2008-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Sid": "", - "Effect": "Allow", - "Principal": { - "Service": "elasticmapreduce.amazonaws.com" - } - } - ] - }, - "RoleId": "AROAJRHC33G6KRX5D5QF2", - "CreateDate": "2014-06-14T01:05:17.464Z", - "RoleName": "EMR_DefaultRole", - "Path": "/", - "Arn": "arn:aws:iam::176430881729:role/EMR_DefaultRole" - } - } - ] + [ + { + "RolePolicy": { + "Statement": [ + { + "Action": [ + "cloudwatch:*", + "dynamodb:*", + "ec2:Describe*", + "elasticmapreduce:Describe*", + "rds:Describe*", + "s3:*", + "sdb:*", + "sns:*", + "sqs:*" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJEATMSYEQGRVYQDX4", + "CreateDate": "2014-06-14T01:05:15.356Z", + "RoleName": "EMR_EC2_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" + } + }, + { + "RolePolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:Describe*", + "ec2:DeleteTags", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:RequestSpotInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListInstanceProfiles", + "s3:Get*", + "s3:List*", + "s3:CreateBucket", + "sdb:BatchPutAttributes", + "sdb:Select" + ], + "Resource": "*", + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "elasticmapreduce.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJRHC33G6KRX5D5QF2", + "CreateDate": "2014-06-14T01:05:17.464Z", + "RoleName": "EMR_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_DefaultRole" + } + } + ] diff --git a/doc/source/conf.py b/doc/source/conf.py index c1c3d0749e78..a91ca5d3c1e0 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.3.' # The full version, including alpha/beta/rc tags. -release = '1.3.22' +release = '1.3.23' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d6f057645227..a3e87200173a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import awscli -requires = ['botocore>=0.56.0,<0.57.0', +requires = ['botocore>=0.57.0,<0.58.0', 'bcdoc>=0.12.0,<0.13.0', 'six>=1.1.0', 'colorama==0.2.5', diff --git a/tests/unit/customizations/emr/test_add_steps.py b/tests/unit/customizations/emr/test_add_steps.py index a35aa3d91492..d81a858ae231 100644 --- a/tests/unit/customizations/emr/test_add_steps.py +++ b/tests/unit/customizations/emr/test_add_steps.py @@ -45,9 +45,11 @@ class TestAddSteps(BaseAWSCommandParamsTest): 's3://elasticmapreduce/samples/hive-ads/libs/model-build.q' HIVE_DEFAULT_HADOOP_JAR_STEP = \ - {'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + {'Jar': + ('s3://us-east-1.elasticmapreduce/' + 'libs/script-runner/script-runner.jar'), 'Args': - ['s3://elasticmapreduce/libs/hive/hive-script', + ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--run-hive-script', '--hive-versions', 'latest', @@ -58,9 +60,11 @@ class TestAddSteps(BaseAWSCommandParamsTest): } HIVE_BASIC_HADOOP_JAR_STEP = \ - {'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + {'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': - ['s3://elasticmapreduce/libs/hive/hive-script', + ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--run-hive-script', '--hive-versions', '0.11.0.1', @@ -74,9 +78,11 @@ class TestAddSteps(BaseAWSCommandParamsTest): 's3://elasticmapreduce/samples/pig-apache/do-reports2.pig' PIG_DEFAULT_HADOOP_JAR_STEP = \ - {'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + {'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': - ['s3://elasticmapreduce/libs/pig/pig-script', + ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--run-pig-script', '--pig-versions', 'latest', @@ -87,9 +93,11 @@ class TestAddSteps(BaseAWSCommandParamsTest): ]} PIG_BASIC_HADOOP_JAR_STEP = \ - {'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + {'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': - ['s3://elasticmapreduce/libs/pig/pig-script', + ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--run-pig-script', '--pig-versions', '0.11.1.0', @@ -104,9 +112,11 @@ class TestAddSteps(BaseAWSCommandParamsTest): '--console-output-path,s3://myimpala/output' IMPALA_BASIC_HADOOP_JAR_STEP = \ - {'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + {'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': - ['s3://elasticmapreduce/libs/impala/setup-impala', + ['s3://us-east-1.elasticmapreduce/libs/impala/setup-impala', '--run-impala-script', '--impala-script', 's3://myimpala/input', diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 3c678258404c..d44025b64da0 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -91,23 +91,43 @@ INSTALL_HIVE_STEP = { 'HadoopJarStep': { - 'Args': ['s3://elasticmapreduce/libs/hive/hive-script', + 'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--install-hive', '--base-path', - 's3://elasticmapreduce/libs/hive', + 's3://us-east-1.elasticmapreduce/libs/hive', '--hive-versions', 'latest'], - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar' + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') }, 'Name': 'Install Hive', 'ActionOnFailure': 'TERMINATE_CLUSTER' } +INSTALL_HIVE_SITE_STEP = { + 'HadoopJarStep': { + 'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', + '--base-path', + 's3://us-east-1.elasticmapreduce/libs/hive', + '--install-hive-site', + '--hive-site=s3://test/hive-conf/hive-site.xml', + '--hive-versions', 'latest'], + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') + }, + 'Name': 'Install Hive Site Configuration', + 'ActionOnFailure': 'CANCEL_AND_WAIT' +} + INSTALL_PIG_STEP = { 'HadoopJarStep': { - 'Args': ['s3://elasticmapreduce/libs/pig/pig-script', + 'Args': ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--install-pig', '--base-path', - 's3://elasticmapreduce/libs/pig', + 's3://us-east-1.elasticmapreduce/libs/pig', '--pig-versions', 'latest'], - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar' + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') }, 'Name': 'Install Pig', 'ActionOnFailure': 'TERMINATE_CLUSTER' @@ -125,22 +145,24 @@ INSTALL_GANGLIA_BA = { 'ScriptBootstrapAction': { - 'Path': 's3://elasticmapreduce/bootstrap-actions/install-ganglia' + 'Path': + ('s3://us-east-1.elasticmapreduce/' + 'bootstrap-actions/install-ganglia') }, 'Name': 'Install Ganglia' } INSTALL_HBASE_BA = { 'ScriptBootstrapAction': { - 'Path': 's3://elasticmapreduce/bootstrap-actions/setup-hbase' + 'Path': 's3://us-east-1.elasticmapreduce/bootstrap-actions/setup-hbase' }, 'Name': 'Install HBase' } INSTALL_IMPALA_BA = { 'ScriptBootstrapAction': { - 'Path': 's3://elasticmapreduce/libs/impala/setup-impala', - 'Args': ['--base-path', 's3://elasticmapreduce', + 'Path': 's3://us-east-1.elasticmapreduce/libs/impala/setup-impala', + 'Args': ['--base-path', 's3://us-east-1.elasticmapreduce', '--impala-version', 'latest'] }, 'Name': 'Install Impala' @@ -198,9 +220,11 @@ 'Name': 'Hive program', 'ActionOnFailure': 'CONTINUE', 'HadoopJarStep': { - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': [ - 's3://elasticmapreduce/libs/hive/hive-script', + 's3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--run-hive-script', '--hive-versions', 'latest', @@ -213,9 +237,11 @@ 'Name': 'HiveBasicStep', 'ActionOnFailure': 'CANCEL_AND_WAIT', 'HadoopJarStep': { - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': [ - 's3://elasticmapreduce/libs/hive/hive-script', + 's3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--run-hive-script', '--hive-versions', '0.11.0.1', @@ -231,9 +257,11 @@ 'Name': 'Pig program', 'ActionOnFailure': 'CONTINUE', 'HadoopJarStep': { - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': [ - 's3://elasticmapreduce/libs/pig/pig-script', + 's3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--run-pig-script', '--pig-versions', 'latest', @@ -247,9 +275,11 @@ 'Name': 'PigBasicStep', 'ActionOnFailure': 'CANCEL_AND_WAIT', 'HadoopJarStep': { - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': [ - 's3://elasticmapreduce/libs/pig/pig-script', + 's3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--run-pig-script', '--pig-versions', '0.11.1.0', @@ -266,9 +296,11 @@ 'Name': 'Impala program', 'ActionOnFailure': 'CONTINUE', 'HadoopJarStep': { - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar', + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar'), 'Args': [ - 's3://elasticmapreduce/libs/impala/setup-impala', + 's3://us-east-1.elasticmapreduce/libs/impala/setup-impala', '--run-impala-script', '--impala-script', 's3://myimpala/input', @@ -432,9 +464,11 @@ def test_enable_debugging(self): [{'Name': 'Setup Hadoop Debugging', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': - {'Args': ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'], + {'Args': + [('s3://us-east-1.elasticmapreduce/libs/' + 'state-pusher/0.1/fetch')], 'Jar': - 's3://elasticmapreduce/libs/' + + 's3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar' } }] @@ -694,6 +728,16 @@ def test_install_hive_with_version(self): result['Steps'] = [steps] self.assert_params_for_cmd(cmd, result) + def test_install_hive_site(self): + cmdline = (DEFAULT_CMD + '--applications Name=Hive,' + 'Args=[--hive-site=s3://test/hive-conf/hive-site.xml]') + result = copy.deepcopy(DEFAULT_RESULT) + result['Steps'] = [INSTALL_HIVE_STEP, INSTALL_HIVE_SITE_STEP] + self.assert_params_for_cmd(cmdline, result) + cmdline = (DEFAULT_CMD + '--applications Name=Hive,' + 'Args=[--hive-site=s3://test/hive-conf/hive-site.xml,k1]') + self.assert_params_for_cmd(cmdline, result) + def test_install_pig_with_defaults(self): cmd = DEFAULT_CMD + '--applications Name=Pig' result = copy.deepcopy(DEFAULT_RESULT) diff --git a/tests/unit/customizations/emr/test_install_applications.py b/tests/unit/customizations/emr/test_install_applications.py index 9029627a2f6d..51a27eeffa0a 100644 --- a/tests/unit/customizations/emr/test_install_applications.py +++ b/tests/unit/customizations/emr/test_install_applications.py @@ -18,23 +18,43 @@ INSTALL_HIVE_STEP = { 'HadoopJarStep': { - 'Args': ['s3://elasticmapreduce/libs/hive/hive-script', + 'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', '--install-hive', '--base-path', - 's3://elasticmapreduce/libs/hive', + 's3://us-east-1.elasticmapreduce/libs/hive', '--hive-versions', 'latest'], - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar' + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') }, 'Name': 'Install Hive', 'ActionOnFailure': 'TERMINATE_CLUSTER' } +INSTALL_HIVE_SITE_STEP = { + 'HadoopJarStep': { + 'Args': ['s3://us-east-1.elasticmapreduce/libs/hive/hive-script', + '--base-path', + 's3://us-east-1.elasticmapreduce/libs/hive', + '--install-hive-site', + '--hive-site=s3://test/hive-conf/hive-site.xml', + '--hive-versions', 'latest'], + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') + }, + 'Name': 'Install Hive Site Configuration', + 'ActionOnFailure': 'CANCEL_AND_WAIT' +} + INSTALL_PIG_STEP = { 'HadoopJarStep': { - 'Args': ['s3://elasticmapreduce/libs/pig/pig-script', + 'Args': ['s3://us-east-1.elasticmapreduce/libs/pig/pig-script', '--install-pig', '--base-path', - 's3://elasticmapreduce/libs/pig', + 's3://us-east-1.elasticmapreduce/libs/pig', '--pig-versions', 'latest'], - 'Jar': 's3://elasticmapreduce/libs/script-runner/script-runner.jar' + 'Jar': + ('s3://us-east-1.elasticmapreduce/libs/' + 'script-runner/script-runner.jar') }, 'Name': 'Install Pig', 'ActionOnFailure': 'TERMINATE_CLUSTER' @@ -42,10 +62,11 @@ class TestInstallApplications(BaseAWSCommandParamsTest): - prefix = 'emr install-applications --cluster-id j-ABC123456' + prefix = ('emr install-applications --cluster-id ' + 'j-ABC123456 --applications ') def test_intall_hive_with_version(self): - cmdline = self.prefix + ' --applications Name=Hive,Version=0.8.1.8' + cmdline = self.prefix + 'Name=Hive,Version=0.8.1.8' step = copy.deepcopy(INSTALL_HIVE_STEP) step['HadoopJarStep']['Args'][5] = '0.8.1.8' @@ -53,8 +74,19 @@ def test_intall_hive_with_version(self): result = {'JobFlowId': 'j-ABC123456', 'Steps': [step]} self.assert_params_for_cmd(cmdline, result) + def test_install_hive_site(self): + cmdline = (self.prefix + 'Name=Hive,' + 'Args=[--hive-site=s3://test/hive-conf/hive-site.xml]') + result = {'JobFlowId': 'j-ABC123456', + 'Steps': [INSTALL_HIVE_STEP, INSTALL_HIVE_SITE_STEP] + } + self.assert_params_for_cmd(cmdline, result) + cmdline = (self.prefix + 'Name=Hive,' + 'Args=[--hive-site=s3://test/hive-conf/hive-site.xml,k1]') + self.assert_params_for_cmd(cmdline, result) + def test_intall_pig_with_version(self): - cmdline = self.prefix + ' --applications Name=Pig,Version=0.9.2.1' + cmdline = self.prefix + 'Name=Pig,Version=0.9.2.1' step = copy.deepcopy(INSTALL_PIG_STEP) step['HadoopJarStep']['Args'][5] = '0.9.2.1' @@ -63,15 +95,13 @@ def test_intall_pig_with_version(self): self.assert_params_for_cmd(cmdline, result) def test_intall_hive_and_pig_without_version(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --applications Name=Hive' +\ - ' Name=Pig' + cmdline = self.prefix + 'Name=Hive Name=Pig' result = {'JobFlowId': 'j-ABC123456', 'Steps': [INSTALL_HIVE_STEP, INSTALL_PIG_STEP]} self.assert_params_for_cmd(cmdline, result) def test_install_impala_error(self): - cmdline = self.prefix + \ - ' --cluster-id j-ABC123456 --applications Name=Impala' + cmdline = self.prefix + ' Name=Impala' expected_error_msg = "\naws: error: Impala cannot be installed on" +\ " a running cluster. 'Name' should be one of the following:" +\ @@ -80,8 +110,7 @@ def test_install_impala_error(self): self.assertEqual(result[1], expected_error_msg) def test_install_unknown_app_error(self): - cmdline = self.prefix + \ - ' --cluster-id j-ABC123456 --applications Name=unknown' + cmdline = self.prefix + 'Name=unknown' expected_error_msg = "\naws: error: Unknown application: unknown." +\ " 'Name' should be one of the following: HIVE, PIG, HBASE," +\ diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index e0a983e90517..d0d5652d1bc9 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -351,5 +351,104 @@ def test_compare_size_only_src_older_than_dest(self): self.assertEqual(sum(1 for _ in files), 0) +class ComparatorExactTimestampsTest(unittest.TestCase): + def setUp(self): + self.comparator = Comparator({'exact_timestamps': True}) + + def test_compare_exact_timestamps_dest_older(self): + """ + Confirm that same-sized files are synced when + the destination is older than the source and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() + time_dst = time_src - datetime.timedelta(days=1) + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + def test_compare_exact_timestamps_src_older(self): + """ + Confirm that same-sized files are synced when + the source is older than the destination and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() - datetime.timedelta(days=1) + time_dst = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + def test_compare_exact_timestamps_same_age_same_size(self): + """ + Confirm that same-sized files are not synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 0) + + def test_compare_exact_timestamps_same_age_diff_size(self): + """ + Confirm that files of differing sizes are synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=20, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + if __name__ == "__main__": unittest.main()