Skip to content

Commit

Permalink
EMR: Added default configuration for create-cluster. Fixed create-clu…
Browse files Browse the repository at this point in the history
…ster.rst format. Added shortcut options --instance-type and --instance-count to --instance-groups.
  • Loading branch information
sdiao committed Aug 4, 2014
1 parent a4a5985 commit fb84aaf
Show file tree
Hide file tree
Showing 8 changed files with 310 additions and 107 deletions.
43 changes: 23 additions & 20 deletions awscli/customizations/emr/createcluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from awscli.customizations.commands import BasicCommand
from awscli.customizations.emr import constants
from awscli.customizations.emr import defaultconfig
from awscli.customizations.emr import emrutils
from awscli.customizations.emr import steputils
from awscli.customizations.emr import hbaseutils
Expand All @@ -32,18 +33,19 @@ class CreateCluster(BasicCommand):
DESCRIPTION = (
'Creates and starts running an EMR cluster.\n'
'\nQuick start:\n'
'\naws emr create-cluster --ami-version 3.1.0 '
' --instance-groups InstanceGroupType=MASTER,InstanceCount=1,'
'InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,'
'InstanceType=m3.xlarge --auto-terminate\n')
'\naws emr create-cluster --ami-version <ami-version> '
'--instance-type <instance-type> --instance-count <instance-count>\n')
ARG_TABLE = [
{'name': 'ami-version',
'required': True,
'help_text': helptext.AMI_VERSION},
'help_text': helptext.AMI_VERSION,
'required': True},
{'name': 'instance-groups',
'required': True,
'schema': argumentschema.INSTANCE_GROUPS_SCHEMA,
'help_text': helptext.INSTANCE_GROUPS},
{'name': 'instance-type',
'help_text': helptext.INSTANCE_TYPE},
{'name': 'instance-count',
'help_text': helptext.INSTANCE_COUNT},
{'name': 'auto-terminate', 'action': 'store_true',
'group_name': 'auto_terminate',
'help_text': helptext.AUTO_TERMINATE},
Expand Down Expand Up @@ -83,7 +85,8 @@ class CreateCluster(BasicCommand):
'schema': argumentschema.BOOTSTRAP_ACTIONS_SCHEMA},
{'name': 'applications',
'help_text': helptext.APPLICATIONS,
'schema': argumentschema.APPLICATIONS_SCHEMA},
'schema': argumentschema.APPLICATIONS_SCHEMA,
'default': defaultconfig.APPLICATIONS},
{'name': 'steps',
'schema': argumentschema.STEPS_SCHEMA,
'help_text': helptext.STEPS},
Expand All @@ -102,10 +105,17 @@ def _run_main(self, parsed_args, parsed_globals):
bootstrap_actions = []
params['Name'] = parsed_args.name

is_valid_ami = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami is None:
raise exceptions.\
InvalidAmiVersionError(ami_version=parsed_args.ami_version)
instances_config = {}
instances_config['InstanceGroups'] = \
instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count)

is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(
ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(
params, 'AdditionalInfo', parsed_args.additional_info)
Expand All @@ -117,18 +127,11 @@ def _run_main(self, parsed_args, parsed_globals):
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
instances_config = {}
instances_config['InstanceGroups'] = \
instancegroupsutils.build_instance_groups(
parsed_args.instance_groups)

if (
parsed_args.no_auto_terminate is False and
parsed_args.auto_terminate is False):
raise exceptions.\
MissingBooleanOptionsError(
true_option='--auto-terminate',
false_option='--no-auto-terminate')
parsed_args.no_auto_terminate = True

instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
Expand Down
27 changes: 27 additions & 0 deletions awscli/customizations/emr/defaultconfig.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

# Declare all the constants used by EMR in this file.


# create-cluster default config

INSTANCE_GROUPS = \
('[{"InstanceGroupType": "MASTER","InstanceCount": 1, '
'"Name": "Master Instance Group","InstanceType": "m3.xlarge"},'
'{"InstanceGroupType": "CORE", "InstanceCount": 2,'
'"Name": "Core Instance Group", "InstanceType": "m3.xlarge"}]')

APPLICATIONS = '[{"Name": "Hive"}, {"Name": "Pig"}]'

RELEASE_LABEL = '3.1.0'
26 changes: 24 additions & 2 deletions awscli/customizations/emr/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,34 @@ class MissingParametersError(EmrError):
'{object_name}: {missing}.')


class MissingRequiredInstanceGroupsError(EmrError):
"""
In create-cluster command, none of --instance-group,
--instance-count nor --instance-type were not supplied.
"""
fmt = ('aws: error: Must specify either --instance-groups or '
'--instance-type with --instance-count(optional) to '
'configure instance groups.')


class InstanceGroupsValidationError(EmrError):
"""
--instance-type and --instance-count are shortcut option
for --instance-groups and they cannot be specified
together with --instance-groups
"""
fmt = ('aws: error: You may not specify --instance-type '
'or --instance-count with --instance-groups, '
'because --instance-type and --instance-count are '
'shortcut options for --instance-groups.')


class InvalidAmiVersionError(EmrError):
"""
The supplied ami-version is invalid.
:ivar ami_version: The provided ami-version.
:ivar ami_version: The provided ami_version.
"""
fmt = ('aws: error: The supplied AMI version {ami_version} is invalid.'
fmt = ('aws: error: The supplied AMI version "{ami_version}" is invalid.'
' Please see AMI Versions Supported in Amazon EMR in '
'Amazon Elastic MapReduce Developer Guide: '
'http://docs.aws.amazon.com/ElasticMapReduce/'
Expand Down
13 changes: 13 additions & 0 deletions awscli/customizations/emr/helptext.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,19 @@
'<code>[Name], InstanceGroupType, InstanceType, InstanceCount,'
' [BidPrice]</code></p>')

INSTANCE_TYPE = (
'<p>Shortcut option for --instance-groups. A specification of the '
'type of Amazon EC2 instances used together with --instance-count '
'(optional) to create instance groups in a cluster. '
'If --instance-count is not provided, a single node cluster '
'will be created by default. </p>')

INSTANCE_COUNT = (
'<p>Shortcut option for --instance-groups. '
'A specification of the number of Amazon EC2 instances used '
'together with --instance-type to create instance groups in '
'a cluster. </p>')

ADDITIONAL_INFO = (
'<p>Specifies additional information during cluster creation</p>')

Expand Down
43 changes: 42 additions & 1 deletion awscli/customizations/emr/instancegroupsutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# language governing permissions and limitations under the License.

from awscli.customizations.emr import constants
from awscli.customizations.emr import exceptions


def build_instance_groups(parsed_instance_groups):
Expand Down Expand Up @@ -39,4 +40,44 @@ def build_instance_groups(parsed_instance_groups):
ig_config['Market'] = constants.ON_DEMAND
instance_groups.append(ig_config)

return instance_groups
return instance_groups


def _build_instance_group(
instance_type, instance_count, instance_group_type):
ig_config = {}
ig_config['InstanceType'] = instance_type
ig_config['InstanceCount'] = instance_count
ig_config['InstanceRole'] = instance_group_type.upper()
ig_config['Name'] = ig_config['InstanceRole']
ig_config['Market'] = constants.ON_DEMAND
return ig_config


def validate_and_build_instance_groups(
instance_groups, instance_type, instance_count):
if (instance_groups is None and instance_type is None):
raise exceptions.MissingRequiredInstanceGroupsError

if (instance_groups is not None and
(instance_type is not None or
instance_count is not None)):
raise exceptions.InstanceGroupsValidationError

if instance_groups is not None:
return build_instance_groups(instance_groups)
else:
instance_groups = []
master_ig = _build_instance_group(
instance_type=instance_type,
instance_count=1,
instance_group_type="MASTER")
instance_groups.append(master_ig)
if instance_count is not None and int(instance_count) > 1:
core_ig = _build_instance_group(
instance_type=instance_type,
instance_count=int(instance_count)-1,
instance_group_type="CORE")
instance_groups.append(core_ig)

return instance_groups
Loading

0 comments on commit fb84aaf

Please sign in to comment.