Merge branch 'support_ecs_fields' into dev

Allows fields specific to an ECS TaskDefinition to be passed into compose
commands via a yaml file.

Closes aws#153.
Closes aws#188.
Addresses most of aws#267.

README.md

@@ -20,6 +20,20 @@ level than those provided by the Amazon ECS CLI. For more information about supp
 services and to download the AWS CLI, see the
 [AWS Command Line Interface](http://aws.amazon.com/cli/) product detail page.
 
+- [Installing](#Installing)
+	- [Latest version](#latest-version)
+		- [Download Links for within China](#download-links-for-within-china)
+	- [Download specific version](#)
+- [Configuring the CLI](#configuring-the-cli)
+- [Using the CLI](#using-the-cli)
+	- [Creating an ECS Cluster](#creating-an-ecs-cluster)
+	- [Starting/Running Tasks](#startingrunning-tasks)
+	- [Creating a Service](#creating-a-service)
+	- [Using ECS parameters](#using-ecs-parameters)
+- [Amazon ECS CLI Commands](#amazon-ecs-cli-commands)
+- [Contributing to the CLI](#contributing-to-the-cli)
+- [License](#license)
+
 ## Installing
 
 Download the binary archive for your platform, decompress the archive, and
@@ -89,9 +103,12 @@ OPTIONS:
 ```
 
 ## Using the CLI
+
+### Creating an ECS Cluster
 After installing the Amazon ECS CLI and configuring your credentials, you are ready to
 create an ECS cluster.
 
+
 ```
 $ ecs-cli help up
 NAME:
@@ -101,7 +118,7 @@ USAGE:
    command up [command options] [arguments...]
 
 OPTIONS:
-   --verbose, --debug                
+   --verbose, --debug
    --keypair value                   Specifies the name of an existing Amazon EC2 key pair to enable SSH access to the EC2 instances in your cluster.
    --capability-iam                  Acknowledges that this command may create IAM resources. Required if --instance-role is not specified.
    --size value                      [Optional] Specifies the number of instances to launch and register to the cluster. Defaults to 1.
@@ -152,11 +169,13 @@ described in the
 Alternatively, you may specify one or more existing security group IDs with the
 `--security-group` option.
 
+### Starting/Running Tasks
 After the cluster is created, you can run tasks – groups of containers – on the
 ECS cluster. First, author a
 [Docker Compose configuration file]( https://docs.docker.com/compose).
-You can run the configuration file locally using Docker Compose. Here is an
-example Docker Compose configuration file that creates a web page:
+You can run the configuration file locally using Docker Compose.
+
+Here is an example Docker Compose configuration file that creates a web page:
 
 ```
 version: '2'
@@ -180,6 +199,7 @@ fd8d5a69-87c5-46a4-80b6-51918092e600/web  RUNNING  54.209.244.64:80->80/tcp  ecs
 Navigate your web browser to the task’s IP address to see the sample app
 running in the ECS cluster.
 
+### Creating a Service
 You can also run tasks as services. The ECS service scheduler ensures that the
 specified number of tasks are constantly running and reschedules tasks when a
 task fails (for example, if the underlying container instance fails for some
@@ -205,6 +225,37 @@ Name                                            State    Ports
 34333aa6-e976-4096-991a-0ec4cd5af5bd/mysql      RUNNING                             ecscompose-wordpress-test:1
 ```
 
+### Using ECS parameters
+
+Since there are certain fields in an ECS task definition that do not correspond to fields in a Docker Composefile, you can specify those values using the `--ecs-params` flag. Currently, the file supports the follow schema:
+
+```
+version: 1
+task_definition:
+  ecs_network_mode: string // supported string values: none, bridge, or host
+  task_role_arn: string
+```
+
+Example `ecs-params.yml` file:
+
+```
+version: 1
+task_definition:
+  ecs_network_mode: host
+  task_role_arn: myCustomRole
+```
+
+You can then start a task by calling:
+```
+ecs-cli compose --ecs-params my-ecs-params.yml up
+```
+
+If you have a file name `ecs-params.yml` in your current directory, `ecs-cli compose` will automatically read it without your having to set the `--ecs-params` flag value explicitly.
+
+```
+ecs-cli compose up
+```
+
 ## Amazon ECS CLI Commands
 
 For a complete list of commands, see the

ecs-cli/modules/cli/compose/entity/entity_helper.go

@@ -32,7 +32,7 @@ func SetupTaskDefinitionCache() cache.Cache {
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error": err,
-		}).Warn("Unable to create cache for task definitions; extranious ones may be registered")
+		}).Warn("Unable to create cache for task definitions; extraneous ones may be registered")
 		tdCache = cache.NewNoopCache()
 	}
 	return tdCache
@@ -53,12 +53,9 @@ func GetOrCreateTaskDefinition(entity ProjectEntity) (*ecs.TaskDefinition, error
 		"TaskDefinition": taskDefinition,
 	}).Debug("Finding task definition in cache or creating if needed")
 
-	resp, err := entity.Context().ECSClient.RegisterTaskDefinitionIfNeeded(&ecs.RegisterTaskDefinitionInput{
-		Family:               taskDefinition.Family,
-		ContainerDefinitions: taskDefinition.ContainerDefinitions,
-		Volumes:              taskDefinition.Volumes,
-		TaskRoleArn:          taskDefinition.TaskRoleArn,
-	}, entity.TaskDefinitionCache())
+	request := createRegisterTaskDefinitionRequest(taskDefinition)
+
+	resp, err := entity.Context().ECSClient.RegisterTaskDefinitionIfNeeded(request, entity.TaskDefinitionCache())
 
 	if err != nil {
 		composeutils.LogError(err, "Create task definition failed")
@@ -74,6 +71,25 @@ func GetOrCreateTaskDefinition(entity ProjectEntity) (*ecs.TaskDefinition, error
 	return resp, nil
 }
 
+func createRegisterTaskDefinitionRequest(taskDefinition *ecs.TaskDefinition) *ecs.RegisterTaskDefinitionInput {
+	// Valid values for network mode are none, host or bridge. If no value
+	// is passed for network mode, ECS will set it to 'bridge' on most
+	// platforms, but Windows has different network modes. Passing nil allows ECS
+	// to do the right thing for each platform.
+	request := &ecs.RegisterTaskDefinitionInput{
+		Family:               taskDefinition.Family,
+		ContainerDefinitions: taskDefinition.ContainerDefinitions,
+		Volumes:              taskDefinition.Volumes,
+		TaskRoleArn:          taskDefinition.TaskRoleArn,
+	}
+
+	if networkMode := taskDefinition.NetworkMode; aws.StringValue(networkMode) != "" {
+		request.NetworkMode = taskDefinition.NetworkMode
+	}
+
+	return request
+}
+
 // Info returns a formatted list of containers (running and stopped) in the current cluster
 // filtered by this project if filterLocal is set to true
 func Info(entity ProjectEntity, filterLocal bool) (project.InfoSet, error) {

ecs-cli/modules/cli/compose/factory/factory.go

@@ -55,17 +55,20 @@ func (projectFactory projectFactory) Create(cliContext *cli.Context, isService b
 	return project, nil
 }
 
-// populateContext sets the required CLI arguments to the context
+// populateContext sets the required CLI arguments to the ECS context
 func (projectFactory projectFactory) populateContext(ecsContext *context.Context, cliContext *cli.Context) error {
 	/*
-	 Populate the following libcompose fields to context
-	 - ComposeFiles: reads from `--file` or `-f` flags. Defaults to `docker-compose.yml` and `docker-compose.override.yml` if no flags are specified.
-	 - ProjectName: reads from `--project-name` or `-p` flags.
+		Populate the following libcompose fields on the ECS context:
+		 - ComposeFiles: reads from `--file` or `-f` flags. Defaults to
+		 `docker-compose.yml` and `docker-compose.override.yml` if no flags are
+		 specified.
+		 - ProjectName: reads from `--project-name` or `-p` flags.
 	*/
 	libcomposecommand.Populate(&ecsContext.Context, cliContext)
 	ecsContext.CLIContext = cliContext
 
-	// reads and sets the parameters (required to create ECS Service Client) from the cli context to ecs context
+	// reads and sets the parameters (required to create ECS Service
+	// Client) from the cli context to ECS context
 	rdwr, err := config.NewReadWriter()
 	if err != nil {
 		utils.LogError(err, "Error loading config")
@@ -86,7 +89,7 @@ func (projectFactory projectFactory) populateContext(ecsContext *context.Context
 	return nil
 }
 
-// populateLibcomposeContext sets the required Libcompose lookup utilities to the context
+// populateLibcomposeContext sets the required Libcompose lookup utilities on the ECS context
 func (projectFactory projectFactory) populateLibcomposeContext(ecsContext *context.Context) error {
 	envLookup, err := utils.GetDefaultEnvironmentLookup()
 	if err != nil {

ecs-cli/modules/cli/compose/project/project.go

@@ -20,10 +20,10 @@ import (
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity/service"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity/task"
 
+	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/commands"
 	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/utils/compose"
 	"github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project"
-	"github.com/aws/amazon-ecs-cli/ecs-cli/modules/commands"
 )
 
 // Project is the starting point for the compose app to interact with and issue commands
@@ -115,28 +115,33 @@ func (p *ecsProject) Parse() error {
 	return p.transformTaskDefinition()
 }
 
-// parseCompose loads and parses the compose yml files
+// parseCompose sets data from the compose files on the ecsProject
 func (p *ecsProject) parseCompose() error {
-	// load the compose files using libcompose
 	logrus.Debug("Parsing the compose yaml...")
+	// libcompose.Project#Parse populates project information based on its
+	// context. It sets up the name, the composefile and the composebytes
+	// (the composefile content). This is where p.ServiceConfigs gets loaded.
 	if err := p.Project.Parse(); err != nil {
 		return err
 	}
 
 	// libcompose sanitizes the project name and removes any non alpha-numeric character.
-	// The following undo-es that and sets the project name as user defined it.
+	// The following undoes that and sets the project name as user defined it.
 	return p.context.SetProjectName()
 }
 
-// transformTaskDefinition converts the compose yml into task definition
+// transformTaskDefinition converts the compose yml and ecs-params yml into an ECS task definition
 func (p *ecsProject) transformTaskDefinition() error {
 	context := p.context
 
 	// convert to task definition
 	logrus.Debug("Transforming yaml to task definition...")
 	taskDefinitionName := utils.GetTaskDefinitionName(context.ECSParams.ComposeProjectNamePrefix, context.Context.ProjectName)
+
 	taskRoleArn := context.CLIContext.GlobalString(command.TaskRoleArnFlag)
-	taskDefinition, err := utils.ConvertToTaskDefinition(taskDefinitionName, &context.Context, p.ServiceConfigs(), taskRoleArn)
+	ecsParamsFileName := context.CLIContext.GlobalString(command.ECSParamsFileNameFlag)
+
+	taskDefinition, err := utils.ConvertToTaskDefinition(taskDefinitionName, &context.Context, p.ServiceConfigs(), taskRoleArn, ecsParamsFileName)
 	if err != nil {
 		return err
 	}

ecs-cli/modules/commands/compose/compose_command.go

@@ -32,18 +32,22 @@ import (
 //   ecs-cli compose create      : creates ECS.TaskDefinition or gets from FS cache
 //   ecs-cli compose start       : invokes ECS.RunTask if count(running tasks) == 0
 //   ecs-cli compose up          : compose create ; compose start and does a deployment of new compose yml if changes were found
+//
 // List containers in or view details of the project:
 //   ecs-cli compose ps          : calls ECS.ListTasks (running and stopped) filtered with Task group: this project
+//
 // Modify containers
 //   ecs-cli compose scale       : calls ECS.RunTask/StopTask based on the count
 //   ecs-cli compose run         : calls ECS.RunTask with overrides
+//
 // Stop and delete the project
 //   ecs-cli compose stop        : calls ECS.StopTask and ECS deletes them (rm)
 //* --------------------------------------------------- */
 
 const (
 	composeFileNameDefaultValue         = "docker-compose.yml"
 	composeOverrideFileNameDefaultValue = "docker-compose.override.yml"
+	ecsParamsFileNameDefaultValue       = "ecs-params.yml"
 	containerNameFlag                   = "name"
 )
 
@@ -96,6 +100,10 @@ func composeFlags() []cli.Flag {
 			Name:  command.TaskRoleArnFlag,
 			Usage: "[Optional] Specifies the short name or full Amazon Resource Name (ARN) of the IAM role that containers in this task can assume. All containers in this task are granted the permissions that are specified in this role.",
 		},
+		cli.StringFlag{
+			Name:  command.ECSParamsFileNameFlag,
+			Usage: "[Optional] Specifies ecs-params file to use. Defaults to " + ecsParamsFileNameDefaultValue + " file, if one exists.",
+		},
 	}
 }
 

ecs-cli/modules/commands/flags.go

@@ -63,9 +63,10 @@ const (
 	UntaggedFlag   = "untagged"
 
 	// Compose
-	ProjectNameFlag     = "project-name"
-	ComposeFileNameFlag = "file"
-	TaskRoleArnFlag     = "task-role-arn"
+	ProjectNameFlag       = "project-name"
+	ComposeFileNameFlag   = "file"
+	TaskRoleArnFlag       = "task-role-arn"
+	ECSParamsFileNameFlag = "ecs-params"
 
 	// Compose Service
 	CreateServiceCommandName                = "create"

ecs-cli/modules/config/params.go

@@ -37,7 +37,7 @@ func (p *CliParams) GetCfnStackName() string {
 	return fmt.Sprintf("%s%s", p.CFNStackNamePrefix, p.Cluster)
 }
 
-// Searches as far up the context as necesarry. This function works no matter
+// Searches as far up the context as necessary. This function works no matter
 // how many layers of nested subcommands there are. It is more powerful
 // than merely calling context.String and context.GlobalString
 func recursiveFlagSearch(context *cli.Context, flag string) string {

ecs-cli/modules/utils/compose/convert_task_definition.go

@@ -15,8 +15,8 @@ package utils
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -27,6 +27,9 @@ import (
 	"github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project"
 	"github.com/docker/libcompose/yaml"
+	"github.com/pkg/errors"
+	goyaml "gopkg.in/yaml.v2"
+	"io/ioutil"
 )
 
 const (
@@ -49,6 +52,16 @@ var supportedComposeYamlOptions = []string{
 
 var supportedComposeYamlOptionsMap = getSupportedComposeYamlOptionsMap()
 
+type ECSParams struct {
+	Version        string
+	TaskDefinition ecsTaskDef `yaml:"task_definition"`
+}
+
+type ecsTaskDef struct {
+	NetworkMode string `yaml:"ecs_network_mode"`
+	TaskRoleArn string `yaml:"task_role_arn"`
+}
+
 type volumes struct {
 	volumeWithHost  map[string]string
 	volumeEmptyHost []string
@@ -64,7 +77,7 @@ func getSupportedComposeYamlOptionsMap() map[string]bool {
 
 // ConvertToTaskDefinition transforms the yaml configs to its ecs equivalent (task definition)
 func ConvertToTaskDefinition(taskDefinitionName string, context *project.Context,
-	serviceConfigs *config.ServiceConfigs, taskRoleArn string) (*ecs.TaskDefinition, error) {
+	serviceConfigs *config.ServiceConfigs, taskRoleArn string, ecsParamsFileName string) (*ecs.TaskDefinition, error) {
 
 	if serviceConfigs.Len() == 0 {
 		return nil, errors.New("cannot create a task definition with no containers; invalid service config")
@@ -92,15 +105,54 @@ func ConvertToTaskDefinition(taskDefinitionName string, context *project.Context
 		containerDefinitions = append(containerDefinitions, containerDef)
 	}
 
+	ecsParams, err := readECSParams(ecsParamsFileName)
+	if err != nil {
+		return nil, err
+	}
+
+	var networkMode string
+	if ecsParams != nil {
+		networkMode = ecsParams.TaskDefinition.NetworkMode
+		// The task-role-arn flag should take precedence over a taskRoleArn value specified in ECS fields file
+		if taskRoleArn == "" {
+			taskRoleArn = ecsParams.TaskDefinition.TaskRoleArn
+		}
+	}
+
 	taskDefinition := &ecs.TaskDefinition{
 		Family:               aws.String(taskDefinitionName),
 		ContainerDefinitions: containerDefinitions,
 		Volumes:              convertToECSVolumes(volumes),
 		TaskRoleArn:          aws.String(taskRoleArn),
+		NetworkMode:          aws.String(networkMode),
 	}
+
 	return taskDefinition, nil
 }
 
+func readECSParams(filename string) (*ECSParams, error) {
+	if filename == "" {
+		defaultFilename := "ecs-params.yml"
+		if _, err := os.Stat(defaultFilename); err == nil {
+			filename = defaultFilename
+		} else {
+			return nil, nil
+		}
+	}
+	ecsParamsData, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error reading file '%v'", filename)
+	}
+
+	ecsParams := &ECSParams{}
+
+	if err = goyaml.Unmarshal([]byte(ecsParamsData), &ecsParams); err != nil {
+		return nil, errors.Wrapf(err, "Error unmarshalling yaml data from ECS params file: %v", filename)
+	}
+
+	return ecsParams, nil
+}
+
 // logUnsupportedConfigFields adds a WARNING to the customer about the fields that are unused.
 func logUnsupportedConfigFields(project *project.Project) {
 	if project.VolumeConfigs != nil && len(project.VolumeConfigs) > 0 {