diff --git a/README.md b/README.md index d99f9e87e..766408e5d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,20 @@ level than those provided by the Amazon ECS CLI. For more information about supp services and to download the AWS CLI, see the [AWS Command Line Interface](http://aws.amazon.com/cli/) product detail page. +- [Installing](#Installing) + - [Latest version](#latest-version) + - [Download Links for within China](#download-links-for-within-china) + - [Download specific version](#) +- [Configuring the CLI](#configuring-the-cli) +- [Using the CLI](#using-the-cli) + - [Creating an ECS Cluster](#creating-an-ecs-cluster) + - [Starting/Running Tasks](#startingrunning-tasks) + - [Creating a Service](#creating-a-service) + - [Using ECS parameters](#using-ecs-parameters) +- [Amazon ECS CLI Commands](#amazon-ecs-cli-commands) +- [Contributing to the CLI](#contributing-to-the-cli) +- [License](#license) + ## Installing Download the binary archive for your platform, decompress the archive, and @@ -89,9 +103,12 @@ OPTIONS: ``` ## Using the CLI + +### Creating an ECS Cluster After installing the Amazon ECS CLI and configuring your credentials, you are ready to create an ECS cluster. + ``` $ ecs-cli help up NAME: @@ -101,7 +118,7 @@ USAGE: command up [command options] [arguments...] OPTIONS: - --verbose, --debug + --verbose, --debug --keypair value Specifies the name of an existing Amazon EC2 key pair to enable SSH access to the EC2 instances in your cluster. --capability-iam Acknowledges that this command may create IAM resources. Required if --instance-role is not specified. --size value [Optional] Specifies the number of instances to launch and register to the cluster. Defaults to 1. @@ -152,11 +169,13 @@ described in the Alternatively, you may specify one or more existing security group IDs with the `--security-group` option. +### Starting/Running Tasks After the cluster is created, you can run tasks – groups of containers – on the ECS cluster. First, author a [Docker Compose configuration file]( https://docs.docker.com/compose). -You can run the configuration file locally using Docker Compose. Here is an -example Docker Compose configuration file that creates a web page: +You can run the configuration file locally using Docker Compose. + +Here is an example Docker Compose configuration file that creates a web page: ``` version: '2' @@ -180,6 +199,7 @@ fd8d5a69-87c5-46a4-80b6-51918092e600/web RUNNING 54.209.244.64:80->80/tcp ecs Navigate your web browser to the task’s IP address to see the sample app running in the ECS cluster. +### Creating a Service You can also run tasks as services. The ECS service scheduler ensures that the specified number of tasks are constantly running and reschedules tasks when a task fails (for example, if the underlying container instance fails for some @@ -205,6 +225,37 @@ Name State Ports 34333aa6-e976-4096-991a-0ec4cd5af5bd/mysql RUNNING ecscompose-wordpress-test:1 ``` +### Using ECS parameters + +Since there are certain fields in an ECS task definition that do not correspond to fields in a Docker Composefile, you can specify those values using the `--ecs-params` flag. Currently, the file supports the follow schema: + +``` +version: 1 +task_definition: + ecs_network_mode: string // supported string values: none, bridge, or host + task_role_arn: string +``` + +Example `ecs-params.yml` file: + +``` +version: 1 +task_definition: + ecs_network_mode: host + task_role_arn: myCustomRole +``` + +You can then start a task by calling: +``` +ecs-cli compose --ecs-params my-ecs-params.yml up +``` + +If you have a file name `ecs-params.yml` in your current directory, `ecs-cli compose` will automatically read it without your having to set the `--ecs-params` flag value explicitly. + +``` +ecs-cli compose up +``` + ## Amazon ECS CLI Commands For a complete list of commands, see the diff --git a/ecs-cli/modules/cli/compose/entity/entity_helper.go b/ecs-cli/modules/cli/compose/entity/entity_helper.go index 9d9bb3dbb..5dea011b8 100644 --- a/ecs-cli/modules/cli/compose/entity/entity_helper.go +++ b/ecs-cli/modules/cli/compose/entity/entity_helper.go @@ -32,7 +32,7 @@ func SetupTaskDefinitionCache() cache.Cache { if err != nil { log.WithFields(log.Fields{ "error": err, - }).Warn("Unable to create cache for task definitions; extranious ones may be registered") + }).Warn("Unable to create cache for task definitions; extraneous ones may be registered") tdCache = cache.NewNoopCache() } return tdCache @@ -53,12 +53,9 @@ func GetOrCreateTaskDefinition(entity ProjectEntity) (*ecs.TaskDefinition, error "TaskDefinition": taskDefinition, }).Debug("Finding task definition in cache or creating if needed") - resp, err := entity.Context().ECSClient.RegisterTaskDefinitionIfNeeded(&ecs.RegisterTaskDefinitionInput{ - Family: taskDefinition.Family, - ContainerDefinitions: taskDefinition.ContainerDefinitions, - Volumes: taskDefinition.Volumes, - TaskRoleArn: taskDefinition.TaskRoleArn, - }, entity.TaskDefinitionCache()) + request := createRegisterTaskDefinitionRequest(taskDefinition) + + resp, err := entity.Context().ECSClient.RegisterTaskDefinitionIfNeeded(request, entity.TaskDefinitionCache()) if err != nil { composeutils.LogError(err, "Create task definition failed") @@ -74,6 +71,25 @@ func GetOrCreateTaskDefinition(entity ProjectEntity) (*ecs.TaskDefinition, error return resp, nil } +func createRegisterTaskDefinitionRequest(taskDefinition *ecs.TaskDefinition) *ecs.RegisterTaskDefinitionInput { + // Valid values for network mode are none, host or bridge. If no value + // is passed for network mode, ECS will set it to 'bridge' on most + // platforms, but Windows has different network modes. Passing nil allows ECS + // to do the right thing for each platform. + request := &ecs.RegisterTaskDefinitionInput{ + Family: taskDefinition.Family, + ContainerDefinitions: taskDefinition.ContainerDefinitions, + Volumes: taskDefinition.Volumes, + TaskRoleArn: taskDefinition.TaskRoleArn, + } + + if networkMode := taskDefinition.NetworkMode; aws.StringValue(networkMode) != "" { + request.NetworkMode = taskDefinition.NetworkMode + } + + return request +} + // Info returns a formatted list of containers (running and stopped) in the current cluster // filtered by this project if filterLocal is set to true func Info(entity ProjectEntity, filterLocal bool) (project.InfoSet, error) { diff --git a/ecs-cli/modules/cli/compose/factory/factory.go b/ecs-cli/modules/cli/compose/factory/factory.go index c7871b056..0eef65f92 100644 --- a/ecs-cli/modules/cli/compose/factory/factory.go +++ b/ecs-cli/modules/cli/compose/factory/factory.go @@ -55,17 +55,20 @@ func (projectFactory projectFactory) Create(cliContext *cli.Context, isService b return project, nil } -// populateContext sets the required CLI arguments to the context +// populateContext sets the required CLI arguments to the ECS context func (projectFactory projectFactory) populateContext(ecsContext *context.Context, cliContext *cli.Context) error { /* - Populate the following libcompose fields to context - - ComposeFiles: reads from `--file` or `-f` flags. Defaults to `docker-compose.yml` and `docker-compose.override.yml` if no flags are specified. - - ProjectName: reads from `--project-name` or `-p` flags. + Populate the following libcompose fields on the ECS context: + - ComposeFiles: reads from `--file` or `-f` flags. Defaults to + `docker-compose.yml` and `docker-compose.override.yml` if no flags are + specified. + - ProjectName: reads from `--project-name` or `-p` flags. */ libcomposecommand.Populate(&ecsContext.Context, cliContext) ecsContext.CLIContext = cliContext - // reads and sets the parameters (required to create ECS Service Client) from the cli context to ecs context + // reads and sets the parameters (required to create ECS Service + // Client) from the cli context to ECS context rdwr, err := config.NewReadWriter() if err != nil { utils.LogError(err, "Error loading config") @@ -86,7 +89,7 @@ func (projectFactory projectFactory) populateContext(ecsContext *context.Context return nil } -// populateLibcomposeContext sets the required Libcompose lookup utilities to the context +// populateLibcomposeContext sets the required Libcompose lookup utilities on the ECS context func (projectFactory projectFactory) populateLibcomposeContext(ecsContext *context.Context) error { envLookup, err := utils.GetDefaultEnvironmentLookup() if err != nil { diff --git a/ecs-cli/modules/cli/compose/project/project.go b/ecs-cli/modules/cli/compose/project/project.go index 5cdc76d11..f07170ff9 100644 --- a/ecs-cli/modules/cli/compose/project/project.go +++ b/ecs-cli/modules/cli/compose/project/project.go @@ -20,10 +20,10 @@ import ( "github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity/service" "github.com/aws/amazon-ecs-cli/ecs-cli/modules/cli/compose/entity/task" + "github.com/aws/amazon-ecs-cli/ecs-cli/modules/commands" "github.com/aws/amazon-ecs-cli/ecs-cli/modules/utils/compose" "github.com/docker/libcompose/config" "github.com/docker/libcompose/project" - "github.com/aws/amazon-ecs-cli/ecs-cli/modules/commands" ) // Project is the starting point for the compose app to interact with and issue commands @@ -115,28 +115,33 @@ func (p *ecsProject) Parse() error { return p.transformTaskDefinition() } -// parseCompose loads and parses the compose yml files +// parseCompose sets data from the compose files on the ecsProject func (p *ecsProject) parseCompose() error { - // load the compose files using libcompose logrus.Debug("Parsing the compose yaml...") + // libcompose.Project#Parse populates project information based on its + // context. It sets up the name, the composefile and the composebytes + // (the composefile content). This is where p.ServiceConfigs gets loaded. if err := p.Project.Parse(); err != nil { return err } // libcompose sanitizes the project name and removes any non alpha-numeric character. - // The following undo-es that and sets the project name as user defined it. + // The following undoes that and sets the project name as user defined it. return p.context.SetProjectName() } -// transformTaskDefinition converts the compose yml into task definition +// transformTaskDefinition converts the compose yml and ecs-params yml into an ECS task definition func (p *ecsProject) transformTaskDefinition() error { context := p.context // convert to task definition logrus.Debug("Transforming yaml to task definition...") taskDefinitionName := utils.GetTaskDefinitionName(context.ECSParams.ComposeProjectNamePrefix, context.Context.ProjectName) + taskRoleArn := context.CLIContext.GlobalString(command.TaskRoleArnFlag) - taskDefinition, err := utils.ConvertToTaskDefinition(taskDefinitionName, &context.Context, p.ServiceConfigs(), taskRoleArn) + ecsParamsFileName := context.CLIContext.GlobalString(command.ECSParamsFileNameFlag) + + taskDefinition, err := utils.ConvertToTaskDefinition(taskDefinitionName, &context.Context, p.ServiceConfigs(), taskRoleArn, ecsParamsFileName) if err != nil { return err } diff --git a/ecs-cli/modules/commands/compose/compose_command.go b/ecs-cli/modules/commands/compose/compose_command.go index 023717d30..5ccc2a806 100644 --- a/ecs-cli/modules/commands/compose/compose_command.go +++ b/ecs-cli/modules/commands/compose/compose_command.go @@ -32,11 +32,14 @@ import ( // ecs-cli compose create : creates ECS.TaskDefinition or gets from FS cache // ecs-cli compose start : invokes ECS.RunTask if count(running tasks) == 0 // ecs-cli compose up : compose create ; compose start and does a deployment of new compose yml if changes were found +// // List containers in or view details of the project: // ecs-cli compose ps : calls ECS.ListTasks (running and stopped) filtered with Task group: this project +// // Modify containers // ecs-cli compose scale : calls ECS.RunTask/StopTask based on the count // ecs-cli compose run : calls ECS.RunTask with overrides +// // Stop and delete the project // ecs-cli compose stop : calls ECS.StopTask and ECS deletes them (rm) //* --------------------------------------------------- */ @@ -44,6 +47,7 @@ import ( const ( composeFileNameDefaultValue = "docker-compose.yml" composeOverrideFileNameDefaultValue = "docker-compose.override.yml" + ecsParamsFileNameDefaultValue = "ecs-params.yml" containerNameFlag = "name" ) @@ -96,6 +100,10 @@ func composeFlags() []cli.Flag { Name: command.TaskRoleArnFlag, Usage: "[Optional] Specifies the short name or full Amazon Resource Name (ARN) of the IAM role that containers in this task can assume. All containers in this task are granted the permissions that are specified in this role.", }, + cli.StringFlag{ + Name: command.ECSParamsFileNameFlag, + Usage: "[Optional] Specifies ecs-params file to use. Defaults to " + ecsParamsFileNameDefaultValue + " file, if one exists.", + }, } } diff --git a/ecs-cli/modules/commands/flags.go b/ecs-cli/modules/commands/flags.go index e4a1729b2..d2ac425f8 100644 --- a/ecs-cli/modules/commands/flags.go +++ b/ecs-cli/modules/commands/flags.go @@ -63,9 +63,10 @@ const ( UntaggedFlag = "untagged" // Compose - ProjectNameFlag = "project-name" - ComposeFileNameFlag = "file" - TaskRoleArnFlag = "task-role-arn" + ProjectNameFlag = "project-name" + ComposeFileNameFlag = "file" + TaskRoleArnFlag = "task-role-arn" + ECSParamsFileNameFlag = "ecs-params" // Compose Service CreateServiceCommandName = "create" diff --git a/ecs-cli/modules/config/params.go b/ecs-cli/modules/config/params.go index 48d7fd6d5..b76ad0008 100644 --- a/ecs-cli/modules/config/params.go +++ b/ecs-cli/modules/config/params.go @@ -37,7 +37,7 @@ func (p *CliParams) GetCfnStackName() string { return fmt.Sprintf("%s%s", p.CFNStackNamePrefix, p.Cluster) } -// Searches as far up the context as necesarry. This function works no matter +// Searches as far up the context as necessary. This function works no matter // how many layers of nested subcommands there are. It is more powerful // than merely calling context.String and context.GlobalString func recursiveFlagSearch(context *cli.Context, flag string) string { diff --git a/ecs-cli/modules/utils/compose/convert_task_definition.go b/ecs-cli/modules/utils/compose/convert_task_definition.go index 77c5bb421..81ce38aaf 100644 --- a/ecs-cli/modules/utils/compose/convert_task_definition.go +++ b/ecs-cli/modules/utils/compose/convert_task_definition.go @@ -15,8 +15,8 @@ package utils import ( "encoding/json" - "errors" "fmt" + "os" "reflect" "strconv" "strings" @@ -27,6 +27,9 @@ import ( "github.com/docker/libcompose/config" "github.com/docker/libcompose/project" "github.com/docker/libcompose/yaml" + "github.com/pkg/errors" + goyaml "gopkg.in/yaml.v2" + "io/ioutil" ) const ( @@ -49,6 +52,16 @@ var supportedComposeYamlOptions = []string{ var supportedComposeYamlOptionsMap = getSupportedComposeYamlOptionsMap() +type ECSParams struct { + Version string + TaskDefinition ecsTaskDef `yaml:"task_definition"` +} + +type ecsTaskDef struct { + NetworkMode string `yaml:"ecs_network_mode"` + TaskRoleArn string `yaml:"task_role_arn"` +} + type volumes struct { volumeWithHost map[string]string volumeEmptyHost []string @@ -64,7 +77,7 @@ func getSupportedComposeYamlOptionsMap() map[string]bool { // ConvertToTaskDefinition transforms the yaml configs to its ecs equivalent (task definition) func ConvertToTaskDefinition(taskDefinitionName string, context *project.Context, - serviceConfigs *config.ServiceConfigs, taskRoleArn string) (*ecs.TaskDefinition, error) { + serviceConfigs *config.ServiceConfigs, taskRoleArn string, ecsParamsFileName string) (*ecs.TaskDefinition, error) { if serviceConfigs.Len() == 0 { return nil, errors.New("cannot create a task definition with no containers; invalid service config") @@ -92,15 +105,54 @@ func ConvertToTaskDefinition(taskDefinitionName string, context *project.Context containerDefinitions = append(containerDefinitions, containerDef) } + ecsParams, err := readECSParams(ecsParamsFileName) + if err != nil { + return nil, err + } + + var networkMode string + if ecsParams != nil { + networkMode = ecsParams.TaskDefinition.NetworkMode + // The task-role-arn flag should take precedence over a taskRoleArn value specified in ECS fields file + if taskRoleArn == "" { + taskRoleArn = ecsParams.TaskDefinition.TaskRoleArn + } + } + taskDefinition := &ecs.TaskDefinition{ Family: aws.String(taskDefinitionName), ContainerDefinitions: containerDefinitions, Volumes: convertToECSVolumes(volumes), TaskRoleArn: aws.String(taskRoleArn), + NetworkMode: aws.String(networkMode), } + return taskDefinition, nil } +func readECSParams(filename string) (*ECSParams, error) { + if filename == "" { + defaultFilename := "ecs-params.yml" + if _, err := os.Stat(defaultFilename); err == nil { + filename = defaultFilename + } else { + return nil, nil + } + } + ecsParamsData, err := ioutil.ReadFile(filename) + if err != nil { + return nil, errors.Wrapf(err, "Error reading file '%v'", filename) + } + + ecsParams := &ECSParams{} + + if err = goyaml.Unmarshal([]byte(ecsParamsData), &ecsParams); err != nil { + return nil, errors.Wrapf(err, "Error unmarshalling yaml data from ECS params file: %v", filename) + } + + return ecsParams, nil +} + // logUnsupportedConfigFields adds a WARNING to the customer about the fields that are unused. func logUnsupportedConfigFields(project *project.Project) { if project.VolumeConfigs != nil && len(project.VolumeConfigs) > 0 { diff --git a/ecs-cli/modules/utils/compose/convert_task_definition_test.go b/ecs-cli/modules/utils/compose/convert_task_definition_test.go index 2cd6cc554..9b565488b 100644 --- a/ecs-cli/modules/utils/compose/convert_task_definition_test.go +++ b/ecs-cli/modules/utils/compose/convert_task_definition_test.go @@ -15,6 +15,7 @@ package utils import ( "fmt" + "io/ioutil" "os" "reflect" "strconv" @@ -114,6 +115,73 @@ func TestConvertToTaskDefinition(t *testing.T) { assert.Equal(t, taskRoleArn, aws.StringValue(taskDefinition.TaskRoleArn), "Expected taskRoleArn to match") } +func TestConvertToTaskDefinitionWithECSParams(t *testing.T) { + ecsParamsString := `version: 1 +task_definition: + ecs_network_mode: host + task_role_arn: arn:aws:iam::123456789012:role/my_role` + + content := []byte(ecsParamsString) + + tmpfile, err := ioutil.TempFile("", "ecs-params") + assert.NoError(t, err, "Could not create ecs fields tempfile") + + defer os.Remove(tmpfile.Name()) + + _, err = tmpfile.Write(content) + assert.NoError(t, err, "Could not write data to ecs fields tempfile") + + err = tmpfile.Close() + assert.NoError(t, err, "Could not close tempfile") + + ecsParamsFileName := tmpfile.Name() + + taskDefinition, err := convertToTaskDefWithEcsParamsInTest(t, "name", &config.ServiceConfig{}, "", ecsParamsFileName) + + if assert.NoError(t, err) { + assert.Equal(t, "host", aws.StringValue(taskDefinition.NetworkMode), "Expected network mode to match") + assert.Equal(t, "arn:aws:iam::123456789012:role/my_role", aws.StringValue(taskDefinition.TaskRoleArn), "Expected task role ARN to match") + } +} + +func TestConvertToTaskDefinitionWithECSParamsAndTaskRoleArnFlag(t *testing.T) { + ecsParamsString := `version: 1 +task_definition: + ecs_network_mode: host + task_role_arn: arn:aws:iam::123456789012:role/tweedledee` + + content := []byte(ecsParamsString) + + tmpfile, err := ioutil.TempFile("", "ecs-params") + assert.NoError(t, err, "Could not create ecs fields tempfile") + + defer os.Remove(tmpfile.Name()) + + _, err = tmpfile.Write(content) + assert.NoError(t, err, "Could not write data to ecs fields tempfile") + + err = tmpfile.Close() + assert.NoError(t, err, "Could not close tempfile") + + ecsParamsFileName := tmpfile.Name() + taskRoleArn := "arn:aws:iam::123456789012:role/tweedledum" + + taskDefinition, err := convertToTaskDefWithEcsParamsInTest(t, "name", &config.ServiceConfig{}, taskRoleArn, ecsParamsFileName) + + if assert.NoError(t, err) { + assert.Equal(t, "host", aws.StringValue(taskDefinition.NetworkMode), "Expected network mode to match") + assert.Equal(t, "arn:aws:iam::123456789012:role/tweedledum", aws.StringValue(taskDefinition.TaskRoleArn), "Expected task role arn to match") + } +} + +func TestConvertToTaskDefinitionWithECSParamsWithNoSuchFileError(t *testing.T) { + ecsParamsFileName := "ecs-params.yml" + taskRoleArn := "arn:aws:iam::123456789012:role/tweedledum" + + _, err := convertToTaskDefWithEcsParamsInTest(t, "name", &config.ServiceConfig{}, taskRoleArn, ecsParamsFileName) + assert.Error(t, err) +} + func TestConvertToTaskDefinitionWithDnsSearch(t *testing.T) { dnsSearchDomains := []string{"search.example.com"} @@ -531,13 +599,39 @@ func convertToTaskDefinitionInTest(t *testing.T, name string, serviceConfig *con EnvironmentLookup: envLookup, ResourceLookup: resourceLookup, } - taskDefinition, err := ConvertToTaskDefinition(taskDefName, context, serviceConfigs, taskRoleArn) + taskDefinition, err := ConvertToTaskDefinition(taskDefName, context, serviceConfigs, taskRoleArn, "") if err != nil { t.Errorf("Expected to convert [%v] serviceConfigs without errors. But got [%v]", serviceConfig, err) } return taskDefinition } +func convertToTaskDefWithEcsParamsInTest(t *testing.T, name string, serviceConfig *config.ServiceConfig, taskRoleArn string, ecsParamsFileName string) (*ecs.TaskDefinition, error) { + serviceConfigs := config.NewServiceConfigs() + serviceConfigs.Add(name, serviceConfig) + + taskDefName := "ProjectName" + envLookup, err := GetDefaultEnvironmentLookup() + if err != nil { + t.Fatal("Unexpected error setting up environment lookup") + } + resourceLookup, err := GetDefaultResourceLookup() + if err != nil { + t.Fatal("Unexpected error setting up resource lookup") + } + context := &project.Context{ + Project: &project.Project{}, + EnvironmentLookup: envLookup, + ResourceLookup: resourceLookup, + } + taskDefinition, err := ConvertToTaskDefinition(taskDefName, context, serviceConfigs, taskRoleArn, ecsParamsFileName) + if err != nil { + return nil, err + } + + return taskDefinition, nil +} + func TestIsZeroForEmptyConfig(t *testing.T) { serviceConfig := &config.ServiceConfig{} @@ -640,7 +734,7 @@ func TestMemReservationHigherThanMemLimit(t *testing.T) { EnvironmentLookup: envLookup, ResourceLookup: resourceLookup, } - _, err = ConvertToTaskDefinition(taskDefName, context, serviceConfigs, "") + _, err = ConvertToTaskDefinition(taskDefName, context, serviceConfigs, "", "") assert.EqualError(t, err, "mem_limit should not be less than mem_reservation") }