bunch of updates

Dockerfile

@@ -11,15 +11,15 @@ RUN apt-get update \
   && apt-get clean
 
 COPY tools /usr/local/bin
-RUN curl -sL "https://releases.hashicorp.com/terraform/0.7.2/terraform_0.7.2_linux_amd64.zip"> terraform.zip \
+RUN curl -sL "https://releases.hashicorp.com/terraform/0.9.11/terraform_0.9.11_linux_amd64.zip"> terraform.zip \
   && unzip terraform.zip \
   && mv terraform /usr/local/bin
 
-RUN curl -sL "https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip" > packer.zip \
+RUN curl -sL "https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip" > packer.zip \
   && unzip packer.zip \
   && mv packer /usr/local/bin
 
-RUN curl -sL -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64 && chmod +x /usr/local/bin/dumb-init
+RUN curl -sL -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 && chmod +x /usr/local/bin/dumb-init
 
 ENTRYPOINT ["/usr/local/bin/dumb-init"]
 

Makefile

@@ -33,8 +33,13 @@ endif
 
 install-tools: $(tools)
 
+ifeq (${platform},Darwin)
 /usr/local/bin/%: ./tools/%
 	install -S -m 0755 $< /usr/local/bin
+else
+/usr/local/bin/%: ./tools/%
+	install -m 0755 $< /usr/local/bin
+endif
 
 amis:
 	pack-ami build -p ./packer -t base -r

Readme.md

@@ -194,7 +194,7 @@ For more complicated service discovery which handles cases like versioning, we'd
 
 ### Bastion
 
-The bastion host acts as the "jump point" for the rest of the infrastructure. Since most of our instances aren't exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
+The bastion host acts as the "jump point" for the rest of the infrastructure. Since most of our instances are not exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
 
 The bastion is provisioned using the key name that you pass to the stack (and hopefully have stored somewhere). If you ever need to access an instance directly, you can do it by "jumping through" the bastion:
 

defaults/main.tf

@@ -23,6 +23,8 @@ variable "cidr" {
 }
 
 variable "default_ecs_ami" {
+  type = "map"
+
   default = {
     us-east-1      = "ami-dde4e6ca"
     us-west-1      = "ami-6d21770d"
@@ -39,6 +41,8 @@ variable "default_ecs_ami" {
 
 # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html#attach-bucket-policy
 variable "default_log_account_ids" {
+  type = "map"
+
   default = {
     us-east-1      = "127311923021"
     us-west-2      = "797873946194"

docs.md

@@ -88,7 +88,7 @@ Usage:
 # bastion
 
 The bastion host acts as the "jump point" for the rest of the infrastructure.
-Since most of our instances aren't exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
+Since most of our instances are not exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
 The bastion is provisioned using the key name that you pass to the stack (and hopefully have stored somewhere).
 If you ever need to access an instance directly, you can do it by "jumping through" the bastion.
 
@@ -130,7 +130,7 @@ Usage:
 # defaults
 
 This module is used to set configuration defaults for the AWS infrastructure.
-It doesn't provide much value when used on its own because terraform makes it
+It does not provide much value when used on its own because terraform makes it
 hard to do dynamic generations of things like subnets, for now it's used as
 a helper module for the stack.
 
@@ -248,7 +248,7 @@ Usage:
 | instance_type | The instance type to use, e.g t2.small | - | yes |
 | instance_ebs_optimized | When set to true the instance will be launched with EBS optimized turned on | `true` | no |
 | min_size | Minimum instance count | `3` | no |
-| max_size | Maxmimum instance count | `100` | no |
+| max_size | Maximum instance count | `100` | no |
 | desired_capacity | Desired instance count | `3` | no |
 | associate_public_ip_address | Should created instances be publicly accessible (if the SG allows) | `false` | no |
 | root_volume_size | Root volume size in GB | `25` | no |

ecs-cluster/main.tf

@@ -93,7 +93,7 @@ variable "desired_capacity" {
 
 variable "associate_public_ip_address" {
   description = "Should created instances be publicly accessible (if the SG allows)"
-  default = false
+  default     = false
 }
 
 variable "root_volume_size" {

iam-role/main.tf

@@ -95,9 +95,9 @@ EOF
 }
 
 resource "aws_iam_instance_profile" "default_ecs" {
-  name  = "ecs-instance-profile-${var.name}-${var.environment}"
-  path  = "/"
-  role  = "${aws_iam_role.default_ecs_role.name}"
+  name = "ecs-instance-profile-${var.name}-${var.environment}"
+  path = "/"
+  role = "${aws_iam_role.default_ecs_role.name}"
 }
 
 output "default_ecs_role_id" {

main.tf

@@ -46,28 +46,31 @@ variable "cidr" {
 }
 
 variable "internal_subnets" {
+  type        = "list"
   description = "a list of CIDRs for internal subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones"
-  default     = ["10.30.0.0/19" ,"10.30.64.0/19", "10.30.128.0/19"]
+  default     = ["10.30.0.0/19", "10.30.64.0/19", "10.30.128.0/19"]
 }
 
 variable "external_subnets" {
+  type        = "list"
   description = "a list of CIDRs for external subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones"
   default     = ["10.30.32.0/20", "10.30.96.0/20", "10.30.160.0/20"]
 }
 
 variable "availability_zones" {
+  type        = "list"
   description = "a comma-separated list of availability zones, defaults to all AZ of the region, if set to something other than the defaults, both internal_subnets and external_subnets have to be defined as well"
   default     = ["us-west-2a", "us-west-2b", "us-west-2c"]
 }
 
 variable "bastion_instance_type" {
   description = "Instance type for the bastion"
-  default = "t2.micro"
+  default     = "t2.micro"
 }
 
 variable "ecs_cluster_name" {
   description = "the name of the cluster, if not specified the variable name will be used"
-  default = ""
+  default     = ""
 }
 
 variable "ecs_instance_type" {
@@ -202,28 +205,28 @@ module "iam_role" {
 }
 
 module "ecs_cluster" {
-  source                 = "./ecs-cluster"
-  name                   = "${coalesce(var.ecs_cluster_name, var.name)}"
-  environment            = "${var.environment}"
-  vpc_id                 = "${module.vpc.id}"
-  image_id               = "${coalesce(var.ecs_ami, module.defaults.ecs_ami)}"
-  subnet_ids             = "${module.vpc.internal_subnets}"
-  key_name               = "${var.key_name}"
-  instance_type          = "${var.ecs_instance_type}"
-  instance_ebs_optimized = "${var.ecs_instance_ebs_optimized}"
-  iam_instance_profile   = "${module.iam_role.profile}"
-  min_size               = "${var.ecs_min_size}"
-  max_size               = "${var.ecs_max_size}"
-  desired_capacity       = "${var.ecs_desired_capacity}"
-  region                 = "${var.region}"
-  availability_zones     = "${module.vpc.availability_zones}"
-  root_volume_size       = "${var.ecs_root_volume_size}"
-  docker_volume_size     = "${var.ecs_docker_volume_size}"
-  docker_auth_type       = "${var.ecs_docker_auth_type}"
-  docker_auth_data       = "${var.ecs_docker_auth_data}"
-  security_groups        = "${coalesce(var.ecs_security_groups, format("%s,%s,%s", module.security_groups.internal_ssh, module.security_groups.internal_elb, module.security_groups.external_elb))}"
-  extra_cloud_config_type     = "${var.extra_cloud_config_type}"
-  extra_cloud_config_content  = "${var.extra_cloud_config_content}"
+  source                     = "./ecs-cluster"
+  name                       = "${coalesce(var.ecs_cluster_name, var.name)}"
+  environment                = "${var.environment}"
+  vpc_id                     = "${module.vpc.id}"
+  image_id                   = "${coalesce(var.ecs_ami, module.defaults.ecs_ami)}"
+  subnet_ids                 = "${module.vpc.internal_subnets}"
+  key_name                   = "${var.key_name}"
+  instance_type              = "${var.ecs_instance_type}"
+  instance_ebs_optimized     = "${var.ecs_instance_ebs_optimized}"
+  iam_instance_profile       = "${module.iam_role.profile}"
+  min_size                   = "${var.ecs_min_size}"
+  max_size                   = "${var.ecs_max_size}"
+  desired_capacity           = "${var.ecs_desired_capacity}"
+  region                     = "${var.region}"
+  availability_zones         = "${module.vpc.availability_zones}"
+  root_volume_size           = "${var.ecs_root_volume_size}"
+  docker_volume_size         = "${var.ecs_docker_volume_size}"
+  docker_auth_type           = "${var.ecs_docker_auth_type}"
+  docker_auth_data           = "${var.ecs_docker_auth_data}"
+  security_groups            = "${coalesce(var.ecs_security_groups, format("%s,%s,%s", module.security_groups.internal_ssh, module.security_groups.internal_elb, module.security_groups.external_elb))}"
+  extra_cloud_config_type    = "${var.extra_cloud_config_type}"
+  extra_cloud_config_content = "${var.extra_cloud_config_content}"
 }
 
 module "s3_logs" {

packer/base/packer.yml

@@ -1,7 +1,26 @@
 ---
+# Latest xenial	16.04 LTS	amd64	hvm:ebs-ssd Releases 
+#  us-gov-west-1	20170619.1	ami-939412f2
+#  us-east-2	    20170619.1	ami-8b92b4ee
+#  sa-east-1	    20170619.1	ami-34afc458
+#  eu-central-1	    20170619.1	ami-1c45e273
+#  us-west-1	    20170619.1	ami-73f7da13
+#  us-west-2	    20170619.1	ami-835b4efa
+#  ap-northeast-2	20170619.1	ami-94d20dfa
+#  ca-central-1	    20170619.1	ami-7ed56a1a
+#  eu-west-2	    20170619.1	ami-cc7066a8
+#  ap-southeast-1	20170619.1	ami-2378f540
+#  eu-west-1	    20170619.1	ami-6d48500b
+#  ap-southeast-2	20170619.1	ami-e94e5e8a
+#  ap-northeast-1	20170619.1	ami-785c491f
+#  us-east-1	    20170619.1	ami-d15a75c7
+#  ap-south-1	    20170619.1	ami-49e59a26
+#  cn-north-1	    20170303	ami-a163b4cc
+
+
 # https://www.packer.io/docs/builders/amazon-ebs.html
 ami:
-  source_ami: ami-e6d5d2f1
+  source_ami: ami-d15a75c7
   region: us-east-1
   instance_type: c4.2xlarge
   ssh_username: ubuntu

packer/base/root/etc/apt/preferences.d/docker-engine.pref

@@ -0,0 +1,3 @@
+Package: *
+Pin: release o=Docker
+Pin-Priority: 900

packer/base/scripts/base.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 set -e
 
+export DEBIAN_FRONTEND=noninteractive
 systemctl disable apt-daily.service
 systemctl disable apt-daily.timer
 
 apt-get update -y
-apt-get upgrade -y
 
 apt-get install -y \
         build-essential  \
@@ -35,8 +35,12 @@ apt-get install -y \
         ntp \
         logrotate \
         dhcping \
+        nfs-common \
+        curl \
+        unzip \
+        jq \
         dhcpdump
 
 pip install awscli
 
-apt-get dist-upgrade -y
+apt-get upgrade -y

packer/base/scripts/docker.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+export DEBIAN_FRONTEND=noninteractive
+
 apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
 echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' > /etc/apt/sources.list.d/docker.list
 
@@ -9,7 +11,6 @@ apt-get purge -y lxc-docker
 apt-cache policy docker-engine
 
 apt-get install -o Dpkg::Options::="--force-confold" -y \
-        linux-image-extra-$(uname -r) \
         docker-engine
 
 gpasswd -a ubuntu docker

packer/base/scripts/ixgbevf.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 set -e
 
-wget -q -N -P /tmp/ "sourceforge.net/projects/e1000/files/ixgbevf stable/3.1.2/ixgbevf-3.1.2.tar.gz"
-tar -xzf /tmp/ixgbevf-3.1.2.tar.gz
-mv ixgbevf-3.1.2 /usr/src/
+wget -q -N -P /tmp/ "sourceforge.net/projects/e1000/files/ixgbevf stable/3.4.3/ixgbevf-3.4.3.tar.gz"
+tar -xzf /tmp/ixgbevf-3.4.3.tar.gz
+mv ixgbevf-3.4.3 /usr/src/
 
-cat <<EOT | tee /usr/src/ixgbevf-3.1.2/dkms.conf
+cat <<EOT | tee /usr/src/ixgbevf-3.4.3/dkms.conf
 PACKAGE_NAME="ixgbevf"
-PACKAGE_VERSION="3.1.2"
+PACKAGE_VERSION="3.4.3"
 CLEAN="cd src/; make clean"
 MAKE="cd src/; make BUILD_KERNEL=\${kernelver}"
 BUILT_MODULE_LOCATION[0]="src/"
@@ -17,9 +17,11 @@ DEST_MODULE_NAME[0]="ixgbevf"
 AUTOINSTALL="yes"
 EOT
 
-dkms add -m ixgbevf -v 3.1.2
-dkms build -m ixgbevf -v 3.1.2
-dkms install -m ixgbevf -v 3.1.2
+sed -i 's/#if UTS_UBUNTU_RELEASE_ABI > 255/#if UTS_UBUNTU_RELEASE_ABI > 99255/' /usr/src/ixgbevf-3.4.3/src/kcompat.h
+dkms remove ixgbevf -v 3.4.3 --all 2>/dev/null || true
+dkms add -m ixgbevf -v 3.4.3
+dkms build -m ixgbevf -v 3.4.3
+dkms install -m ixgbevf -v 3.4.3
 update-initramfs -c -k all
 
 echo "options ixgbevf InterruptThrottleRate=1,1,1,1,1,1,1,1" | tee /etc/modprobe.d/ixgbevf.conf

packer/ecs/root/etc/systemd/system/ecs-agent.service

@@ -14,7 +14,7 @@ ExecStartPre=/bin/mkdir -p /var/lib/ecs/data
 ExecStartPre=/bin/mkdir -p /var/log/ecs
 ExecStartPre=-/usr/bin/docker kill ecs-agent
 ExecStartPre=-/usr/bin/docker rm ecs-agent
-ExecStartPre=-/usr/bin/docker pull amazon/amazon-ecs-agent:v1.14.1
+ExecStartPre=-/usr/bin/docker pull amazon/amazon-ecs-agent:latest
 ExecStart=/usr/bin/docker run \
     --name=ecs-agent \
     --restart=on-failure:10 \
@@ -27,7 +27,7 @@ ExecStart=/usr/bin/docker run \
     --publish=127.0.0.1:51679:51679 \
     --env-file=/etc/ecs/ecs.config \
     --env=ECS_CLUSTER=${SERVER_GROUP} \
-    amazon/amazon-ecs-agent:v1.14.1
+    amazon/amazon-ecs-agent:latest
 ExecStop=-/usr/bin/docker stop ecs-agent
 
 [Install]

packer/ecs/root/etc/systemd/system/ecs-logs.service

@@ -11,12 +11,12 @@ RestartPreventExitStatus=5
 SyslogIdentifier=ecs-logs
 ExecStartPre=-/usr/bin/docker kill ecs-logs
 ExecStartPre=-/usr/bin/docker rm ecs-logs
-ExecStartPre=-/usr/bin/docker pull segment/ecs-logs:0.1.1
+ExecStartPre=-/usr/bin/docker pull segment/ecs-logs:0.1.5
 ExecStart=/usr/bin/docker run \
     --name=ecs-logs \
     --restart=on-failure:10 \
     --volume=/run/log/journal:/run/log/journal:ro \
-    segment/ecs-logs:0.1.1 -src journald -dst cloudwatchlogs
+    segment/ecs-logs:0.1.5 -src journald -dst cloudwatchlogs
 ExecStop=-/usr/bin/docker stop ecs-logs
 
 [Install]

s3-logs/main.tf

@@ -1,11 +1,8 @@
-variable "name" {
-}
+variable "name" {}
 
-variable "environment" {
-}
+variable "environment" {}
 
-variable "account_id" {
-}
+variable "account_id" {}
 
 variable "logs_expiration_enabled" {
   default = false
@@ -28,8 +25,8 @@ resource "aws_s3_bucket" "logs" {
   bucket = "${var.name}-${var.environment}-logs"
 
   lifecycle_rule {
-    id = "logs-expiration"
-    prefix = ""
+    id      = "logs-expiration"
+    prefix  = ""
     enabled = "${var.logs_expiration_enabled}"
 
     expiration {

scripts/docs.sh

@@ -12,10 +12,10 @@ echo "Generating docs for stack"
 printf "# Stack\n\n" >> docs.md
 terraform-docs md . >> docs.md
 
-for m in $modules; do
+for m in ${modules}; do
   if [[ "$m" != "iam-role" ]]; then
     echo "generating docs for $m"
     printf "# $m\n\n" >> docs.md
-    terraform-docs md $m >> docs.md
+    terraform-docs md ${m} >> docs.md
   fi
 done

scripts/test.sh

@@ -4,6 +4,6 @@ modules=$(find -mindepth 2 -name *.tf -printf '%P\n' | xargs -I % dirname %)
 
 (terraform validate . && echo "√ stack") || exit 1
 
-for m in $modules; do
-  (terraform validate $m && echo "√ $m") || exit 1
+for m in ${modules}; do
+  (terraform validate ${m} && echo "√ $m") || exit 1
 done

tools/pack-ami

@@ -386,7 +386,7 @@ class Template(object):
             packer.wait()
 
             if packer.returncode != 0:
-                raise Exception("Buliding %s failed: see logs in %s" % (self.name, pwd))
+                raise Exception("Building %s failed: see logs in %s" % (self.name, pwd))
 
             try:
                 ami = subprocess.check_output("grep ami- %s  | tail -n 1 | cut -d' ' -f2" % os.path.join(pwd, 'stdout.log'), shell=True)