Add initial windows packer templates for ami (#155)

* Add initial windows packer templates for ami Signed-off-by: Peter Zhu <[email protected]> * Tweak readme Signed-off-by: Peter Zhu <[email protected]> * Make changes based on the PR comments Signed-off-by: Peter Zhu <[email protected]> * Disable windows internal firewall and allow Security Group to take care of ports Signed-off-by: Peter Zhu <[email protected]> * Change templates to use gp3 as default ebs Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Jul 19, 2022 · 25e235e · 25e235e
1 parent 75b5ce9
commit 25e235e
Show file tree

Hide file tree

Showing 13 changed files with 511 additions and 0 deletions.
diff --git a/packer/README.md b/packer/README.md
@@ -0,0 +1,39 @@
+## Packer Templates for Creating EC2 AMI
+
+### Structure
+
+* **config:** This folder contains all the configuration files.
+* **scripts:** This folder contains all the running scripts during the image creation.
+* **.json:** All templates are now in JSON format, we have not converted them into HCL2 yet.
+
+### Templates
+* jenkins-agent-win2016-x64.json: Windows 2016 Server.
+* jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended).
+* jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3.
+
+### Usages
+
+* You need to install `packer` on your host as a pre-requisite.
+```
+# Needs to be run in this directory
+$ cd packer/
+
+# Run build AMI:
+$ packer build <template json name>
+
+# Run build AMI with debug mode:
+$ packer build -debug <template json name>
+```
+
+### Notes
+
+* Run packer outside of VPN as port 5985/5986, 22, 445 might be blocked for winrm/ssh/smb during the provision, corresponding security group must have the same rules.
+* Make sure the variable section in the template file is filled up, as well as the configs in config folder.
+* Must use a public subnet for packer to connect to the hosts.
+* You can choose to use fixed AMI ID instead of using the AMI filter to find the IDs.
+* EC2Launch vs EC2Launchv2 have a lot of differences, all the templates here are using EC2Launch on Windows AMI.
+* If the process get interrupted in the middle of the run, you need to log onto AWS console to cleanup everything starts with `packer` prefix.
+
+### Thanks
+
+* The Windows templates here are based on the original work by [Ross Derewianko](https://github.com/rderewianko). Thanks for his contribution and post to make it work.
diff --git a/packer/config/windows/launch-config.json b/packer/config/windows/launch-config.json
@@ -0,0 +1,8 @@
+{
+  "setComputerName": false,
+  "setWallpaper": true,
+  "addDnsSuffixList": true,
+  "extendBootVolumeSize": true,
+  "adminPasswordType": "Specify",
+  "adminPassword":  "EnterYourPassWordMustMatchAwsPasswordRequirements"
+}
diff --git a/packer/jenkins-agent-win2016-x64.json b/packer/jenkins-agent-win2016-x64.json
@@ -0,0 +1,86 @@
+{
+  "variables":{
+    "name-base":"Jenkins-Agent-Windows2016",
+    "build-region":"us-east-1",
+    "build-vpc":"vpc-<>",
+    "build-subnet":"subnet-<>",
+    "build-secgrp":"sg-<>",
+    "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}",
+    "aws_ami_region":"us-east-1"
+  },
+  "builders":[
+    {
+      "name":"{{user `name-base`}}-{{user `build-time`}}" ,
+      "ami_description":"{{user `name-base`}}",
+      "type":"amazon-ebs",
+      "encrypt_boot":"false",
+      "region":"{{user `build-region`}}",
+      "ami_regions":"{{user `aws_ami_region`}}",
+      "instance_type":"c5.4xlarge",
+      "ami_name":"{{user `name-base`}}-{{user `build-time`}}",
+      "vpc_id":"{{user `build-vpc`}}",
+      "subnet_id":"{{user `build-subnet`}}",
+      "security_group_ids":"{{user `build-secgrp`}}",
+      "source_ami_filter":{
+        "filters":{
+          "virtualization-type":"hvm",
+          "architecture":"x86_64",
+          "name": "Windows_Server-2016-English-Full-Base-20*",
+          "block-device-mapping.volume-type":"gp2",
+          "root-device-type":"ebs"
+        },
+        "owners":[
+          "amazon"
+        ],
+        "most_recent":true
+      },
+      "user_data_file":"scripts/windows/userdata.ps1",
+      "associate_public_ip_address":false,
+      "communicator":"winrm",
+      "winrm_username":"Administrator",
+      "winrm_timeout":"40m",
+      "windows_password_timeout":"30m",
+      "winrm_insecure":true,
+      "tags":{
+        "Name": "{{user `name-base`}}-{{user `build-time`}}",
+        "OS_Version":"{{user `name-base`}}",
+        "User":"Packer",
+        "Encrypted_AMI":"False",
+        "Created":"{{user `build-time`}}"
+      },
+      "launch_block_device_mappings":[
+        {
+          "device_name":"/dev/sda1",
+          "volume_size":100,
+          "delete_on_termination":true,
+          "volume_type":"gp3"
+        }
+      ]
+    }
+  ],
+  "provisioners":[
+    {
+      "type":"powershell",
+      "scripts": [
+        "scripts/windows/smb-setup.ps1",
+        "scripts/windows/scoop-setup.ps1",
+        "scripts/windows/scoop-install-commons.ps1",
+        "scripts/windows/pip-install.ps1"
+      ],
+      "max_retries": 3
+    },
+    {
+      "type":"file",
+      "source":"config/windows/launch-config.json",
+      "destination":"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Config\\LaunchConfig.json"
+    },
+    {
+      "type":"powershell",
+      "inline":"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule"
+    },
+    {
+      "type":"powershell",
+      "inline":"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1"
+    }
+  ]
+}
diff --git a/packer/jenkins-agent-win2019-x64-alpine-wsl.json b/packer/jenkins-agent-win2019-x64-alpine-wsl.json
@@ -0,0 +1,95 @@
+{
+  "variables": {
+    "name-base":"Jenkins-Agent-Windows2019",
+    "build-region":"us-east-1",
+    "build-vpc":"vpc-<>",
+    "build-subnet":"subnet-<>",
+    "build-secgrp":"sg-<>",
+    "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}",
+    "aws_ami_region":"us-east-1"
+  },
+  "builders": [
+    {
+      "name":"{{user `name-base`}}-{{user `build-time`}}" ,
+      "ami_description":"{{user `name-base`}}",
+      "type":"amazon-ebs",
+      "encrypt_boot":"false",
+      "region":"{{user `build-region`}}",
+      "ami_regions":"{{user `aws_ami_region`}}",
+      "instance_type":"c5.4xlarge",
+      "ami_name":"{{user `name-base`}}-{{user `build-time`}}",
+      "vpc_id":"{{user `build-vpc`}}",
+      "subnet_id":"{{user `build-subnet`}}",
+      "security_group_ids":"{{user `build-secgrp`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type":"hvm",
+          "architecture":"x86_64",
+          "name": "Windows_Server-2019-English-Full-Base-20*",
+          "block-device-mapping.volume-type":"gp2",
+          "root-device-type":"ebs"
+        },
+        "owners": [
+          "amazon"
+        ],
+        "most_recent":true
+      },
+      "user_data_file":"scripts/windows/userdata.ps1",
+      "associate_public_ip_address":false,
+      "communicator":"winrm",
+      "winrm_username":"Administrator",
+      "winrm_timeout":"40m",
+      "windows_password_timeout":"30m",
+      "winrm_insecure":true,
+      "tags": {
+        "Name": "{{user `name-base`}}-{{user `build-time`}}",
+        "OS_Version":"{{user `name-base`}}",
+        "User":"Packer",
+        "Encrypted_AMI":"False",
+        "Created":"{{user `build-time`}}"
+      },
+      "launch_block_device_mappings": [
+        {
+          "device_name":"/dev/sda1",
+          "volume_size":100,
+          "delete_on_termination":true,
+          "volume_type":"gp3"
+        }
+      ]
+    }
+  ],
+  "provisioners": [
+    {
+      "type":"powershell",
+      "scripts": [
+        "scripts/windows/smb-setup-2019-plus.ps1",
+        "scripts/windows/scoop-setup.ps1",
+        "scripts/windows/scoop-install-commons.ps1",
+        "scripts/windows/pip-install.ps1",
+        "scripts/windows/wsl-setup.ps1"
+      ],
+      "max_retries": 3
+    },
+    {
+      "type":"file",
+      "source":"config/windows/launch-config.json",
+      "destination":"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Config\\LaunchConfig.json"
+    },
+    {
+      "type": "windows-restart"
+    },
+    {
+      "type":"powershell",
+      "scripts": [
+        "scripts/windows/scoop-install-alpinewsl.ps1"
+      ]
+    },
+    {
+      "type":"powershell",
+      "inline": [
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule",
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown"
+      ]
+    }
+  ]
+}
diff --git a/packer/jenkins-agent-win2019-x64.json b/packer/jenkins-agent-win2019-x64.json
@@ -0,0 +1,88 @@
+{
+  "variables": {
+    "name-base":"Jenkins-Agent-Windows2019",
+    "build-region":"us-east-1",
+    "build-vpc":"vpc-<>",
+    "build-subnet":"subnet-<>",
+    "build-secgrp":"sg-<>",
+    "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}",
+    "aws_ami_region":"us-east-1"
+  },
+  "builders": [
+    {
+      "name":"{{user `name-base`}}-{{user `build-time`}}" ,
+      "ami_description":"{{user `name-base`}}",
+      "type":"amazon-ebs",
+      "encrypt_boot":"false",
+      "region":"{{user `build-region`}}",
+      "ami_regions":"{{user `aws_ami_region`}}",
+      "instance_type":"c5.4xlarge",
+      "ami_name":"{{user `name-base`}}-{{user `build-time`}}",
+      "vpc_id":"{{user `build-vpc`}}",
+      "subnet_id":"{{user `build-subnet`}}",
+      "security_group_ids":"{{user `build-secgrp`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type":"hvm",
+          "architecture":"x86_64",
+          "name": "Windows_Server-2019-English-Full-Base-20*",
+          "block-device-mapping.volume-type":"gp2",
+          "root-device-type":"ebs"
+        },
+        "owners": [
+          "amazon"
+        ],
+        "most_recent":true
+      },
+      "user_data_file":"scripts/windows/userdata.ps1",
+      "associate_public_ip_address":false,
+      "communicator":"winrm",
+      "winrm_username":"Administrator",
+      "winrm_timeout":"40m",
+      "windows_password_timeout":"30m",
+      "winrm_insecure":true,
+      "tags": {
+        "Name": "{{user `name-base`}}-{{user `build-time`}}",
+        "OS_Version":"{{user `name-base`}}",
+        "User":"Packer",
+        "Encrypted_AMI":"False",
+        "Created":"{{user `build-time`}}"
+      },
+      "launch_block_device_mappings": [
+        {
+          "device_name":"/dev/sda1",
+          "volume_size":100,
+          "delete_on_termination":true,
+          "volume_type":"gp3"
+        }
+      ]
+    }
+  ],
+  "provisioners": [
+    {
+      "type":"powershell",
+      "scripts": [
+        "scripts/windows/smb-setup-2019-plus.ps1",
+        "scripts/windows/scoop-setup.ps1",
+        "scripts/windows/scoop-install-commons.ps1",
+        "scripts/windows/pip-install.ps1"
+      ],
+      "max_retries": 3
+    },
+    {
+      "type":"file",
+      "source":"config/windows/launch-config.json",
+      "destination":"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Config\\LaunchConfig.json"
+    },
+    {
+      "type": "windows-restart"
+    },
+    {
+      "type":"powershell",
+      "inline": [
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule",
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown"
+      ]
+    }
+  ]
+}
diff --git a/packer/scripts/windows/pip-install.ps1 b/packer/scripts/windows/pip-install.ps1
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This needs to be repeated more than twice (sometimes) to actually install packages without --user
+# After applying the install-pep-514.reg from scoop for Python specifically
+
+# Need TLS12 in order to install pip correctly
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+wget https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py
+python get-pip.py
+pip --version
+
+# Install pipenv
+pip install pipenv
+pipenv --version
+
+# Install awscli
+pip install awscli
+aws --version
+
+# Cleanup
+Remove-Item 'get-pip.py' -Force
diff --git a/packer/scripts/windows/scoop-install-alpinewsl.ps1 b/packer/scripts/windows/scoop-install-alpinewsl.ps1
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# Disable "current" alias directory as it is not preserved after AMI creation
+# Use static path in environment variable
+scoop config NO_JUNCTIONS true
+
+# Install alpine
+scoop bucket add extras
+scoop install alpinewsl