From 547b7daf3649db67b6028650924f5706c730a150 Mon Sep 17 00:00:00 2001 From: echohack Date: Thu, 6 Jul 2017 21:25:55 -0700 Subject: [PATCH] added load testing resource in testing.tf and updated ami method Signed-off-by: echohack --- .chef/delivery-validator.pem.pub | 1 + chef_load.conf.tpl | 97 ++++++++++++++++++++++++++++++++ files/chef_load.service | 15 +++++ files/installer.sh | 3 +- main.tf | 32 ++++++++--- testing.tf | 94 +++++++++++++++++++++++++++++++ variables.tf | 4 -- 7 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 .chef/delivery-validator.pem.pub create mode 100644 chef_load.conf.tpl create mode 100644 files/chef_load.service create mode 100644 testing.tf diff --git a/.chef/delivery-validator.pem.pub b/.chef/delivery-validator.pem.pub new file mode 100644 index 0000000..682e7aa --- /dev/null +++ b/.chef/delivery-validator.pem.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjXesfQjFFI8s5HgiexeuEQ0JFpMNc01/dC9/llaimceWqKdLk/KDeYCb0DiQNixKCwo6ah+5gRx/s70aPyfxeqmRlSf1WCuPWpPL4oPM6NaLRG0hJYjubVIumrwzsF5rL1JHGLwRmw3zxZ8LaSufIS+rp2r65oGZnyAGXJx0okWatZVcDie00ZhR2qRzfReyE/R50sgDfmujQwmJdKcangj6S6z/mbRhxn535dobOzNi9mT94Gj42FV65D7/q5TLEG7c9A1qEsKNPyFORInX3qE40RcA48S5u9TMxFeu+3IUD+YLH+LgOUzqe7hCUPk6+1qRTf1AF1PAu1x1hO0ED echohack@echohack diff --git a/chef_load.conf.tpl b/chef_load.conf.tpl new file mode 100644 index 0000000..33a5792 --- /dev/null +++ b/chef_load.conf.tpl @@ -0,0 +1,97 @@ +# The chef_server_url, client_name and client_key parameters must be set if you want +# to make API requests to a Chef Server. +# +# Be sure to include the organization name +# For example: chef_server_url = "https://chef.example.com/organizations/demo/" +chef_server_url = "https://${chef_server_fqdn}/organizations/delivery/" +# +# The client defined by client_name needs to be an admin user of the Chef Server org. +client_name = "delivery-validator" +client_key = "./delivery-validator.pem" + +# The data_collector_url must be set if you want to make API requests to an Automate server. +# For example: data_collector_url = "https://automate.example.org/data-collector/v0/" +data_collector_url = "https://${automate_server_fqdn}/data-collector/v0/" + +# The Authorization token for the Automate server. +# The following default value is sufficient unless you set your own token in your Automate server. +data_collector_token = "93a49a4f2482c64126f7b6015e6b0f30284287ee4054ff8807fb63d9cbd1c506" + +# Ohai data will be loaded from this file and used for the nodes' automatic attributes. +# See the chef-load README for instructions for creating an ohai JSON file. +ohai_json_file = "/home/centos/chef-load/sample-data/example-ohai.json" + +# Data from a converge status report will be loaded from this file and used +# for each node's converge status report that is sent to the Automate server. +# See the chef-load README for instructions for creating a converge status JSON file. +converge_status_json_file = "/home/centos/chef-load/sample-data/example-converge-status.json" + +# Data from a compliance status report will be loaded from this file and used +# for each node's compliance status report that is sent to the Automate server. +# See the chef-load README for instructions for creating a compliance status JSON file. +compliance_status_json_file = "/home/centos/chef-load/sample-data/example-compliance-status.json" + +# The number of Chef Client runs to be made per minute +runs_per_minute = 1 + +# Number of minutes between a node's chef-client runs +interval = 30 + +# This prefix will go at the beginning of each node name. +# This enables running multiple instances of chef-load without affecting each others' nodes +# For example, a value of "chef-load" will result in nodes named "chef-load-1", "chef-load-2", ... +node_name_prefix = "chef-load" + +# Chef environment used for each node +chef_environment = "_default" + +# run_list is the run list used for each node. It should be a list of strings. +# For example: run_list = [ "role[role_name]", "recipe_name", "recipe[different_recipe_name@1.0.0]" ] +# The default value is an empty run_list. +run_list = [ ] + +# sleep_duration is an optional setting that is available to provide a delay to simulate +# the amount of time a Chef Client takes actually converging all of the run list's resources. +# sleep_duration is measured in seconds +sleep_duration = 0 + +# download_cookbooks controls which chef-client run downloads cookbook files. +# Options are: "never", "first" (first chef-client run only), "always" +# +# Downloading cookbooks can significantly increase the number of API requests that chef-load +# makes depending on the run_list. +# +# Normal TCP protocol requires the connections to be in TIME-WAIT for about two minutes and it is +# recommended that the system's TIME-WAIT parameter's do not get changed. +# Ref: http://vincent.bernat.im/en/blog/2014-tcp-time-wait-state-linux.html +# +# If chef-load makes enough API requests then the number of connections can increase until +# the system runs out of ephemeral ports resulting in connect(2) error EADDRNOTAVAIL. +# Ref: http://manpages.ubuntu.com/manpages/trusty/en/man2/connect.2freebsd.html +# Ref: http://manpages.ubuntu.com/manpages/trusty/en/man7/ip.7.html +# +# If you aren't concerned with simulating the download of cookbook files then the recommendation is +# to use "never" or "first". +# +# If you want to use "always" and you run out of ephemeral ports then consider increasing the range of +# ephemeral ports or reducing load by reducing the requests_per_minute setting. +# Ref: http://www.cyberciti.biz/tips/linux-increase-outgoing-network-sockets-range.html +# +download_cookbooks = "never" + +# api_get_requests is an optional list of API GET requests that are made during the chef-client run. +# This is used to simulate the API requests that the cookbooks would make. +# For example, it can make Chef Search or data bag item requests. +# The values can be either full URLs that include the chef_server_url portion or just the portion of +# the URL that comes after the chef_server_url. +# For example, to make a Chef Search API request that searches for all nodes you can use either of the +# following values: +# +# "https://chef.example.com/organizations/orgname/search/node?q=*%253A*&sort=X_CHEF_id_CHEF_X%20asc&start=0" +# or +# "search/node?q=*%253A*&sort=X_CHEF_id_CHEF_X%20asc&start=0" +# +api_get_requests = [ ] + +# Send data to the Chef server's Reporting service +enable_reporting = false diff --git a/files/chef_load.service b/files/chef_load.service new file mode 100644 index 0000000..0a02b86 --- /dev/null +++ b/files/chef_load.service @@ -0,0 +1,15 @@ +[Unit] +Description=Chef load testing tool +After=network.target + +[Service] +ExecStart=/home/centos/chef-load-1.0.0 -config /home/centos/chef_load.conf +Type=forking +PIDFile=/tmp/chef_load.pid +Restart=always +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/files/installer.sh b/files/installer.sh index 0e85b8f..2f1bf92 100644 --- a/files/installer.sh +++ b/files/installer.sh @@ -91,7 +91,8 @@ source 'https://supermarket.chef.io' cookbook 'chef-services', git: 'https://github.com/itmustbejj/chef-services.git', branch: 'deltron-changes' cookbook 'chef-ingredient', git: 'https://github.com/itmustbejj/chef-ingredient', branch: 'debug-branch' -cookbook 'elasticsearch', git: 'https://github.com/elastic/cookbook-elasticsearch', branch: '2.x.x' +cookbook 'collect_metrics', git: 'https://github.com/yzl/collect_metrics.git' +cookbook 'elasticsearch', git: 'https://github.com/itmustbejj/cookbook-elasticsearch', branch: '2.x.x' cookbook 'java' cookbook 'sysctl' cookbook 'backend_search_cluster', git: 'https://github.com/itmustbejj/backend_search_cluster' diff --git a/main.tf b/main.tf index 8092b3d..16876b4 100644 --- a/main.tf +++ b/main.tf @@ -15,6 +15,22 @@ data "aws_subnet_ids" "automate" { vpc_id = "${var.automate_vpc}" } +data "aws_ami" "centos" { + most_recent = true + + filter { + name = "name" + values = ["chef-highperf-centos7-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["446539779517"] +} + # Chef Server resource "aws_instance" "chef_server" { connection { @@ -22,13 +38,13 @@ resource "aws_instance" "chef_server" { private_key = "${file("${var.aws_key_pair_file}")}" } - ami = "${var.aws_ami_rhel}" - instance_type = "${var.aws_instance_type}" - key_name = "${var.aws_key_pair_name}" - subnet_id = "${data.aws_subnet_ids.automate.ids[1]}" - vpc_security_group_ids = ["${aws_security_group.chef_server.id}"] + ami = "${data.aws_ami.centos.id}" + instance_type = "${var.aws_instance_type}" + key_name = "${var.aws_key_pair_name}" + subnet_id = "${data.aws_subnet_ids.automate.ids[1]}" + vpc_security_group_ids = ["${aws_security_group.chef_server.id}"] associate_public_ip_address = true - ebs_optimized = true + ebs_optimized = true root_block_device { delete_on_termination = true @@ -104,7 +120,7 @@ resource "aws_instance" "chef_automate" { private_key = "${file("${var.aws_key_pair_file}")}" } - ami = "${var.aws_ami_rhel}" + ami = "${data.aws_ami.centos.id}" instance_type = "${var.aws_instance_type}" key_name = "${var.aws_key_pair_name}" subnet_id = "${data.aws_subnet_ids.automate.ids[1]}" @@ -193,7 +209,7 @@ resource "aws_instance" "build_nodes" { private_key = "${file("${var.aws_key_pair_file}")}" } - ami = "${var.aws_ami_rhel}" + ami = "${data.aws_ami.centos.id}" instance_type = "t2.medium" key_name = "${var.aws_key_pair_name}" subnet_id = "${data.aws_subnet_ids.automate.ids[1]}" diff --git a/testing.tf b/testing.tf new file mode 100644 index 0000000..43defcb --- /dev/null +++ b/testing.tf @@ -0,0 +1,94 @@ +data "template_file" "chef_load_conf" { + template = "${file("./chef_load.conf.tpl")}" + + vars { + chef_server_fqdn = "${aws_instance.chef_server.public_dns}" + automate_server_fqdn = "${aws_instance.chef_automate.public_dns}" + } +} + +resource "aws_instance" "chef_load" { + connection { + user = "${var.aws_ami_user}" + private_key = "${file("${var.aws_key_pair_file}")}" + } + + count = 0 + ami = "${data.aws_ami.centos.id}" + instance_type = "${var.aws_instance_type}" + key_name = "${var.aws_key_pair_name}" + subnet_id = "${data.aws_subnet_ids.automate.ids[1]}" + vpc_security_group_ids = ["${aws_security_group.chef_automate.id}"] + associate_public_ip_address = true + ebs_optimized = true + + root_block_device { + delete_on_termination = true + volume_size = 20 + volume_type = "gp2" + + #iops = 1000 + } + + ebs_block_device { + device_name = "/dev/sdb" + volume_type = "io1" + iops = 5000 # iops = volume_size * 50 + volume_size = 100 + delete_on_termination = true + } + + tags { + Name = "${format("${var.tag_automate}_chef_load_%02d_${random_id.automate_instance_id.hex}", count.index + 1)}" + X-Dept = "${var.tag_dept}" + X-Contact = "${var.tag_contact}" + } + + # Set hostname in separate connection. + # Transient hostname doesn't set correctly in time otherwise. + provisioner "remote-exec" { + inline = [ + "sudo hostnamectl set-hostname ${aws_instance.chef_automate.public_dns}", + "sudo mkdir /etc/chef/", + ] + } + + # mount the EBS volume + provisioner "file" { + source = "mount_data_volume" + destination = "/tmp/mount_data_volume" + } + + provisioner "file" { + content = "${data.template_file.delivery_validator.rendered}" + destination = "/home/centos/delivery-validator.pem" + } + + provisioner "file" { + content = "${data.template_file.chef_load_conf.rendered}" + destination = "/home/centos/chef_load.conf" + } + + provisioner "file" { + source = "./files/chef_load.service" + destination = "/tmp/chef_load.service" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv /tmp/chef_load.service /etc/systemd/system/chef_load.service", + ] + } + + provisioner "remote-exec" { + inline = [ + "sudo bash -ex /tmp/mount_data_volume", + "sudo yum install git -y", + "cd && git clone https://github.com/jeremiahsnapp/chef-load.git", + "wget https://github.com/chef/chef-load/releases/download/v1.0.0/chef-load_1.0.0_Linux_64bit -O chef-load-1.0.0", + "chmod +x chef-load-1.0.0", + "chmod 600 delivery-validator.pem", + "knife ssl fetch https://${aws_instance.chef_server.public_dns}", + ] + } +} diff --git a/variables.tf b/variables.tf index e0aab82..c4c3f9e 100644 --- a/variables.tf +++ b/variables.tf @@ -41,7 +41,3 @@ variable "aws_instance_type" { variable "aws_ami_user" { default = "centos" } - -variable "aws_ami_rhel" { - default = "ami-e6963186" -}