From 356a641caa790796bd34c0ecd8496dbcbfd38b14 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:49:59 +0100 Subject: [PATCH 01/23] set up rspec --- Gemfile | 8 ++++++++ Guardfile | 9 +++++++++ spec/spec_helper.rb | 17 +++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 Guardfile create mode 100644 spec/spec_helper.rb diff --git a/Gemfile b/Gemfile index dc3b5ba..8aae5bc 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,11 @@ source "https://rubygems.org" gemfile gemspec + +group :development do + gem 'chef' + gem 'chef-metal' + gem 'guard' + gem 'guard-rspec' + gem 'rb-readline' +end diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..59bc9e5 --- /dev/null +++ b/Guardfile @@ -0,0 +1,9 @@ +# A sample Guardfile +# More info at https://github.com/guard/guard#readme + +guard :rspec do + watch(%r{^spec/.+_spec\.rb$}) + watch(%r{^lib/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" } + watch('spec/spec_helper.rb') { "spec" } +end + diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..324f142 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,17 @@ +$:.unshift File.expand_path('../../lib', __FILE__) +require 'fog' +require 'chef_metal' +require 'chef_metal_fog' + +RSpec.configure do |config| + config.run_all_when_everything_filtered = true + config.filter_run :focus + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = 'random' +end + +Fog.mock! From 76334ea60da93157813e94cd13d03c259eb92668 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:54:13 +0100 Subject: [PATCH 02/23] Select and load the correct driver based on URI --- lib/chef_metal_fog/fog_driver.rb | 15 +++++++++++++++ spec/unit/fog_driver_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 spec/unit/fog_driver_spec.rb diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 7cd2f97..c78140f 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -102,6 +102,21 @@ class FogDriver < ChefMetal::Driver :ssh_timeout => 20 } + class << self + alias :__new__ :new + + def inherited(klass) + class << klass + alias :new :__new__ + end + end + end + + def self.new(driver_url, config) + driver = driver_url.split(':')[1] + ChefMetalFog::Drivers.const_get(driver).new(driver_url, config) + end + # Passed in a driver_url, and a config in the format of Driver.config. def self.from_url(driver_url, config) FogDriver.new(driver_url, config) diff --git a/spec/unit/fog_driver_spec.rb b/spec/unit/fog_driver_spec.rb new file mode 100644 index 0000000..14baad5 --- /dev/null +++ b/spec/unit/fog_driver_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' +require 'chef_metal_fog/fog_driver' + +describe ChefMetalFog::FogDriver do + module ChefMetalFog::Drivers + class TestDriver < ChefMetalFog::FogDriver + attr_reader :config + def initialize(driver_url, config) + super + end + end + end + + describe "when creating a new class" do + it "should return the correct class" do + test = ChefMetalFog::FogDriver.new('fog:TestDriver:foo', {}) + expect(test).to be_an_instance_of ChefMetalFog::Drivers::TestDriver + end + + it "should populate config" do + test = ChefMetalFog::FogDriver.new('fog:TestDriver:foo', {test: "metal"}) + expect(test.config[:test]).to eq "metal" + end + end +end From 8fbc5071465eb081bc24dfa6bcf526f52ec1e43c Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:55:27 +0100 Subject: [PATCH 03/23] ignore generated cruft --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4c07b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +pkg/ +*.lock +.*~ +tags From a13ab5d46010b993ef20e3fbaeb2ecadd48522b5 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:57:31 +0100 Subject: [PATCH 04/23] add rackspace driver --- lib/chef_metal_fog.rb | 2 +- lib/chef_metal_fog/drivers/rackspace.rb | 36 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 lib/chef_metal_fog/drivers/rackspace.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index 1aea216..0796a07 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -1,3 +1,3 @@ require 'chef_metal' -require 'chef_metal_fog/fog_driver' +require 'chef_metal_fog/drivers/rackspace' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/rackspace.rb b/lib/chef_metal_fog/drivers/rackspace.rb new file mode 100644 index 0000000..0af3d74 --- /dev/null +++ b/lib/chef_metal_fog/drivers/rackspace.rb @@ -0,0 +1,36 @@ +require 'chef_metal_fog/fog_driver' +module ChefMetalFog + module Drivers + class Rackspace < ChefMetalFog::FogDriver + + def creator + compute_options[:rackspace_username] + end + + def self.compute_options_for(provider, id, config) + new_compute_options = {} + new_compute_options[:provider] = provider + new_config = { :driver_options => { :compute_options => new_compute_options }} + new_defaults = { + :driver_options => { :compute_options => {} }, + :machine_options => { :bootstrap_options => {} } + } + result = Cheffish::MergedConfig.new(new_config, config, new_defaults) + + new_compute_options[:rackspace_auth_url] = id if (id && id != '') + credential = Fog.credentials + + new_compute_options[:rackspace_username] ||= credential[:rackspace_username] + new_compute_options[:rackspace_api_key] ||= credential[:rackspace_api_key] + new_compute_options[:rackspace_auth_url] ||= credential[:rackspace_auth_url] + new_compute_options[:rackspace_region] ||= credential[:rackspace_region] + new_compute_options[:rackspace_endpoint] ||= credential[:rackspace_endpoint] + + id = result[:driver_options][:compute_options][:rackspace_auth_url] + + [result, id] + end + + end + end +end From 95804d1ffd6e92ae35a6a188cbd99445d6fbcb41 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:58:08 +0100 Subject: [PATCH 05/23] add openstack driver --- lib/chef_metal_fog.rb | 1 + lib/chef_metal_fog/drivers/openstack.rb | 35 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 lib/chef_metal_fog/drivers/openstack.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index 0796a07..336bc6a 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -1,3 +1,4 @@ require 'chef_metal' require 'chef_metal_fog/drivers/rackspace' +require 'chef_metal_fog/drivers/openstack' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/openstack.rb b/lib/chef_metal_fog/drivers/openstack.rb new file mode 100644 index 0000000..c891d00 --- /dev/null +++ b/lib/chef_metal_fog/drivers/openstack.rb @@ -0,0 +1,35 @@ +require 'chef_metal_fog/fog_driver' +module ChefMetalFog + module Drivers + class OpenStack < ChefMetalFog::FogDriver + + def creator + compute_options[:openstack_username] + end + + def self.compute_options_for(provider, id, config) + new_compute_options = {} + new_compute_options[:provider] = provider + new_config = { :driver_options => { :compute_options => new_compute_options }} + new_defaults = { + :driver_options => { :compute_options => {} }, + :machine_options => { :bootstrap_options => {} } + } + result = Cheffish::MergedConfig.new(new_config, config, new_defaults) + + new_compute_options[:openstack_auth_url] = id if (id && id != '') + credential = Fog.credentials + + new_compute_options[:openstack_username] ||= credential[:openstack_username] + new_compute_options[:openstack_api_key] ||= credential[:openstack_api_key] + new_compute_options[:openstack_auth_url] ||= credential[:openstack_auth_url] + new_compute_options[:openstack_tenant] ||= credential[:openstack_tenant] + + id = result[:driver_options][:compute_options][:openstack_auth_url] + + [result, id] + end + + end + end +end From 26206117055b71320b2024f6a623aeb7e548ade3 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 16:59:45 +0100 Subject: [PATCH 06/23] add DigitalOcean driver --- lib/chef_metal_fog.rb | 1 + lib/chef_metal_fog/drivers/digital_ocean.rb | 99 +++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 lib/chef_metal_fog/drivers/digital_ocean.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index 336bc6a..7a8fc79 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -1,4 +1,5 @@ require 'chef_metal' require 'chef_metal_fog/drivers/rackspace' require 'chef_metal_fog/drivers/openstack' +require 'chef_metal_fog/drivers/digital_ocean' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/digital_ocean.rb b/lib/chef_metal_fog/drivers/digital_ocean.rb new file mode 100644 index 0000000..4117786 --- /dev/null +++ b/lib/chef_metal_fog/drivers/digital_ocean.rb @@ -0,0 +1,99 @@ +require 'chef_metal_fog/fog_driver' + +# fog:DigitalOcean: +module ChefMetalFog + module Drivers + class DigitalOcean < ChefMetalFog::FogDriver + + def bootstrap_options_for(action_handler, machine_spec, machine_options) + bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) + unless bootstrap_options[:key_name] + bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) + end + + tags = { + 'Name' => machine_spec.name, + 'BootstrapId' => machine_spec.id, + 'BootstrapHost' => Socket.gethostname, + 'BootstrapUser' => Etc.getlogin + } + # User-defined tags override the ones we set + tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] + bootstrap_options.merge!({ :tags => tags }) + + if !bootstrap_options[:image_id] + bootstrap_options[:image_name] ||= 'CentOS 6.4 x32' + bootstrap_options[:image_id] = compute.images.select { |image| image.name == bootstrap_options[:image_name] }.first.id + end + if !bootstrap_options[:flavor_id] + bootstrap_options[:flavor_name] ||= '512MB' + bootstrap_options[:flavor_id] = compute.flavors.select { |flavor| flavor.name == bootstrap_options[:flavor_name] }.first.id + end + if !bootstrap_options[:region_id] + bootstrap_options[:region_name] ||= 'San Francisco 1' + bootstrap_options[:region_id] = compute.regions.select { |region| region.name == bootstrap_options[:region_name] }.first.id + end + found_key = compute.ssh_keys.select { |k| k.name == bootstrap_options[:key_name] }.first + if !found_key + raise "Could not find key named '#{bootstrap_options[:key_name]}' on #{driver_url}" + end + bootstrap_options[:ssh_key_ids] ||= [ found_key.id ] + + # You don't get to specify name yourself + bootstrap_options[:name] = machine_spec.name + bootstrap_options[:name] ||= machine_spec.name + + bootstrap_options + end + + def self.compute_options_for(provider, id, config) + new_compute_options = {} + new_compute_options[:provider] = provider + new_config = { :driver_options => { :compute_options => new_compute_options }} + new_defaults = { + :driver_options => { :compute_options => {} }, + :machine_options => { :bootstrap_options => {} } + } + result = Cheffish::MergedConfig.new(new_config, config, new_defaults) + + new_compute_options[:digitalocean_client_id] = id if (id && id != '') + + # This uses ~/.tugboat, generated by "tugboat authorize" - see https://github.com/pearkes/tugboat + tugboat_file = File.expand_path('~/.tugboat') + if File.exist?(tugboat_file) + tugboat_data = YAML.load(IO.read(tugboat_file)) + new_compute_options.merge!( + :digitalocean_client_id => tugboat_data['authentication']['client_key'], + :digitalocean_api_key => tugboat_data['authentication']['api_key'] + ) + new_defaults[:machine_options].merge!( + #:ssh_username => tugboat_data['ssh']['ssh_user'], + :ssh_options => { + :port => tugboat_data['ssh']['ssh_port'], + # TODO we ignore ssh_key_path in favor of ssh_key / key_name stuff + #:key_data => [ IO.read(tugboat_data['ssh']['ssh_key_path']) ] # TODO use paths, not data? + } + ) + + # TODO verify that the key_name exists and matches the ssh key path + + new_defaults[:machine_options][:bootstrap_options].merge!( + :region_id => tugboat_data['defaults']['region'].to_i, + :image_id => tugboat_data['defaults']['image'].to_i, + :size_id => tugboat_data['defaults']['region'].to_i, + :private_networking => tugboat_data['defaults']['private_networking'] == 'true', + :backups_enabled => tugboat_data['defaults']['backups_enabled'] == 'true', + ) + ssh_key = tugboat_data['defaults']['ssh_key'] + if ssh_key && ssh_key.size > 0 + new_defaults[:machine_options][:bootstrap_options][:key_name] = ssh_key + end + end + id = result[:driver_options][:compute_options][:digitalocean_client_id] + + [result, id] + end + + end + end +end From b0f468454b1a1fe6f6f4fb2f5355feb4d06be860 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:24:13 +0100 Subject: [PATCH 07/23] add aws driver --- lib/chef_metal_fog.rb | 1 + lib/chef_metal_fog/drivers/aws.rb | 61 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 lib/chef_metal_fog/drivers/aws.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index 7a8fc79..ecabce4 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -2,4 +2,5 @@ require 'chef_metal_fog/drivers/rackspace' require 'chef_metal_fog/drivers/openstack' require 'chef_metal_fog/drivers/digital_ocean' +require 'chef_metal_fog/drivers/aws' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/aws.rb b/lib/chef_metal_fog/drivers/aws.rb new file mode 100644 index 0000000..980c564 --- /dev/null +++ b/lib/chef_metal_fog/drivers/aws.rb @@ -0,0 +1,61 @@ +require 'chef_metal_fog/fog_driver' + +# fog:AWS:: +# fog:AWS: +# fog:AWS:: +module ChefMetalFog + module Drivers + class AWS < ChefMetalFog::FogDriver + + def creator + driver_options[:aws_account_info][:aws_username] + end + + def ssh_username + machine_spec.location['ssh_username'] || 'ubuntu' + end + + def self.compute_options_for(provider, id, config) + new_compute_options = {} + new_compute_options[:provider] = provider + new_config = { :driver_options => { :compute_options => new_compute_options }} + new_defaults = { + :driver_options => { :compute_options => {} }, + :machine_options => { :bootstrap_options => {} } + } + result = Cheffish::MergedConfig.new(new_config, config, new_defaults) + + if id && id != '' + # AWS canonical URLs are of the form fog:AWS: + if id =~ /^(\d{12})(:(.+))?$/ + if $2 + id = $1 + new_compute_options[:region] = $3 + else + Chef::Log.warn("Old-style AWS URL #{id} from an early beta of chef-metal (before 0.11-final) found. If you have servers in multiple regions on this account, you may see odd behavior like servers being recreated. To fix, edit any nodes with attribute metal.location.driver_url to include the region like so: fog:AWS:#{id}: (e.g. us-east-1)") + end + else + # Assume it is a profile name, and set that. + aws_profile, region = id.split(':', 2) + new_config[:driver_options][:aws_profile] = aws_profile + new_compute_options[:region] = region + id = nil + end + end + + aws_profile = FogDriverAWS.get_aws_profile(result[:driver_options], id) + new_compute_options[:aws_access_key_id] = aws_profile[:aws_access_key_id] + new_compute_options[:aws_secret_access_key] = aws_profile[:aws_secret_access_key] + new_compute_options[:aws_session_token] = aws_profile[:aws_security_token] + new_defaults[:driver_options][:compute_options][:region] = aws_profile[:region] + + account_info = FogDriverAWS.aws_account_info_for(result[:driver_options][:compute_options]) + new_config[:driver_options][:aws_account_info] = account_info + id = "#{account_info[:aws_account_id]}:#{result[:driver_options][:compute_options][:region]}" + + [result, id] + end + + end + end +end From c674ea99c8924c36586456aebb4f7c3f674e9e8a Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:30:50 +0100 Subject: [PATCH 08/23] add cloudstack driver --- lib/chef_metal_fog.rb | 1 + lib/chef_metal_fog/drivers/cloudstack.rb | 35 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 lib/chef_metal_fog/drivers/cloudstack.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index ecabce4..85a9962 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -3,4 +3,5 @@ require 'chef_metal_fog/drivers/openstack' require 'chef_metal_fog/drivers/digital_ocean' require 'chef_metal_fog/drivers/aws' +require 'chef_metal_fog/drivers/cloudstack' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/cloudstack.rb b/lib/chef_metal_fog/drivers/cloudstack.rb new file mode 100644 index 0000000..9a3f99b --- /dev/null +++ b/lib/chef_metal_fog/drivers/cloudstack.rb @@ -0,0 +1,35 @@ +require 'chef_metal_fog/fog_driver' +module ChefMetalFog + module Drivers + class CloudStack < ChefMetalFog::FogDriver + + def self.compute_options_for(provider, id, config) + new_compute_options = {} + new_compute_options[:provider] = provider + new_config = { :driver_options => { :compute_options => new_compute_options }} + new_defaults = { + :driver_options => { :compute_options => {} }, + :machine_options => { :bootstrap_options => {} } + } + result = Cheffish::MergedConfig.new(new_config, config, new_defaults) + + if id && id != '' + cloudstack_uri = URI.parse(id) + new_compute_options[:cloudstack_scheme] = cloudstack_uri.scheme + new_compute_options[:cloudstack_host] = cloudstack_uri.host + new_compute_options[:cloudstack_port] = cloudstack_uri.port + new_compute_options[:cloudstack_path] = cloudstack_uri.path + end + + host = result[:driver_options][:compute_options][:cloudstack_host] + path = result[:driver_options][:compute_options][:cloudstack_path] || '/client/api' + port = result[:driver_options][:compute_options][:cloudstack_port] || 443 + scheme = result[:driver_options][:compute_options][:cloudstack_scheme] || 'https' + id = URI.scheme_list[scheme.upcase].build(:host => host, :port => port, :path => path).to_s + + [result, id] + end + + end + end +end From c9c9402e29e441ffd67f1623f7097eb9f7af49b2 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:36:54 +0100 Subject: [PATCH 09/23] allow drivers to override default ssh username --- lib/chef_metal_fog/fog_driver.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index c78140f..ada5001 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -527,19 +527,17 @@ def ssh_options_for(machine_spec, machine_options, server) result end + def ssh_username + machine_spec.location['ssh_username'] || 'root' + end + def create_ssh_transport(machine_spec, machine_options, server) ssh_options = ssh_options_for(machine_spec, machine_options, server) - # If we're on AWS, the default is to use ubuntu, not root - if provider == 'AWS' - username = machine_spec.location['ssh_username'] || 'ubuntu' - else - username = machine_spec.location['ssh_username'] || 'root' - end if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.location['ssh_username'] Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.location['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}. Using #{machine_spec.location['ssh_username']}. Please edit the node and change the metal.location.ssh_username attribute if you want to change it.") end options = {} - if machine_spec.location[:sudo] || (!machine_spec.location.has_key?(:sudo) && username != 'root') + if machine_spec.location[:sudo] || (!machine_spec.location.has_key?(:sudo) && ssh_username != 'root') options[:prefix] = 'sudo ' end @@ -559,7 +557,7 @@ def create_ssh_transport(machine_spec, machine_options, server) options[:ssh_pty_enable] = true options[:ssh_gateway] = machine_spec.location['ssh_gateway'] if machine_spec.location.has_key?('ssh_gateway') - ChefMetal::Transport::SSH.new(remote_host, username, ssh_options, options, config) + ChefMetal::Transport::SSH.new(remote_host, ssh_username, ssh_options, options, config) end def self.compute_options_for(provider, id, config) From c6d6094748efb0dadaa8a493b1b069abb7e24d5b Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:37:36 +0100 Subject: [PATCH 10/23] allow drivers to pull out creator --- lib/chef_metal_fog/fog_driver.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index ada5001..7072457 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -245,6 +245,10 @@ def option_for(machine_options, key) machine_options[key] || DEFAULT_OPTIONS[key] end + def creator + '' + end + def create_server(action_handler, machine_spec, machine_options) if machine_spec.location if machine_spec.location['driver_url'] != driver_url @@ -270,14 +274,6 @@ def create_server(action_handler, machine_spec, machine_options) server = nil action_handler.report_progress description if action_handler.should_perform_actions - creator = case provider - when 'AWS' - driver_options[:aws_account_info][:aws_username] - when 'OpenStack' - compute_options[:openstack_username] - when 'Rackspace' - compute_options[:rackspace_username] - end server = compute.servers.create(bootstrap_options) machine_spec.location = { 'driver_url' => driver_url, From 996608a9fa1a549d1c02437e0175519ba24f7cf9 Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:37:57 +0100 Subject: [PATCH 11/23] remove driver specific code --- lib/chef_metal_fog/fog_driver.rb | 166 +------------------------------ 1 file changed, 3 insertions(+), 163 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 7072457..ce30856 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -25,14 +25,7 @@ module ChefMetalFog # ## Fog Driver URLs # # All Metal drivers use URLs to uniquely identify a driver's "bucket" of machines. - # Fog URLs are of the form fog::: - # - # fog:AWS:: - # fog:AWS: - # fog:AWS:: - # fog:OpenStack:https://identityHost:portNumber/v2.0 - # fog:DigitalOcean: - # fog:Rackspace:https://identity.api.rackspacecloud.com/v2.0 + # Fog URLs are of the form fog:: # # Identifier is generally something uniquely identifying the account. If multiple # users can access the account, the identifier should be the same for all of @@ -429,7 +422,7 @@ def overwrite_default_key_willy_nilly(action_handler) def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) - if (provider == 'DigitalOcean' || provider == 'AWS') && !bootstrap_options[:key_name] + if provider == 'AWS' && !bootstrap_options[:key_name] bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) end tags = { @@ -442,30 +435,6 @@ def bootstrap_options_for(action_handler, machine_spec, machine_options) tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] bootstrap_options.merge!({ :tags => tags }) - # Provide reasonable defaults for DigitalOcean - if provider == 'DigitalOcean' - if !bootstrap_options[:image_id] - bootstrap_options[:image_name] ||= 'CentOS 6.4 x32' - bootstrap_options[:image_id] = compute.images.select { |image| image.name == bootstrap_options[:image_name] }.first.id - end - if !bootstrap_options[:flavor_id] - bootstrap_options[:flavor_name] ||= '512MB' - bootstrap_options[:flavor_id] = compute.flavors.select { |flavor| flavor.name == bootstrap_options[:flavor_name] }.first.id - end - if !bootstrap_options[:region_id] - bootstrap_options[:region_name] ||= 'San Francisco 1' - bootstrap_options[:region_id] = compute.regions.select { |region| region.name == bootstrap_options[:region_name] }.first.id - end - found_key = compute.ssh_keys.select { |k| k.name == bootstrap_options[:key_name] }.first - if !found_key - raise "Could not find key named '#{bootstrap_options[:key_name]}' on #{driver_url}" - end - bootstrap_options[:ssh_key_ids] ||= [ found_key.id ] - - # You don't get to specify name yourself - bootstrap_options[:name] = machine_spec.name - end - bootstrap_options[:name] ||= machine_spec.name bootstrap_options @@ -557,136 +526,7 @@ def create_ssh_transport(machine_spec, machine_options, server) end def self.compute_options_for(provider, id, config) - driver_options = config[:driver_options] || {} - compute_options = driver_options[:compute_options] || {} - new_compute_options = {} - new_compute_options[:provider] = provider - new_config = { :driver_options => { :compute_options => new_compute_options }} - new_defaults = { - :driver_options => { :compute_options => {} }, - :machine_options => { :bootstrap_options => {} } - } - result = Cheffish::MergedConfig.new(new_config, config, new_defaults) - - # Get data from the identifier in the URL - if id && id != '' - case provider - when 'AWS' - # AWS canonical URLs are of the form fog:AWS: - if id =~ /^(\d{12})(:(.+))?$/ - if $2 - id = $1 - new_compute_options[:region] = $3 - else - Chef::Log.warn("Old-style AWS URL #{id} from an early beta of chef-metal (before 0.11-final) found. If you have servers in multiple regions on this account, you may see odd behavior like servers being recreated. To fix, edit any nodes with attribute metal.location.driver_url to include the region like so: fog:AWS:#{id}: (e.g. us-east-1)") - end - else - # Assume it is a profile name, and set that. - aws_profile, region = id.split(':', 2) - new_config[:driver_options][:aws_profile] = aws_profile - new_compute_options[:region] = region - id = nil - end - when 'DigitalOcean' - new_compute_options[:digitalocean_client_id] = id - when 'OpenStack' - new_compute_options[:openstack_auth_url] = id - when 'Rackspace' - new_compute_options[:rackspace_auth_url] = id - when 'CloudStack' - cloudstack_uri = URI.parse(id) - new_compute_options[:cloudstack_scheme] = cloudstack_uri.scheme - new_compute_options[:cloudstack_host] = cloudstack_uri.host - new_compute_options[:cloudstack_port] = cloudstack_uri.port - new_compute_options[:cloudstack_path] = cloudstack_uri.path - else - raise "unsupported fog provider #{provider}" - end - end - - # Set auth info from environment - case provider - when 'AWS' - # Grab the profile - aws_profile = FogDriverAWS.get_aws_profile(result[:driver_options], id) - new_compute_options[:aws_access_key_id] = aws_profile[:aws_access_key_id] - new_compute_options[:aws_secret_access_key] = aws_profile[:aws_secret_access_key] - new_compute_options[:aws_session_token] = aws_profile[:aws_security_token] - new_defaults[:driver_options][:compute_options][:region] = aws_profile[:region] - when 'OpenStack' - # TODO it is supposed to be unnecessary to load credentials from fog this way; - # why are we doing it? - # TODO support http://docs.openstack.org/cli-reference/content/cli_openrc.html - credential = Fog.credentials - - new_compute_options[:openstack_username] ||= credential[:openstack_username] - new_compute_options[:openstack_api_key] ||= credential[:openstack_api_key] - new_compute_options[:openstack_auth_url] ||= credential[:openstack_auth_url] - new_compute_options[:openstack_tenant] ||= credential[:openstack_tenant] - when 'Rackspace' - credential = Fog.credentials - - new_compute_options[:rackspace_username] ||= credential[:rackspace_username] - new_compute_options[:rackspace_api_key] ||= credential[:rackspace_api_key] - new_compute_options[:rackspace_auth_url] ||= credential[:rackspace_auth_url] - new_compute_options[:rackspace_region] ||= credential[:rackspace_region] - new_compute_options[:rackspace_endpoint] ||= credential[:rackspace_endpoint] - new_compute_options[:rackspace_compute_url] ||= credential[:rackspace_compute_url] - when 'DigitalOcean' - # This uses ~/.tugboat, generated by "tugboat authorize" - see https://github.com/pearkes/tugboat - tugboat_file = File.expand_path('~/.tugboat') - if File.exist?(tugboat_file) - tugboat_data = YAML.load(IO.read(tugboat_file)) - new_compute_options.merge!( - :digitalocean_client_id => tugboat_data['authentication']['client_key'], - :digitalocean_api_key => tugboat_data['authentication']['api_key'] - ) - new_defaults[:machine_options].merge!( - #:ssh_username => tugboat_data['ssh']['ssh_user'], - :ssh_options => { - :port => tugboat_data['ssh']['ssh_port'], - # TODO we ignore ssh_key_path in favor of ssh_key / key_name stuff - #:key_data => [ IO.read(tugboat_data['ssh']['ssh_key_path']) ] # TODO use paths, not data? - } - ) - - # TODO verify that the key_name exists and matches the ssh key path - - new_defaults[:machine_options][:bootstrap_options].merge!( - :region_id => tugboat_data['defaults']['region'].to_i, - :image_id => tugboat_data['defaults']['image'].to_i, - :size_id => tugboat_data['defaults']['region'].to_i, - :private_networking => tugboat_data['defaults']['private_networking'] == 'true', - :backups_enabled => tugboat_data['defaults']['backups_enabled'] == 'true', - ) - ssh_key = tugboat_data['defaults']['ssh_key'] - if ssh_key && ssh_key.size > 0 - new_defaults[:machine_options][:bootstrap_options][:key_name] = ssh_key - end - end - end - - id = case provider - when 'AWS' - account_info = FogDriverAWS.aws_account_info_for(result[:driver_options][:compute_options]) - new_config[:driver_options][:aws_account_info] = account_info - "#{account_info[:aws_account_id]}:#{result[:driver_options][:compute_options][:region]}" - when 'DigitalOcean' - result[:driver_options][:compute_options][:digitalocean_client_id] - when 'OpenStack' - result[:driver_options][:compute_options][:openstack_auth_url] - when 'Rackspace' - result[:driver_options][:compute_options][:rackspace_auth_url] - when 'CloudStack' - host = result[:driver_options][:compute_options][:cloudstack_host] - path = result[:driver_options][:compute_options][:cloudstack_path] || '/client/api' - port = result[:driver_options][:compute_options][:cloudstack_port] || 443 - scheme = result[:driver_options][:compute_options][:cloudstack_scheme] || 'https' - - URI.scheme_list[scheme.upcase].build(:host => host, :port => port, :path => path).to_s - end - - [ result, id ] + raise "unsupported fog provider #{provider}" end end end From 6a5fffa29e460765ce0b6b5af65b2c4be89e718b Mon Sep 17 00:00:00 2001 From: Thom May Date: Wed, 11 Jun 2014 17:45:08 +0100 Subject: [PATCH 12/23] make ssh username selection actually work --- lib/chef_metal_fog/drivers/aws.rb | 2 +- lib/chef_metal_fog/fog_driver.rb | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/chef_metal_fog/drivers/aws.rb b/lib/chef_metal_fog/drivers/aws.rb index 980c564..f70bddd 100644 --- a/lib/chef_metal_fog/drivers/aws.rb +++ b/lib/chef_metal_fog/drivers/aws.rb @@ -12,7 +12,7 @@ def creator end def ssh_username - machine_spec.location['ssh_username'] || 'ubuntu' + 'ubuntu' end def self.compute_options_for(provider, id, config) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index ce30856..1a603ee 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -493,16 +493,17 @@ def ssh_options_for(machine_spec, machine_options, server) end def ssh_username - machine_spec.location['ssh_username'] || 'root' + 'root' end def create_ssh_transport(machine_spec, machine_options, server) ssh_options = ssh_options_for(machine_spec, machine_options, server) + username = machine_spec.location['ssh_username'] || ssh_username if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.location['ssh_username'] Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.location['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}. Using #{machine_spec.location['ssh_username']}. Please edit the node and change the metal.location.ssh_username attribute if you want to change it.") end options = {} - if machine_spec.location[:sudo] || (!machine_spec.location.has_key?(:sudo) && ssh_username != 'root') + if machine_spec.location[:sudo] || (!machine_spec.location.has_key?(:sudo) && username != 'root') options[:prefix] = 'sudo ' end @@ -522,7 +523,7 @@ def create_ssh_transport(machine_spec, machine_options, server) options[:ssh_pty_enable] = true options[:ssh_gateway] = machine_spec.location['ssh_gateway'] if machine_spec.location.has_key?('ssh_gateway') - ChefMetal::Transport::SSH.new(remote_host, ssh_username, ssh_options, options, config) + ChefMetal::Transport::SSH.new(remote_host, username, ssh_options, options, config) end def self.compute_options_for(provider, id, config) From 2976a35cf19a83e7c576df6366ad6a8afa18cd2b Mon Sep 17 00:00:00 2001 From: Thom May Date: Fri, 13 Jun 2014 08:48:52 +0100 Subject: [PATCH 13/23] load providers correctly shift to loading providers on demand, and maintain a registry of providers --- lib/chef_metal_fog/fog_driver.rb | 17 +++++++++++---- spec/spec_helper.rb | 1 + .../chef_metal_fog/providers/testdriver.rb | 14 +++++++++++++ spec/unit/fog_driver_spec.rb | 21 ++++++++++++------- 4 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 spec/support/chef_metal_fog/providers/testdriver.rb diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 1a603ee..ad3df0f 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -105,9 +105,15 @@ class << klass end end + @@registered_provider_classes = {} + def self.register_provider_class(name, driver) + @@registered_provider_classes[name] = driver + end + def self.new(driver_url, config) - driver = driver_url.split(':')[1] - ChefMetalFog::Drivers.const_get(driver).new(driver_url, config) + provider = driver_url.split(':')[1] + require "chef_metal_fog/providers/#{provider.downcase}" + @@registered_provider_classes[provider].new(driver_url, config) end # Passed in a driver_url, and a config in the format of Driver.config. @@ -117,15 +123,18 @@ def self.from_url(driver_url, config) def self.canonicalize_url(driver_url, config) _, provider, id = driver_url.split(':', 3) - config, id = compute_options_for(provider, id, config) + config, id = @@registered_provider_classes[provider].compute_options_for(provider, id, config) [ "fog:#{provider}:#{id}", config ] end # Passed in a config which is *not* merged with driver_url (because we don't # know what it is yet) but which has the same keys def self.from_provider(provider, config) + + require "chef_metal_fog/providers/#{provider.downcase}" + # Figure out the options and merge them into the config - config, id = compute_options_for(provider, nil, config) + config, id = @@registered_provider_classes[provider].compute_options_for(provider, nil, config) driver_url = "fog:#{provider}:#{id}" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 324f142..2c6310f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,5 @@ $:.unshift File.expand_path('../../lib', __FILE__) +$:.unshift File.expand_path('../support', __FILE__) require 'fog' require 'chef_metal' require 'chef_metal_fog' diff --git a/spec/support/chef_metal_fog/providers/testdriver.rb b/spec/support/chef_metal_fog/providers/testdriver.rb new file mode 100644 index 0000000..1a51b92 --- /dev/null +++ b/spec/support/chef_metal_fog/providers/testdriver.rb @@ -0,0 +1,14 @@ +module ChefMetalFog::Providers + class TestDriver < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('TestDriver', ChefMetalFog::Providers::TestDriver) + + attr_reader :config + def initialize(driver_url, config) + super + end + + def self.compute_options_for(provider, id, config) + [config, 'test'] + end + end +end diff --git a/spec/unit/fog_driver_spec.rb b/spec/unit/fog_driver_spec.rb index 14baad5..98271f8 100644 --- a/spec/unit/fog_driver_spec.rb +++ b/spec/unit/fog_driver_spec.rb @@ -2,19 +2,26 @@ require 'chef_metal_fog/fog_driver' describe ChefMetalFog::FogDriver do - module ChefMetalFog::Drivers - class TestDriver < ChefMetalFog::FogDriver - attr_reader :config - def initialize(driver_url, config) - super - end + + describe ".from_url" do + subject { ChefMetalFog::FogDriver.from_provider('TestDriver', {}) } + + it "should return the correct class" do + expect(subject).to be_an_instance_of ChefMetalFog::Providers::TestDriver end + + it "should call the target compute_options_for" do + expect(ChefMetalFog::Providers::TestDriver).to receive(:compute_options_for) + .with('TestDriver', anything, {}).and_return([{}, 'test']).twice + subject + end + end describe "when creating a new class" do it "should return the correct class" do test = ChefMetalFog::FogDriver.new('fog:TestDriver:foo', {}) - expect(test).to be_an_instance_of ChefMetalFog::Drivers::TestDriver + expect(test).to be_an_instance_of ChefMetalFog::Providers::TestDriver end it "should populate config" do From 05ba6eb546ad31190da1194614088305afb321a1 Mon Sep 17 00:00:00 2001 From: Thom May Date: Fri, 13 Jun 2014 10:39:51 +0100 Subject: [PATCH 14/23] rename drivers to providers --- lib/chef_metal_fog.rb | 6 +----- lib/chef_metal_fog/{drivers => providers}/aws.rb | 6 +++--- .../{drivers => providers}/cloudstack.rb | 5 +++-- .../digitalocean.rb} | 6 +++--- .../{drivers => providers}/openstack.rb | 5 +++-- .../{drivers => providers}/rackspace.rb | 5 +++-- spec/unit/providers/rackspace_spec.rb | 16 ++++++++++++++++ 7 files changed, 32 insertions(+), 17 deletions(-) rename lib/chef_metal_fog/{drivers => providers}/aws.rb (95%) rename lib/chef_metal_fog/{drivers => providers}/cloudstack.rb (91%) rename lib/chef_metal_fog/{drivers/digital_ocean.rb => providers/digitalocean.rb} (97%) rename lib/chef_metal_fog/{drivers => providers}/openstack.rb (90%) rename lib/chef_metal_fog/{drivers => providers}/rackspace.rb (91%) create mode 100644 spec/unit/providers/rackspace_spec.rb diff --git a/lib/chef_metal_fog.rb b/lib/chef_metal_fog.rb index 85a9962..1aea216 100644 --- a/lib/chef_metal_fog.rb +++ b/lib/chef_metal_fog.rb @@ -1,7 +1,3 @@ require 'chef_metal' -require 'chef_metal_fog/drivers/rackspace' -require 'chef_metal_fog/drivers/openstack' -require 'chef_metal_fog/drivers/digital_ocean' -require 'chef_metal_fog/drivers/aws' -require 'chef_metal_fog/drivers/cloudstack' +require 'chef_metal_fog/fog_driver' require 'chef_metal_fog/recipe_dsl' diff --git a/lib/chef_metal_fog/drivers/aws.rb b/lib/chef_metal_fog/providers/aws.rb similarity index 95% rename from lib/chef_metal_fog/drivers/aws.rb rename to lib/chef_metal_fog/providers/aws.rb index f70bddd..6aee0b4 100644 --- a/lib/chef_metal_fog/drivers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -1,12 +1,12 @@ -require 'chef_metal_fog/fog_driver' - # fog:AWS:: # fog:AWS: # fog:AWS:: module ChefMetalFog - module Drivers + module Providers class AWS < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('AWS', ChefMetalFog::Providers::AWS) + def creator driver_options[:aws_account_info][:aws_username] end diff --git a/lib/chef_metal_fog/drivers/cloudstack.rb b/lib/chef_metal_fog/providers/cloudstack.rb similarity index 91% rename from lib/chef_metal_fog/drivers/cloudstack.rb rename to lib/chef_metal_fog/providers/cloudstack.rb index 9a3f99b..93c7310 100644 --- a/lib/chef_metal_fog/drivers/cloudstack.rb +++ b/lib/chef_metal_fog/providers/cloudstack.rb @@ -1,8 +1,9 @@ -require 'chef_metal_fog/fog_driver' module ChefMetalFog - module Drivers + module Providers class CloudStack < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('CloudStack', ChefMetalFog::Providers::CloudStack) + def self.compute_options_for(provider, id, config) new_compute_options = {} new_compute_options[:provider] = provider diff --git a/lib/chef_metal_fog/drivers/digital_ocean.rb b/lib/chef_metal_fog/providers/digitalocean.rb similarity index 97% rename from lib/chef_metal_fog/drivers/digital_ocean.rb rename to lib/chef_metal_fog/providers/digitalocean.rb index 4117786..a79b38c 100644 --- a/lib/chef_metal_fog/drivers/digital_ocean.rb +++ b/lib/chef_metal_fog/providers/digitalocean.rb @@ -1,10 +1,10 @@ -require 'chef_metal_fog/fog_driver' - # fog:DigitalOcean: module ChefMetalFog - module Drivers + module Providers class DigitalOcean < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('DigitalOcean', ChefMetalFog::Providers::DigitalOcean) + def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) unless bootstrap_options[:key_name] diff --git a/lib/chef_metal_fog/drivers/openstack.rb b/lib/chef_metal_fog/providers/openstack.rb similarity index 90% rename from lib/chef_metal_fog/drivers/openstack.rb rename to lib/chef_metal_fog/providers/openstack.rb index c891d00..7fae094 100644 --- a/lib/chef_metal_fog/drivers/openstack.rb +++ b/lib/chef_metal_fog/providers/openstack.rb @@ -1,8 +1,9 @@ -require 'chef_metal_fog/fog_driver' module ChefMetalFog - module Drivers + module Providers class OpenStack < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('OpenStack', ChefMetalFog::Providers::OpenStack) + def creator compute_options[:openstack_username] end diff --git a/lib/chef_metal_fog/drivers/rackspace.rb b/lib/chef_metal_fog/providers/rackspace.rb similarity index 91% rename from lib/chef_metal_fog/drivers/rackspace.rb rename to lib/chef_metal_fog/providers/rackspace.rb index 0af3d74..3ca8d27 100644 --- a/lib/chef_metal_fog/drivers/rackspace.rb +++ b/lib/chef_metal_fog/providers/rackspace.rb @@ -1,8 +1,9 @@ -require 'chef_metal_fog/fog_driver' module ChefMetalFog - module Drivers + module Providers class Rackspace < ChefMetalFog::FogDriver + ChefMetalFog::FogDriver.register_provider_class('Rackspace', ChefMetalFog::Providers::Rackspace) + def creator compute_options[:rackspace_username] end diff --git a/spec/unit/providers/rackspace_spec.rb b/spec/unit/providers/rackspace_spec.rb new file mode 100644 index 0000000..4db7aac --- /dev/null +++ b/spec/unit/providers/rackspace_spec.rb @@ -0,0 +1,16 @@ +require 'chef_metal_fog/fog_driver' +require 'chef_metal_fog/providers/rackspace' + +describe ChefMetalFog::Providers::Rackspace do + subject { ChefMetalFog::FogDriver.from_provider('Rackspace',{}) } + + it "returns the correct driver" do + expect(subject).to be_an_instance_of ChefMetalFog::Providers::Rackspace + end + + it "has a fog backend" do + pending unless Fog.mock? + expect(subject.compute).to be_an_instance_of Fog::Compute::RackspaceV2::Mock + end + +end From db15d5384786270c0451c6e417dd9ce94b8b7a1e Mon Sep 17 00:00:00 2001 From: Thom May Date: Fri, 13 Jun 2014 13:09:04 +0100 Subject: [PATCH 15/23] shuffle aws credentials loader --- lib/chef_metal_fog/aws_credentials.rb | 67 ----------------- .../providers/aws/credentials.rb | 71 +++++++++++++++++++ spec/support/aws/config-file.csv | 2 + spec/support/aws/ini-file.ini | 7 ++ spec/unit/providers/aws/credentials_spec.rb | 42 +++++++++++ 5 files changed, 122 insertions(+), 67 deletions(-) delete mode 100644 lib/chef_metal_fog/aws_credentials.rb create mode 100644 lib/chef_metal_fog/providers/aws/credentials.rb create mode 100644 spec/support/aws/config-file.csv create mode 100644 spec/support/aws/ini-file.ini create mode 100644 spec/unit/providers/aws/credentials_spec.rb diff --git a/lib/chef_metal_fog/aws_credentials.rb b/lib/chef_metal_fog/aws_credentials.rb deleted file mode 100644 index 226ae21..0000000 --- a/lib/chef_metal_fog/aws_credentials.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'inifile' -require 'csv' - -module ChefMetalFog - # Reads in a credentials file in Amazon's download format and presents the credentials to you - class AWSCredentials - def initialize - @credentials = {} - end - - include Enumerable - - def default - @credentials[ENV['AWS_DEFAULT_PROFILE'] || 'default'] || @credentials.first[1] - end - - def keys - @credentials.keys - end - - def [](name) - @credentials[name] - end - - def each(&block) - @credentials.each(&block) - end - - def load_ini(credentials_ini_file) - inifile = IniFile.load(File.expand_path(credentials_ini_file)) - inifile.each_section do |section| - if section =~ /^\s*profile\s+(.+)$/ || section =~ /^\s*(default)\s*/ - profile_name = $1.strip - profile = inifile[section].inject({}) do |result, pair| - result[pair[0].to_sym] = pair[1] - result - end - profile[:name] = profile_name - @credentials[profile_name] = profile - end - end - end - - def load_csv(credentials_csv_file) - CSV.new(File.open(credentials_csv_file), :headers => :first_row).each do |row| - @credentials[row['User Name']] = { - :name => row['User Name'], - :user_name => row['User Name'], - :aws_access_key_id => row['Access Key Id'], - :aws_secret_access_key => row['Secret Access Key'] - } - end - end - - def load_default - load_ini(ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config')) - end - - def self.method_missing(name, *args, &block) - singleton.send(name, *args, &block) - end - - def self.singleton - @aws_credentials ||= AWSCredentials.new - end - end -end diff --git a/lib/chef_metal_fog/providers/aws/credentials.rb b/lib/chef_metal_fog/providers/aws/credentials.rb new file mode 100644 index 0000000..80d9df1 --- /dev/null +++ b/lib/chef_metal_fog/providers/aws/credentials.rb @@ -0,0 +1,71 @@ +require 'inifile' +require 'csv' + +module ChefMetalFog + module Providers + class AWS + # Reads in a credentials file in Amazon's download format and presents the credentials to you + class Credentials + def initialize + @credentials = {} + end + + include Enumerable + + def default + @credentials[ENV['AWS_DEFAULT_PROFILE'] || 'default'] || @credentials.first[1] + end + + def keys + @credentials.keys + end + + def [](name) + @credentials[name] + end + + def each(&block) + @credentials.each(&block) + end + + def load_ini(credentials_ini_file) + inifile = IniFile.load(File.expand_path(credentials_ini_file)) + inifile.each_section do |section| + if section =~ /^\s*profile\s+(.+)$/ || section =~ /^\s*(default)\s*/ + profile_name = $1.strip + profile = inifile[section].inject({}) do |result, pair| + result[pair[0].to_sym] = pair[1] + result + end + profile[:name] = profile_name + @credentials[profile_name] = profile + end + end + end + + def load_csv(credentials_csv_file) + CSV.new(File.open(credentials_csv_file), :headers => :first_row).each do |row| + @credentials[row['User Name']] = { + :name => row['User Name'], + :user_name => row['User Name'], + :aws_access_key_id => row['Access Key Id'], + :aws_secret_access_key => row['Secret Access Key'] + } + end + end + + def load_default + load_ini(ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config')) + end + + def self.method_missing(name, *args, &block) + singleton.send(name, *args, &block) + end + + def self.singleton + @aws_credentials ||= Credentials.new + end + end + end + end +end diff --git a/spec/support/aws/config-file.csv b/spec/support/aws/config-file.csv new file mode 100644 index 0000000..5bef454 --- /dev/null +++ b/spec/support/aws/config-file.csv @@ -0,0 +1,2 @@ +User Name,Access Key Id,Secret Access Key +test,12345,abcde diff --git a/spec/support/aws/ini-file.ini b/spec/support/aws/ini-file.ini new file mode 100644 index 0000000..e619a4f --- /dev/null +++ b/spec/support/aws/ini-file.ini @@ -0,0 +1,7 @@ +[default] + aws_access_key_id = 12345 + aws_secret_access_key = abcde + +[profile test] + aws_access_key_id = foobar + aws_secret_access_key = canteloupe diff --git a/spec/unit/providers/aws/credentials_spec.rb b/spec/unit/providers/aws/credentials_spec.rb new file mode 100644 index 0000000..b682654 --- /dev/null +++ b/spec/unit/providers/aws/credentials_spec.rb @@ -0,0 +1,42 @@ +require 'chef_metal_fog/providers/aws/credentials' + +describe ChefMetalFog::Providers::AWS::Credentials do + describe "#load_ini" do + let(:aws_credentials_ini_file) { File.join(File.expand_path('../../../../support', __FILE__), 'aws/ini-file.ini') } + + before do + described_class.load_ini(aws_credentials_ini_file) + end + + it "should load a default profile" do + expect(described_class['default']).to include(:aws_access_key_id) + end + + it "should load the correct values" do + expect(described_class['default'][:aws_access_key_id]).to eq "12345" + end + + it "should load several profiles" do + expect(described_class.keys.length).to eq 2 + end + end + + describe "#load_csv" do + let(:aws_credentials_csv_file) { File.join(File.expand_path('../../../../support', __FILE__), 'aws/config-file.csv') } + before do + described_class.load_csv(aws_credentials_csv_file) + end + + it "should load a single profile" do + expect(described_class['default']).to include(:aws_access_key_id) + end + + it "should load the correct values" do + expect(described_class['default'][:aws_access_key_id]).to eq "12345" + end + + it "should load several profiles" do + expect(described_class.keys.length).to eq 2 + end + end +end From 249f0e4d63b10e4a4ec621c25d2e4b179257099f Mon Sep 17 00:00:00 2001 From: Thom May Date: Fri, 13 Jun 2014 13:11:30 +0100 Subject: [PATCH 16/23] push aws specific stuff into the provider --- lib/chef_metal_fog/fog_driver.rb | 2 - lib/chef_metal_fog/fog_driver_aws.rb | 142 -------------------------- lib/chef_metal_fog/providers/aws.rb | 145 ++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 147 deletions(-) delete mode 100644 lib/chef_metal_fog/fog_driver_aws.rb diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index ad3df0f..9b6e447 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -8,11 +8,9 @@ require 'chef_metal/convergence_strategy/no_converge' require 'chef_metal/transport/ssh' require 'chef_metal_fog/version' -require 'chef_metal_fog/fog_driver_aws' require 'fog' require 'fog/core' require 'fog/compute' -require 'fog/aws' require 'socket' require 'etc' require 'time' diff --git a/lib/chef_metal_fog/fog_driver_aws.rb b/lib/chef_metal_fog/fog_driver_aws.rb deleted file mode 100644 index 0b40118..0000000 --- a/lib/chef_metal_fog/fog_driver_aws.rb +++ /dev/null @@ -1,142 +0,0 @@ -require 'chef_metal_fog/aws_credentials' -require 'chef/log' -require 'fog/aws' - -module ChefMetalFog - module FogDriverAWS - def self.get_aws_profile(driver_options, aws_account_id) - aws_credentials = get_aws_credentials(driver_options) - compute_options = driver_options[:compute_options] || {} - - # Order of operations: - # compute_options[:aws_access_key_id] / compute_options[:aws_secret_access_key] / compute_options[:aws_security_token] / compute_options[:region] - # compute_options[:aws_profile] - # ENV['AWS_ACCESS_KEY_ID'] / ENV['AWS_SECRET_ACCESS_KEY'] / ENV['AWS_SECURITY_TOKEN'] / ENV['AWS_REGION'] - # ENV['AWS_PROFILE'] - # ENV['DEFAULT_PROFILE'] - # 'default' - aws_profile = if compute_options[:aws_access_key_id] - Chef::Log.debug("Using AWS driver access key options") - { - :aws_access_key_id => compute_options[:aws_access_key_id], - :aws_secret_access_key => compute_options[:aws_secret_access_key], - :aws_security_token => compute_options[:aws_session_token], - :region => compute_options[:region] - } - elsif driver_options[:aws_profile] - Chef::Log.debug("Using AWS profile #{driver_options[:aws_profile]}") - aws_credentials[driver_options[:aws_profile]] - elsif ENV['AWS_ACCESS_KEY_ID'] - Chef::Log.debug("Using AWS environment variable access keys") - { - :aws_access_key_id => ENV['AWS_ACCESS_KEY_ID'], - :aws_secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'], - :aws_security_token => ENV['AWS_SECURITY_TOKEN'], - :region => ENV['AWS_REGION'] - } - elsif ENV['AWS_PROFILE'] - Chef::Log.debug("Using AWS profile #{ENV['AWS_PROFILE']} from AWS_PROFILE environment variable") - aws_credentials[ENV['AWS_PROFILE']] - else - Chef::Log.debug("Using AWS default profile") - aws_credentials.default - end - - # Merge in account info for profile - if aws_profile - aws_profile = aws_profile.merge(aws_account_info_for(aws_profile)) - end - - # If no profile is found (or the profile is not the right account), search - # for a profile that matches the given account ID - if aws_account_id && (!aws_profile || aws_profile[:aws_account_id] != aws_account_id) - aws_profile = find_aws_profile_for_account_id(aws_credentials, aws_account_id) - end - - if !aws_profile - raise "No AWS profile specified! Are you missing something in the Chef config or ~/.aws/config?" - end - - aws_profile.delete_if { |key, value| value.nil? } - aws_profile - end - - def self.find_aws_profile_for_account_id(aws_credentials, aws_account_id) - aws_profile = nil - aws_credentials.each do |profile_name, profile| - begin - aws_account_info = aws_account_info_for(profile) - rescue - Chef::Log.warn("Could not connect to AWS profile #{aws_credentials[:name]}: #{$!}") - Chef::Log.debug($!.backtrace.join("\n")) - next - end - if aws_account_info[:aws_account_id] == aws_account_id - aws_profile = profile - aws_profile[:name] = profile_name - aws_profile = aws_profile.merge(aws_account_info) - break - end - end - if aws_profile - Chef::Log.info("Discovered AWS profile #{aws_profile[:name]} pointing at account #{aws_account_id}. Using ...") - else - raise "No AWS profile leads to account ##{aws_account_id}. Do you need to add profiles to ~/.aws/config?" - end - aws_profile - end - - def self.aws_account_info_for(aws_profile) - @@aws_account_info ||= {} - @@aws_account_info[aws_profile[:aws_access_key_id]] ||= begin - options = { - :aws_access_key_id => aws_profile[:aws_access_key_id], - :aws_secret_access_key => aws_profile[:aws_secret_access_key], - :aws_session_token => aws_profile[:aws_security_token] - } - options.delete_if { |key, value| value.nil? } - - iam = Fog::AWS::IAM.new(options) - arn = begin - # TODO it would be nice if Fog let you do this normally ... - iam.send(:request, { - 'Action' => 'GetUser', - :parser => Fog::Parsers::AWS::IAM::GetUser.new - }).body['User']['Arn'] - rescue Fog::AWS::IAM::Error - # TODO Someone tell me there is a better way to find out your current - # user ID than this! This is what happens when you use an IAM user - # with default privileges. - if $!.message =~ /AccessDenied.+(arn:aws:iam::\d+:\S+)/ - arn = $1 - else - raise - end - end - arn_split = arn.split(':', 6) - { - :aws_account_id => arn_split[4], - :aws_username => arn_split[5], - :aws_user_arn => arn - } - end - end - - def self.get_aws_credentials(driver_options) - # Grab the list of possible credentials - if driver_options[:aws_credentials] - aws_credentials = driver_options[:aws_credentials] - else - aws_credentials = AWSCredentials.new - if driver_options[:aws_config_file] - aws_credentials.load_ini(driver_options.delete(:aws_config_file)) - elsif driver_options[:aws_csv_file] - aws_credentials.load_csv(driver_options.delete(:aws_csv_file)) - else - aws_credentials.load_default - end - end - aws_credentials - end - end -end diff --git a/lib/chef_metal_fog/providers/aws.rb b/lib/chef_metal_fog/providers/aws.rb index 6aee0b4..b7e8867 100644 --- a/lib/chef_metal_fog/providers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -1,3 +1,6 @@ +require 'chef/log' +require 'fog/aws' + # fog:AWS:: # fog:AWS: # fog:AWS:: @@ -5,6 +8,8 @@ module ChefMetalFog module Providers class AWS < ChefMetalFog::FogDriver + require_relative 'aws/credentials' + ChefMetalFog::FogDriver.register_provider_class('AWS', ChefMetalFog::Providers::AWS) def creator @@ -15,6 +20,141 @@ def ssh_username 'ubuntu' end + def self.get_aws_profile(driver_options, aws_account_id) + aws_credentials = get_aws_credentials(driver_options) + compute_options = driver_options[:compute_options] || {} + + # Order of operations: + # compute_options[:aws_access_key_id] / compute_options[:aws_secret_access_key] / compute_options[:aws_security_token] / compute_options[:region] + # compute_options[:aws_profile] + # ENV['AWS_ACCESS_KEY_ID'] / ENV['AWS_SECRET_ACCESS_KEY'] / ENV['AWS_SECURITY_TOKEN'] / ENV['AWS_REGION'] + # ENV['AWS_PROFILE'] + # ENV['DEFAULT_PROFILE'] + # 'default' + aws_profile = if compute_options[:aws_access_key_id] + Chef::Log.debug("Using AWS driver access key options") + { + :aws_access_key_id => compute_options[:aws_access_key_id], + :aws_secret_access_key => compute_options[:aws_secret_access_key], + :aws_security_token => compute_options[:aws_session_token], + :region => compute_options[:region] + } + elsif driver_options[:aws_profile] + Chef::Log.debug("Using AWS profile #{driver_options[:aws_profile]}") + aws_credentials[driver_options[:aws_profile]] + elsif ENV['AWS_ACCESS_KEY_ID'] + Chef::Log.debug("Using AWS environment variable access keys") + { + :aws_access_key_id => ENV['AWS_ACCESS_KEY_ID'], + :aws_secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'], + :aws_security_token => ENV['AWS_SECURITY_TOKEN'], + :region => ENV['AWS_REGION'] + } + elsif ENV['AWS_PROFILE'] + Chef::Log.debug("Using AWS profile #{ENV['AWS_PROFILE']} from AWS_PROFILE environment variable") + aws_credentials[ENV['AWS_PROFILE']] + else + Chef::Log.debug("Using AWS default profile") + aws_credentials.default + end + + # Merge in account info for profile + if aws_profile + aws_profile = aws_profile.merge(aws_account_info_for(aws_profile)) + end + + # If no profile is found (or the profile is not the right account), search + # for a profile that matches the given account ID + if aws_account_id && (!aws_profile || aws_profile[:aws_account_id] != aws_account_id) + aws_profile = find_aws_profile_for_account_id(aws_credentials, aws_account_id) + end + + if !aws_profile + raise "No AWS profile specified! Are you missing something in the Chef config or ~/.aws/config?" + end + + aws_profile.delete_if { |key, value| value.nil? } + aws_profile + end + + def self.find_aws_profile_for_account_id(aws_credentials, aws_account_id) + aws_profile = nil + aws_credentials.each do |profile_name, profile| + begin + aws_account_info = aws_account_info_for(profile) + rescue + Chef::Log.warn("Could not connect to AWS profile #{aws_credentials[:name]}: #{$!}") + Chef::Log.debug($!.backtrace.join("\n")) + next + end + if aws_account_info[:aws_account_id] == aws_account_id + aws_profile = profile + aws_profile[:name] = profile_name + aws_profile = aws_profile.merge(aws_account_info) + break + end + end + if aws_profile + Chef::Log.info("Discovered AWS profile #{aws_profile[:name]} pointing at account #{aws_account_id}. Using ...") + else + raise "No AWS profile leads to account ##{aws_account_id}. Do you need to add profiles to ~/.aws/config?" + end + aws_profile + end + + def self.aws_account_info_for(aws_profile) + @@aws_account_info ||= {} + @@aws_account_info[aws_profile[:aws_access_key_id]] ||= begin + options = { + :aws_access_key_id => aws_profile[:aws_access_key_id], + :aws_secret_access_key => aws_profile[:aws_secret_access_key], + :aws_session_token => aws_profile[:aws_security_token] + } + options.delete_if { |key, value| value.nil? } + + iam = Fog::AWS::IAM.new(options) + arn = begin + # TODO it would be nice if Fog let you do this normally ... + iam.send(:request, { + 'Action' => 'GetUser', + :parser => Fog::Parsers::AWS::IAM::GetUser.new + }).body['User']['Arn'] + rescue Fog::AWS::IAM::Error + # TODO Someone tell me there is a better way to find out your current + # user ID than this! This is what happens when you use an IAM user + # with default privileges. + if $!.message =~ /AccessDenied.+(arn:aws:iam::\d+:\S+)/ + arn = $1 + else + raise + end + end + arn_split = arn.split(':', 6) + { + :aws_account_id => arn_split[4], + :aws_username => arn_split[5], + :aws_user_arn => arn + } + end + end + + def self.get_aws_credentials(driver_options) + # Grab the list of possible credentials + if driver_options[:aws_credentials] + aws_credentials = driver_options[:aws_credentials] + else + aws_credentials = AWSCredentials.new + if driver_options[:aws_config_file] + aws_credentials.load_ini(driver_options.delete(:aws_config_file)) + elsif driver_options[:aws_csv_file] + aws_credentials.load_csv(driver_options.delete(:aws_csv_file)) + else + aws_credentials.load_default + end + end + aws_credentials + end + def self.compute_options_for(provider, id, config) new_compute_options = {} new_compute_options[:provider] = provider @@ -43,19 +183,18 @@ def self.compute_options_for(provider, id, config) end end - aws_profile = FogDriverAWS.get_aws_profile(result[:driver_options], id) + aws_profile = get_aws_profile(result[:driver_options], id) new_compute_options[:aws_access_key_id] = aws_profile[:aws_access_key_id] new_compute_options[:aws_secret_access_key] = aws_profile[:aws_secret_access_key] new_compute_options[:aws_session_token] = aws_profile[:aws_security_token] new_defaults[:driver_options][:compute_options][:region] = aws_profile[:region] - account_info = FogDriverAWS.aws_account_info_for(result[:driver_options][:compute_options]) + account_info = aws_account_info_for(result[:driver_options][:compute_options]) new_config[:driver_options][:aws_account_info] = account_info id = "#{account_info[:aws_account_id]}:#{result[:driver_options][:compute_options][:region]}" [result, id] end - end end end From 20c26b4b1e7525093baa4c6adbdb8b8945438b0b Mon Sep 17 00:00:00 2001 From: Thom May Date: Mon, 16 Jun 2014 18:49:31 +0100 Subject: [PATCH 17/23] DRY up loading the correct provider class --- lib/chef_metal_fog/fog_driver.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 9b6e447..d2dfc73 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -108,10 +108,14 @@ def self.register_provider_class(name, driver) @@registered_provider_classes[name] = driver end + def self.provider_class_for(provider) + require "chef_metal_fog/providers/#{provider.downcase}" + @@registered_provider_classes[provider] + end + def self.new(driver_url, config) provider = driver_url.split(':')[1] - require "chef_metal_fog/providers/#{provider.downcase}" - @@registered_provider_classes[provider].new(driver_url, config) + provider_class_for(provider).new(driver_url, config) end # Passed in a driver_url, and a config in the format of Driver.config. @@ -121,18 +125,15 @@ def self.from_url(driver_url, config) def self.canonicalize_url(driver_url, config) _, provider, id = driver_url.split(':', 3) - config, id = @@registered_provider_classes[provider].compute_options_for(provider, id, config) + config, id = provider_class_for(provider).compute_options_for(provider, id, config) [ "fog:#{provider}:#{id}", config ] end # Passed in a config which is *not* merged with driver_url (because we don't # know what it is yet) but which has the same keys def self.from_provider(provider, config) - - require "chef_metal_fog/providers/#{provider.downcase}" - # Figure out the options and merge them into the config - config, id = @@registered_provider_classes[provider].compute_options_for(provider, nil, config) + config, id = provider_class_for(provider).compute_options_for(provider, nil, config) driver_url = "fog:#{provider}:#{id}" From d6f7293cb9ec135342c8ea6cc4e9e49bc07c76fc Mon Sep 17 00:00:00 2001 From: Thom May Date: Mon, 16 Jun 2014 18:54:24 +0100 Subject: [PATCH 18/23] note sample fog urls --- lib/chef_metal_fog/fog_driver.rb | 3 ++- lib/chef_metal_fog/providers/openstack.rb | 1 + lib/chef_metal_fog/providers/rackspace.rb | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index d2dfc73..eb6bd30 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -23,7 +23,8 @@ module ChefMetalFog # ## Fog Driver URLs # # All Metal drivers use URLs to uniquely identify a driver's "bucket" of machines. - # Fog URLs are of the form fog:: + # Fog URLs are of the form fog:: - see individual providers + # for sample URLs. # # Identifier is generally something uniquely identifying the account. If multiple # users can access the account, the identifier should be the same for all of diff --git a/lib/chef_metal_fog/providers/openstack.rb b/lib/chef_metal_fog/providers/openstack.rb index 7fae094..8af0339 100644 --- a/lib/chef_metal_fog/providers/openstack.rb +++ b/lib/chef_metal_fog/providers/openstack.rb @@ -1,3 +1,4 @@ +# fog:OpenStack:https://identifyhost:portNumber/v2.0 module ChefMetalFog module Providers class OpenStack < ChefMetalFog::FogDriver diff --git a/lib/chef_metal_fog/providers/rackspace.rb b/lib/chef_metal_fog/providers/rackspace.rb index 3ca8d27..f5943f3 100644 --- a/lib/chef_metal_fog/providers/rackspace.rb +++ b/lib/chef_metal_fog/providers/rackspace.rb @@ -1,3 +1,4 @@ +# fog:Rackspace:https://identity.api.rackspacecloud.com/v2.0 module ChefMetalFog module Providers class Rackspace < ChefMetalFog::FogDriver From 1c260fb93d294583ac393e5d666935fc73398d74 Mon Sep 17 00:00:00 2001 From: Thom May Date: Mon, 16 Jun 2014 18:59:59 +0100 Subject: [PATCH 19/23] make #creator raise and ensure digital ocean returns an empty string --- lib/chef_metal_fog/fog_driver.rb | 2 +- lib/chef_metal_fog/providers/digitalocean.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index eb6bd30..b6cdced 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -248,7 +248,7 @@ def option_for(machine_options, key) end def creator - '' + raise "unsupported fog provider #{provider} (please implement #creator)" end def create_server(action_handler, machine_spec, machine_options) diff --git a/lib/chef_metal_fog/providers/digitalocean.rb b/lib/chef_metal_fog/providers/digitalocean.rb index a79b38c..faa1354 100644 --- a/lib/chef_metal_fog/providers/digitalocean.rb +++ b/lib/chef_metal_fog/providers/digitalocean.rb @@ -5,6 +5,10 @@ class DigitalOcean < ChefMetalFog::FogDriver ChefMetalFog::FogDriver.register_provider_class('DigitalOcean', ChefMetalFog::Providers::DigitalOcean) + def creator + '' + end + def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) unless bootstrap_options[:key_name] From ebc499b1d23e2f71d6a4977ab1417f8a02b58379 Mon Sep 17 00:00:00 2001 From: Thom May Date: Mon, 16 Jun 2014 19:01:34 +0100 Subject: [PATCH 20/23] rename ssh_username to reveal intentions better --- lib/chef_metal_fog/fog_driver.rb | 4 ++-- lib/chef_metal_fog/providers/aws.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index b6cdced..269235f 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -501,13 +501,13 @@ def ssh_options_for(machine_spec, machine_options, server) result end - def ssh_username + def default_ssh_username 'root' end def create_ssh_transport(machine_spec, machine_options, server) ssh_options = ssh_options_for(machine_spec, machine_options, server) - username = machine_spec.location['ssh_username'] || ssh_username + username = machine_spec.location['ssh_username'] || default_ssh_username if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.location['ssh_username'] Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.location['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}. Using #{machine_spec.location['ssh_username']}. Please edit the node and change the metal.location.ssh_username attribute if you want to change it.") end diff --git a/lib/chef_metal_fog/providers/aws.rb b/lib/chef_metal_fog/providers/aws.rb index b7e8867..81e0823 100644 --- a/lib/chef_metal_fog/providers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -16,7 +16,7 @@ def creator driver_options[:aws_account_info][:aws_username] end - def ssh_username + def default_ssh_username 'ubuntu' end From ae0540d406a138c33bc0c6392462ce1f1b19e9d8 Mon Sep 17 00:00:00 2001 From: Thom May Date: Mon, 16 Jun 2014 19:12:35 +0100 Subject: [PATCH 21/23] insert a bootstrap helper for customization so that drivers can add additional bootstrapping options --- lib/chef_metal_fog/fog_driver.rb | 8 +++++--- lib/chef_metal_fog/providers/aws.rb | 7 +++++++ lib/chef_metal_fog/providers/digitalocean.rb | 13 +------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 269235f..dc24b1c 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -429,11 +429,13 @@ def overwrite_default_key_willy_nilly(action_handler) 'metal_default' end + def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) + bootstrap_options + end + def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) - if provider == 'AWS' && !bootstrap_options[:key_name] - bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) - end + bootstrap_options = bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) tags = { 'Name' => machine_spec.name, 'BootstrapId' => machine_spec.id, diff --git a/lib/chef_metal_fog/providers/aws.rb b/lib/chef_metal_fog/providers/aws.rb index 81e0823..df5deb9 100644 --- a/lib/chef_metal_fog/providers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -20,6 +20,13 @@ def default_ssh_username 'ubuntu' end + def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) + unless bootstrap_options[:key_name] + bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) + end + bootstrap_options + end + def self.get_aws_profile(driver_options, aws_account_id) aws_credentials = get_aws_credentials(driver_options) compute_options = driver_options[:compute_options] || {} diff --git a/lib/chef_metal_fog/providers/digitalocean.rb b/lib/chef_metal_fog/providers/digitalocean.rb index faa1354..810056f 100644 --- a/lib/chef_metal_fog/providers/digitalocean.rb +++ b/lib/chef_metal_fog/providers/digitalocean.rb @@ -9,22 +9,11 @@ def creator '' end - def bootstrap_options_for(action_handler, machine_spec, machine_options) - bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) + def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) unless bootstrap_options[:key_name] bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) end - tags = { - 'Name' => machine_spec.name, - 'BootstrapId' => machine_spec.id, - 'BootstrapHost' => Socket.gethostname, - 'BootstrapUser' => Etc.getlogin - } - # User-defined tags override the ones we set - tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] - bootstrap_options.merge!({ :tags => tags }) - if !bootstrap_options[:image_id] bootstrap_options[:image_name] ||= 'CentOS 6.4 x32' bootstrap_options[:image_id] = compute.images.select { |image| image.name == bootstrap_options[:image_name] }.first.id From 99f4acb9dcb4be7d08e3785fc8775ceea98cc1bd Mon Sep 17 00:00:00 2001 From: Thom May Date: Tue, 17 Jun 2014 13:42:22 +0100 Subject: [PATCH 22/23] Revert "insert a bootstrap helper for customization" This reverts commit ae0540d406a138c33bc0c6392462ce1f1b19e9d8. --- lib/chef_metal_fog/fog_driver.rb | 8 +++----- lib/chef_metal_fog/providers/aws.rb | 7 ------- lib/chef_metal_fog/providers/digitalocean.rb | 13 ++++++++++++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index dc24b1c..269235f 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -429,13 +429,11 @@ def overwrite_default_key_willy_nilly(action_handler) 'metal_default' end - def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) - bootstrap_options - end - def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) - bootstrap_options = bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) + if provider == 'AWS' && !bootstrap_options[:key_name] + bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) + end tags = { 'Name' => machine_spec.name, 'BootstrapId' => machine_spec.id, diff --git a/lib/chef_metal_fog/providers/aws.rb b/lib/chef_metal_fog/providers/aws.rb index df5deb9..81e0823 100644 --- a/lib/chef_metal_fog/providers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -20,13 +20,6 @@ def default_ssh_username 'ubuntu' end - def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) - unless bootstrap_options[:key_name] - bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) - end - bootstrap_options - end - def self.get_aws_profile(driver_options, aws_account_id) aws_credentials = get_aws_credentials(driver_options) compute_options = driver_options[:compute_options] || {} diff --git a/lib/chef_metal_fog/providers/digitalocean.rb b/lib/chef_metal_fog/providers/digitalocean.rb index 810056f..faa1354 100644 --- a/lib/chef_metal_fog/providers/digitalocean.rb +++ b/lib/chef_metal_fog/providers/digitalocean.rb @@ -9,11 +9,22 @@ def creator '' end - def bootstrap_helper(action_handler, machine_spec, machine_options, bootstrap_options) + def bootstrap_options_for(action_handler, machine_spec, machine_options) + bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) unless bootstrap_options[:key_name] bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) end + tags = { + 'Name' => machine_spec.name, + 'BootstrapId' => machine_spec.id, + 'BootstrapHost' => Socket.gethostname, + 'BootstrapUser' => Etc.getlogin + } + # User-defined tags override the ones we set + tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] + bootstrap_options.merge!({ :tags => tags }) + if !bootstrap_options[:image_id] bootstrap_options[:image_name] ||= 'CentOS 6.4 x32' bootstrap_options[:image_id] = compute.images.select { |image| image.name == bootstrap_options[:image_name] }.first.id From eb60dc3ec7444e0f7ae08cc8bb9c719baa99550e Mon Sep 17 00:00:00 2001 From: Thom May Date: Tue, 17 Jun 2014 15:26:40 +0100 Subject: [PATCH 23/23] shuffle bootstrap options around --- lib/chef_metal_fog/fog_driver.rb | 19 ++++++++++--------- lib/chef_metal_fog/providers/aws.rb | 14 ++++++++++++++ lib/chef_metal_fog/providers/digitalocean.rb | 11 +---------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/lib/chef_metal_fog/fog_driver.rb b/lib/chef_metal_fog/fog_driver.rb index 269235f..85d7015 100644 --- a/lib/chef_metal_fog/fog_driver.rb +++ b/lib/chef_metal_fog/fog_driver.rb @@ -431,9 +431,15 @@ def overwrite_default_key_willy_nilly(action_handler) def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) - if provider == 'AWS' && !bootstrap_options[:key_name] - bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) - end + + bootstrap_options[:tags] = default_tags(machine_spec, bootstrap_options[:tags] || {}) + + bootstrap_options[:name] ||= machine_spec.name + + bootstrap_options + end + + def default_tags(machine_spec, bootstrap_tags = {}) tags = { 'Name' => machine_spec.name, 'BootstrapId' => machine_spec.id, @@ -441,12 +447,7 @@ def bootstrap_options_for(action_handler, machine_spec, machine_options) 'BootstrapUser' => Etc.getlogin } # User-defined tags override the ones we set - tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] - bootstrap_options.merge!({ :tags => tags }) - - bootstrap_options[:name] ||= machine_spec.name - - bootstrap_options + tags.merge(bootstrap_tags) end def machine_for(machine_spec, machine_options, server = nil) diff --git a/lib/chef_metal_fog/providers/aws.rb b/lib/chef_metal_fog/providers/aws.rb index 81e0823..61fd95e 100644 --- a/lib/chef_metal_fog/providers/aws.rb +++ b/lib/chef_metal_fog/providers/aws.rb @@ -20,6 +20,20 @@ def default_ssh_username 'ubuntu' end + def bootstrap_options_for(action_handler, machine_spec, machine_options) + bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {}) + + unless !bootstrap_options[:key_name] + bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) + end + + bootstrap_options[:tags] = default_tags(machine_spec, bootstrap_options[:tags] || {}) + + bootstrap_options[:name] ||= machine_spec.name + + bootstrap_options + end + def self.get_aws_profile(driver_options, aws_account_id) aws_credentials = get_aws_credentials(driver_options) compute_options = driver_options[:compute_options] || {} diff --git a/lib/chef_metal_fog/providers/digitalocean.rb b/lib/chef_metal_fog/providers/digitalocean.rb index faa1354..3cd40f1 100644 --- a/lib/chef_metal_fog/providers/digitalocean.rb +++ b/lib/chef_metal_fog/providers/digitalocean.rb @@ -15,15 +15,7 @@ def bootstrap_options_for(action_handler, machine_spec, machine_options) bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler) end - tags = { - 'Name' => machine_spec.name, - 'BootstrapId' => machine_spec.id, - 'BootstrapHost' => Socket.gethostname, - 'BootstrapUser' => Etc.getlogin - } - # User-defined tags override the ones we set - tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags] - bootstrap_options.merge!({ :tags => tags }) + bootstrap_options[:tags] = default_tags(machine_spec, bootstrap_options[:tags] || {}) if !bootstrap_options[:image_id] bootstrap_options[:image_name] ||= 'CentOS 6.4 x32' @@ -45,7 +37,6 @@ def bootstrap_options_for(action_handler, machine_spec, machine_options) # You don't get to specify name yourself bootstrap_options[:name] = machine_spec.name - bootstrap_options[:name] ||= machine_spec.name bootstrap_options end