From 1144fd9c52357b591d814520cc4f7ad1a64d7001 Mon Sep 17 00:00:00 2001 From: Clayton O'Neill Date: Thu, 11 Jun 2015 16:44:00 +0000 Subject: [PATCH] Enable parallel diffs using 'parallel' gem If doing diffs of directories full of catalogs and the parallel gem is installed, then use that for multi-process catalog diff parallelization instead of threads. Since MRI can't take advantage of multiple CPUs, the existing threading doesn't speed up catalog diffs of directories. --- README.md | 6 +++++ lib/puppet/face/catalog/diff.rb | 42 +++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 52d17c9..8a6314d 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,12 @@ to the `--threads` option. This will balence the catalogs evenly on the old and masters. This option defaults to 10 and in testing 50 threads seemed correct for 4 masters with two load balancers. +Note: When using catalog diff to compare directories, one thread per catalog +comparison will be created. However, since Ruby cannot take advantage of +multiple CPUs this may be of limited use comparing local catalogs. If the +'parallel' gem is installed, then one process will be forked off per CPU on the +system, allowing use of all CPUs. + ## Fact search You can pass `--fact_search` to filter the list of nodes based on a single fact value. This currently defaults to `kernel=Linux` if you do not pass it. The yaml cache will be diff --git a/lib/puppet/face/catalog/diff.rb b/lib/puppet/face/catalog/diff.rb index 984ad70..0b25dae 100644 --- a/lib/puppet/face/catalog/diff.rb +++ b/lib/puppet/face/catalog/diff.rb @@ -1,6 +1,14 @@ require 'puppet/face' require 'thread' require 'json' + +begin + require 'parallel' + HAS_PARALLEL_GEM = true +rescue LoadError + HAS_PARALLEL_GEM = false +end + Puppet::Face.define(:catalog, '0.0.1') do action :diff do @@ -103,19 +111,29 @@ found_catalogs = Puppet::CatalogDiff::FindCatalogs.new(catalog1,catalog2).return_catalogs(options) new_catalogs = found_catalogs.keys - thread_count = 1 - mutex = Mutex.new - - thread_count.times.map { - Thread.new(nodes,new_catalogs,options) do |nodes,new_catalogs,options| - while new_catalog = mutex.synchronize { new_catalogs.pop } - node_name = File.basename(new_catalog,File.extname(new_catalog)) - old_catalog = found_catalogs[new_catalog] - node_summary = Puppet::CatalogDiff::Differ.new(old_catalog, new_catalog).diff(options) - mutex.synchronize { nodes[node_name] = node_summary } - end + if HAS_PARALLEL_GEM + results = Parallel.map(new_catalogs) do |new_catalog| + node_name = File.basename(new_catalog,File.extname(new_catalog)) + old_catalog = found_catalogs[new_catalog] + node_summary = Puppet::CatalogDiff::Differ.new(old_catalog, new_catalog).diff(options) + [ node_name, node_summary ] end - }.each(&:join) + nodes = Hash[results] + else + thread_count = 1 + mutex = Mutex.new + + thread_count.times.map { + Thread.new(nodes,new_catalogs,options) do |nodes,new_catalogs,options| + while new_catalog = mutex.synchronize { new_catalogs.pop } + node_name = File.basename(new_catalog,File.extname(new_catalog)) + old_catalog = found_catalogs[new_catalog] + node_summary = Puppet::CatalogDiff::Differ.new(old_catalog, new_catalog).diff(options) + mutex.synchronize { nodes[node_name] = node_summary } + end + end + }.each(&:join) + end elsif File.file?(catalog1) && File.file?(catalog2) # User passed us two files node_name = File.basename(catalog2,File.extname(catalog2))