Merge pull request #16 from gds-operations/carbon_aggregator

Optional support for carbon-aggregator

manifests/config.pp

@@ -8,6 +8,13 @@
   $port = $::graphite::port
   $root_dir = $::graphite::root_dir
 
+  if ($::graphite::aggregation_rules_source == undef and
+      $::graphite::aggregation_rules_content == undef) {
+    $aggregation_rules_ensure = absent
+  } else {
+    $aggregation_rules_ensure = present
+  }
+
   if ($::graphite::storage_aggregation_source == undef and
       $::graphite::storage_aggregation_content == undef) {
     $storage_aggregation_source = 'puppet:///modules/graphite/storage-aggregation.conf'
@@ -37,13 +44,20 @@
 
   file {
   [
+    '/etc/init.d/carbon-aggregator',
     '/etc/init.d/carbon-cache',
     '/etc/init.d/graphite-web'
   ]:
     ensure => link,
     target => '/lib/init/upstart-job',
   }
 
+  file { '/etc/init/carbon-aggregator.conf':
+    ensure  => present,
+    content => template('graphite/upstart/carbon-aggregator.conf'),
+    mode    => '0555',
+  }
+
   file { '/etc/init/carbon-cache.conf':
     ensure  => present,
     content => template('graphite/upstart/carbon-cache.conf'),
@@ -62,6 +76,12 @@
     source    => $carbon_source,
   }
 
+  file { "${root_dir}/conf/aggregation-rules.conf":
+    ensure    => $aggregation_rules_ensure,
+    content   => $::graphite::aggregation_rules_content,
+    source    => $::graphite::aggregation_rules_source,
+  }
+
   file { "${root_dir}/conf/storage-aggregation.conf":
     ensure    => present,
     content   => $storage_aggregation_content,

manifests/init.pp

@@ -14,29 +14,41 @@
 # [*root_dir*]
 #   Where to install Graphite.
 #
+# [*carbon_aggregator*]
+#   Optional: Boolean, whether to run carbon-aggregator. You will need to
+#   provide an appropriate `carbon_content` or `carbon_source` config.
+#
+# [*aggregation_rules_content*]
+#   Optional: the content of the aggregation-rules.conf file.
+#
+# [*aggregation_rules_source*]
+#   Optional: the source of the aggregation-rules.conf file.
+#
 # [*storage_aggregation_content*]
 #   Optional: the content of the storage-aggregation.conf file.
 #
 # [*storage_aggregation_source*]
 #   Optional: the source of the storage-aggregation.conf file.
-
+#
 # [*storage_schemas_content*]
 #   Optional: the content of the storage-schemas.conf file.
 #
 # [*storage_schemas_source*]
 #   Optional: the source of the storage-schemas.conf file.
 #
-# [*carbon_conf_content*]
+# [*carbon_content*]
 #   Optional: the content of the carbon.conf file.
 #
-# [*carbon_conf_source*]
+# [*carbon_source*]
 #   Optional: the source of the carbon.conf file.
-
-
+#
 class graphite(
   $admin_password = $graphite::params::admin_password,
   $port = $graphite::params::port,
   $root_dir = $graphite::params::root_dir,
+  $carbon_aggregator = false,
+  $aggregation_rules_content = undef,
+  $aggregation_rules_source = undef,
   $storage_aggregation_content = undef,
   $storage_aggregation_source = undef,
   $storage_schemas_content = undef,

manifests/service.pp

@@ -3,12 +3,24 @@
 # Class to start carbon-cache and graphite-web processes
 #
 class graphite::service {
+  $aggregator_ensure = $::graphite::carbon_aggregator ? {
+    true    => running,
+    default => stopped,
+  }
+
+  service { 'carbon-aggregator':
+    ensure     => $aggregator_ensure,
+    hasstatus  => true,
+    hasrestart => true,
+    provider   => upstart,
+  } ->
   service { 'carbon-cache':
     ensure     => running,
     hasstatus  => true,
     hasrestart => true,
     provider   => upstart,
   }
+
   service { 'graphite-web':
     ensure     => running,
     hasstatus  => true,

spec/classes/graphite/graphite__config_spec.rb

@@ -1,48 +1,82 @@
 require 'spec_helper'
 
 describe 'graphite', :type => :class do
+  it { should contain_file('/etc/init.d/carbon-aggregator').with_ensure('link').
+       with_target('/lib/init/upstart-job') }
   it { should contain_file('/etc/init.d/carbon-cache').with_ensure('link').
        with_target('/lib/init/upstart-job') }
   it { should contain_file('/etc/init.d/graphite-web').with_ensure('link').
        with_target('/lib/init/upstart-job') }
+
   it { should contain_file('/opt/graphite/storage').with_owner('www-data').
        with_mode('0775') }
   it { should contain_file('/opt/graphite/storage/whisper').
        with_owner('www-data').with_mode('0775') }
 
-  describe "carbon-cache.conf" do
+  context "root_dir" do
     let(:params) {{ :root_dir => '/this/is/root' }}
-    it { should contain_file('/etc/init/carbon-cache.conf').with_ensure('present').
-         with_content(/chdir '\/this\/is\/root'/).
-         with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
-         with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
-         with_content(/python '\/this\/is\/root\/bin\/carbon-cache.py'/).
-         with_mode('0555') }
-  end
 
-  describe "graphite-web.conf" do
-    let(:params) {{ :root_dir => '/this/is/root' }}
-    it { should contain_file('/etc/init/graphite-web.conf').with_ensure('present').
-         with_content(/chdir '\/this\/is\/root\/webapp'/).
-         with_content(/PYTHONPATH='\/this\/is\/root\/webapp'/).
-         with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
-         with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
-         with_mode('0555') }
-  end
+    describe "intial_data.json" do
+      it { should contain_file('/this/is/root/webapp/graphite/initial_data.json') }
+    end
+
+    describe "carbon-aggregator.conf" do
+      it { should contain_file('/etc/init/carbon-aggregator.conf').with_ensure('present').
+           with_content(/chdir '\/this\/is\/root'/).
+           with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
+           with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
+           with_content(/python '\/this\/is\/root\/bin\/carbon-aggregator.py'/).
+           with_mode('0555') }
+    end
+
+    describe "carbon-cache.conf" do
+      it { should contain_file('/etc/init/carbon-cache.conf').with_ensure('present').
+           with_content(/chdir '\/this\/is\/root'/).
+           with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
+           with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
+           with_content(/python '\/this\/is\/root\/bin\/carbon-cache.py'/).
+           with_mode('0555') }
+    end
+
+    describe "graphite-web.conf" do
+      it { should contain_file('/etc/init/graphite-web.conf').with_ensure('present').
+           with_content(/chdir '\/this\/is\/root\/webapp'/).
+           with_content(/PYTHONPATH='\/this\/is\/root\/webapp'/).
+           with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
+           with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
+           with_mode('0555') }
+    end
 
-  describe "carbon.conf" do
-    context 'with unconfigured carbon contents' do
-      let(:params) {{ :root_dir => '/this/is/root' }}
+    describe "carbon.conf" do
       it { should contain_file('/this/is/root/conf/carbon.conf').
            with_content(/LOCAL_DATA_DIR = \/this\/is\/root\/storage\/whisper\//) }
-      end
+    end
+  end
+
+  context "carbon_content" do
+    let(:params) {{ :root_dir => '/this/is/root', :carbon_content => 'SOMEVAR=SOMECONTENT' }}
 
     context 'with configured carbon contents' do
-      let(:params) {{ :root_dir => '/this/is/root', :carbon_content => 'SOMEVAR=SOMECONTENT' }}
       it { should contain_file('/this/is/root/conf/carbon.conf').with_ensure('present').
            with_content(/SOMECONTENT/) }
     end
+  end
 
+  describe "aggregation-rules.conf" do
+    context 'with unconfigured aggregation rules' do
+      it { should contain_file('/opt/graphite/conf/aggregation-rules.conf').
+           with_ensure('absent') }
+      it { should_not contain_file('/etc/init/carbon-aggregator.conf').
+           with_content(/--rules/) }
+    end
+
+    context 'with configured aggregation rules' do
+      let(:params) { {'aggregation_rules_content' => "Elephants and giraffes!" } }
+      it { should contain_file('/opt/graphite/conf/aggregation-rules.conf').
+           with_content("Elephants and giraffes!") }
+      it { should contain_file('/etc/init/carbon-aggregator.conf').
+           with_content(/--rules='\/opt\/graphite\/conf\/aggregation-rules\.conf'/) }
+    end
   end
 
   describe "storage-aggregation.conf" do
@@ -71,6 +105,11 @@
     end
   end
 
+  context 'with admin password' do
+    let(:params) { {'admin_password' => 'should be a hash' }}
+    it { should contain_file('/opt/graphite/webapp/graphite/initial_data.json').with_content(/should be a hash/) }
+  end
+
   it {
     should contain_exec('init-db').with_command('/usr/bin/python manage.py syncdb --noinput').
     with_cwd('/opt/graphite/webapp/graphite')

spec/classes/graphite/graphite__install_spec.rb

@@ -3,6 +3,7 @@
 describe 'graphite', :type => :class do
   let(:version) { '0.9.12' }
 
+  it { should contain_package('whisper').with_ensure(version) }
   it { should contain_exec('graphite/install carbon').
        with_command("/usr/bin/pip install --install-option=\"--prefix=/opt/graphite\" --install-option=\"--install-lib=/opt/graphite/lib\" carbon==#{version}") }
   it { should contain_exec('graphite/install graphite-web').

spec/classes/graphite/graphite__service_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe 'graphite', :type => :class do
+  context 'param defaults' do
+    it { should contain_service('carbon-aggregator').with_ensure('stopped') }
+    it { should contain_service('carbon-cache') }
+    it { should contain_service('graphite-web') }
+  end
+
+  context 'carbon_aggregator' do
+    let(:params) {{
+      :carbon_aggregator => true,
+    }}
+
+    it { should contain_service('carbon-aggregator').with_ensure('running') }
+  end
+end

spec/classes/graphite/graphite_spec.rb

@@ -6,23 +6,5 @@
   it { should create_class('graphite::install')}
   it { should create_class('graphite::service')}
 
-  it { should contain_package('whisper')}
-  it { should contain_exec('graphite/install carbon')}
-  it { should contain_exec('graphite/install graphite-web')}
-
-  it { should contain_service('carbon-cache') }
-  it { should contain_service('graphite-web') }
-
   it { should contain_class('graphite::deps') }
-
-  context 'with admin password' do
-    let(:params) { {'admin_password' => 'should be a hash' }}
-    it { should contain_file('/opt/graphite/webapp/graphite/initial_data.json').with_content(/should be a hash/) }
-  end
-
-  context 'with different root dir' do
-    let(:params) { {'root_dir' => '/var/lib/graphite' }}
-    it { should contain_file('/var/lib/graphite/webapp/graphite/initial_data.json') }
-  end
-
 end

templates/upstart/carbon-aggregator.conf

@@ -0,0 +1,17 @@
+description "Carbon aggregator service for Graphite"
+
+start on (net-device-up
+          and local-filesystems
+          and runlevel [2345])
+stop on runlevel [!2345]
+
+respawn
+
+chdir '<%= @root_dir %>'
+env GRAPHITE_STORAGE_DIR='<%= @root_dir %>/storage'
+env GRAPHITE_CONF_DIR='<%= @root_dir %>/conf'
+exec python '<%= @root_dir %>/bin/carbon-aggregator.py' \
+<% if @aggregation_rules_ensure == 'present' -%>
+    --rules='<%= @root_dir %>/conf/aggregation-rules.conf' \
+<% end -%>
+    --debug start