Merge pull request #17 from gds-operations/virtualenv_tests_with_aggr…

…egator

Install to Python virtualenv

.fixtures.yml

@@ -1,5 +1,6 @@
 fixtures:
   repositories:
     stdlib: git://github.com/puppetlabs/puppetlabs-stdlib.git
+    python: git://github.com/stankevich/puppet-python.git
   symlinks:
     graphite: "#{source_dir}"

Modulefile

@@ -5,4 +5,6 @@ license       'MIT'
 summary       'Module to manage the Graphite monitoring tool'
 source        'https://github.com/gds-operations/puppet-graphite'
 project_page  'https://github.com/gds-operations/puppet-graphite'
+
 dependency    'puppetlabs/stdlib', '>= 2.5.0'
+dependency    'stankevich/python', '>= 1.2.0'

README.md

@@ -6,9 +6,23 @@ Status](https://secure.travis-ci.org/gds-operations/puppet-graphite.png)](http:/
 
 # Usage
 
-Nice and simple, mainly because it's not yet very configurable.
+You will need Python, Python's development headers/libs, pip and virtualenv
+installed. If you're not already managing these you can use the `python`
+module, which is included as a dependency:
 
-    include graphite
+```puppet
+class { 'python':
+  pip        => true,
+  dev        => true,
+  virtualenv => true,
+}
+```
+
+Then for the simplest possible configuration:
+
+```puppet
+include graphite
+```
 
 ## Configuration
 

Vagrantfile

@@ -9,9 +9,19 @@ DEBIAN_FRONTEND=noninteractive apt-get -qy update
 if [ ! -e /etc/puppet/modules/stdlib ]; then
   puppet module install puppetlabs-stdlib --modulepath /etc/puppet/modules
 fi
+if [ ! -e /etc/puppet/modules/python ]; then
+  puppet module install stankevich-python --modulepath /etc/puppet/modules
+fi
 
 puppet apply --verbose -e "node default { \
-  class { 'graphite': root_dir => '/var/lib/graphite' } \
+  class { 'python': \
+    pip        => true, \
+    dev        => true, \
+    virtualenv => true, \
+  } \
+  class { 'graphite': \
+    root_dir => '/var/lib/graphite', \
+  } \
 }"
 EOM
 

manifests/config.pp

@@ -100,7 +100,7 @@
   }
 
   exec { 'init-db':
-    command   => '/usr/bin/python manage.py syncdb --noinput',
+    command   => "${root_dir}/bin/python manage.py syncdb --noinput",
     cwd       => "${root_dir}/webapp/graphite",
     creates   => "${root_dir}/storage/graphite.db",
     subscribe => File["${root_dir}/storage"],
@@ -123,7 +123,6 @@
     ensure    => 'directory',
     owner     => 'www-data',
     mode      => '0775',
-    subscribe => Exec['graphite/install graphite-web'],
   }
 
   file { "${root_dir}/webapp/graphite/local_settings.py":

manifests/deps.pp

@@ -1,21 +1,36 @@
 # == Class: graphite::deps
 #
-# Class to install all graphite dependency packages
+# Class to create a Python virtualenv and install all graphite dependencies
+# within that sandbox.
+#
+# With the exception of pycairo which can't be installed by pip. It is
+# installed as a system package and symlinked into the virtualenv. This is a
+# slightly hacky alternative to `--system-site-packages` which would mess
+# with Graphite's other dependencies.
 #
 class graphite::deps {
+  $root_dir = $::graphite::root_dir
 
-  ensure_packages([
-    'python-ldap',
-    'python-cairo',
-    'python-django',
-    'python-twisted',
-    'python-django-tagging',
-    'python-simplejson',
-    'python-memcache',
-    'python-pysqlite2',
-    'python-support',
-    'python-pip',
+  python::virtualenv { $root_dir: } ->
+  python::pip { [
     'gunicorn',
-  ])
+    'twisted==11.1.0',
+    'django==1.4.10',
+    'django-tagging==0.3.1',
+    'python-memcached==1.47',
+    'simplejson==2.1.6',
+  ]:
+    virtualenv => $root_dir,
+  }
+
+  ensure_packages(['python-cairo'])
 
+  file { "${root_dir}/lib/python2.7/site-packages/cairo":
+    ensure  => link,
+    target  => '/usr/lib/python2.7/dist-packages/cairo',
+    require => [
+      Python::Virtualenv[$root_dir],
+      Package['python-cairo'],
+    ],
+  }
 }

manifests/install.pp

@@ -6,37 +6,13 @@
   $root_dir = $::graphite::root_dir
   $ver = $::graphite::version
 
-  package { 'whisper':
-    ensure   => $ver,
-    provider => pip,
-  }
-
-  $carbon_pip_args = [
-    "--install-option=\"--prefix=${root_dir}\"",
-    "--install-option=\"--install-lib=${root_dir}/lib\"",
-  ]
-  $carbon_args = join($carbon_pip_args, ' ')
-
-  exec { 'graphite/install carbon':
-    command => "/usr/bin/pip install ${carbon_args} carbon==${ver}",
-    creates => "${root_dir}/bin/carbon-cache.py",
-  }
-
-  $graphite_pip_args = [
-    "--install-option=\"--prefix=${root_dir}\"",
-    "--install-option=\"--install-lib=${root_dir}/webapp\""
-  ]
-  $graphite_web_args = join($graphite_pip_args, ' ')
-
-  exec { 'graphite/install graphite-web':
-    command => "/usr/bin/pip install ${graphite_web_args} graphite-web==${ver}",
-    creates => "${root_dir}/webapp/graphite/manage.py",
-  }
-
-  file { '/var/log/carbon':
-    ensure => directory,
-    owner  => www-data,
-    group  => www-data,
+  python::pip { [
+    "whisper==${ver}",
+    "carbon==${ver}",
+    "graphite-web==${ver}"
+  ]:
+    virtualenv  => $root_dir,
+    environment => ["PYTHONPATH=${root_dir}/lib:${root_dir}/webapp"],
   }
 
 }

spec/classes/graphite/graphite__config_spec.rb

@@ -25,7 +25,7 @@
            with_content(/chdir '\/this\/is\/root'/).
            with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
            with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
-           with_content(/python '\/this\/is\/root\/bin\/carbon-aggregator.py'/).
+           with_content(/exec \/this\/is\/root\/bin\/carbon-aggregator.py/).
            with_mode('0555') }
     end
 
@@ -34,7 +34,7 @@
            with_content(/chdir '\/this\/is\/root'/).
            with_content(/GRAPHITE_STORAGE_DIR='\/this\/is\/root\/storage'/).
            with_content(/GRAPHITE_CONF_DIR='\/this\/is\/root\/conf'/).
-           with_content(/python '\/this\/is\/root\/bin\/carbon-cache.py'/).
+           with_content(/exec \/this\/is\/root\/bin\/carbon-cache.py/).
            with_mode('0555') }
     end
 
@@ -111,7 +111,7 @@
   end
 
   it {
-    should contain_exec('init-db').with_command('/usr/bin/python manage.py syncdb --noinput').
+    should contain_exec('init-db').with_command('/opt/graphite/bin/python manage.py syncdb --noinput').
     with_cwd('/opt/graphite/webapp/graphite')
   }
 

spec/classes/graphite/graphite__deps_spec.rb

@@ -1,22 +1,19 @@
 require 'spec_helper'
 
-describe 'graphite::deps', :type => :class do
+describe 'graphite', :type => :class do
 
-  [
-    'python-ldap',
-    'python-cairo',
-    'python-django',
-    'python-twisted',
-    'python-django-tagging',
-    'python-simplejson',
-    'python-memcache',
-    'python-pysqlite2',
-    'python-support',
-    'python-pip',
-    'gunicorn',
-  ].each do |pkg|
-    it {
-      should contain_package(pkg)
-    }
+  context 'root_dir' do
+    let(:params) {{
+      :root_dir => '/this/is/root',
+    }}
+
+    it { should contain_python__virtualenv('/this/is/root') }
+    it { should contain_python__pip('gunicorn').with_virtualenv('/this/is/root') }
+
+    it { should contain_package('python-cairo').with_ensure('present') }
+    it { should contain_file('/this/is/root/lib/python2.7/site-packages/cairo').
+         with_ensure('link').
+         with_target('/usr/lib/python2.7/dist-packages/cairo').
+         with_require(['Python::Virtualenv[/this/is/root]', 'Package[python-cairo]']) }
   end
 end

spec/classes/graphite/graphite__install_spec.rb

@@ -1,11 +1,20 @@
 require 'spec_helper'
 
+shared_examples "pip_package" do |package|
+  it { should contain_python__pip("#{package}==1.2.3").
+       with_virtualenv('/this/is/root').
+       with_environment(["PYTHONPATH=/this/is/root/lib:/this/is/root/webapp"]) }
+end
+
 describe 'graphite', :type => :class do
-  let(:version) { '0.9.12' }
+  context 'root_dir and version' do
+    let(:params) {{
+      :version  => '1.2.3',
+      :root_dir => '/this/is/root',
+    }}
 
-  it { should contain_package('whisper').with_ensure(version) }
-  it { should contain_exec('graphite/install carbon').
-       with_command("/usr/bin/pip install --install-option=\"--prefix=/opt/graphite\" --install-option=\"--install-lib=/opt/graphite/lib\" carbon==#{version}") }
-  it { should contain_exec('graphite/install graphite-web').
-       with_command("/usr/bin/pip install --install-option=\"--prefix=/opt/graphite\" --install-option=\"--install-lib=/opt/graphite/webapp\" graphite-web==#{version}") }
+    it_should_behave_like "pip_package", "whisper"
+    it_should_behave_like "pip_package", "carbon"
+    it_should_behave_like "pip_package", "graphite-web"
+  end
 end

templates/upstart/carbon-aggregator.conf

@@ -10,7 +10,7 @@ respawn
 chdir '<%= @root_dir %>'
 env GRAPHITE_STORAGE_DIR='<%= @root_dir %>/storage'
 env GRAPHITE_CONF_DIR='<%= @root_dir %>/conf'
-exec python '<%= @root_dir %>/bin/carbon-aggregator.py' \
+exec <%= @root_dir %>/bin/carbon-aggregator.py \
 <% if @aggregation_rules_ensure == 'present' -%>
     --rules='<%= @root_dir %>/conf/aggregation-rules.conf' \
 <% end -%>

templates/upstart/carbon-cache.conf

@@ -12,4 +12,4 @@ pre-start exec rm -f '<%= @root_dir %>'/storage/carbon-cache.pid
 chdir '<%= @root_dir %>'
 env GRAPHITE_STORAGE_DIR='<%= @root_dir %>/storage'
 env GRAPHITE_CONF_DIR='<%= @root_dir %>/conf'
-exec python '<%= @root_dir %>/bin/carbon-cache.py' --debug start
+exec <%= @root_dir %>/bin/carbon-cache.py --debug start

templates/upstart/graphite-web.conf

@@ -11,4 +11,4 @@ chdir '<%= @root_dir %>/webapp'
 env PYTHONPATH='<%= @root_dir %>/webapp'
 env GRAPHITE_STORAGE_DIR='<%= @root_dir %>/storage'
 env GRAPHITE_CONF_DIR='<%= @root_dir %>/conf'
-exec /usr/bin/gunicorn_django -b127.0.0.1:<%= @port %> -w2 graphite/settings.py
+exec <%= @root_dir %>/bin/gunicorn_django -b127.0.0.1:<%= @port %> -w2 graphite/settings.py