diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14c9201e..74932199 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,12 +40,12 @@ jobs: strategy: matrix: os: - - 'amazonlinux' - 'amazonlinux-2' - - 'debian-8' - 'debian-9' + - 'debian-10' - 'ubuntu-1604' - 'ubuntu-1804' + - 'ubuntu-2004' - 'centos-7' suite: - 'default' diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e404b2a..730455cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,12 @@ All notable changes to this project will be documented in this file. -## Unreleased +## 2.0.0 (2020-08-05) -- resolved cookstyle error: libraries/provider_carbon_conf_accumulator.rb:37:7 refactor: `ChefModernize/ActionMethodInResource` -- resolved cookstyle error: libraries/provider_storage_conf_accumulator.rb:37:7 refactor: `ChefModernize/ActionMethodInResource` -- Fix frozen method error in libraries/chef_graphite_carbon_config_converter.rb +- Replace poise-python cookbook with sous-chef pyenv cookbook ([#350]) +- Default Python version installed is latest (v3.8.5) +- Default Graphite version installed is latest (v1.1.7) +- Default Django version installed is latest (v3.0.9) ## 1.3.0 (2020-06-06) diff --git a/README.md b/README.md index 6f0cc737..3a2538ff 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ This cookbook is maintained by the Sous Chefs. The Sous Chefs are a community of ### Cookbooks -- poise-python +- pyenv ## Attributes @@ -79,7 +79,7 @@ Some of this weirdness may not really be needed, so send us a PR if you fix it b ### uwsgi -Start a uwsgi runit service for graphite-web. That's it. +Start a uwsgi systemd service for graphite-web. That's it. ### Various internal recipes @@ -93,7 +93,7 @@ It's like a treasure hunt. Management for the various [Carbon](https://github.com/graphite-project/carbon) services which receive your metrics and write them to disk. -- `graphite_service`: sets up a carbon service with runit, essentially a glorified `runit_service`. Carbon configuration should be defined first with one of the `graphite_carbon_*` resources. Multiple daemons can be run by using multiple resources with names such as `cache:a`, `cache:b`, etc.. +- `graphite_service`: sets up a carbon service with systemd, essentially a glorified `systemd_unit`. Carbon configuration should be defined first with one of the `graphite_carbon_*` resources. Multiple daemons can be run by using multiple resources with names such as `cache:a`, `cache:b`, etc.. - `graphite_carbon_aggregator`: data driven resource for carbon-aggregator configuration - `graphite_carbon_cache`: data driven resource for carbon-cache configuration - `graphite_carbon_relay`: data driven resource for carbon-cache configuration @@ -113,7 +113,7 @@ Write the configuration file for [Graphite Web](https://github.com/graphite-proj Yes it's [writing python via ruby](https://github.com/sous-chefs/graphite/blob/master/libraries/chef_graphite_python.rb#L14). -A runit service definition is provided to [start a uwsgi process](https://github.com/sous-chefs/graphite/blob/master/example/graphite_example/recipes/single_node.rb#L105), but note that choice of web server for proxying to the application server is left up to you. No more hard Apache dependency! +A systemd service unit definition is provided to [start a uwsgi process](https://github.com/sous-chefs/graphite/blob/master/recipes/uwsgi.rb), but note that choice of web server for proxying to the application server is left up to you. No more hard Apache dependency! ### Accumulators diff --git a/attributes/default.rb b/attributes/default.rb index cec1818b..9db14259 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -17,7 +17,7 @@ # limitations under the License. # -default['graphite']['version'] = '1.1.3' +default['graphite']['version'] = '1.1.7' # You may set versions of Twisted and Django packages explicitly, otherwise it # installs actual versions of these packages as dependecies default['graphite']['twisted_version'] = '' @@ -66,7 +66,7 @@ case node['platform_family'] when 'debian' %w(libcairo2-dev libffi-dev python-rrdtool) - when 'rhel' + when 'rhel', 'amazon' %w(cairo-devel libffi-devel python-rrdtool bitmap-fonts) else [] diff --git a/attributes/uwsgi.rb b/attributes/uwsgi.rb index b51a65a7..8dce8e07 100644 --- a/attributes/uwsgi.rb +++ b/attributes/uwsgi.rb @@ -25,5 +25,4 @@ default['graphite']['uwsgi']['carbon'] = '127.0.0.1:2003' default['graphite']['uwsgi']['listen_http'] = false default['graphite']['uwsgi']['port'] = 8080 -default['graphite']['uwsgi']['service_type'] = 'runit' default['graphite']['uwsgi']['buffer-size'] = '4096' diff --git a/kitchen.dokken.yml b/kitchen.dokken.yml index 436bd28c..2b9de91d 100644 --- a/kitchen.dokken.yml +++ b/kitchen.dokken.yml @@ -16,26 +16,21 @@ verifier: name: inspec platforms: - - name: amazonlinux - driver: - image: dokken/amazonlinux - pid_one_command: /sbin/init - - name: amazonlinux-2 driver: image: dokken/amazonlinux-2 pid_one_command: /usr/lib/systemd/systemd - - name: debian-8 + - name: debian-9 driver: - image: dokken/debian-8 + image: dokken/debian-9 pid_one_command: /bin/systemd intermediate_instructions: - RUN /usr/bin/apt-get update - - name: debian-9 + - name: debian-10 driver: - image: dokken/debian-9 + image: dokken/debian-10 pid_one_command: /bin/systemd intermediate_instructions: - RUN /usr/bin/apt-get update @@ -58,3 +53,10 @@ platforms: pid_one_command: /bin/systemd intermediate_instructions: - RUN /usr/bin/apt-get update + + - name: ubuntu-20.04 + driver: + image: dokken/ubuntu-20.04 + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update diff --git a/kitchen.yml b/kitchen.yml index 202fe74b..fbe67c46 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -10,15 +10,13 @@ verifier: name: inspec platforms: - - name: amazonlinux - driver_config: - box: mvbcoding/awslinux + - name: amazonlinux-2 - name: centos-7 - - name: debian-8 - name: debian-9 - - name: ubuntu-14.04 + - name: debian-10 - name: ubuntu-16.04 - name: ubuntu-18.04 + - name: ubuntu-20.04 suites: - name: default @@ -26,7 +24,6 @@ suites: - recipe[test::single_node] attributes: graphite: - listen_port: "8080" uwsgi: workers: 4 listen_http: true diff --git a/metadata.rb b/metadata.rb index 0d55d0b3..0bbbb6fa 100644 --- a/metadata.rb +++ b/metadata.rb @@ -4,17 +4,15 @@ license 'Apache-2.0' description 'Installs/Configures graphite' -version '1.3.0' +version '2.0.0' -supports 'ubuntu' -supports 'debian' -supports 'redhat' -supports 'centos' -supports 'scientific' -supports 'oracle' +supports 'ubuntu', '>= 16.0' +supports 'debian', '>= 9.0' +supports 'redhat', '>= 7.0' +supports 'centos', '>= 7.0' -depends 'poise-python', '>= 1.5' +depends 'pyenv', '>= 3.1' source_url 'https://github.com/sous-chefs/graphite' issues_url 'https://github.com/sous-chefs/graphite/issues' -chef_version '>= 13' +chef_version '>= 14' diff --git a/recipes/_carbon_packages.rb b/recipes/_carbon_packages.rb index c9057cf1..51181ec0 100644 --- a/recipes/_carbon_packages.rb +++ b/recipes/_carbon_packages.rb @@ -23,22 +23,21 @@ # Compliler is needed to build Twisted gem on this step package platform_family?('debian') ? 'build-essential' : 'gcc' -python_package 'Twisted' do +pyenv_pip 'twisted' do + virtualenv node['graphite']['base_dir'] user node['graphite']['user'] - group node['graphite']['group'] version node['graphite']['twisted_version'] - virtualenv node['graphite']['base_dir'] end -python_package 'carbon' do +pyenv_pip 'carbon' do package_name lazy { node['graphite']['package_names']['carbon'][node['graphite']['install_type']] } + + virtualenv node['graphite']['base_dir'] + user node['graphite']['user'] version lazy { node['graphite']['install_type'] == 'package' ? node['graphite']['version'] : nil } - user node['graphite']['user'] - group node['graphite']['group'] - install_options '--no-binary=:all:' - virtualenv node['graphite']['base_dir'] + options '--no-binary=:all:' end diff --git a/recipes/_directories.rb b/recipes/_directories.rb index 87e6ac41..b7b6edef 100644 --- a/recipes/_directories.rb +++ b/recipes/_directories.rb @@ -17,6 +17,11 @@ # limitations under the License. # +directory node['graphite']['base_dir'] do + owner node['graphite']['user'] + group node['graphite']['group'] +end + directory node['graphite']['storage_dir'] do owner node['graphite']['user'] group node['graphite']['group'] diff --git a/recipes/_web_packages.rb b/recipes/_web_packages.rb index f6622555..3b34b6f6 100644 --- a/recipes/_web_packages.rb +++ b/recipes/_web_packages.rb @@ -19,30 +19,28 @@ package Array(node['graphite']['system_packages']) -python_package 'django' do +pyenv_pip 'django' do + virtualenv node['graphite']['base_dir'] user node['graphite']['user'] - group node['graphite']['group'] version node['graphite']['django_version'] - virtualenv node['graphite']['base_dir'] end -python_package 'uwsgi' do +pyenv_pip 'uwsgi' do + virtualenv node['graphite']['base_dir'] user node['graphite']['user'] - group node['graphite']['group'] options '--isolated' - virtualenv node['graphite']['base_dir'] end -python_package 'graphite_web' do +pyenv_pip 'graphite_web' do package_name lazy { key = node['graphite']['install_type'] node['graphite']['package_names']['graphite_web'][key] } + + virtualenv node['graphite']['base_dir'] + user node['graphite']['user'] version lazy { node['graphite']['version'] if node['graphite']['install_type'] == 'package' } - user node['graphite']['user'] - group node['graphite']['group'] - install_options '--no-binary=:all:' - virtualenv node['graphite']['base_dir'] + options '--no-binary=:all:' end diff --git a/recipes/carbon.rb b/recipes/carbon.rb index a5516371..b3d74672 100644 --- a/recipes/carbon.rb +++ b/recipes/carbon.rb @@ -17,25 +17,10 @@ # limitations under the License. # -python_runtime 'carbons_python' do - provider :system - version '2.7' - options pip_version: true -end - include_recipe 'graphite::_user' +include_recipe 'graphite::_directories' -directory node['graphite']['base_dir'] do - owner node['graphite']['user'] - group node['graphite']['group'] -end - -python_virtualenv node['graphite']['base_dir'] do - user node['graphite']['user'] - group node['graphite']['group'] -end +graphite_python 'carbons_python' -include_recipe 'graphite::_user' include_recipe 'graphite::_carbon_packages' -include_recipe 'graphite::_directories' include_recipe 'graphite::_carbon_config' diff --git a/recipes/web.rb b/recipes/web.rb index 59421402..d8acc00b 100644 --- a/recipes/web.rb +++ b/recipes/web.rb @@ -17,17 +17,12 @@ # limitations under the License. # -python_runtime 'webs_python' do - provider :system - version '2.7' - options pip_version: true -end +include_recipe 'graphite::_user' +include_recipe 'graphite::_directories' -python_virtualenv node['graphite']['base_dir'] +graphite_python 'webs_python' -include_recipe 'graphite::_user' include_recipe 'graphite::_web_packages' -include_recipe 'graphite::_directories' basedir = node['graphite']['base_dir'] docroot = node['graphite']['doc_root'] diff --git a/resources/carbon_cache.rb b/resources/carbon_cache.rb index 2e76004e..19315c54 100644 --- a/resources/carbon_cache.rb +++ b/resources/carbon_cache.rb @@ -21,14 +21,10 @@ property :config, [Hash, nil], default: nil action :create do - python_package backend_name do - backend_attributes.each { |attr, value| send(attr, value) } - Chef::Log.info "Installing storage backend: #{package_name}" - action :install - user node['graphite']['user'] - group node['graphite']['group'] - install_options '--no-binary=:all:' + pyenv_pip backend_name do virtualenv node['graphite']['base_dir'] + user node['graphite']['user'] + options '--no-binary=:all:' end end diff --git a/resources/python.rb b/resources/python.rb new file mode 100644 index 00000000..97055dc7 --- /dev/null +++ b/resources/python.rb @@ -0,0 +1,27 @@ +property :pyenv_name, String, name_property: true +property :python_version, String, default: '3.8.5' +property :user, String, default: lazy { node['graphite']['user'] } +property :graphite_dir, String, default: lazy { node['graphite']['base_dir'] } + +action :install do + pyenv_user_install new_resource.pyenv_name do + user new_resource.user + end + + pyenv_python new_resource.python_version do + user new_resource.user + end + + pyenv_global new_resource.python_version do + user new_resource.user + end + + pyenv_pip 'virtualenv' do + user new_resource.user + end + + pyenv_script 'setup graphite virtualenv' do + code "virtualenv #{new_resource.graphite_dir}" + user new_resource.user + end +end diff --git a/resources/storage.rb b/resources/storage.rb index 0dd73ede..b2be8c40 100644 --- a/resources/storage.rb +++ b/resources/storage.rb @@ -39,14 +39,14 @@ action_class do def manage_python_pip(resource_action) - python_package new_resource.package_name do - version new_resource.version if new_resource.version - Chef::Log.info 'Installing whisper pip package' + pyenv_pip new_resource.package_name do action resource_action - user node['graphite']['user'] - group node['graphite']['group'] - install_options '--no-binary=:all:' + virtualenv node['graphite']['base_dir'] + user node['graphite']['user'] + version new_resource.version if new_resource.version + Chef::Log.info 'Installing whisper pip package' + options '--no-binary=:all:' end end diff --git a/spec/recipes/_carbon_package_spec.rb b/spec/recipes/_carbon_package_spec.rb index fce76f1d..c0bc07d5 100644 --- a/spec/recipes/_carbon_package_spec.rb +++ b/spec/recipes/_carbon_package_spec.rb @@ -4,6 +4,6 @@ let(:chef_run) { ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '16.04').converge(described_recipe) } it 'installs python runtime' do - expect(chef_run).to install_python_runtime('carbons_python') + expect(chef_run).to install_graphite_python('carbons_python') end end diff --git a/spec/recipes/_web_packages_spec.rb b/spec/recipes/_web_packages_spec.rb index 6084b177..ebdac84b 100644 --- a/spec/recipes/_web_packages_spec.rb +++ b/spec/recipes/_web_packages_spec.rb @@ -13,13 +13,13 @@ end it 'installs a django python package' do - expect(chef_run).to install_python_package('django') + expect(chef_run).to install_pyenv_pip('django') end it 'installs a specific version of graphite_web python package' do - expect(chef_run).to install_python_package('graphite_web').with( + expect(chef_run).to install_pyenv_pip('graphite_web').with( package_name: 'graphite-web', - version: '1.1.3' + version: '1.1.7' ) end end diff --git a/test/fixtures/cookbooks/test/recipes/single_node.rb b/test/fixtures/cookbooks/test/recipes/single_node.rb index 01b6ff32..af7700c5 100644 --- a/test/fixtures/cookbooks/test/recipes/single_node.rb +++ b/test/fixtures/cookbooks/test/recipes/single_node.rb @@ -1,4 +1,8 @@ apt_update 'update' if platform_family?('debian') + +node.default['graphite']['twisted_version'] = '19.10.0' +node.default['graphite']['django_version'] = '3.0.9' + include_recipe 'graphite::carbon' include_recipe 'graphite::web' @@ -44,7 +48,7 @@ base_dir = (node['graphite']['base_dir']).to_s -graphite_web_config "#{base_dir}/webapp/graphite/local_settings.py" do +graphite_web_config "#{base_dir}/lib/python3.8/site-packages/graphite/local_settings.py" do config(secret_key: 'a_very_secret_key_jeah!', time_zone: 'America/Chicago', conf_dir: "#{base_dir}/conf", @@ -63,13 +67,9 @@ notifies :restart, 'service[graphite-web]', :delayed end -python_execute 'bin/django-admin.py migrate --run-syncdb --settings="graphite.settings" --pythonpath=webapp' do +pyenv_script 'django migrate' do + code "#{node['graphite']['base_dir']}/bin/python #{node['graphite']['base_dir']}/bin/django-admin.py migrate --run-syncdb --settings=\"graphite.settings\" --pythonpath=#{node['graphite']['base_dir']}/webapp" user node['graphite']['user'] - group node['graphite']['group'] - cwd node['graphite']['base_dir'] - python 'webs_python' - virtualenv node['graphite']['base_dir'] - creates "#{storage_dir}/graphite.db" end include_recipe 'graphite::uwsgi' diff --git a/test/integration/default/carbon_cache_spec.rb b/test/integration/default/carbon_cache_spec.rb new file mode 100644 index 00000000..33556fbc --- /dev/null +++ b/test/integration/default/carbon_cache_spec.rb @@ -0,0 +1,4 @@ +describe systemd_service('carbon-cache') do + it { should be_enabled } + it { should be_running } +end diff --git a/test/integration/default/default_spec.rb b/test/integration/default/default_spec.rb index fa105d8d..09022adb 100644 --- a/test/integration/default/default_spec.rb +++ b/test/integration/default/default_spec.rb @@ -14,3 +14,7 @@ it { should be_listening } its('protocols') { should include 'tcp' } end + +describe http('http://localhost:8080/metrics/find?query=carbon.*', enable_remote_worker: true) do + its('status') { should cmp 200 } +end diff --git a/test/integration/default/graphite_web_spec.rb b/test/integration/default/graphite_web_spec.rb new file mode 100644 index 00000000..7d8dbd28 --- /dev/null +++ b/test/integration/default/graphite_web_spec.rb @@ -0,0 +1,4 @@ +describe systemd_service('graphite-web') do + it { should be_enabled } + it { should be_running } +end diff --git a/test/integration/default/python_install_spec.rb b/test/integration/default/python_install_spec.rb new file mode 100644 index 00000000..50ce0778 --- /dev/null +++ b/test/integration/default/python_install_spec.rb @@ -0,0 +1,12 @@ +describe directory('/var/lib/graphite/.pyenv') do + it { should exist } +end + +describe file('/opt/graphite/bin/python') do + it { should be_executable } + its(:owner) { should eq('graphite') } +end + +describe command('/opt/graphite/bin/python --version') do + its(:stdout) { should match(/3\.8\.5/) } +end