From 876af73aeef68c7e890da7191dd159ffecef6acb Mon Sep 17 00:00:00 2001 From: Yann Mahe <yann@datadoghq.com> Date: Thu, 10 Dec 2015 17:03:14 -0500 Subject: [PATCH] [cassandra] YAML configuration template versioning * Enhance `DatadogMonitor` resource with a `version` attribute. * Use `version` attribute in `cassandra` recipe to version, i.e. select the appriopriate YAML configuration file. Two versions are currently available * `1` (Default): Legacy YAML configuration file, compatible with Cassandra < 2.2. * `2`: Required for Cassandra > 2.2, use the YAML configuration file * introduced with https://github.com/DataDog/dd-agent/commit/2df7566d2d1881ec3a2722a0bde1e3a0a99b3d9a * Enable `cassandra_aliasing` option to comply with https://github.com/DataDog/dd-agent/pull/2035 More information: https://github.com/DataDog/dd-agent/pull/2142 --- .kitchen.yml | 15 ++ attributes/cassandra.rb | 1 + providers/monitor.rb | 3 +- recipes/cassandra.rb | 60 ++++--- resources/monitor.rb | 1 + templates/default/cassandra.yaml.erb | 87 ++++++++++ .../serverspec/cassandra_spec.rb | 3 +- .../serverspec/Gemfile | 1 + .../serverspec/cassandra_spec.rb | 152 ++++++++++++++++++ 9 files changed, 297 insertions(+), 26 deletions(-) create mode 100644 attributes/cassandra.rb create mode 120000 test/integration/datadog_cassandra_greater_22/serverspec/Gemfile create mode 100644 test/integration/datadog_cassandra_greater_22/serverspec/cassandra_spec.rb diff --git a/.kitchen.yml b/.kitchen.yml index 5c596c96..8d866959 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -135,6 +135,21 @@ suites: password: somepass process_name_regex: .*cassandra.* +- name: datadog_cassandra_version + run_list: + - recipe[datadog::cassandra] + attributes: + datadog: + <<: *DATADOG + cassandra: + version: 2 + instances: + - host: localhost + port: 7199 + user: someuser + password: somepass + process_name_regex: .*cassandra.* + - name: datadog_couchdb run_list: - recipe[datadog::couchdb] diff --git a/attributes/cassandra.rb b/attributes/cassandra.rb new file mode 100644 index 00000000..3df57b70 --- /dev/null +++ b/attributes/cassandra.rb @@ -0,0 +1 @@ +default['datadog']['cassandra']['version'] = 1 diff --git a/providers/monitor.rb b/providers/monitor.rb index 493a562d..2de9fe24 100644 --- a/providers/monitor.rb +++ b/providers/monitor.rb @@ -20,7 +20,8 @@ def whyrun_supported? end variables( :init_config => new_resource.init_config, - :instances => new_resource.instances + :instances => new_resource.instances, + :version => new_resource.version ) cookbook new_resource.cookbook sensitive true if Chef::Resource.instance_methods(false).include?(:sensitive) diff --git a/recipes/cassandra.rb b/recipes/cassandra.rb index fbf68f2f..c4323b9d 100644 --- a/recipes/cassandra.rb +++ b/recipes/cassandra.rb @@ -1,29 +1,41 @@ include_recipe 'datadog::dd-agent' -# Monitor cassandra -# -# Assuming you have 2 clusters "test" and "prod", -# one with and one without authentication -# you need to set up the following attributes -# -# node['datadog']['cassandra']['instances'] = [ -# { -# host: 'localhost', -# port: 7199, -# name: 'prod', -# user: 'username', -# password: 'secret' -# }, -# { -# server: 'localhost', -# port: 8199, -# name: 'test' -# } -# ] -# -# See here for more configuration values: -# https://github.com/DataDog/dd-agent/blob/master/conf.d/cassandra.yaml.example -# +# Monitor Cassandra + +# Set the following attributes +# * `instances` (required) +# List of Cassandra clusters to monitor. Each cluster is generally a dictionary with a `host`, `port` and a `name`. +# More attributes are available. For more information, please refer to : https://github.com/DataDog/dd-agent/blob/master/conf.d/cassandra.yaml.example +# * `version` (optional) +# Select the appropriate configuration file template. Available options are: +# * `1` (Default, recommended for Cassandra < 2.2). +# Use Cassandra legacy metrics, i.e. https://github.com/DataDog/dd-agent/blob/5.6.x/conf.d/cassandra.yaml.example#L23-L74 +# * `2` (recommended for Cassandra >= 2.2). +# Use Cassandra expanded metrics (CASSANDRA-4009) introduced in 1.2 (https://wiki.apache.org/cassandra/Metrics), +# i.e. https://github.com/DataDog/dd-agent/blob/master/conf.d/cassandra.yaml.example#L23-L102 + +# Example: + +# ``` +# node['datadog']['cassandra'] = +# instances: [ +# { +# host: 'localhost', +# port: 7199, +# name: 'prod', +# user: 'username', +# password: 'secret' +# }, +# { +# server: 'localhost', +# port: 8199, +# name: 'test' +# }], +# version: 2 +# } +# ``` + datadog_monitor 'cassandra' do instances node['datadog']['cassandra']['instances'] + version node['datadog']['cassandra']['version'] end diff --git a/resources/monitor.rb b/resources/monitor.rb index 03250cb8..b85b8148 100644 --- a/resources/monitor.rb +++ b/resources/monitor.rb @@ -11,3 +11,4 @@ # is evaluated. attribute :init_config, :kind_of => Hash, :required => false, :default => {} attribute :instances, :kind_of => Array, :required => false, :default => [] +attribute :version, :kind_of => Integer, :required => false, :default => nil diff --git a/templates/default/cassandra.yaml.erb b/templates/default/cassandra.yaml.erb index 9c8ce6c3..5dd22bf7 100644 --- a/templates/default/cassandra.yaml.erb +++ b/templates/default/cassandra.yaml.erb @@ -4,8 +4,94 @@ instances: <% (i.keys - ['host']).each do |key| -%> <%= key %>: <%= i[key] %> <% end -%> + cassandra_aliasing: true <% end -%> +<% if @version == 2 %> +init_config: + # List of metrics to be collected by the integration + # Read http://docs.datadoghq.com/integrations/java/ to learn how to customize it + conf: + - include: + domain: org.apache.cassandra.metrics + type: ClientRequest + scope: + - Read + - Write + name: + - Latency + - Timeouts + - Unavailables + attribute: + - Count + - OneMinuteRate + - include: + domain: org.apache.cassandra.metrics + type: ClientRequest + scope: + - Read + - Write + name: + - TotalLatency + - include: + domain: org.apache.cassandra.metrics + type: Storage + name: + - Load + - Exceptions + - include: + domain: org.apache.cassandra.metrics + type: ColumnFamily + name: + - TotalDiskSpaceUsed + - BloomFilterDiskSpaceUsed + - BloomFilterFalsePositives + - BloomFilterFalseRatio + - CompressionRatio + - LiveDiskSpaceUsed + - LiveSSTableCount + - MaxRowSize + - MeanRowSize + - MemtableColumnsCount + - MemtableLiveDataSize + - MemtableSwitchCount + - MinRowSize + exclude: + keyspace: + - system + - system_auth + - system_distributed + - system_traces + - include: + domain: org.apache.cassandra.metrics + type: Cache + name: + - Capacity + - Size + attribute: + - Value + - include: + domain: org.apache.cassandra.metrics + type: Cache + name: + - Hits + - Requests + attribute: + - Count + - include: + domain: org.apache.cassandra.metrics + type: ThreadPools + path: request + name: + - ActiveTasks + - CompletedTasks + - PendingTasks + - CurrentlyBlockedTasks + - include: + domain: org.apache.cassandra.db + attribute: + - UpdateInterval +<% else %> # List of metrics to be collected. init_config: conf: @@ -57,3 +143,4 @@ init_config: domain: org.apache.cassandra.net attribute: - TotalTimeouts +<% end %> diff --git a/test/integration/datadog_cassandra/serverspec/cassandra_spec.rb b/test/integration/datadog_cassandra/serverspec/cassandra_spec.rb index 4821a781..9b88426a 100644 --- a/test/integration/datadog_cassandra/serverspec/cassandra_spec.rb +++ b/test/integration/datadog_cassandra/serverspec/cassandra_spec.rb @@ -20,7 +20,8 @@ port: 7199, user: 'someuser', password: 'somepass', - process_name_regex: '.*cassandra.*' + process_name_regex: '.*cassandra.*', + cassandra_aliasing: true } ], 'init_config' => { diff --git a/test/integration/datadog_cassandra_greater_22/serverspec/Gemfile b/test/integration/datadog_cassandra_greater_22/serverspec/Gemfile new file mode 120000 index 00000000..74f9789f --- /dev/null +++ b/test/integration/datadog_cassandra_greater_22/serverspec/Gemfile @@ -0,0 +1 @@ +../../helpers/serverspec/Gemfile \ No newline at end of file diff --git a/test/integration/datadog_cassandra_greater_22/serverspec/cassandra_spec.rb b/test/integration/datadog_cassandra_greater_22/serverspec/cassandra_spec.rb new file mode 100644 index 00000000..e513d1d5 --- /dev/null +++ b/test/integration/datadog_cassandra_greater_22/serverspec/cassandra_spec.rb @@ -0,0 +1,152 @@ +# Encoding: utf-8 +require 'spec_helper' + +AGENT_CONFIG = File.join(@agent_config_dir, 'conf.d/cassandra.yaml') + +describe service(@agent_service_name) do + it { should be_running } +end + +describe file(AGENT_CONFIG) do + it { should be_a_file } + + it 'is valid yaml matching input values' do + generated = YAML.load_file(AGENT_CONFIG) + + expected = { + 'instances' => [ + { + host: 'localhost', + port: 7199, + user: 'someuser', + password: 'somepass', + process_name_regex: '.*cassandra.*', + cassandra_aliasing: true + } + ], + 'init_config' => { + conf: [ + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'ClientRequest', + scope: [ + 'Read', + 'Write' + ], + name: [ + 'Latency', + 'Timeouts', + 'Unavailables' + ], + attribute: [ + 'Count', + 'OneMinuteRate' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'ClientRequest', + scope: [ + 'Read', + 'Write' + ], + name: [ + 'TotalLatency' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'Storage', + name: [ + 'Load', + 'Exceptions' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'ColumnFamily', + name: %w( + TotalDiskSpaceUsed + BloomFilterDiskSpaceUsed + BloomFilterFalsePositives + BloomFilterFalseRatio + CompressionRatio + LiveDiskSpaceUsed + LiveSSTableCount + MaxRowSize + MeanRowSize + MemtableColumnsCount + MemtableLiveDataSize + MemtableSwitchCount + MinRowSize) + }, + exclude: { + keyspace: [ + 'system', + 'system_auth', + 'system_distributed', + 'system_traces' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'Cache', + name: [ + 'Capacity', + 'Size' + ], + attribute: [ + 'Value' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'Cache', + name: [ + 'Hits', + 'Requests' + ], + attribute: [ + 'Count' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.metrics', + type: 'ThreadPools', + path: 'request', + name: [ + 'ActiveTasks', + 'CompletedTasks', + 'PendingTasks', + 'CurrentlyBlockedTasks' + ] + } + }, + { + include: { + domain: 'org.apache.cassandra.db', + attribute: [ + 'UpdateInterval' + ] + } + } + ] + } + } + + expect(generated.to_json).to be_json_eql expected.to_json + end +end