diff --git a/Puppetfile b/Puppetfile index d3fce8fe9..67c6342e1 100644 --- a/Puppetfile +++ b/Puppetfile @@ -38,6 +38,10 @@ mod 'firewall', :commit => 'e7121fe12f7582695ef774b51742b65a37d462db', :git => 'https://github.com/puppetlabs/puppetlabs-firewall.git' +mod 'fluentd', + :commit => '37b764532a911a36cb365eb54aa18e0880f805ba', + :git => 'https://github.com/directxman12/puppet-fluentd.git' + mod 'galera', :commit => '92463ad0567c066796b7fac4a1466e6a60621f6c', :git => 'https://github.com/redhat-openstack/puppet-galera.git' @@ -233,3 +237,4 @@ mod 'vswitch', mod 'xinetd', :commit => '4f16fc824e04d724a486634bd9c26ef549f10ff5', :git => 'https://github.com/puppetlabs/puppetlabs-xinetd.git' + diff --git a/fluentd/.fixtures.yml b/fluentd/.fixtures.yml new file mode 100644 index 000000000..3360eb4ea --- /dev/null +++ b/fluentd/.fixtures.yml @@ -0,0 +1,7 @@ +fixtures: + repositories: + apt: "https://github.com/puppetlabs/puppetlabs-apt.git" + stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" + symlinks: + "fluentd": "#{source_dir}" + diff --git a/fluentd/.gemfile b/fluentd/.gemfile new file mode 100644 index 000000000..e7ea9dbb9 --- /dev/null +++ b/fluentd/.gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +group :development, :test do + gem 'rake', :require => false + gem 'rspec-puppet', :require => false + gem 'puppetlabs_spec_helper', :require => false + gem 'rspec-system', :require => false + gem 'rspec-system-puppet', :require => false + gem 'rspec-system-serverspec', :require => false +end + +if puppetversion = ENV['PUPPET_GEM_VERSION'] + gem 'puppet', puppetversion, :require => false +else + gem 'puppet', :require => false +end diff --git a/fluentd/.gitignore b/fluentd/.gitignore new file mode 100644 index 000000000..e7beff00f --- /dev/null +++ b/fluentd/.gitignore @@ -0,0 +1,7 @@ +pkg/ +Gemfile.lock +vendor/ +spec/fixtures/modules/ +.vagrant/ +.bundle/ +coverage/ diff --git a/fluentd/.travis.yml b/fluentd/.travis.yml new file mode 100644 index 000000000..0a8b188f4 --- /dev/null +++ b/fluentd/.travis.yml @@ -0,0 +1,36 @@ +--- +branches: + only: + - master +language: ruby +bundler_args: --without development +script: bundle exec rake spec SPEC_OPTS='--format documentation' +rvm: + - 1.9.3 + - 2.0.0 +gemfile: .gemfile +env: + matrix: + - PUPPET_GEM_VERSION="~> 2.7.0" + - PUPPET_GEM_VERSION="~> 3.1.0" + - PUPPET_GEM_VERSION="~> 3.2.0" + - PUPPET_GEM_VERSION="~> 3.3.0" + - PUPPET_GEM_VERSION="~> 3.4.0" + - PUPPET_GEM_VERSION="~> 3.5.0" + - PUPPET_GEM_VERSION="~> 3.6.0" +matrix: + exclude: + - rvm: 1.9.3 + env: PUPPET_GEM_VERSION="~> 2.7.0" + gemfile: .gemfile + - rvm: 1.9.3 + env: PUPPET_GEM_VERSION="~> 3.1.0" + gemfile: .gemfile + - rvm: 2.0.0 + env: PUPPET_GEM_VERSION="~> 2.7.0" + gemfile: .gemfile + - rvm: 2.0.0 + env: PUPPET_GEM_VERSION="~> 3.1.0" + gemfile: .gemfile +notifications: + email: gianni.carafa@srf.ch diff --git a/fluentd/CHANGELOG b/fluentd/CHANGELOG new file mode 100644 index 000000000..d26195091 --- /dev/null +++ b/fluentd/CHANGELOG @@ -0,0 +1,29 @@ +## puppet-fluentd changelog + +Release notes for the srfmmz/puppet-fluentd module. + +------------------------------------------ + +#### 0.2.0 +IMPORTANT: this version will break your old configuration. + +* CHANGE remove concattination of files + + +#### 0.1.4 - 2014/08/05 + +* FEATURE added filter functionality (by softecspa) +* FIX configfile syntax fix for td-agent v2 (by Molezeem) +* CHANGE changed the plugin installation to fluentgem provider (by saleksee) + +#### 0.1.3 - 2014/07/06 + +Please update to get the new repository of Treasuredata. + +Detailed Changes: + +* FEATURE : added time_format for source configuration +* CHANGE : Added new treasuredata repositories. + - for more info see here : https://groups.google.com/forum/#!topic/fluentd/46FnWIoLTrU +* CHANGE : Restart td-agent when config changed +* FEATURE : Startetd this changelog diff --git a/fluentd/Gemfile b/fluentd/Gemfile new file mode 100644 index 000000000..e30fed0ea --- /dev/null +++ b/fluentd/Gemfile @@ -0,0 +1,21 @@ +# Gemfile to run bundler command +# usagen: bundler exec rake spec + +source 'https://rubygems.org' + +puppetversion = ENV.key?('PUPPET_VERSION') ? "~> #{ENV['PUPPET_VERSION']}" : ['>= 3.2.1'] +gem 'puppet', puppetversion + +if puppetversion =~ /^3/ + ## rspec-hiera-puppet is puppet 3 only + gem 'rspec-hiera-puppet', '>=1.0.0' +end + +facterversion = ENV.key?('FACTER_VERSION') ? "~> #{ENV['FACTER_VERSION']}" : ['>= 1.7.1'] + +gem 'facter', facterversion + +gem 'rake' +gem 'rspec' +gem 'rspec-puppet', '>=0.1.6' +gem 'puppetlabs_spec_helper', '>=0.4.1' diff --git a/fluentd/LICENSE b/fluentd/LICENSE new file mode 100644 index 000000000..e06d20818 --- /dev/null +++ b/fluentd/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/fluentd/Modulefile b/fluentd/Modulefile new file mode 100644 index 000000000..4b247c5dd --- /dev/null +++ b/fluentd/Modulefile @@ -0,0 +1,12 @@ +name 'srf-fluentd' +version '0.2.0' + +dependency 'puppetlabs/apt', '>= 1.4.0' +dependency 'puppetlabs/stdlib', '>= 4.1.0' + +summary 'fluentd module' +description 'Manages Fluentd installation and configuration with Puppet using the td-agent.' +source 'https://github.com/mmz-srf/puppet-fluentd.git' +project_page 'https://github.com/mmz-srf/puppet-fluentd' +author 'Gianni Carafa' +license 'Apache License, Version 2.0' diff --git a/fluentd/README.md b/fluentd/README.md new file mode 100644 index 000000000..79bd6d9ec --- /dev/null +++ b/fluentd/README.md @@ -0,0 +1,201 @@ +puppet-fluentd +============== + +[![Build Status](https://travis-ci.org/mmz-srf/puppet-fluentd.png?branch=master)](https://travis-ci.org/mmz-srf/puppet-fluentd) + +Manage Fluentd installation, configuration and Plugin-management with Puppet using the td-agent. + +## Supported Operating Systems + +- Debian (tested on Debian 7.5) +- Ubuntu +- Redhat +- CentOS (tested on CentOS 6.4) + +## Used Modules + +- apt: "https://github.com/puppetlabs/puppetlabs-apt.git" (Only for Debian) +- stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" + +## Contributing + +- Fork it +- Create a feature branch (`git checkout -b my-new-feature`) +- Run rspec tests (`bundle exec rake spec` or `rake spec`) +- Commit your changes (`git commit -am 'Added some feature'`) +- Push to the branch (`git push origin my-new-feature`) +- Create new Pull Request + +### Branches in this module +- **master** tested and working. Latest Version on http://forge.puppetlabs.com +- **developmemt** includes the newewst features. works too, mostly. + +## Configuration + +How to configure a Agent to send data to a centralised Fluentd-Server + +### Install a Plugin + +Install your fluentd plugin. (Check [here](http://fluentd.org/plugin/) for the +right plugin name.) + +You can choose from a file or gem based installation. + +``` +include ::fluentd + +fluentd::install_plugin { 'elasticsearch': + plugin_type => 'gem', + plugin_name => 'fluent-plugin-elasticsearch', +} +``` + +### Create an Agent + +The agent watches over your logfiles and sends its content to the collector. + +``` +include ::fluentd + +fluentd::source { 'apache': + config => { + 'format' => 'apache2', + 'path' => '/var/log/apache2/access.log', + 'pos_file' => '/var/tmp/fluentd.pos', + 'tag' => 'apache.access_log', + 'type' => 'tail', + }, +} + +fluentd::source { 'syslog': + config => { + 'format' => 'syslog', + 'path' => '/var/log/syslog', + 'pos_file' => '/tmp/td-agent.syslog.pos', + 'tag' => 'system.syslog', + 'type' => 'tail', + }, +} + +fluentd::match { 'forward': + pattern => '**', + priority => '80', + config => { + 'type' => 'forward', + 'servers' => [ + { 'host' => 'fluentd.example.com', 'port' => '24224' } + ], + }, +} + +# ensure an old config is no longer there +fluentd::source { 'some_source': + ensure => 'absent', +} +``` + +...creates the following files: + +``` +/etc/td-agent/ + ├── config.d + │   ├── 50-source-apache.conf + │   ├── 50-source-syslog.conf + │   └── 80-match-forward.conf + ├── ... + ... +``` + +### Create a Collector + +The Collector collects all the data from the Agents. He now stores the data in +files, Elasticsearch or elsewhere. + +``` +include ::fluentd + +fluentd::source { 'collector': + priority => '10', + config => { + 'type' => 'forward', + } +} + +fluentd::match { 'collector': + pattern => '**', + config => { + 'type' => 'elasticsearch', + 'logstash_format' => true, + }, +} + +# all rsyslog daemons on the clients sends their messages to 5140 +fluentd::source { 'rsyslog': + type => 'syslog', + config => { + 'port' => '5140', + 'bind' => '0.0.0.0', + 'tag' => 'system.local', + }, +} +``` + +...creates the following files: + +``` +/etc/td-agent/ + ├── config.d + │   ├── 10-source-collector.conf + │   ├── 50-match-collector.conf + │   └── 50-source-rsyslog.conf + ├── ... + ... +``` + +### Copy output to multiple stores + +An array of configurations implies type "copy". + +```` +$logger=[ { 'host' => 'logger-sample01', 'port' => '24224'}, + { 'host' => 'logger-example01', 'port' => '24224', 'standby' => ''} ] + +fluentd::match { 'forward_to_logger': + pattern => 'alocal', + config => [ + { + 'type' => 'forward', + 'send_timeout' => '60s', + 'recover_wait' => '10s', + 'heartbeat_interval' => '1s', + 'phi_threshold' => 8, + 'hard_timeout' => '60s', + 'flush_interval' => '5s', + 'servers' => $logger, + }, + { + 'type' => 'stdout', + 'output_type' => 'json', + } + ], +} +``` + +### add a filter +``` + fluentd::filter { 'myfilter': + pattern => '**', + config => { + 'type' => 'grep', + 'input_key' => 'key', + 'regexp' => '/*.foo.*/', + 'exclude' => 'baar', + 'output_tag' => 'mytag', + 'add_tag_prefix' => 'pre_', + 'remove_tag_prefix' => 'remove_', + 'add_tag_suffix' => '_after', + 'remove_tag_suffix' => '_remove', + 'customvalue' => true, + } + } +``` diff --git a/fluentd/Rakefile b/fluentd/Rakefile new file mode 100644 index 000000000..bbfcc8b9c --- /dev/null +++ b/fluentd/Rakefile @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/rake_tasks' \ No newline at end of file diff --git a/fluentd/files/packages.treasure-data.com.key b/fluentd/files/packages.treasure-data.com.key new file mode 100644 index 000000000..c86c35489 --- /dev/null +++ b/fluentd/files/packages.treasure-data.com.key @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.5 (GNU/Linux) + +mQGiBFKuy6MRBACJccrkWHfYSKSbchD4ywYLGkIKITSLzEXrlpy4Sp6Mr9G1OGIv +buIUIxksnev6h6wlFgbFYTLngpod+Jn4DpcGgXqzZ3lhOY1fvAaQ4SJ4RUELm1JJ +SEWE0uYHuoI4+TrLtod3U02ETD+Lf6LWO3IfU1AYsA467tDYpVXR0GHeAwCgqU/V +M+ypNOYFHVs1/aX83wdg0K8D+wf+tcvTHb/i2lND5DVfTWtC54WYNIG6QO8DgGdu +EYregcMKRRNM4nSfUISzCgrjnpLGjvB9NKONCrimDiuz4TH7KDcEsKGA+zTzJLCy +mcE6tKEHV5yS7cK0inim4nxAKFB70306CkhHyL2TrB42ppNFJwG1igO4BJ+55b9X +Dd03A/9o6ONFle6khG+UE82WYTayy2pAshKTSLnOlB7CIxwt3SrzFXTuNosajVgZ +S0EIB8c6SjFN6P1jHjd7PYOmBxd2xptERTg8CJMoTUI+P7LypWuvmagIdEwe571d +ZMQ26HNuHg7/IMg+FQxhl/DFU6Aq+2CBJ3q8gO8Us4/UYc1PhrRUVHJlYXN1cmUg +RGF0YSwgSW5jIChUcmVhc3VyZSBBZ2VudCBPZmZpY2lhbCBTaWduaW5nIGtleSkg +PHN1cHBvcnRAdHJlYXN1cmUtZGF0YS5jb20+iGAEExECACAFAlKuy6MCGwMGCwkI +BwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAQk9tFoS4gb3HgAKCBCDNT/cWzkKRQxvnL +LiGvPetVSACeLLKnt2twfiPc9ZEA/X8MKkPjWqc= +=e5Fe +-----END PGP PUBLIC KEY BLOCK----- diff --git a/fluentd/lib/puppet/provider/package/fluentgem.rb b/fluentd/lib/puppet/provider/package/fluentgem.rb new file mode 100644 index 000000000..9c5165481 --- /dev/null +++ b/fluentd/lib/puppet/provider/package/fluentgem.rb @@ -0,0 +1,137 @@ +require 'puppet/provider/package' +require 'uri' + +# Ruby gems support. +Puppet::Type.type(:package).provide :fluentgem, :parent => Puppet::Provider::Package do + desc "Install gem via fluent-gem (included by td-agent). Ruby Gem support. If a URL is passed via `source`, then that URL is used as the + remote gem repository; if a source is present but is not a valid URL, it will be + interpreted as the path to a local gem file. If source is not present at all, + the gem will be installed from the default gem repositories. + + This provider supports the `install_options` attribute, which allows command-line flags to be passed to the gem command. + These options should be specified as a string (e.g. '--flag'), a hash (e.g. {'--flag' => 'value'}), + or an array where each element is either a string or a hash." + + has_feature :versionable, :install_options + + ENV['PATH'] = "#{ENV['PATH']}:/usr/lib64/fluent/ruby/bin:/usr/lib/fluent/ruby/bin" + + commands :gemcmd => "fluent-gem" + + def self.gemlist(options) + gem_list_command = [command(:gemcmd), "list"] + + if options[:local] + gem_list_command << "--local" + else + gem_list_command << "--remote" + end + if options[:source] + gem_list_command << "--source" << options[:source] + end + if name = options[:justme] + gem_list_command << "^" + name + "$" + end + + begin + list = execute(gem_list_command).lines. + map {|set| gemsplit(set) }. + reject {|x| x.nil? } + rescue Puppet::ExecutionFailure => detail + raise Puppet::Error, "Could not list gems: #{detail}", detail.backtrace + end + + if options[:justme] + return list.shift + else + return list + end + end + + def self.gemsplit(desc) + # `gem list` when output console has a line like: + # *** LOCAL GEMS *** + # but when it's not to the console that line + # and all blank lines are stripped + # so we don't need to check for them + + if desc =~ /^(\S+)\s+\((.+)\)/ + name = $1 + versions = $2.split(/,\s*/) + { + :name => name, + :ensure => versions.map{|v| v.split[0]}, + :provider => :gem + } + else + Puppet.warning "Could not match #{desc}" unless desc.chomp.empty? + nil + end + end + + def self.instances(justme = false) + gemlist(:local => true).collect do |hash| + new(hash) + end + end + + def install(useversion = true) + command = [command(:gemcmd), "install"] + command << "-v" << resource[:ensure] if (! resource[:ensure].is_a? Symbol) and useversion + + if source = resource[:source] + begin + uri = URI.parse(source) + rescue => detail + self.fail Puppet::Error, "Invalid source '#{uri}': #{detail}", detail + end + + case uri.scheme + when nil + # no URI scheme => interpret the source as a local file + command << source + when /file/i + command << uri.path + when 'puppet' + # we don't support puppet:// URLs (yet) + raise Puppet::Error.new("puppet:// URLs are not supported as gem sources") + else + # interpret it as a gem repository + command << "--source" << "#{source}" << resource[:name] + end + else + command << "--no-rdoc" << "--no-ri" << resource[:name] + end + + command += install_options if resource[:install_options] + + output = execute(command) + # Apparently some stupid gem versions don't exit non-0 on failure + self.fail "Could not install: #{output.chomp}" if output.include?("ERROR") + end + + def latest + # This always gets the latest version available. + gemlist_options = {:justme => resource[:name]} + gemlist_options.merge!({:source => resource[:source]}) unless resource[:source].nil? + hash = self.class.gemlist(gemlist_options) + + hash[:ensure][0] + end + + def query + self.class.gemlist(:justme => resource[:name], :local => true) + end + + def uninstall + gemcmd "uninstall", "-x", "-a", resource[:name] + end + + def update + self.install(false) + end + + def install_options + join_options(resource[:install_options]) + end +end diff --git a/fluentd/manifests/config.pp b/fluentd/manifests/config.pp new file mode 100644 index 000000000..b2051f540 --- /dev/null +++ b/fluentd/manifests/config.pp @@ -0,0 +1,25 @@ +# Class: fluentd::config() +# +# +class fluentd::config() { + if $fluentd::product_name == 'fluentd' { + $config_file = 'fluent' + } else { + $config_file = $fluentd::product_name + } + + file { "/etc/${fluentd::product_name}/${config_file}.conf" : + ensure => file, + owner => 'root', + group => 'root', + content => template('fluentd/td-agent.conf.erb'), + notify => Class['fluentd::service'], + } + + file {"/etc/${fluentd::product_name}/config.d": + ensure => 'directory', + owner => "${fluentd::product_name}", + group => "${fluentd::product_name}", + mode => '0750', + } +} diff --git a/fluentd/manifests/configfile.pp b/fluentd/manifests/configfile.pp new file mode 100644 index 000000000..297e4e621 --- /dev/null +++ b/fluentd/manifests/configfile.pp @@ -0,0 +1,36 @@ +# == definition fluentd::configfile +define fluentd::configfile( + $ensure = present, + $content, + $priority = 50, +) { + $base_name = "${name}.conf" + $conf_name = "${priority}-${base_name}" + $conf_path = "/etc/${fluentd::product_name}/config.d/${conf_name}" + $wildcard_path = "/etc/${fluentd::product_name}/config.d/*-${base_name}" + + # clean up in case of a priority change + exec { "rm ${wildcard_path}": + onlyif => "test \$(ls ${wildcard_path} | grep -v ${conf_name} | wc -l) -gt 0", + path => ['/usr/bin', '/usr/sbin', '/bin'], + before => File[$conf_path], + notify => Class['fluentd::service'], + } + + if $ensure == 'absent' { + file { $conf_path: + ensure => $ensure, + notify => Class['fluentd::service'], + } + } else { + file { $conf_path: + ensure => $ensure, + content => $content, + owner => "${fluentd::product_name}", + group => "${fluentd::product_name}", + mode => '0644', + require => Class['fluentd::packages'], + notify => Class['fluentd::service'], + } + } +} diff --git a/fluentd/manifests/filter.pp b/fluentd/manifests/filter.pp new file mode 100644 index 000000000..64cebd89c --- /dev/null +++ b/fluentd/manifests/filter.pp @@ -0,0 +1,29 @@ +# == definition fluentd::match +define fluentd::filter ( + $ensure = present, + $config = { + 'type' => 'grep', + }, + $priority = 50, + $pattern, +) { + + if ($config['type'] == 'grep') { + if !defined(Fluentd::Install_plugin['fluent-plugin-grep']) { + fluentd::install_plugin{'fluent-plugin-grep': + plugin_type => 'gem' + } + } + } + + if ((($regexp != '') or ($exclude != '')) and ($input_key == '')) or (($input_key != '') and ($regexp == '') and ($exclude == '')) { + fail ('regexp, exlude and input_key must be used in conjuction') + } + + fluentd::configfile { "filter-${name}": + ensure => $ensure, + content => template( 'fluentd/filter.erb' ), + priority => $priority, + } + +} diff --git a/fluentd/manifests/forest_match.pp b/fluentd/manifests/forest_match.pp new file mode 100644 index 000000000..c28e8ba11 --- /dev/null +++ b/fluentd/manifests/forest_match.pp @@ -0,0 +1,15 @@ +# == definition fluentd::forest_match +define fluentd::forest_match ( + $configfile, + $type, + $pattern, + $config = {}, + $servers = [], +) { + + concat::fragment { "match_${title}": + target => "/etc/${fluentd::product_name}/config.d/${configfile}.conf", + require => Package["${fluentd::package_name}"], + content => template('fluentd/forest_match.erb'), + } +} diff --git a/fluentd/manifests/init.pp b/fluentd/manifests/init.pp new file mode 100644 index 000000000..db5383e1a --- /dev/null +++ b/fluentd/manifests/init.pp @@ -0,0 +1,21 @@ +# == class fluentd +class fluentd ( + $version = '1', + $package_name = $fluentd::params::package_name, + $install_repo = $fluentd::params::install_repo, + $package_ensure = $fluentd::params::package_ensure, + $service_enable = $fluentd::params::service_enable, + $service_ensure = $fluentd::params::service_ensure, + $service_name = $fluentd::params::service_name +) inherits fluentd::params { + # fluentd vs td-agent + $product_name = $service_name + + class{'fluentd::packages': } + class{'fluentd::config': } + class{'fluentd::service': } + + validate_bool($install_repo, $service_enable) + + Class['Fluentd::Packages'] -> Class['Fluentd::Config'] -> Class['Fluentd::Service'] +} diff --git a/fluentd/manifests/install_plugin.pp b/fluentd/manifests/install_plugin.pp new file mode 100644 index 000000000..a395b7fde --- /dev/null +++ b/fluentd/manifests/install_plugin.pp @@ -0,0 +1,40 @@ +# == fluentd::install_plugin +# +# install a plugin with either /usr/lib/fluent/ruby/bin/fluent-gem or a file +# +# you need to sepecify wich one of theese options plus the name. +# +# Parameters: +# the name of this ressource reflects the either the filename of the plugin, which must +# be copied to fluentd/files or the name of the gem +# +# plugin_type: specify "file" to copy that file to /etc/fluentd/plugins +# specify "gem" to try to install the plugin with fluent-gem +# +# ensure: "present"(default) or "absent", install or uninstall a plugin +# +define fluentd::install_plugin ( + $plugin_type, + $ensure = 'present', + $plugin_name = $name, +) { + case $plugin_type { + 'file': { + fluentd::install_plugin::file { + [$plugin_name]: + ensure => $ensure, + require => Class['Fluentd::Packages'] + } + } + 'gem': { + fluentd::install_plugin::gem { + [$plugin_name]: + ensure => $ensure, + require => Class['Fluentd::Packages'] + } + } + default: { + fail("plugin_type => ${plugin_type} is currently not supportet by this module") + } + } +} diff --git a/fluentd/manifests/install_plugin/file.pp b/fluentd/manifests/install_plugin/file.pp new file mode 100644 index 000000000..20f405331 --- /dev/null +++ b/fluentd/manifests/install_plugin/file.pp @@ -0,0 +1,24 @@ +# == fluentd::install_plugin::file +# +# install a plugin with by copying a file to /etc/${fluentd::product_name}/plugins +# +# Parameters: +# the name of this ressource reflects the filename of the plugin, which must +# be copied to fluentd/files +# +# ensure: "present"(default) or "absent", install or uninstall a plugin +# +define fluentd::install_plugin::file ( + $ensure = 'present', + $plugin_name = $name, +) { + file { + "/etc/${fluentd::product_name}/plugin/${plugin_name}": + ensure => $ensure, + owner => ${fluentd::product_name}, + group => ${fluentd::product_name}, + mode => '0640', + source => "puppet:///fluentd/plugins/${plugin_name}", + notify => Service["${::fluentd::service_name}"]; + } +} diff --git a/fluentd/manifests/install_plugin/gem.pp b/fluentd/manifests/install_plugin/gem.pp new file mode 100644 index 000000000..bd8e14508 --- /dev/null +++ b/fluentd/manifests/install_plugin/gem.pp @@ -0,0 +1,21 @@ +# == fluentd::install_plugin::gem +# +# install a plugin with /usr/lib/fluent/ruby/bin/fluent-gem +# +# Parameters: +# the name of this ressource reflects the name of the gem +# +# ensure: "present"(default) or "absent", install or uninstall a plugin +# +define fluentd::install_plugin::gem ( + $ensure = 'latest', + $plugin_name = $name, +) { + + package { $plugin_name: + ensure => $ensure, + provider => 'fluentgem', + notify => Service["${fluentd::service_name}"]; + } + +} diff --git a/fluentd/manifests/install_repo/apt.pp b/fluentd/manifests/install_repo/apt.pp new file mode 100644 index 000000000..e2eadbfc1 --- /dev/null +++ b/fluentd/manifests/install_repo/apt.pp @@ -0,0 +1,24 @@ +##apt.pp + +# Class: fluentd::install_repo::apt () +# +# +class fluentd::install_repo::apt () { + + apt::source { 'treasure-data': + location => "http://packages.treasuredata.com/debian", + release => "lucid", + repos => "contrib", + include_src => false, + } + + file { '/tmp/packages.treasure-data.com.key': + ensure => file, + source => 'puppet:///modules/fluentd/packages.treasure-data.com.key' + }-> + exec { "import gpg key Treasure Data": + command => "/bin/cat /tmp/packages.treasure-data.com.key | apt-key add -", + unless => "/usr/bin/apt-key list | grep -q 'Treasure Data'", + notify => Class['::apt::update'], + } +} diff --git a/fluentd/manifests/install_repo/yum.pp b/fluentd/manifests/install_repo/yum.pp new file mode 100644 index 000000000..790781465 --- /dev/null +++ b/fluentd/manifests/install_repo/yum.pp @@ -0,0 +1,19 @@ +# yum.pp + +# Class: fluentd::install_repo::yum () +# +# +class fluentd::install_repo::yum ( + $key = $fluentd::params::yum_key_url, + ) { + + # Sorry for the different naming of the Repository between debian and redhat. + # But I dont want rename it to avoid a duplication. + yumrepo { 'treasuredata': + descr => 'Treasure Data', + baseurl => 'http://packages.treasuredata.com/redhat/$basearch', + gpgkey => 'http://packages.treasuredata.com/redhat/RPM-GPG-KEY-td-agent', + gpgcheck => 1, + } + +} diff --git a/fluentd/manifests/match.pp b/fluentd/manifests/match.pp new file mode 100644 index 000000000..bd6789baa --- /dev/null +++ b/fluentd/manifests/match.pp @@ -0,0 +1,13 @@ +# == definition fluentd::match +define fluentd::match ( + $ensure = present, + $config = {}, + $priority = 50, + $pattern, +) { + fluentd::configfile { "match-${name}": + ensure => $ensure, + content => template( 'fluentd/match.erb' ), + priority => $priority, + } +} diff --git a/fluentd/manifests/packages.pp b/fluentd/manifests/packages.pp new file mode 100644 index 000000000..38070f781 --- /dev/null +++ b/fluentd/manifests/packages.pp @@ -0,0 +1,50 @@ +# == class fluentd::packages +class fluentd::packages ( + $package_name = $fluentd::package_name, + $install_repo = $fluentd::install_repo, + $package_ensure = $fluentd::package_ensure +){ + if $install_repo { + case $::osfamily { + 'redhat': { + class{'fluentd::install_repo::yum': + before => Package[$package_name], + } + } + 'debian': { + class{'fluentd::install_repo::apt': + before => Package[$package_name], + } + } + default: { + fail("Unsupported osfamily ${::osfamily}") + } + } + } + package { "$package_name": + ensure => $package_ensure + } + +# extra bits... why this is required isn't quite clear. + case $::osfamily { + 'debian': { + package{[ + 'libxslt1.1', + 'libyaml-0-2', + ]: + before => Package[$package_name], + ensure => $package_ensure + } + exec {"add user ${fluentd::product_name} to group adm": + provider => shell, + unless => "/bin/grep -q \"adm\\S*${fluentd::product_name}\" /etc/group", + command => "/usr/sbin/usermod -aG adm ${fluentd::product_name}", + subscribe => Package[$package_name], + } + } + default: { + info("No required fluentd::packages extra bits for ${::osfamily}") + } + } + +} diff --git a/fluentd/manifests/params.pp b/fluentd/manifests/params.pp new file mode 100644 index 000000000..aed3b9a31 --- /dev/null +++ b/fluentd/manifests/params.pp @@ -0,0 +1,12 @@ +# Class: fluentd::params +# +# +class fluentd::params { + $package_name = 'td-agent' + $package_ensure = 'installed' + $install_repo = true + $service_ensure = 'running' + $service_enable = true + $service_name = 'td-agent' + $yum_key_url = "http://packages.treasure-data.com/redhat/RPM-GPG-KEY-td-agent" +} \ No newline at end of file diff --git a/fluentd/manifests/service.pp b/fluentd/manifests/service.pp new file mode 100644 index 000000000..89760aab4 --- /dev/null +++ b/fluentd/manifests/service.pp @@ -0,0 +1,12 @@ +# == class fluentd::service +class fluentd::service ( + $service_ensure = $fluentd::service_ensure, + $service_enable = $fluentd::service_enable, +) { + include fluentd::params + service {"${fluentd::service_name}": + ensure => $service_ensure, + enable => $service_enable, + hasstatus => true + } +} diff --git a/fluentd/manifests/source.pp b/fluentd/manifests/source.pp new file mode 100644 index 000000000..0104dbbbd --- /dev/null +++ b/fluentd/manifests/source.pp @@ -0,0 +1,12 @@ +# == definition fluentd::source +define fluentd::source ( + $ensure = present, + $priority = 50, + $config = {}, +) { + fluentd::configfile { "source-${name}": + ensure => $ensure, + content => template( 'fluentd/source.erb' ), + priority => $priority, + } +} diff --git a/fluentd/spec/classes/fluentd_config_spec.rb b/fluentd/spec/classes/fluentd_config_spec.rb new file mode 100644 index 000000000..cb744b569 --- /dev/null +++ b/fluentd/spec/classes/fluentd_config_spec.rb @@ -0,0 +1,58 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::config', :type => :class do + context "On a Debian OS, config files should be written" do + let :facts do + { + :osfamily => 'Debian', + :operatingsystemrelease => '7', + :concat_basedir => '/tmp', + :lsbdistid => 'Debian', + } + end + + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + end + context "On a Redhat OS, config files should be written" do + let :facts do + { + :osfamily => 'RedHat', + :operatingsystemrelease => '6.5', + :concat_basedir => '/tmp', + :lsbdistid => 'CentOS', + } + end + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + end +end diff --git a/fluentd/spec/classes/fluentd_packages_spec.rb b/fluentd/spec/classes/fluentd_packages_spec.rb new file mode 100644 index 000000000..bad912abb --- /dev/null +++ b/fluentd/spec/classes/fluentd_packages_spec.rb @@ -0,0 +1,70 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::packages', :type => :class do + let (:params) {{:package_name => 'td-agent', :package_ensure => 'installed'}} + context "On a Debian OS" do + let :facts do + { + :osfamily => 'Debian', + :operatingsystemrelease => '7', + :lsbdistid => 'Debian', + } + end + + context "with install_repo=>true" do + let(:params) { {:install_repo => true} } + it do + should contain_apt__source("treasure-data").with( + 'location' => 'http://packages.treasuredata.com/debian' + ) + end + end + it { should contain_package("libxslt1.1").with( + 'ensure' => 'installed' + ) + } + it { should contain_package("libyaml-0-2").with( + 'ensure' => 'installed' + ) + } + it { should contain_package("td-agent").with( + 'ensure' => 'installed' + ) + } + end + + context "On a RedHat/Centos OS" do + + let :params do + { + :package_name => 'td-agent', + :package_ensure => 'running', + } + end + + let :facts do + { + :osfamily => 'Redhat', + } + end + + it { should contain_class('fluentd::packages')} + + context "with install_repo=>true" do + let(:params) { {:install_repo => true} } + it do + should contain_yumrepo('treasuredata').with( + 'baseurl' => 'http://packages.treasuredata.com/redhat/$basearch', + 'gpgkey' => 'http://packages.treasuredata.com/redhat/RPM-GPG-KEY-td-agent', + 'gpgcheck' => 1 + ) + end + end + + it { should contain_package("td-agent").with( + 'ensure' => 'running' + ) + } + end +end diff --git a/fluentd/spec/classes/fluentd_service_spec.rb b/fluentd/spec/classes/fluentd_service_spec.rb new file mode 100644 index 000000000..c9ec2bbaf --- /dev/null +++ b/fluentd/spec/classes/fluentd_service_spec.rb @@ -0,0 +1,79 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::service', :type => :class do + context "On a Debian OS" do + let :facts do + { + :osfamily => 'Debian', + :operatingsystemrelease => '7', + :concat_basedir => '/tmp', + :lsbdistid => 'Debian', + } + end + context "td-agent running and enabled" do + let (:params) {{:service_ensure => 'running', :service_enable => true}} + it { should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + } + end + context "td-agent stopped but enabled" do + let (:params) {{:service_ensure => 'stopped', :service_enable => true}} + it { should contain_service("td-agent").with( + 'ensure' => 'stopped', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + } + end + context "td-agent stopped and disabled" do + let (:params) {{:service_ensure => 'stopped', :service_enable => false}} + it { should contain_service("td-agent").with( + 'ensure' => 'stopped', + 'enable' => 'false', + 'hasstatus' => 'true' + ) + } + end + end + context "On a Redhat OS" do + let :facts do + { + :osfamily => 'RedHat', + :operatingsystemrelease => '6.5', + :concat_basedir => '/tmp', + :lsbdistid => 'CentOs', + } + end + context "td-agent running and enabled" do + let (:params) {{:service_ensure => 'running', :service_enable => true}} + it { should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + } + end + context "td-agent stopped but enabled" do + let (:params) {{:service_ensure => 'stopped', :service_enable => true}} + it { should contain_service("td-agent").with( + 'ensure' => 'stopped', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + } + end + context "td-agent stopped and disabled" do + let (:params) {{:service_ensure => 'stopped', :service_enable => false}} + it { should contain_service("td-agent").with( + 'ensure' => 'stopped', + 'enable' => 'false', + 'hasstatus' => 'true' + ) + } + end + end +end diff --git a/fluentd/spec/classes/fluentd_spec.rb b/fluentd/spec/classes/fluentd_spec.rb new file mode 100644 index 000000000..0d5c7cb4d --- /dev/null +++ b/fluentd/spec/classes/fluentd_spec.rb @@ -0,0 +1,233 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd', :type => :class do + context "On a Debian OS " do + let :facts do + { + :osfamily => 'Debian', + :operatingsystemrelease => '7', + :concat_basedir => '/tmp', + :lsbdistid => 'Debian', + } + end + context "Debian with repo installed" do + let (:params) {{ + :package_name => 'td-agent', + :package_ensure => 'installed', + :install_repo => true, + :service_enable => true, + :service_ensure => 'running' + }} + it "has fluentd::packages" do + should contain_class('fluentd::packages') + end + it "has fluentd::config class" do + should contain_class('fluentd::config') + end + it "has fluentd::service class" do + should contain_class('fluentd::service') + end + it "the apt repo file exists" do + should contain_apt__source("treasure-data").with( + 'location' => 'http://packages.treasuredata.com/debian' + ) + end + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + it "td-agent package should be installed" do + should contain_package("td-agent").with( + 'ensure' => 'installed' + ) + end + it "td-agent is running" do + should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + end + end + context "Debian without repo installed" do + let (:params) {{ + :package_name => 'td-agent', + :package_ensure => 'installed', + :install_repo => false, + :service_enable => true, + :service_ensure => 'running' + }} + it "has fluentd::packages" do + should contain_class('fluentd::packages') + end + it "has fluentd::config class" do + should contain_class('fluentd::config') + end + it "has fluentd::service class" do + should contain_class('fluentd::service') + end + it "the apt repo file DOES NOT exists" do + should_not contain_apt__source("treasure-data").with( + 'location' => 'http://packages.treasure-data.com/debian' + ) + end + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + it "td-agent package should be installed" do + should contain_package("td-agent").with( + 'ensure' => 'installed' + ) + end + it "td-agent is running" do + should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + end + end + end + context "On a Redhat-like OS with repo installed" do + let (:params) {{ + :package_name => 'td-agent', + :package_ensure => 'installed', + :install_repo => true, + :service_enable => true, + :service_ensure => 'running' + }} + let :facts do + { + :osfamily => 'RedHat', + :operatingsystemrelease => '6.5', + :concat_basedir => '/tmp', + :lsbdistid => 'CentOs', + } + end + + it "has fluentd::packages" do + should contain_class('fluentd::packages') + end + it "has fluentd::config class" do + should contain_class('fluentd::config') + end + it "has fluentd::service class" do + should contain_class('fluentd::service') + end + it "the yum repo file exists" do + should contain_yumrepo('treasuredata') + end + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + it "td-agent package should be installed" do + should contain_package("td-agent").with( + 'ensure' => 'installed' + ) + end + it "td-agent is running" do + should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + end + end + context "On a Redhat-like OS without repo installed" do + let (:params) {{ + :package_name => 'td-agent', + :package_ensure => 'installed', + :install_repo => false, + :service_enable => true, + :service_ensure => 'running' + }} + let :facts do + { + :osfamily => 'RedHat', + :operatingsystemrelease => '6.5', + :concat_basedir => '/tmp', + :lsbdistid => 'CentOs', + } + end + + it "has fluentd::packages" do + should contain_class('fluentd::packages') + end + it "has fluentd::config class" do + should contain_class('fluentd::config') + end + it "has fluentd::service class" do + should contain_class('fluentd::service') + end + it "the yum repo file DOES NOT exists" do + should_not contain_file('/etc/yum.repos.d/td.repo').with_content(/^name=TreasureData$/) + end + it "/etc/td-agent/td-agent.conf should be in place" do + should contain_file("/etc/td-agent/td-agent.conf").with( + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'notify' => 'Class[Fluentd::Service]' + ).with_content(/^# Include.*config.d.*/m) + end + it "/etc/td-agent/config.d is created" do + should contain_file("/etc/td-agent/config.d").with( + 'ensure' => 'directory', + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0750' + ) + end + it "td-agent package should be installed" do + should contain_package("td-agent").with( + 'ensure' => 'installed' + ) + end + it "td-agent is running" do + should contain_service("td-agent").with( + 'ensure' => 'running', + 'enable' => 'true', + 'hasstatus' => 'true' + ) + end + end +end diff --git a/fluentd/spec/defines/fluent_filter_spec.rb b/fluentd/spec/defines/fluent_filter_spec.rb new file mode 100644 index 000000000..0f49c0403 --- /dev/null +++ b/fluentd/spec/defines/fluent_filter_spec.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::filter' do + let(:title) {'bar'} + + let (:facts) {{ + :osfamily => 'Debian', + :lsbdistid => 'Debian', + }} + + context "when type is grep" do + let(:params) {{ + :pattern => 'baz', + :config => { + 'type' => 'grep', + 'input_key' => 'input_key', + 'regexp' => '/regex/', + 'exclude' => 'exclude', + 'output_tag' => 'output_tag', + 'add_tag_prefix' => 'add_tag_prefix', + 'remove_tag_prefix' => 'remove_tag_prefix', + 'add_tag_suffix' => 'add_tag_suffix', + 'remove_tag_suffix' => 'remove_tag_suffix', + }, + }} + + it "should install the grep plugin" do + should contain_fluentd__install_plugin('fluent-plugin-grep') + end + + + it "should create matcher single segment" do + should contain_fluentd__configfile('filter-bar').with_content(/.*type.*input_key.*regexp.*exclude.*output_tag.*add_tag_prefix.*remove_tag_prefix.*add_tag_suffix.*remove_tag_suffix.*<\/match>/m) + end + end + +end \ No newline at end of file diff --git a/fluentd/spec/defines/fluentd_configfile_spec.rb b/fluentd/spec/defines/fluentd_configfile_spec.rb new file mode 100644 index 000000000..77a64e08b --- /dev/null +++ b/fluentd/spec/defines/fluentd_configfile_spec.rb @@ -0,0 +1,31 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::configfile', :type => :define do + let :pre_condition do + 'include fluentd' + end + let(:params) { { :content => "" } } + + ['Debian','RedHat'].each do |osfam| + context "When on an #{osfam} system" do + let (:facts) {{ + :osfamily => osfam, + :lsbdistid => 'Debian', # Concatlib needs this value. Works for RedHat too. + }} + context 'when fed no parameters' do + let (:title) { 'MyBaconBringsAllTheBoysToTheYard'} + context 'provides a stub config' do + it { should contain_class('fluentd') } + it { should contain_file("/etc/td-agent/config.d/50-#{title}.conf").with( + 'owner' => 'td-agent', + 'group' => 'td-agent', + 'mode' => '0644', + 'require' => 'Class[Fluentd::Packages]' + ) + } + end + end + end + end +end \ No newline at end of file diff --git a/fluentd/spec/defines/fluentd_match_spec.rb b/fluentd/spec/defines/fluentd_match_spec.rb new file mode 100644 index 000000000..6aff94ca3 --- /dev/null +++ b/fluentd/spec/defines/fluentd_match_spec.rb @@ -0,0 +1,71 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +describe 'fluentd::match' do + let(:title) {'bar'} + + let (:facts) {{ + :osfamily => 'Debian', + :lsbdistid => 'Debian', + }} + + context "when no servers or out_copy" do + let(:params) {{ + :pattern => 'baz', + :config => { + 'type' => 'file', + 'time_slice_wait' => '10m', + 'compress' => 'gzip', + } + }} + + it "should create matcher single segment" do + should contain_fluentd__configfile('match-bar').with_content(/.*type.*file.*time_slice_wait.*10m.*compress.*gzip.*<\/match>/m) + should_not contain_fluentd__configfile('match-bar').with_content(/server/) + should_not contain_fluentd__configfile('match-bar').with_content(/store/) + end + end + + context "when servers but no out_copy" do + let(:params) {{ + :pattern => 'baz', + :config => { + 'servers' => [{ 'host' => 'kelis', 'port' => '24224'}, { 'host' => 'bossy', 'port' => '24224'}], + 'type' => 'file', + 'time_slice_wait' => '10m', + 'compress' => 'gzip', + } + }} + + it "should create matcher with server" do + should contain_fluentd__configfile('match-bar').with_content(/.*.*host kelis.*port.*24224.*<\/server>.*.*host.*bossy.*port.*24224.*<\/server>.*type.*file.*time_slice_wait.*10m.*compress.*gzip.*<\/match>/m) + should contain_fluentd__configfile('match-bar').with_content(/server/) + should_not contain_fluentd__configfile('match-bar').with_content(/store/) + end + end + + context "when out_copy" do + let(:params) {{ + :pattern => 'baz', + :config => [ + { + 'type' => 'file', + 'compress' => 'gzip', + 'servers' => [{ 'host' => 'kelis', 'port' => '24224'}, { 'host' => 'bossy', 'port' => '24224'}], + }, + { + 'type' => 'mongo', + 'database' => 'dummy', + } + ] + }} + + it "should create matcher with server" do + should contain_fluentd__configfile('match-bar').with_content(/.*type.*copy.*.*type.*file.*compress.*gzip.*.*host.*kelis.*port.*24224.*<\/server>.*.*host.*bossy.*port.*24224.*<\/server>.*<\/store>.*.*type.*mongo.*database.*dummy.*<\/store>.*<\/match>/m) + end + end + + +end + + diff --git a/fluentd/spec/fixtures/unit/provider/fluentgem/gem-list-single-package b/fluentd/spec/fixtures/unit/provider/fluentgem/gem-list-single-package new file mode 100644 index 000000000..41576e3e6 --- /dev/null +++ b/fluentd/spec/fixtures/unit/provider/fluentgem/gem-list-single-package @@ -0,0 +1,4 @@ + +*** REMOTE GEMS *** + +bundler (1.6.2) diff --git a/fluentd/spec/fixtures/unit/provider/fluentgem/line-with-1.8.5-warning b/fluentd/spec/fixtures/unit/provider/fluentgem/line-with-1.8.5-warning new file mode 100644 index 000000000..07a27e7b8 --- /dev/null +++ b/fluentd/spec/fixtures/unit/provider/fluentgem/line-with-1.8.5-warning @@ -0,0 +1,14 @@ +/home/jenkins/.rvm/gems/ruby-1.8.5-p231@global/gems/rubygems-bundler-0.9.0/lib/rubygems-bundler/regenerate_binstubs_command.rb:34: warning: parenthesize argument(s) for future version + +*** LOCAL GEMS *** + +columnize (0.3.2) +diff-lcs (1.1.3) +metaclass (0.0.1) +mocha (0.10.5) +rake (0.8.7) +rspec-core (2.9.0) +rspec-expectations (2.9.1) +rspec-mocks (2.9.0) +rubygems-bundler (0.9.0) +rvm (1.11.3.3) diff --git a/fluentd/spec/spec.opts b/fluentd/spec/spec.opts new file mode 100644 index 000000000..de653df4b --- /dev/null +++ b/fluentd/spec/spec.opts @@ -0,0 +1,4 @@ +--format s +--colour +--loadby mtime +--backtrace diff --git a/fluentd/spec/spec_helper.rb b/fluentd/spec/spec_helper.rb new file mode 100644 index 000000000..479ed6922 --- /dev/null +++ b/fluentd/spec/spec_helper.rb @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/module_spec_helper' \ No newline at end of file diff --git a/fluentd/spec/unit/provider/fluentgem_spec.rb b/fluentd/spec/unit/provider/fluentgem_spec.rb new file mode 100644 index 000000000..17d9a9a93 --- /dev/null +++ b/fluentd/spec/unit/provider/fluentgem_spec.rb @@ -0,0 +1,176 @@ +#! /usr/bin/env ruby +require 'spec_helper' + +provider_class = Puppet::Type.type(:package).provider(:fluentgem) + +describe provider_class do + let(:resource) do + Puppet::Type.type(:package).new( + :name => 'myresource', + :ensure => :installed + ) + end + + let(:provider) do + provider = provider_class.new + provider.resource = resource + provider + end + + before :each do + resource.provider = provider + end + + describe "when installing" do + it "should use the path to the gem" do + provider_class.stubs(:command).with(:gemcmd).returns "/my/gem" + provider.expects(:execute).with { |args| args[0] == "/my/gem" }.returns "" + provider.install + end + + it "should specify that the gem is being installed" do + provider.expects(:execute).with { |args| args[1] == "install" }.returns "" + provider.install + end + + it "should specify that documentation should not be included" do + provider.expects(:execute).with { |args| args[2] == "--no-rdoc" }.returns "" + provider.install + end + + it "should specify that RI should not be included" do + provider.expects(:execute).with { |args| args[3] == "--no-ri" }.returns "" + provider.install + end + + it "should specify the package name" do + provider.expects(:execute).with { |args| args[4] == "myresource" }.returns "" + provider.install + end + + describe "when a source is specified" do + describe "as a normal file" do + it "should use the file name instead of the gem name" do + resource[:source] = "/my/file" + provider.expects(:execute).with { |args| args[2] == "/my/file" }.returns "" + provider.install + end + end + describe "as a file url" do + it "should use the file name instead of the gem name" do + resource[:source] = "file:///my/file" + provider.expects(:execute).with { |args| args[2] == "/my/file" }.returns "" + provider.install + end + end + describe "as a puppet url" do + it "should fail" do + resource[:source] = "puppet://my/file" + lambda { provider.install }.should raise_error(Puppet::Error) + end + end + describe "as a non-file and non-puppet url" do + it "should treat the source as a gem repository" do + resource[:source] = "http://host/my/file" + provider.expects(:execute).with { |args| args[2..4] == ["--source", "http://host/my/file", "myresource"] }.returns "" + provider.install + end + end + describe "with an invalid uri" do + it "should fail" do + URI.expects(:parse).raises(ArgumentError) + resource[:source] = "http:::::uppet:/:/my/file" + lambda { provider.install }.should raise_error(Puppet::Error) + end + end + end + end + + describe "#latest" do + it "should return a single value for 'latest'" do + #gemlist is used for retrieving both local and remote version numbers, and there are cases + # (particularly local) where it makes sense for it to return an array. That doesn't make + # sense for '#latest', though. + provider.class.expects(:gemlist).with({ :justme => 'myresource'}).returns({ + :name => 'myresource', + :ensure => ["3.0"], + :provider => :gem, + }) + provider.latest.should == "3.0" + end + + it "should list from the specified source repository" do + resource[:source] = "http://foo.bar.baz/gems" + provider.class.expects(:gemlist). + with({:justme => 'myresource', :source => "http://foo.bar.baz/gems"}). + returns({ + :name => 'myresource', + :ensure => ["3.0"], + :provider => :gem, + }) + provider.latest.should == "3.0" + end + end + + describe "#instances" do + before do + provider_class.stubs(:command).with(:gemcmd).returns "/my/gem" + end + + it "should return an empty array when no gems installed" do + provider_class.expects(:execute).with(%w{/my/gem list --local}).returns("\n") + provider_class.instances.should == [] + end + + it "should return ensure values as an array of installed versions" do + provider_class.expects(:execute).with(%w{/my/gem list --local}).returns <<-HEREDOC.gsub(/ /, '') + systemu (1.2.0) + vagrant (0.8.7, 0.6.9) + HEREDOC + + provider_class.instances.map {|p| p.properties}.should == [ + {:ensure => ["1.2.0"], :provider => :gem, :name => 'systemu'}, + {:ensure => ["0.8.7", "0.6.9"], :provider => :gem, :name => 'vagrant'} + ] + end + + it "should ignore platform specifications" do + provider_class.expects(:execute).with(%w{/my/gem list --local}).returns <<-HEREDOC.gsub(/ /, '') + systemu (1.2.0) + nokogiri (1.6.1, 1.4.4.1) + HEREDOC + + provider_class.instances.map {|p| p.properties}.should == [ + {:ensure => ["1.2.0"], :provider => :gem, :name => 'systemu'}, + {:ensure => ["1.6.1", "1.4.4.1"], :provider => :gem, :name => 'nokogiri'} + ] + end + + it "should not fail when an unmatched line is returned" do + provider_class.expects(:execute).with(%w{/my/gem list --local}). + returns(File.read(my_fixture('line-with-1.8.5-warning'))) + + provider_class.instances.map {|p| p.properties}. + should == [{:provider=>:gem, :ensure=>["0.3.2"], :name=>"columnize"}, + {:provider=>:gem, :ensure=>["1.1.3"], :name=>"diff-lcs"}, + {:provider=>:gem, :ensure=>["0.0.1"], :name=>"metaclass"}, + {:provider=>:gem, :ensure=>["0.10.5"], :name=>"mocha"}, + {:provider=>:gem, :ensure=>["0.8.7"], :name=>"rake"}, + {:provider=>:gem, :ensure=>["2.9.0"], :name=>"rspec-core"}, + {:provider=>:gem, :ensure=>["2.9.1"], :name=>"rspec-expectations"}, + {:provider=>:gem, :ensure=>["2.9.0"], :name=>"rspec-mocks"}, + {:provider=>:gem, :ensure=>["0.9.0"], :name=>"rubygems-bundler"}, + {:provider=>:gem, :ensure=>["1.11.3.3"], :name=>"rvm"}] + end + end + + #describe "listing gems" do + # describe "searching for a single package" do + # it "searches for an exact match" do + # provider_class.expects(:execute).with(includes('^bundler$')).returns(File.read(my_fixture('gem-list-single-package'))) + # expected = {:name => 'bundler', :ensure => %w[1.6.2], :provider => :fluentgem} + # expect(provider_class.gemlist({:justme => 'bundler'})).to eq(expected) + # end + # end + #end +end diff --git a/fluentd/spec_helper.rb b/fluentd/spec_helper.rb new file mode 100644 index 000000000..3d9200524 --- /dev/null +++ b/fluentd/spec_helper.rb @@ -0,0 +1 @@ +require 'puppetlabs_spec_helper/module_spec_helper' \ No newline at end of file diff --git a/fluentd/templates/filter.erb b/fluentd/templates/filter.erb new file mode 100644 index 000000000..e6bf92246 --- /dev/null +++ b/fluentd/templates/filter.erb @@ -0,0 +1,5 @@ +> +<% @config.each do |key, val| -%> + <%= key %> <%= val %> +<% end -%> + \ No newline at end of file diff --git a/fluentd/templates/forest_match.erb b/fluentd/templates/forest_match.erb new file mode 100644 index 000000000..2b7053b25 --- /dev/null +++ b/fluentd/templates/forest_match.erb @@ -0,0 +1,21 @@ +> + type <%= @type %> +<% @config.sort_by{|key,val|key}.each do |key, val| -%> + <% if val.class == Hash && key == 'template' %> + + <% else -%> + <%= key %> <%= val %> + <% end -%> +<% end -%> +<% @servers.each do |server| -%> + +<% @server.sort_by{|key,val|key}.each do |key, val| -%> + <%= key %> <%= val %> +<% end -%> + +<% end -%> + \ No newline at end of file diff --git a/fluentd/templates/match.erb b/fluentd/templates/match.erb new file mode 100644 index 000000000..f0543acaf --- /dev/null +++ b/fluentd/templates/match.erb @@ -0,0 +1,36 @@ +> +<%- if @config.is_a?( Array ) -%> + type copy + <%- @config.each do |config| -%> + + <%- config.each do |key, val| -%> + <%- if key == 'servers' -%> + <%- val.each do |server| -%> + + <%- server.each do |k,v| -%> + <%= k %> <%= v %> + <%- end -%> + + <%- end -%> + <%- else -%> + <%= key %> <%= val %> + <%- end -%> + <%- end -%> + + <%- end -%> +<%- else -%> + <%- @config.each do |key, val| -%> + <%- if key == 'servers' -%> + <%- val.each do |server| -%> + + <%- server.each do |k,v| -%> + <%= k %> <%= v %> + <%- end -%> + + <%- end -%> + <%- else -%> + <%= key %> <%= val %> + <%- end -%> + <%- end -%> +<%- end -%> + diff --git a/fluentd/templates/source.erb b/fluentd/templates/source.erb new file mode 100644 index 000000000..d120bacaf --- /dev/null +++ b/fluentd/templates/source.erb @@ -0,0 +1,5 @@ + +<% @config.sort_by{|key,val|key}.each do |key, val| -%> + <%= key %> <%= val %> +<% end -%> + diff --git a/fluentd/templates/td-agent.conf.erb b/fluentd/templates/td-agent.conf.erb new file mode 100644 index 000000000..dfced4343 --- /dev/null +++ b/fluentd/templates/td-agent.conf.erb @@ -0,0 +1,2 @@ +# Include config files in the ./config.d directory +<% if @version == '2' -%>@<% end %>include config.d/*.conf