From 19fd15cd83bc96e3016f4432fb3ce24f238f263f Mon Sep 17 00:00:00 2001 From: Jason Guiditta Date: Wed, 16 Dec 2015 08:53:02 -0500 Subject: [PATCH] Add soylent/konstantin-fluentd module. The upstream module is now: https://github.com/soylent/konstantin-fluentd This module update commit was generated by Bade. For more info please check https://github.com/paramite/bade This commit is setting modules to following state: fluentd - initial commit: b462da7d1c6290afba38fd7e64226990ecf795c7 Change-Id: I2e6af8e44d382cf1e5baa5fabf4b2fc0e542d357 --- Puppetfile | 4 + fluentd/.fixtures.yml | 6 + fluentd/.gitignore | 5 + fluentd/CHANGELOG.md | 28 +++ fluentd/Gemfile | 9 + fluentd/LICENSE | 176 +++++++++++++++ fluentd/README.md | 212 ++++++++++++++++++ fluentd/Rakefile | 17 ++ fluentd/examples/init.pp | 19 ++ fluentd/files/td-agent.conf | 2 + .../puppet/parser/functions/fluent_config.rb | 29 +++ .../lib/puppet/provider/package/tdagent.rb | 3 + fluentd/manifests/config.pp | 10 + fluentd/manifests/init.pp | 46 ++++ fluentd/manifests/install.pp | 18 ++ fluentd/manifests/install_repo.pp | 47 ++++ fluentd/manifests/params.pp | 41 ++++ fluentd/manifests/plugin.pp | 12 + fluentd/manifests/service.pp | 10 + fluentd/metadata.json | 47 ++++ fluentd/spec/acceptance/class_spec.rb | 20 ++ .../acceptance/nodesets/debian-78-x64.yml | 11 + fluentd/spec/acceptance/nodesets/default.yml | 11 + .../nodesets/ubuntu-server-1404-x64.yml | 11 + fluentd/spec/classes/init_spec.rb | 24 ++ fluentd/spec/classes/install_spec.rb | 33 +++ fluentd/spec/classes/service_spec.rb | 17 ++ fluentd/spec/defines/config_spec.rb | 66 ++++++ fluentd/spec/defines/plugin_spec.rb | 20 ++ fluentd/spec/spec_helper.rb | 1 + fluentd/spec/spec_helper_acceptance.rb | 19 ++ 31 files changed, 974 insertions(+) create mode 100644 fluentd/.fixtures.yml create mode 100644 fluentd/.gitignore create mode 100644 fluentd/CHANGELOG.md create mode 100644 fluentd/Gemfile create mode 100644 fluentd/LICENSE create mode 100644 fluentd/README.md create mode 100644 fluentd/Rakefile create mode 100644 fluentd/examples/init.pp create mode 100644 fluentd/files/td-agent.conf create mode 100644 fluentd/lib/puppet/parser/functions/fluent_config.rb create mode 100644 fluentd/lib/puppet/provider/package/tdagent.rb create mode 100644 fluentd/manifests/config.pp create mode 100644 fluentd/manifests/init.pp create mode 100644 fluentd/manifests/install.pp create mode 100644 fluentd/manifests/install_repo.pp create mode 100644 fluentd/manifests/params.pp create mode 100644 fluentd/manifests/plugin.pp create mode 100644 fluentd/manifests/service.pp create mode 100644 fluentd/metadata.json create mode 100644 fluentd/spec/acceptance/class_spec.rb create mode 100644 fluentd/spec/acceptance/nodesets/debian-78-x64.yml create mode 100644 fluentd/spec/acceptance/nodesets/default.yml create mode 100644 fluentd/spec/acceptance/nodesets/ubuntu-server-1404-x64.yml create mode 100644 fluentd/spec/classes/init_spec.rb create mode 100644 fluentd/spec/classes/install_spec.rb create mode 100644 fluentd/spec/classes/service_spec.rb create mode 100644 fluentd/spec/defines/config_spec.rb create mode 100644 fluentd/spec/defines/plugin_spec.rb create mode 100644 fluentd/spec/spec_helper.rb create mode 100644 fluentd/spec/spec_helper_acceptance.rb diff --git a/Puppetfile b/Puppetfile index 185fb2c8c..6746cafa5 100644 --- a/Puppetfile +++ b/Puppetfile @@ -58,6 +58,10 @@ mod 'firewall', :commit => '1b6cc9192150d9521cc70301d0452daf189a63f7', :git => 'https://github.com/puppetlabs/puppetlabs-firewall.git' +mod 'fluentd', + :commit => 'b462da7d1c6290afba38fd7e64226990ecf795c7', + :git => 'https://github.com/soylent/konstantin-fluentd.git' + mod 'galera', :commit => 'a2ecf273aef9ef9193bba35c235fb7b8b821a0c3', :git => 'https://github.com/redhat-openstack/puppet-galera.git' diff --git a/fluentd/.fixtures.yml b/fluentd/.fixtures.yml new file mode 100644 index 000000000..d73a0c5d5 --- /dev/null +++ b/fluentd/.fixtures.yml @@ -0,0 +1,6 @@ +fixtures: + repositories: + stdlib: "git://github.com/puppetlabs/puppetlabs-stdlib.git" + apt: "git://github.com/puppetlabs/puppetlabs-apt.git" + symlinks: + fluentd: "#{source_dir}" diff --git a/fluentd/.gitignore b/fluentd/.gitignore new file mode 100644 index 000000000..1ac0a277b --- /dev/null +++ b/fluentd/.gitignore @@ -0,0 +1,5 @@ +.vagrant/ +Gemfile.lock +log/ +pkg/ +spec/fixtures/ diff --git a/fluentd/CHANGELOG.md b/fluentd/CHANGELOG.md new file mode 100644 index 000000000..3989a7df5 --- /dev/null +++ b/fluentd/CHANGELOG.md @@ -0,0 +1,28 @@ +## Unreleased + +## 2015-12-02 - Release v. 0.3.1 + + - Add param `repo_desc` + +## 2015-10-28 - Release v. 0.3.0 + + - Remove class `fluentd::config` + - Add defined type `fluentd::config` + - Add defined type `fluentd::plugin` + +## 2015-10-22 - Release v. 0.2.0 + + - Add param `service_manage` + - Add param `repo_gpgkeyid` + - Add param `repo_install` + - Add param `plugin_source` + - Rename param `repo_baseurl` to `repo_url` + - Remove param `config_template` + - Param validation + - Support Ubuntu 14.04 + - Support Debian 7.8 + - Support nested config tags + +## 2015-10-19 - Release v. 0.1.0 + + - Initial release diff --git a/fluentd/Gemfile b/fluentd/Gemfile new file mode 100644 index 000000000..b14d3f3a6 --- /dev/null +++ b/fluentd/Gemfile @@ -0,0 +1,9 @@ +source 'https://rubygems.org' + +gem 'puppet', ENV.fetch('PUPPET_VERSION', '>= 3.3') +gem 'rake' +gem 'puppet-lint', '>= 1.0.0' +gem 'puppetlabs_spec_helper', '>= 0.8.2' +gem 'beaker-rspec' +gem 'beaker-puppet_install_helper' +gem 'metadata-json-lint' diff --git a/fluentd/LICENSE b/fluentd/LICENSE new file mode 100644 index 000000000..d0381d6d0 --- /dev/null +++ b/fluentd/LICENSE @@ -0,0 +1,176 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/fluentd/README.md b/fluentd/README.md new file mode 100644 index 000000000..40283d33b --- /dev/null +++ b/fluentd/README.md @@ -0,0 +1,212 @@ +# Fluentd + +Install, configure, and manage Fluentd data collector. + +## Module Description + +* Installs `td-agent` package +* Generates configuration file `td-agent.conf` +* Generates custom configuration files and saves them to `config.d/` +* Manages `td-agent` service +* Installs Fluentd gem plugins + +## Usage + +### Routing events to Elasticsearch + +```puppet +include fluentd + +fluentd::plugin { 'fluent-plugin-elasticsearch': } + +fluentd::config { '500_elasticsearch.conf': + config => { + 'source' => { + 'type' => 'unix', + 'path' => '/tmp/td-agent/td-agent.sock', + }, + 'match' => { + 'tag_pattern' => '**', + 'type' => 'elasticsearch', + 'index_name' => 'foo', + 'type_name' => 'bar', + 'logstash_format' => true, + } + } +} +``` + +### Forwarding events to Fluentd aggregator + +```puppet +include fluentd + +fluentd::config { '600_forwarding.conf': + config => { + 'source' => { + 'type' => unix, + 'path' => '/tmp/td-agent/td-agent.sock', + }, + 'match' => { + 'tag_pattern' => '**', + 'type' => forward, + 'server' => [ + { 'host' => 'example1.com', 'port' => 24224 }, + { 'host' => 'example2.com', 'port' => 24224 }, + ] + } + } +} +``` + +### Config File Naming + +All configs employ a numbering system in the resource's title that is used for +ordering. When titling your config, make sure you prefix the filename with a +number, for example, '999_catch_all.conf', '500_elasticsearch.conf'. 999 has +smaller priority than 500. + +## Reference + +### Classes + +#### Public Classes + +* `fluentd`: Main class, includes all other classes. + +#### Private Classes + +* `fluentd::install`: Handles the packages. +* `fluentd::service`: Handles the service. + +### Parameters + +The following parameters are available in the `fluentd` class: + +#### `repo_install` + +Default value: true + +#### `repo_name` + +Default value: 'treasuredata' + +#### `repo_desc` + +Default value: 'TreasureData' + +#### `repo_url` + +Default value: 'http://packages.treasuredata.com/2/redhat/$releasever/$basearch' + +#### `repo_enabled` + +Default value: true + +#### `repo_gpgcheck` + +Default value: true + +#### `repo_gpgkey` + +Default value: 'https://packages.treasuredata.com/GPG-KEY-td-agent' + +#### `repo_gpgkeyid` + +Default value: 'C901622B5EC4AF820C38AB861093DB45A12E206F' + +#### `package_name` + +Default value: 'td-agent' + +#### `package_ensure` + +Default value: present + +#### `service_name` + +Default value: 'td-agent' + +#### `service_ensure` + +Default value: running + +#### `service_enable` + +Default value: true + +#### `service_manage` + +Default value: true + +#### `config_file` + +Default value: '/etc/td-agent/td-agent.conf' + +### Public Defines + +* `fluentd::config`: Generates custom configuration files. +* `fluentd::plugin`: Installs plugins. + +The following parameters are available in the `fluentd::plugin` defined type: + +#### `title` + +Plugin name + +#### `plugin_ensure` + +Default value: present + +#### `plugin_source` + +Default value: 'https://rubygems.org' + +The following parameters are available in the `fluentd::config` defined type: + +#### `title` + +Config filename + +#### `config` + +Config Hash, please see usage examples. + +## Limitations + +Tested only on CentOS 7, Ubuntu 14.04, Debian 7.8 + +## Development + +Bug reports and pull requests are welcome! + +### Running Tests + + $ bundle install + $ bundle exec rspec + $ bundle exec rake beaker BEAKER_set=debian-78-x64 + $ bundle exec rake beaker BEAKER_set=ubuntu-server-1404-x64 + $ bundle exec rake lint + +Relevant Beaker docs: https://github.com/puppetlabs/beaker/blob/master/docs/How-to-Write-a-Beaker-Test-for-a-Module.md + +### TODO: + +* Remove `rubygems` package dependency + +## License + +Copyright 2015 SPB TV AG + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and limitations +under the License. diff --git a/fluentd/Rakefile b/fluentd/Rakefile new file mode 100644 index 000000000..150a669d9 --- /dev/null +++ b/fluentd/Rakefile @@ -0,0 +1,17 @@ +require 'puppetlabs_spec_helper/rake_tasks' +require 'puppet-lint/tasks/puppet-lint' + +PuppetLint.configuration.ignore_paths = ['spec/**/*.pp', 'pkg/**/*.pp'] + +desc 'Validate manifests, templates, and ruby files' +task :validate do + Dir['manifests/**/*.pp'].each do |manifest| + sh "puppet parser validate --noop #{manifest}" + end + Dir['spec/**/*.rb','lib/**/*.rb'].each do |ruby_file| + sh "ruby -c #{ruby_file}" unless ruby_file =~ /spec\/fixtures/ + end + Dir['templates/**/*.erb'].each do |template| + sh "erb -P -x -T '-' #{template} | ruby -c" + end +end diff --git a/fluentd/examples/init.pp b/fluentd/examples/init.pp new file mode 100644 index 000000000..c21eb9c7d --- /dev/null +++ b/fluentd/examples/init.pp @@ -0,0 +1,19 @@ +include fluentd + +fluentd::plugin { 'fluent-plugin-elasticsearch': } + +fluentd::config { '500_elasticsearch.conf': + config => { + 'source' => { + 'type' => 'unix', + 'path' => '/tmp/td-agent/td-agent.sock', + }, + 'match' => { + 'tag_pattern' => '**', + 'type' => 'elasticsearch', + 'index_name' => 'foo', + 'type_name' => 'bar', + 'logstash_format' => true, + } + } +} diff --git a/fluentd/files/td-agent.conf b/fluentd/files/td-agent.conf new file mode 100644 index 000000000..c66251959 --- /dev/null +++ b/fluentd/files/td-agent.conf @@ -0,0 +1,2 @@ +# td-agent.conf, managed by Puppet. +@include 'config.d/*.conf' diff --git a/fluentd/lib/puppet/parser/functions/fluent_config.rb b/fluentd/lib/puppet/parser/functions/fluent_config.rb new file mode 100644 index 000000000..ac86d86c4 --- /dev/null +++ b/fluentd/lib/puppet/parser/functions/fluent_config.rb @@ -0,0 +1,29 @@ +module Puppet::Parser::Functions + newfunction(:fluent_plugin_config, type: :rvalue) do |args| + plugin_type = args[0] + plugin_config = args[1] + + tag_pattern = plugin_config.delete('tag_pattern') + + config_body = plugin_config.each_with_object('') do |(key, value), result| + if value.is_a?(Array) + value.each do |plugin_sub_config| + result << function_fluent_plugin_config([key, plugin_sub_config]) + end + else + result << [key, value].join(' ') << "\n" + end + end + + "<#{plugin_type} #{tag_pattern}>\n#{config_body}\n\n" + end + + newfunction(:fluent_config, type: :rvalue) do |args| + config = args[0] + + header = "# Managed by Puppet.\n" + config.each_with_object(header) do |(plugin_type, plugin_config), result| + result << function_fluent_plugin_config([plugin_type, plugin_config]) + end.chomp + end +end diff --git a/fluentd/lib/puppet/provider/package/tdagent.rb b/fluentd/lib/puppet/provider/package/tdagent.rb new file mode 100644 index 000000000..ddcdeed52 --- /dev/null +++ b/fluentd/lib/puppet/provider/package/tdagent.rb @@ -0,0 +1,3 @@ +Puppet::Type.type(:package).provide(:tdagent, parent: :gem, source: :gem) do + commands gemcmd: '/opt/td-agent/usr/sbin/td-agent-gem' +end diff --git a/fluentd/manifests/config.pp b/fluentd/manifests/config.pp new file mode 100644 index 000000000..ce7d386ff --- /dev/null +++ b/fluentd/manifests/config.pp @@ -0,0 +1,10 @@ +define fluentd::config($config) { + $path = sprintf('/etc/td-agent/config.d/%s', $title) + + file { $path: + ensure => present, + content => fluent_config($config), + require => Class['Fluentd::Install'], + notify => Class['Fluentd::Service'], + } +} diff --git a/fluentd/manifests/init.pp b/fluentd/manifests/init.pp new file mode 100644 index 000000000..ca8ee064d --- /dev/null +++ b/fluentd/manifests/init.pp @@ -0,0 +1,46 @@ +class fluentd ( + $repo_install = $fluentd::repo_install, + $repo_name = $fluentd::repo_name, + $repo_desc = $fluentd::repo_desc, + $repo_url = $fluentd::repo_url, + $repo_enabled = $fluentd::repo_enabled, + $repo_gpgcheck = $fluentd::repo_gpgcheck, + $repo_gpgkey = $fluentd::repo_gpgkey, + $repo_gpgkeyid = $fluentd::repo_gpgkeyid, + $package_name = $fluentd::package_name, + $package_ensure = $fluentd::package_ensure, + $plugin_names = $fluentd::plugin_names, + $plugin_ensure = $fluentd::plugin_ensure, + $plugin_source = $fluentd::plugin_source, + $service_name = $fluentd::service_name, + $service_ensure = $fluentd::service_ensure, + $service_enable = $fluentd::service_enable, + $service_manage = $fluentd::service_manage, + $config_file = $fluentd::config_file, +) inherits fluentd::params { + + # Param validations + validate_bool($repo_install) + validate_string($repo_name) + validate_string($repo_url) + validate_bool($repo_enabled) + validate_bool($repo_gpgcheck) + validate_string($repo_gpgkey) + validate_string($repo_gpgkeyid) + validate_string($package_name) + validate_string($package_ensure) + validate_array($plugin_names) + validate_string($plugin_ensure) + validate_string($plugin_source) + validate_string($service_name) + validate_string($service_ensure) + validate_bool($service_enable) + validate_bool($service_manage) + validate_absolute_path($config_file) + + contain fluentd::install + contain fluentd::service + + Class['Fluentd::Install'] -> + Class['Fluentd::Service'] +} diff --git a/fluentd/manifests/install.pp b/fluentd/manifests/install.pp new file mode 100644 index 000000000..e5e859774 --- /dev/null +++ b/fluentd/manifests/install.pp @@ -0,0 +1,18 @@ +class fluentd::install inherits fluentd { + if $fluentd::repo_install { + require fluentd::install_repo + } + + package { $fluentd::package_name: + ensure => $fluentd::package_ensure, + } -> + + file { $fluentd::config_path: + ensure => directory, + } -> + + file { $fluentd::config_file: + ensure => present, + content => file('fluentd/td-agent.conf'), + } +} diff --git a/fluentd/manifests/install_repo.pp b/fluentd/manifests/install_repo.pp new file mode 100644 index 000000000..40309641e --- /dev/null +++ b/fluentd/manifests/install_repo.pp @@ -0,0 +1,47 @@ +class fluentd::install_repo inherits fluentd { + case $::osfamily { + 'redhat': { + yumrepo { $fluentd::repo_name: + descr => $fluentd::repo_desc, + baseurl => $fluentd::repo_url, + enabled => $fluentd::repo_enabled, + gpgcheck => $fluentd::repo_gpgcheck, + gpgkey => $fluentd::repo_gpgkey, + notify => Exec['rpmkey'], + } + + exec { 'rpmkey': + command => "rpm --import ${fluentd::repo_gpgkey}", + path => '/usr/bin', + refreshonly => true, + } + + # TODO: Remove this dependency. Gem provider requires this package. + package { 'rubygems': + ensure => present, + } + } + + 'debian': { + apt::source { $fluentd::repo_name: + location => $fluentd::repo_url, + comment => $fluentd::repo_desc, + repos => 'contrib', + architecture => 'amd64', + release => $fluentd::distro_codename, + key => { + id => $fluentd::repo_gpgkeyid, + source => $fluentd::repo_gpgkey, + }, + include => { + 'src' => false, + 'deb' => true, + }, + } + } + + default: { + fail("Unsupported osfamily ${::osfamily}") + } + } +} diff --git a/fluentd/manifests/params.pp b/fluentd/manifests/params.pp new file mode 100644 index 000000000..70572bbb8 --- /dev/null +++ b/fluentd/manifests/params.pp @@ -0,0 +1,41 @@ +class fluentd::params { + $repo_install = true + $repo_name = 'treasuredata' + $repo_desc = 'TreasureData' + + case $::osfamily { + 'redhat': { + $repo_url = 'http://packages.treasuredata.com/2/redhat/$releasever/$basearch' + } + + 'debian': { + $distro_id = downcase($::lsbdistid) + $distro_codename = $::lsbdistcodename + $repo_url = "http://packages.treasuredata.com/2/${distro_id}/${distro_codename}/" + } + + default: { + fail("Unsupported osfamily ${::osfamily}") + } + } + + $repo_enabled = true + $repo_gpgcheck = true + $repo_gpgkey = 'https://packages.treasuredata.com/GPG-KEY-td-agent' + $repo_gpgkeyid = 'C901622B5EC4AF820C38AB861093DB45A12E206F' + + $package_name = 'td-agent' + $package_ensure = present + + $plugin_names = [] + $plugin_ensure = present + $plugin_source = 'https://rubygems.org' + + $service_name = 'td-agent' + $service_ensure = running + $service_enable = true + $service_manage = true + + $config_file = '/etc/td-agent/td-agent.conf' + $config_path = '/etc/td-agent/config.d' +} diff --git a/fluentd/manifests/plugin.pp b/fluentd/manifests/plugin.pp new file mode 100644 index 000000000..f591fbd73 --- /dev/null +++ b/fluentd/manifests/plugin.pp @@ -0,0 +1,12 @@ +define fluentd::plugin( + $plugin_ensure = $fluentd::plugin_ensure, + $plugin_source = $fluentd::plugin_source, +) { + package { $title: + ensure => $plugin_ensure, + source => $plugin_source, + provider => tdagent, + notify => Class['Fluentd::Service'], + require => Class['Fluentd::Install'], + } +} diff --git a/fluentd/manifests/service.pp b/fluentd/manifests/service.pp new file mode 100644 index 000000000..6eafeb584 --- /dev/null +++ b/fluentd/manifests/service.pp @@ -0,0 +1,10 @@ +class fluentd::service inherits fluentd { + if $fluentd::service_manage { + service { $fluentd::service_name: + ensure => $fluentd::service_ensure, + enable => $fluentd::service_enable, + hasstatus => true, + hasrestart => true, + } + } +} diff --git a/fluentd/metadata.json b/fluentd/metadata.json new file mode 100644 index 000000000..91ca80dc5 --- /dev/null +++ b/fluentd/metadata.json @@ -0,0 +1,47 @@ +{ + "author": "konstantin", + "dependencies": [ + { + "name": "puppetlabs/stdlib", + "version_requirement": ">= 1.0.0" + }, + { + "name": "puppetlabs/apt", + "version_requirement": ">= 1.0.0" + } + ], + "issues_url": "https://github.com/soylent/konstantin-fluentd/issues", + "license": "Apache-2.0", + "name": "konstantin-fluentd", + "operatingsystem_support": [ + { + "operatingsystem": "CentOS", + "operatingsystemrelease": [ + "7.0" + ] + }, + { + "operatingsystem": "Ubuntu", + "operatingsystemrelease": [ + "14.04" + ] + }, + { + "operatingsystem": "Debian", + "operatingsystemrelease": [ + "7" + ] + } + ], + "project_page": "https://github.com/soylent/konstantin-fluentd", + "source": "https://github.com/soylent/konstantin-fluentd", + "summary": "Installs, configures, and manages the Fluentd service", + "tags": [ + "fluentd", + "fluent", + "td-agent", + "treasuredata", + "logging" + ], + "version": "0.3.1" +} diff --git a/fluentd/spec/acceptance/class_spec.rb b/fluentd/spec/acceptance/class_spec.rb new file mode 100644 index 000000000..d9d5694ce --- /dev/null +++ b/fluentd/spec/acceptance/class_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper_acceptance' + +RSpec.describe 'fluentd' do + it 'runs successfully' do + manifest = File.read(File.expand_path('../../examples/init.pp', __dir__)) + + # Run it twice and test for idempotency + apply_manifest(manifest, catch_failures: true) + expect(apply_manifest(manifest, catch_failures: true).exit_code).to be_zero + end + + describe package('td-agent') do + it { is_expected.to be_installed } + end + + describe service('td-agent') do + it { is_expected.to be_enabled.with_level(3) } + it { is_expected.to be_running } + end +end diff --git a/fluentd/spec/acceptance/nodesets/debian-78-x64.yml b/fluentd/spec/acceptance/nodesets/debian-78-x64.yml new file mode 100644 index 000000000..762164d67 --- /dev/null +++ b/fluentd/spec/acceptance/nodesets/debian-78-x64.yml @@ -0,0 +1,11 @@ +HOSTS: + debian-78-x64: + roles: + - master + platform: debian-7-amd64 + box: puppetlabs/debian-7.8-64-nocm + box_url: https://vagrantcloud.com/puppetlabs/boxes/debian-7.8-64-nocm + hypervisor: vagrant +CONFIG: + log_level: verbose + type: foss diff --git a/fluentd/spec/acceptance/nodesets/default.yml b/fluentd/spec/acceptance/nodesets/default.yml new file mode 100644 index 000000000..ec395054d --- /dev/null +++ b/fluentd/spec/acceptance/nodesets/default.yml @@ -0,0 +1,11 @@ +HOSTS: + centos-70-x64: + roles: + - master + platform: el-7-x86_64 + box: puppetlabs/centos-7.0-64-nocm + box_url: https://atlas.hashicorp.com/puppetlabs/boxes/centos-7.0-64-nocm + hypervisor: vagrant +CONFIG: + log_level: verbose + type: foss diff --git a/fluentd/spec/acceptance/nodesets/ubuntu-server-1404-x64.yml b/fluentd/spec/acceptance/nodesets/ubuntu-server-1404-x64.yml new file mode 100644 index 000000000..7e389e71e --- /dev/null +++ b/fluentd/spec/acceptance/nodesets/ubuntu-server-1404-x64.yml @@ -0,0 +1,11 @@ +HOSTS: + ubuntu-server-1404-x64: + roles: + - master + platform: ubuntu-14.04-amd64 + box: puppetlabs/ubuntu-14.04-64-nocm + box_url: https://vagrantcloud.com/puppetlabs/boxes/ubuntu-14.04-64-nocm + hypervisor: vagrant +CONFIG: + log_level: verbose + type: foss diff --git a/fluentd/spec/classes/init_spec.rb b/fluentd/spec/classes/init_spec.rb new file mode 100644 index 000000000..22a2cc64b --- /dev/null +++ b/fluentd/spec/classes/init_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +RSpec.describe 'fluentd' do + shared_examples 'works' do + it { is_expected.to compile.with_all_deps } + it { is_expected.to contain_class('fluentd') } + it { is_expected.to contain_class('fluentd::install') } + it { is_expected.to contain_class('fluentd::service') } + end + + context 'when osfamily is debian' do + let(:facts) do + { osfamily: 'Debian', lsbdistid: 'Ubuntu', lsbdistcodename: 'trusty' } + end + + include_examples 'works' + end + + context 'with defaults for all parameters' do + let(:facts) { { osfamily: 'RedHat' } } + + include_examples 'works' + end +end diff --git a/fluentd/spec/classes/install_spec.rb b/fluentd/spec/classes/install_spec.rb new file mode 100644 index 000000000..a5e91563d --- /dev/null +++ b/fluentd/spec/classes/install_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +RSpec.describe 'fluentd::install' do + shared_examples 'package and configs' do + it { is_expected.to contain_package('td-agent') } + it { is_expected.to contain_file('/etc/td-agent/td-agent.conf') } + it { is_expected.to contain_file('/etc/td-agent/config.d') } + end + + context 'on RedHat based system' do + let(:facts) { { osfamily: 'RedHat' } } + + it { is_expected.to contain_yumrepo('treasuredata') } + + include_examples 'package and configs' + end + + context 'on Debian based system' do + let(:facts) do + { osfamily: 'Debian', lsbdistid: 'Ubuntu', lsbdistcodename: 'trusty' } + end + + it { is_expected.to contain_apt__source('treasuredata') } + + include_examples 'package and configs' + end + + context 'on unsupported system' do + let(:facts) { { osfamily: 'Darwin' } } + + it { is_expected.not_to compile } + end +end diff --git a/fluentd/spec/classes/service_spec.rb b/fluentd/spec/classes/service_spec.rb new file mode 100644 index 000000000..47d3fe69b --- /dev/null +++ b/fluentd/spec/classes/service_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +RSpec.describe 'fluentd::service' do + context 'on RedHat based system' do + let(:facts) { { osfamily: 'RedHat' } } + + it { is_expected.to contain_service('td-agent') } + end + + context 'on Debian based system' do + let(:facts) do + { osfamily: 'Debian', lsbdistid: 'Ubuntu', lsbdistcodename: 'trusty' } + end + + it { is_expected.to contain_service('td-agent') } + end +end diff --git a/fluentd/spec/defines/config_spec.rb b/fluentd/spec/defines/config_spec.rb new file mode 100644 index 000000000..2d2f86cb6 --- /dev/null +++ b/fluentd/spec/defines/config_spec.rb @@ -0,0 +1,66 @@ +require 'spec_helper' + +RSpec.describe 'fluentd::config' do + let(:pre_condition) { 'include fluentd' } + + context 'on RedHat based system' do + let(:facts) { { osfamily: 'RedHat' } } + + let(:title) { 'stdout.conf' } + + context 'when config does not contain nested hashes' do + let(:params) do + { + config: { + 'source' => { + 'type' => 'forward' + }, + 'match' => { + 'tag_pattern' => '**', + 'type' => 'stdout' + } + } + } + end + + it do + is_expected.to contain_file('/etc/td-agent/config.d/stdout.conf'). + with_content(//). + with_content(/type forward/). + with_content(/<\/source>/). + with_content(//). + with_content(/type stdout/). + with_content(/<\/match>/) + end + end + + context 'when config contains nested hashes' do + let(:params) do + { + config: { + 'match' => { + 'tag_pattern' => '**', + 'type' => 'forward', + 'server' => [ + { 'host' => 'example1.com', 'port' => 24224 }, + { 'host' => 'example2.com', 'port' => 24224 } + ] + } + } + } + end + + it do + is_expected.to contain_file('/etc/td-agent/config.d/stdout.conf'). + with_content(//). + with_content(/type forward/). + with_content(//). + with_content(/host example1.com/). + with_content(/port 24224/). + with_content(/<\/server>/). + with_content(/host example2.com/). + with_content(/<\/match>/) + end + end + end +end diff --git a/fluentd/spec/defines/plugin_spec.rb b/fluentd/spec/defines/plugin_spec.rb new file mode 100644 index 000000000..8483d6e81 --- /dev/null +++ b/fluentd/spec/defines/plugin_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +RSpec.describe 'fluentd::plugin' do + let(:pre_condition) { 'include fluentd' } + let(:title) { 'fluent-plugin-test' } + + context 'on RedHat based system' do + let(:facts) { { osfamily: 'RedHat' } } + + it { is_expected.to contain_package(title).with(provider: 'tdagent') } + end + + context 'on Debian based system' do + let(:facts) do + { osfamily: 'Debian', lsbdistid: 'Ubuntu', lsbdistcodename: 'trusty' } + end + + it { is_expected.to contain_package(title).with(provider: 'tdagent') } + end +end diff --git a/fluentd/spec/spec_helper.rb b/fluentd/spec/spec_helper.rb new file mode 100644 index 000000000..2c6f56649 --- /dev/null +++ b/fluentd/spec/spec_helper.rb @@ -0,0 +1 @@ +require 'puppetlabs_spec_helper/module_spec_helper' diff --git a/fluentd/spec/spec_helper_acceptance.rb b/fluentd/spec/spec_helper_acceptance.rb new file mode 100644 index 000000000..24570f7ad --- /dev/null +++ b/fluentd/spec/spec_helper_acceptance.rb @@ -0,0 +1,19 @@ +require 'beaker-rspec' +require 'beaker/puppet_install_helper' + +run_puppet_install_helper unless ENV['BEAKER_provision'] == 'no' + +RSpec.configure do |c| + module_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) + + c.formatter = :documentation + + c.before :suite do + puppet_module_install(source: module_root, module_name: 'fluentd') + + hosts.each do |host| + on host, puppet('module', 'install', 'puppetlabs-stdlib'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'puppetlabs-apt'), acceptable_exit_codes: [0, 1] + end + end +end