Update mongodb to e2383ef854707f1f2a92196b943f377770b2b7ae

e2383ef854707f1f2a92196b943f377770b2b7ae Merge pull request redhat-openstack#224 from nibalizer/pr_204
4560185bce759aa9abad0323593e14e57b2136ad Merge pull request redhat-openstack#219 from vorlock/master
6fad8b44495aabe87db5dd3548f432ad0c04ec87 Merge pull request redhat-openstack#217 from iberezovskiy/master
3cf6f329556a20fbf5adf316674e1aeacfe0b230 Merge pull request redhat-openstack#216 from sielaq/master
1e235c267ed3a62ce6aab955ede21afc2ec454ef fix 'Undefined variable' by adding 'mongodb::globals::mongos_service_[enable|ensure|status]' and 'mongodb::params::[pidfilepath|journal]' for Debian"
7d1d7383fce0c15b421b6408e9f903df335d809f Merge pull request redhat-openstack#208 from mvernimmen-CG/MODULE-1835/mongoDB_3_engine_selection_support
e90818a30dcbcb05cdb68118a5a8ce5703037445 Merge pull request redhat-openstack#188 from andyroyle/noprealloc-negation
a7fe0e247346c6390b1d2455d9665cbbef1e2226 Merge pull request redhat-openstack#186 from serverbiz/master
20e3bc6b0be617875c77432347620b7a48b402d2 Merge pull request redhat-openstack#223 from nibalizer/pr_185
c6160d75a7bcb6d1b8d08796a2686e1d6f32c3f4 ensure that the client install does not start before the repo setup
e73dc2212c8b6fa8a692cdb541a27b015c43e310 Added specific service provider for Debian
7dcb230e18c9856b3031e6cad9585161bde4b919 Initiate replica set creation from localhost if auth is enabled
eca75727cb5ccadf168891f3db15e79397942e8a Implement retries for MongoDB shell commands
0d59bea14dd94b2a6830dbf85d53beb8eb48d71c Make distinguish between repo and package mgmt
01ba34f34cc040dd34c342deeaaf0f095bf98a15 (MODULES-1835) Add mongoDB 3.x storage engine selection support. Allows user to set the attribute with empty value without affecting the mongod.conf Without this it is not possible to use the wiredTiger engine in MongoDB 3.x. This change is backwards compatible with MongoDB 2.x Add test for mongoDB 3.x engine support
85d632552be517e16caf1ac75cb53a479f3301bc Make distinguish between repo and package mgmt
04c9d2dd97f5df352a6c248277fbf7fe830968af prealloc setting needs to be negated
4b99ab2ac49b64e4d5979d293d6a2d1fd65dac14 Fix replset not working on mongo 3.x due to the output containing an ObjectID

Change-Id: Ibb82567ec7f29afa33b1c20d2c9207f80693a05f

Puppetfile

@@ -103,7 +103,7 @@ mod 'module-data',
   :git => 'https://github.com/ripienaar/puppet-module-data.git'
 
 mod 'mongodb',
-  :commit => '1caa046c5bf51a29ff1732e1aef54f0d59086e52',
+  :commit => 'e2383ef854707f1f2a92196b943f377770b2b7ae',
   :git => 'https://github.com/puppetlabs/puppetlabs-mongodb.git'
 
 mod 'mysql',

mongodb/README.md

@@ -92,6 +92,19 @@ class {'::mongodb::server': }->
 class {'::mongodb::client': }
 ```
 
+Having a local copy of MongoDB repository (that is managed by your private modules)
+you can still enjoy the charms of `mongodb::params` that manage packages.
+To disable managing of repository, but still enable managing packages:
+
+```puppet
+class {'::mongodb::globals':
+  manage_package_repo => false,
+  manage_package      => true,
+}->
+class {'::mongodb::server': }->
+class {'::mongodb::client': }
+```
+
 ## Usage
 
 Most of the interaction for the server is done via `mongodb::server`. For
@@ -434,6 +447,12 @@ Default: <>
 Whether or not the MongoDB service resource should be part of the catalog.
 Default: true
 
+#####`storage_engine`
+Only needed for MongoDB 3.x versions, where it's possible to select the
+'wiredTiger' engine in addition to the default 'mmapv1' engine. If not set, the
+config is left out and mongo will default to 'mmapv1'.
+You should not set this for MongoDB versions < 3.x
+
 #####`restart`
 Specifies whether the service should be restarted on config changes. Default: 'true'
 

mongodb/lib/puppet/provider/mongodb.rb

@@ -76,11 +76,27 @@ def self.get_conn_string
 
   # Mongo Command Wrapper
   def self.mongo_eval(cmd, db = 'admin')
+    retry_count = 10
+    retry_sleep = 3
     if mongorc_file
         cmd = mongorc_file + cmd
     end
 
-    out = mongo([db, '--quiet', '--host', get_conn_string, '--eval', cmd])
+    out = nil
+    retry_count.times do |n|
+      begin
+        out = mongo([db, '--quiet', '--host', get_conn_string, '--eval', cmd])
+      rescue => e
+        debug "Request failed: '#{e.message}' Retry: '#{n}'"
+        sleep retry_sleep
+        next
+      end
+      break
+    end
+
+    if !out
+     fail "Could not evalute MongoDB shell command: #{cmd}"
+    end
 
     out.gsub!(/ObjectId\(([^)]*)\)/, '\1')
     out

mongodb/lib/puppet/provider/mongodb_replset/mongo.rb

@@ -94,6 +94,10 @@ def rs_add_arbiter(host, master)
     mongo_command("rs.addArb(\"#{host}\")", master)
   end
 
+  def auth_enabled
+    @resource[:auth_enabled]
+  end
+
   def master_host(hosts)
     hosts.each do |host|
       status = db_ismaster(host)
@@ -135,31 +139,36 @@ def self.get_replset_properties
   end
 
   def alive_members(hosts)
+    alive = []
     hosts.select do |host|
       begin
         Puppet.debug "Checking replicaset member #{host} ..."
         status = rs_status(host)
         if status.has_key?('errmsg') and status['errmsg'] == 'not running with --replSet'
           raise Puppet::Error, "Can't configure replicaset #{self.name}, host #{host} is not supposed to be part of a replicaset."
         end
+
+        if auth_enabled and status.has_key?('errmsg') and (status['errmsg'].include? "unauthorized" or status['errmsg'].include? "not authorized")
+          Puppet.warning "Host #{host} is available, but you are unauthorized because of authentication is enabled: #{auth_enabled}"
+          alive.push(host)
+        end
+
         if status.has_key?('set')
           if status['set'] != self.name
             raise Puppet::Error, "Can't configure replicaset #{self.name}, host #{host} is already part of another replicaset."
           end
 
           # This node is alive and supposed to be a member of our set
           Puppet.debug "Host #{host} is available for replset #{status['set']}"
-          true
+          alive.push(host)
         elsif status.has_key?('info')
           Puppet.debug "Host #{host} is alive but unconfigured: #{status['info']}"
-          true
         end
       rescue Puppet::ExecutionFailure
         Puppet.warning "Can't connect to replicaset member #{host}."
-
-        false
       end
     end
+    return alive
   end
 
   def set_members
@@ -225,10 +234,17 @@ def set_members
   end
 
   def mongo_command(command, host, retries=4)
-    self.class.mongo_command(command,host,retries)
+    self.class.mongo_command(command,host,retries,auth_enabled)
   end
 
-  def self.mongo_command(command, host=nil, retries=4)
+  def self.mongo_command(command, host=nil, retries=4, auth_enabled=false)
+    if auth_enabled and command =~ 'rs.initiate'
+      # We can't setup replica from any hosts except localhost
+      # if authentication is enabled
+      # User can't be created before replica set initialization
+      # So we can't use user credentials for auth
+      host = '127.0.0.1'
+    end
     # Allow waiting for mongod to become ready
     # Wait for 2 seconds initially and double the delay at each retry
     wait = 2
@@ -252,6 +268,7 @@ def self.mongo_command(command, host=nil, retries=4)
     # Dirty hack to remove JavaScript objects
     output.gsub!(/ISODate\((.+?)\)/, '\1 ')
     output.gsub!(/Timestamp\((.+?)\)/, '[\1]')
+    output.gsub!(/ObjectId\(([^)]*)\)/, '\1')
 
     #Hack to avoid non-json empty sets
     output = "{}" if output == "null\n"

mongodb/lib/puppet/type/mongodb_replset.rb

@@ -17,6 +17,11 @@
     desc "The name of the replicaSet"
   end
 
+  newparam(:auth_enabled) do
+    desc "Check authentication enabled"
+    defaultto false
+  end
+
   newparam(:arbiter) do
     desc "The replicaSet arbiter"
   end

mongodb/manifests/client.pp

@@ -12,5 +12,7 @@
   $ensure       = $mongodb::params::package_ensure_client,
   $package_name = $mongodb::params::client_package_name,
 ) inherits mongodb::params {
-  class { '::mongodb::client::install': }
+  anchor { '::mongodb::client::start': } ->
+  class { '::mongodb::client::install': } ->
+  anchor { '::mongodb::client::end': }
 }

mongodb/manifests/globals.pp

@@ -11,6 +11,9 @@
   $service_ensure        = undef,
   $service_name          = undef,
   $mongos_service_manage = undef,
+  $mongos_service_enable = undef,
+  $mongos_service_ensure = undef,
+  $mongos_service_status = undef,
   $mongos_service_name   = undef,
   $service_provider      = undef,
   $service_status        = undef,
@@ -23,6 +26,7 @@
   $version               = undef,
 
   $manage_package_repo   = undef,
+  $manage_package        = undef,
 
   $use_enterprise_repo   = undef,
 ) {

mongodb/manifests/params.pp

@@ -16,11 +16,13 @@
   $mongos_configdb       = '127.0.0.1:27019'
   $mongos_restart        = true
 
+  $manage_package        = pick($mongodb::globals::manage_package, $mongodb::globals::manage_package_repo, false)
+
   # Amazon Linux's OS Family is 'Linux', operating system 'Amazon'.
   case $::osfamily {
     'RedHat', 'Linux': {
 
-      if $mongodb::globals::manage_package_repo {
+      if $manage_package {
         $user        = pick($::mongodb::globals::user, 'mongod')
         $group       = pick($::mongodb::globals::group, 'mongod')
         if ($::mongodb::globals::version == undef) {
@@ -105,7 +107,7 @@
       }
     }
     'Debian': {
-      if $::mongodb::globals::manage_package_repo {
+      if $manage_package {
         $user  = pick($::mongodb::globals::user, 'mongodb')
         $group = pick($::mongodb::globals::group, 'mongodb')
         if ($::mongodb::globals::version == undef) {
@@ -142,6 +144,7 @@
         $mongos_config           = '/etc/mongodb-shard.conf'
         $dbpath                  = '/var/lib/mongodb'
         $logpath                 = '/var/log/mongodb/mongodb.log'
+        $pidfilepath             = '/var/run/mongod.pid'
         $bind_ip                 = pick($::mongodb::globals::bind_ip, ['127.0.0.1'])
       } else {
         # although we are living in a free world,
@@ -173,6 +176,7 @@
       }
       # avoid using fork because of the init scripts design
       $fork                    = undef
+      $journal                 = undef
       $mongos_pidfilepath      = undef
       $mongos_unixsocketprefix = undef
       $mongos_logpath          = undef
@@ -184,12 +188,21 @@
   }
 
   case $::operatingsystem {
+    'Debian': {
+      case $::operatingsystemmajrelease {
+        '8': {
+          $service_provider = pick($service_provider, 'systemd')
+        }
+        default: {
+          $service_provider = pick($service_provider, 'debian')
+        }
+      }
+    }
     'Ubuntu': {
       $service_provider = pick($service_provider, 'upstart')
     }
     default: {
       $service_provider = undef
     }
   }
-
 }

mongodb/manifests/server.pp

@@ -65,6 +65,7 @@
   $ssl_key         = undef,
   $ssl_ca          = undef,
   $restart         = $mongodb::params::restart,
+  $storage_engine  = undef,
 
   # Deprecated parameters
   $master          = undef,

mongodb/manifests/server/config.pp

@@ -55,6 +55,8 @@
   $ssl             = $mongodb::server::ssl
   $ssl_key         = $mongodb::server::ssl_key
   $ssl_ca          = $mongodb::server::ssl_ca
+  $storage_engine  = $mongodb::server::storage_engine
+  $version         = $mongodb::server::version
 
   File {
     owner => $user,
@@ -83,10 +85,17 @@
       }
     }
 
+    if empty($storage_engine) {
+      $storage_engine_internal = undef
+    } else {
+      $storage_engine_internal = $storage_engine
+    }
+
+
     #Pick which config content to use
     if $config_content {
       $cfg_content = $config_content
-    } elsif (versioncmp($mongodb::globals::version, '2.6.0') >= 0) {
+    } elsif (versioncmp($version, '2.6.0') >= 0) {
       # Template uses:
       # - $auth
       # - $bind_ip
@@ -123,6 +132,7 @@
       # - $verbositylevel
       $cfg_content = template('mongodb/mongodb.conf.2.6.erb')
     } else {
+      # Fall back to oldest most basic config
       # Template uses:
       # - $auth
       # - $bind_ip
@@ -170,6 +180,7 @@
       # - $ssl
       # - $ssl_ca
       # - $ssl_key
+      # - storage_engine_internal
       # - $syslog
       # - $verbose
       # - $verbositylevel

mongodb/spec/classes/server_config_spec.rb

@@ -39,6 +39,14 @@
 
   end
 
+  describe 'when specifying storage_engine' do
+    let(:pre_condition) { ["class mongodb::server { $config = '/etc/mongod.conf' $dbpath = '/var/lib/mongo' $ensure = present $version='3.0.3' $storage_engine = 'SomeEngine' $storage_engine_internal = 'SomeEngine' $user = 'mongod' $group = 'mongod' $port = 29017 $bind_ip = ['0.0.0.0'] $fork = true $logpath ='/var/log/mongo/mongod.log' $logappend = true}", "include mongodb::server"]}
+
+    it {
+      is_expected.to contain_file('/etc/mongod.conf').with_content(/storage.engine:\sSomeEngine/)
+    }
+  end
+
   describe 'with specific bind_ip values and ipv6' do
     let(:pre_condition) { ["class mongodb::server { $config = '/etc/mongod.conf' $dbpath = '/var/lib/mongo' $ensure = present $bind_ip = ['127.0.0.1', 'fd00:beef:dead:55::143'] $ipv6 = true }", "include mongodb::server"]}
 

mongodb/templates/mongodb.conf.2.6.erb

@@ -44,7 +44,7 @@ storage.journal.enabled: false
 storage.journal.enabled: true
 <% end -%>
 <% if @noprealloc -%>
-storage.preallocDataFiles: <%= @noprealloc %>
+storage.preallocDataFiles: <%= !@noprealloc %>
 <% end -%>
 <% if @nssize -%>
 storage.nsSize: <%= @nssize %>
@@ -61,6 +61,9 @@ storage.quota.enforced: <%= @quota %>
 storage.quota.maxFilesPerDB: <%= @quotafiles %>
 <% end -%>
 <% end -%>
+<% if @storage_engine_internal -%>
+storage.engine: <%= @storage_engine_internal %>
+<% end -%>
 
 
 #Security