Merge pull request #50059 from garethgreenaway/allow_multiple_engine_…

…instances

[develop] Initial work to allow running multiple instances of a Salt engine

doc/topics/engines/index.rst

@@ -27,6 +27,24 @@ Salt engines are configured under an ``engines`` top-level section in your Salt
          port: 5959
          proto: tcp
 
+.. versionadded:: Neon
+
+Multiple copies of a particular Salt engine can be configured by including the ``engine_module`` parameter in the engine configuration.
+
+.. code-block:: yaml
+
+   engines:
+     - production_logstash:
+         host: production_log.my_network.com
+         port: 5959
+         proto: tcp
+         engine_module: logstash
+     - develop_logstash:
+         host: develop_log.my_network.com
+         port: 5959
+         proto: tcp
+         engine_module: logstash
+
 Salt engines must be in the Salt path, or you can add the ``engines_dirs`` option in your Salt master configuration with a list of directories under which Salt attempts to find Salt engines. This option should be formatted as a list of directories to search, such as:
 
 .. code-block:: yaml

doc/topics/releases/neon.rst

@@ -257,6 +257,26 @@ Module Changes
   module now supports perstant changes with ``persist=True`` by calling the
   :py:func:`selinux.fcontext_add_policy <salt.modules.selinux.fcontext_add_policy>` module.
 
+Enhancements to Engines
+=======================
+
+Multiple copies of a particular Salt engine can be configured by including
+the ``engine_module`` parameter in the engine configuration.
+
+.. code-block:: yaml
+
+   engines:
+     - production_logstash:
+         host: production_log.my_network.com
+         port: 5959
+         proto: tcp
+         engine_module: logstash
+     - develop_logstash:
+         host: develop_log.my_network.com
+         port: 5959
+         proto: tcp
+         engine_module: logstash
+
 Salt Cloud Features
 ===================
 

salt/engines/__init__.py

@@ -46,10 +46,22 @@ def start_engines(opts, proc_mgr, proxy=None):
             engine, engine_opts = next(iter(engine.items()))
         else:
             engine_opts = None
-        fun = '{0}.start'.format(engine)
+        engine_name = None
+        if engine_opts is not None and 'engine_module' in engine_opts:
+            fun = '{0}.start'.format(engine_opts['engine_module'])
+            engine_name = engine
+            del engine_opts['engine_module']
+        else:
+            fun = '{0}.start'.format(engine)
         if fun in engines:
             start_func = engines[fun]
-            name = '{0}.Engine({1})'.format(__name__, start_func.__module__)
+            if engine_name:
+                name = '{0}.Engine({1}-{2})'.format(__name__,
+                                                    start_func.__module__,
+                                                    engine_name)
+            else:
+                name = '{0}.Engine({1})'.format(__name__,
+                                                start_func.__module__)
             log.info('Starting Engine %s', name)
             proc_mgr.add_process(
                     Engine,

tests/unit/test_engines.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+'''
+unit tests for the Salt engines
+'''
+# Import Python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import skipIf, TestCase
+from tests.support.mock import (
+    NO_MOCK,
+    NO_MOCK_REASON,
+    patch)
+
+# Import Salt Libs
+import salt.engines as engines
+import salt.config
+import salt.utils.process
+
+# Import 3rd-party libs
+from salt.ext import six
+
+import logging
+log = logging.getLogger(__name__)
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class EngineTestCase(TestCase, LoaderModuleMockMixin):
+    '''
+    Test cases for salt.engine.sqs_events
+    '''
+
+    def setup_loader_modules(self):
+        return {engines: {}}
+
+    def test_engine_module(self):
+        '''
+        Test
+        '''
+        mock_opts = salt.config.DEFAULT_MINION_OPTS
+        mock_opts['__role'] = 'minion'
+        mock_opts['engines'] = [{'test_one': {'engine_module': 'test'}},
+                                {'test_two': {'engine_module': 'test'}}]
+
+        process_manager = salt.utils.process.ProcessManager()
+        with patch.dict(engines.__opts__, mock_opts):
+            salt.engines.start_engines(mock_opts, process_manager)
+            process_map = process_manager._process_map
+            count = 0
+            for proc in six.iterkeys(process_map):
+                count += 1
+                fun = process_map[proc]['Process'].fun
+
+                # Ensure function is start from the test engine
+                self.assertEqual(fun, 'test.start')
+
+            # Ensure there were two engine started
+            self.assertEqual(count, len(mock_opts['engines']))