From ddc119764707a355069ff93350ab8e06f2a688ea Mon Sep 17 00:00:00 2001 From: jeanluc Date: Mon, 6 Nov 2023 12:53:50 +0100 Subject: [PATCH] Add `slsutil` SSH wrapper --- changelog/50196.fixed.md | 1 + changelog/61143.fixed.md | 1 + changelog/65067.fixed.md | 1 + salt/client/ssh/wrapper/slsutil.py | 450 ++++++++++++++++++ tests/pytests/integration/ssh/test_slsutil.py | 94 ++++ .../unit/client/ssh/wrapper/test_slsutil.py | 166 +++++++ 6 files changed, 713 insertions(+) create mode 100644 changelog/50196.fixed.md create mode 100644 changelog/61143.fixed.md create mode 100644 changelog/65067.fixed.md create mode 100644 salt/client/ssh/wrapper/slsutil.py create mode 100644 tests/pytests/integration/ssh/test_slsutil.py create mode 100644 tests/pytests/unit/client/ssh/wrapper/test_slsutil.py diff --git a/changelog/50196.fixed.md b/changelog/50196.fixed.md new file mode 100644 index 000000000000..979411a640da --- /dev/null +++ b/changelog/50196.fixed.md @@ -0,0 +1 @@ +Made slsutil.renderer work with salt-ssh diff --git a/changelog/61143.fixed.md b/changelog/61143.fixed.md new file mode 100644 index 000000000000..08a62c9d8b10 --- /dev/null +++ b/changelog/61143.fixed.md @@ -0,0 +1 @@ +Made slsutil.findup work with salt-ssh diff --git a/changelog/65067.fixed.md b/changelog/65067.fixed.md new file mode 100644 index 000000000000..d6de87b5bc15 --- /dev/null +++ b/changelog/65067.fixed.md @@ -0,0 +1 @@ +Fixed slsutil.update with salt-ssh during template rendering diff --git a/salt/client/ssh/wrapper/slsutil.py b/salt/client/ssh/wrapper/slsutil.py new file mode 100644 index 000000000000..e09ca1c29845 --- /dev/null +++ b/salt/client/ssh/wrapper/slsutil.py @@ -0,0 +1,450 @@ +import os.path +import posixpath + +import salt.exceptions +import salt.loader +import salt.template +import salt.utils.args +import salt.utils.dictupdate +import salt.utils.stringio + +CONTEXT_BASE = "slsutil" + + +def update(dest, upd, recursive_update=True, merge_lists=False): + """ + Merge ``upd`` recursively into ``dest`` + + If ``merge_lists=True``, will aggregate list object types instead of + replacing. This behavior is only activated when ``recursive_update=True``. + + CLI Example: + + .. code-block:: shell + + salt '*' slsutil.update '{foo: Foo}' '{bar: Bar}' + + """ + return salt.utils.dictupdate.update(dest, upd, recursive_update, merge_lists) + + +def merge(obj_a, obj_b, strategy="smart", renderer="yaml", merge_lists=False): + """ + Merge a data structure into another by choosing a merge strategy + + Strategies: + + * aggregate + * list + * overwrite + * recurse + * smart + + CLI Example: + + .. code-block:: shell + + salt '*' slsutil.merge '{foo: Foo}' '{bar: Bar}' + """ + return salt.utils.dictupdate.merge(obj_a, obj_b, strategy, renderer, merge_lists) + + +def merge_all(lst, strategy="smart", renderer="yaml", merge_lists=False): + """ + .. versionadded:: 2019.2.0 + + Merge a list of objects into each other in order + + :type lst: Iterable + :param lst: List of objects to be merged. + + :type strategy: String + :param strategy: Merge strategy. See utils.dictupdate. + + :type renderer: String + :param renderer: + Renderer type. Used to determine strategy when strategy is 'smart'. + + :type merge_lists: Bool + :param merge_lists: Defines whether to merge embedded object lists. + + CLI Example: + + .. code-block:: shell + + $ salt-call --output=txt slsutil.merge_all '[{foo: Foo}, {foo: Bar}]' + local: {u'foo': u'Bar'} + """ + + ret = {} + for obj in lst: + ret = salt.utils.dictupdate.merge(ret, obj, strategy, renderer, merge_lists) + + return ret + + +def renderer(path=None, string=None, default_renderer="jinja|yaml", **kwargs): + """ + Parse a string or file through Salt's renderer system + + .. versionchanged:: 2018.3.0 + Add support for Salt fileserver URIs. + + This is an open-ended function and can be used for a variety of tasks. It + makes use of Salt's "renderer pipes" system to run a string or file through + a pipe of any of the loaded renderer modules. + + :param path: The path to a file on Salt's fileserver (any URIs supported by + :py:func:`cp.get_url `) or on the local file + system. + :param string: An inline string to be used as the file to send through the + renderer system. Note, not all renderer modules can work with strings; + the 'py' renderer requires a file, for example. + :param default_renderer: The renderer pipe to send the file through; this + is overridden by a "she-bang" at the top of the file. + :param kwargs: Keyword args to pass to Salt's compile_template() function. + + Keep in mind the goal of each renderer when choosing a render-pipe; for + example, the Jinja renderer processes a text file and produces a string, + however the YAML renderer processes a text file and produces a data + structure. + + One possible use is to allow writing "map files", as are commonly seen in + Salt formulas, but without tying the renderer of the map file to the + renderer used in the other sls files. In other words, a map file could use + the Python renderer and still be included and used by an sls file that uses + the default 'jinja|yaml' renderer. + + For example, the two following map files produce identical results but one + is written using the normal 'jinja|yaml' and the other is using 'py': + + .. code-block:: jinja + + #!jinja|yaml + {% set apache = salt.grains.filter_by({ + ...normal jinja map file here... + }, merge=salt.pillar.get('apache:lookup')) %} + {{ apache | yaml() }} + + .. code-block:: python + + #!py + def run(): + apache = __salt__.grains.filter_by({ + ...normal map here but as a python dict... + }, merge=__salt__.pillar.get('apache:lookup')) + return apache + + Regardless of which of the above map files is used, it can be accessed from + any other sls file by calling this function. The following is a usage + example in Jinja: + + .. code-block:: jinja + + {% set apache = salt.slsutil.renderer('map.sls') %} + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.renderer salt://path/to/file + salt '*' slsutil.renderer /path/to/file + salt '*' slsutil.renderer /path/to/file.jinja default_renderer='jinja' + salt '*' slsutil.renderer /path/to/file.sls default_renderer='jinja|yaml' + salt '*' slsutil.renderer string='Inline template! {{ saltenv }}' + salt '*' slsutil.renderer string='Hello, {{ name }}.' name='world' + """ + if not path and not string: + raise salt.exceptions.SaltInvocationError("Must pass either path or string") + + renderers = salt.loader.render(__opts__, __salt__) + + if path: + path_or_string = __context__["fileclient"].get_url( + path, "", saltenv=kwargs.get("saltenv", "base") + ) + elif string: + path_or_string = ":string:" + kwargs["input_data"] = string + + ret = salt.template.compile_template( + path_or_string, + renderers, + default_renderer, + __opts__["renderer_blacklist"], + __opts__["renderer_whitelist"], + **kwargs + ) + return ret.read() if salt.utils.stringio.is_readable(ret) else ret + + +def _get_serialize_fn(serializer, fn_name): + serializers = salt.loader.serializers(__opts__) + fns = getattr(serializers, serializer, None) + fn = getattr(fns, fn_name, None) + + if not fns: + raise salt.exceptions.CommandExecutionError( + "Serializer '{}' not found.".format(serializer) + ) + + if not fn: + raise salt.exceptions.CommandExecutionError( + "Serializer '{}' does not implement {}.".format(serializer, fn_name) + ) + + return fn + + +def serialize(serializer, obj, **mod_kwargs): + """ + Serialize a Python object using one of the available + :ref:`all-salt.serializers`. + + CLI Example: + + .. code-block:: bash + + salt '*' --no-parse=obj slsutil.serialize 'json' obj="{'foo': 'Foo!'} + + Jinja Example: + + .. code-block:: jinja + + {% set json_string = salt.slsutil.serialize('json', + {'foo': 'Foo!'}) %} + """ + kwargs = salt.utils.args.clean_kwargs(**mod_kwargs) + return _get_serialize_fn(serializer, "serialize")(obj, **kwargs) + + +def deserialize(serializer, stream_or_string, **mod_kwargs): + """ + Deserialize a Python object using one of the available + :ref:`all-salt.serializers`. + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.deserialize 'json' '{"foo": "Foo!"}' + salt '*' --no-parse=stream_or_string slsutil.deserialize 'json' \\ + stream_or_string='{"foo": "Foo!"}' + + Jinja Example: + + .. code-block:: jinja + + {% set python_object = salt.slsutil.deserialize('json', + '{"foo": "Foo!"}') %} + """ + kwargs = salt.utils.args.clean_kwargs(**mod_kwargs) + return _get_serialize_fn(serializer, "deserialize")(stream_or_string, **kwargs) + + +def boolstr(value, true="true", false="false"): + """ + Convert a boolean value into a string. This function is + intended to be used from within file templates to provide + an easy way to take boolean values stored in Pillars or + Grains, and write them out in the appropriate syntax for + a particular file template. + + :param value: The boolean value to be converted + :param true: The value to return if ``value`` is ``True`` + :param false: The value to return if ``value`` is ``False`` + + In this example, a pillar named ``smtp:encrypted`` stores a boolean + value, but the template that uses that value needs ``yes`` or ``no`` + to be written, based on the boolean value. + + *Note: this is written on two lines for clarity. The same result + could be achieved in one line.* + + .. code-block:: jinja + + {% set encrypted = salt[pillar.get]('smtp:encrypted', false) %} + use_tls: {{ salt['slsutil.boolstr'](encrypted, 'yes', 'no') }} + + Result (assuming the value is ``True``): + + .. code-block:: none + + use_tls: yes + + """ + + if value: + return true + + return false + + +def _set_context(keys, function, fun_args=None, fun_kwargs=None, force=False): + """ + Convenience function to set a value in the ``__context__`` dictionary. + + .. versionadded:: 3004 + + :param keys: The list of keys specifying the dictionary path to set. This + list can be of arbitrary length and the path will be created + in the dictionary if it does not exist. + + :param function: A python function to be called if the specified path does + not exist, if the force parameter is ``True``. + + :param fun_args: A list of positional arguments to the function. + + :param fun_kwargs: A dictionary of keyword arguments to the function. + + :param force: If ``True``, force the ```__context__`` path to be updated. + Otherwise, only create it if it does not exist. + """ + + target = __context__ + + # Build each level of the dictionary as needed + for key in keys[:-1]: + if key not in target: + target[key] = {} + target = target[key] + + # Call the supplied function to populate the dictionary + if force or keys[-1] not in target: + if not fun_args: + fun_args = [] + + if not fun_kwargs: + fun_kwargs = {} + + target[keys[-1]] = function(*fun_args, *fun_kwargs) + + +def file_exists(path, saltenv="base"): + """ + Return ``True`` if a file exists in the state tree, ``False`` otherwise. + + .. versionadded:: 3004 + + :param str path: The fully qualified path to a file in the state tree. + :param str saltenv: The fileserver environment to search. Default: ``base`` + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.file_exists nginx/defaults.yaml + """ + + _set_context( + [CONTEXT_BASE, saltenv, "file_list"], __salt__["cp.list_master"], [saltenv] + ) + return path in __context__[CONTEXT_BASE][saltenv]["file_list"] + + +def dir_exists(path, saltenv="base"): + """ + Return ``True`` if a directory exists in the state tree, ``False`` otherwise. + + :param str path: The fully qualified path to a directory in the state tree. + :param str saltenv: The fileserver environment to search. Default: ``base`` + + .. versionadded:: 3004 + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.dir_exists nginx/files + """ + + _set_context( + [CONTEXT_BASE, saltenv, "dir_list"], __salt__["cp.list_master_dirs"], [saltenv] + ) + return path in __context__[CONTEXT_BASE][saltenv]["dir_list"] + + +def path_exists(path, saltenv="base"): + """ + Return ``True`` if a path exists in the state tree, ``False`` otherwise. The path + could refer to a file or directory. + + .. versionadded:: 3004 + + :param str path: The fully qualified path to a file or directory in the state tree. + :param str saltenv: The fileserver environment to search. Default: ``base`` + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.path_exists nginx/defaults.yaml + """ + + return file_exists(path, saltenv) or dir_exists(path, saltenv) + + +def findup(startpath, filenames, saltenv="base"): + """ + Find the first path matching a filename or list of filenames in a specified + directory or the nearest ancestor directory. Returns the full path to the + first file found. + + .. versionadded:: 3004 + + :param str startpath: The fileserver path from which to begin the search. + An empty string refers to the state tree root. + :param filenames: A filename or list of filenames to search for. Searching for + directory names is also supported. + :param str saltenv: The fileserver environment to search. Default: ``base`` + + Example: return the path to ``defaults.yaml``, walking up the tree from the + state file currently being processed. + + .. code-block:: jinja + + {{ salt["slsutil.findup"](tplfile, "defaults.yaml") }} + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.findup formulas/shared/nginx map.jinja + """ + + # Normalize the path + if startpath: + startpath = posixpath.normpath(startpath) + + # Verify the cwd is a valid path in the state tree + if startpath and not path_exists(startpath, saltenv): + raise salt.exceptions.SaltInvocationError( + "Starting path not found in the state tree: {}".format(startpath) + ) + + # Ensure that patterns is a string or list of strings + if isinstance(filenames, str): + filenames = [filenames] + if not isinstance(filenames, list): + raise salt.exceptions.SaltInvocationError( + "Filenames argument must be a string or list of strings" + ) + + while True: + + # Loop over filenames, looking for one at the current path level + for filename in filenames: + fullname = salt.utils.path.join( + startpath or "", filename, use_posixpath=True + ) + if path_exists(fullname, saltenv): + return fullname + + # If the root path was just checked, raise an error + if not startpath: + raise salt.exceptions.CommandExecutionError( + "File pattern(s) not found in path ancestry" + ) + + # Move up one level in the ancestry + startpath = os.path.dirname(startpath) diff --git a/tests/pytests/integration/ssh/test_slsutil.py b/tests/pytests/integration/ssh/test_slsutil.py new file mode 100644 index 000000000000..4ac9ed59a825 --- /dev/null +++ b/tests/pytests/integration/ssh/test_slsutil.py @@ -0,0 +1,94 @@ +import json + +import pytest + + +@pytest.mark.usefixtures("state_tree") +def test_renderer_file(salt_ssh_cli): + ret = salt_ssh_cli.run("slsutil.renderer", "salt://test.sls") + assert ret.returncode == 0 + assert isinstance(ret.data, dict) + assert "Ok with def" in ret.data + + +def test_renderer_string(salt_ssh_cli): + rend = "{{ salt['test.echo']('foo') }}: {{ pillar['ext_spam'] }}" + ret = salt_ssh_cli.run("slsutil.renderer", string=rend) + assert ret.returncode == 0 + assert isinstance(ret.data, dict) + assert ret.data == {"foo": "eggs"} + + +def test_serialize(salt_ssh_cli): + obj = {"foo": "bar"} + ret = salt_ssh_cli.run("slsutil.serialize", "json", obj) + assert ret.returncode == 0 + assert isinstance(ret.data, str) + assert ret.data == json.dumps(obj) + + +def test_deserialize(salt_ssh_cli): + obj = {"foo": "bar"} + data = json.dumps(obj) + # Need to quote it, otherwise it's deserialized by the + # test wrapper + ret = salt_ssh_cli.run("slsutil.deserialize", "json", f"'{data}'") + assert ret.returncode == 0 + assert isinstance(ret.data, type(obj)) + assert ret.data == obj + + +@pytest.mark.parametrize( + "path,expected", + [ + ("test_deep", True), + ("test_deep/test.sls", False), + ("test_deep/b/2", True), + ("does_not/ex/ist", False), + ], +) +def test_dir_exists(salt_ssh_cli, path, expected): + ret = salt_ssh_cli.run("slsutil.dir_exists", path) + assert ret.returncode == 0 + assert isinstance(ret.data, bool) + assert ret.data is expected + + +@pytest.mark.parametrize( + "path,expected", [("test_deep", False), ("test_deep/test.sls", True)] +) +def test_file_exists(salt_ssh_cli, path, expected): + ret = salt_ssh_cli.run("slsutil.file_exists", path) + assert ret.returncode == 0 + assert isinstance(ret.data, bool) + assert ret.data is expected + + +@pytest.mark.parametrize( + "start,name,expected", + [ + ("test_deep/b/2", "test.sls", "test_deep/b/2/test.sls"), + ("test_deep/b/2", "cheese", "cheese"), + ], +) +def test_findup(salt_ssh_cli, start, name, expected): + ret = salt_ssh_cli.run("slsutil.findup", start, name) + assert ret.returncode == 0 + assert isinstance(ret.data, str) + assert ret.data == expected + + +@pytest.mark.parametrize( + "path,expected", + [ + ("test_deep", True), + ("test_deep/test.sls", True), + ("test_deep/b/2", True), + ("does_not/ex/ist", False), + ], +) +def test_path_exists(salt_ssh_cli, path, expected): + ret = salt_ssh_cli.run("slsutil.path_exists", path) + assert ret.returncode == 0 + assert isinstance(ret.data, bool) + assert ret.data is expected diff --git a/tests/pytests/unit/client/ssh/wrapper/test_slsutil.py b/tests/pytests/unit/client/ssh/wrapper/test_slsutil.py new file mode 100644 index 000000000000..558d5ee74dcf --- /dev/null +++ b/tests/pytests/unit/client/ssh/wrapper/test_slsutil.py @@ -0,0 +1,166 @@ +import contextlib +import logging + +import pytest + +import salt.client.ssh.wrapper.slsutil as slsutil +from salt.exceptions import CommandExecutionError, SaltInvocationError +from tests.support.mock import MagicMock + +log = logging.getLogger(__name__) + + +# --- These tests are adapted from tests.pytests.unit.utils.slsutil + + +@pytest.fixture +def configure_loader_modules(master_dirs, master_files): + return { + slsutil: { + "__salt__": { + "cp.list_master": MagicMock(return_value=master_files), + "cp.list_master_dirs": MagicMock(return_value=master_dirs), + }, + "__opts__": { + "renderer": "jinja|yaml", + "renderer_blacklist": [], + "renderer_whitelist": [], + }, + } + } + + +@pytest.fixture +def master_dirs(): + return ["red", "red/files", "blue", "blue/files"] + + +@pytest.fixture +def master_files(): + return [ + "top.sls", + "red/init.sls", + "red/files/default.conf", + "blue/init.sls", + "blue/files/default.conf", + ] + + +@pytest.mark.parametrize("inpt,expected", ((True, "yes"), (False, "no"))) +def test_boolstr(inpt, expected): + assert slsutil.boolstr(inpt, true="yes", false="no") == expected + + +@pytest.mark.parametrize( + "inpt,expected", (("red/init.sls", True), ("green/init.sls", False)) +) +def test_file_exists(inpt, expected): + assert slsutil.file_exists(inpt) is expected + + +@pytest.mark.parametrize("inpt,expected", (("red", True), ("green", False))) +def test_dir_exists(inpt, expected): + assert slsutil.dir_exists(inpt) is expected + + +@pytest.mark.parametrize( + "inpt,expected", + ( + ("red", True), + ("green", False), + ("red/init.sls", True), + ("green/init.sls", False), + ), +) +def test_path_exists(inpt, expected): + assert slsutil.path_exists(inpt) is expected + + +@pytest.mark.parametrize( + "inpt,expected,raises", + [ + (("red/files", "init.sls"), "red/init.sls", None), + (("red/files", ["top.sls"]), "top.sls", None), + (("", "top.sls"), "top.sls", None), + ((None, "top.sls"), "top.sls", None), + (("red/files", ["top.sls", "init.sls"]), "red/init.sls", None), + ( + ("red/files", "notfound"), + None, + pytest.raises( + CommandExecutionError, match=r"File pattern\(s\) not found.*" + ), + ), + ( + ("red", "default.conf"), + None, + pytest.raises( + CommandExecutionError, match=r"File pattern\(s\) not found.*" + ), + ), + ( + ("green", "notfound"), + None, + pytest.raises(SaltInvocationError, match="Starting path not found.*"), + ), + ( + ("red", 1234), + None, + pytest.raises( + SaltInvocationError, match=".*must be a string or list of strings.*" + ), + ), + ], +) +def test_findup(inpt, expected, raises): + if raises is None: + raises = contextlib.nullcontext() + with raises: + res = slsutil.findup(*inpt) + assert res == expected + + +@pytest.mark.parametrize( + "a,b,merge_lists,expected", + [ + ( + {"foo": {"bar": "baz", "hi": "there", "some": ["list"]}}, + {"foo": {"baz": "quux", "bar": "hi", "some": ["other_list"]}}, + False, + { + "foo": { + "baz": "quux", + "bar": "hi", + "hi": "there", + "some": ["other_list"], + } + }, + ), + ( + {"foo": {"bar": "baz", "hi": "there", "some": ["list"]}}, + {"foo": {"baz": "quux", "bar": "hi", "some": ["other_list"]}}, + True, + { + "foo": { + "baz": "quux", + "bar": "hi", + "hi": "there", + "some": ["list", "other_list"], + } + }, + ), + ], +) +@pytest.mark.parametrize("func", ("update", "merge", "merge_all")) +def test_update_merge(a, b, merge_lists, expected, func): + arg = (a, b) + if func == "merge_all": + arg = ([a, b],) + res = getattr(slsutil, func)(*arg, merge_lists=merge_lists) + assert res == expected + assert (a is res) is (func == "update") + + +def test_renderer_requires_either_path_or_string(): + with pytest.raises(SaltInvocationError, match=".*either path or string.*"): + slsutil.renderer()