Merge pull request #62674 from lkubb/vault-pillar-templating

Improve Vault templating (policies, ext_pillar paths)

changelog/43287.added

@@ -0,0 +1 @@
+Added pillar templating to vault policies

salt/modules/vault.py

@@ -130,6 +130,12 @@
         values, eg ``my-policies/{grains[os]}``. ``{minion}`` is shorthand for
         ``grains[id]``, eg ``saltstack/minion/{minion}``.
 
+        .. versionadded:: 3006
+
+            Policies can be templated with pillar values as well: ``salt_role_{pillar[roles]}``
+            Make sure to only reference pillars that are not sourced from Vault since the latter
+            ones might be unavailable during policy rendering.
+
         .. important::
 
             See :ref:`Is Targeting using Grain Data Secure?
@@ -160,6 +166,29 @@
         Optional. If policies is not configured, ``saltstack/minions`` and
         ``saltstack/{minion}`` are used as defaults.
 
+    policies_refresh_pillar
+        Whether to refresh the pillar data when rendering templated policies.
+        When unset (=null/None), will only refresh when the cached data
+        is unavailable, boolean values force one behavior always.
+
+        .. note::
+
+            Using cached pillar data only (policies_refresh_pillar=False)
+            might cause the policies to be out of sync. If there is no cached pillar
+            data available for the minion, pillar templates will fail to render at all.
+
+            If you use pillar values for templating policies and do not disable
+            refreshing pillar data, make sure the relevant values are not sourced
+            from Vault (ext_pillar, sdb) or from a pillar sls file that uses the vault
+            execution module. Although this will often work when cached pillar data is
+            available, if the master needs to compile the pillar data during policy rendering,
+            all Vault modules will be broken to prevent an infinite loop.
+
+    policies_cache_time
+        Policy computation can be heavy in case pillar data is used in templated policies and
+        it has not been cached. Therefore, a short-lived cache specifically for rendered policies
+        is used. This specifies the expiration timeout in seconds. Defaults to 60.
+
     keys
         List of keys to use to unseal vault server with the vault.unseal runner.
 

salt/pillar/vault.py

@@ -52,6 +52,7 @@
       - vault: path=secret/salt
       - vault: path=secret/root
       - vault: path=secret/minions/{minion}/pass
+      - vault: path=secret/roles/{pillar[roles]}/pass
 
 You can also use nesting here as well.  Identical nesting keys will get merged.
 
@@ -118,13 +119,40 @@
                 minion-passwd:
                     minionbadpasswd1
 
+.. versionadded:: 3006
+
+    Pillar values from previously rendered pillars can be used to template
+    vault ext_pillar paths.
+
+Using pillar values to template vault pillar paths requires them to be defined
+before the vault ext_pillar is called. Especially consider the significancy
+of :conf_master:`ext_pillar_first <ext_pillar_first>` master config setting.
+
+If a pillar pattern matches multiple paths, the results are merged according to
+the master configuration values :conf_master:`pillar_source_merging_strategy <pillar_source_merging_strategy>`
+and :conf_master:`pillar_merge_lists <pillar_merge_lists>` by default.
+
+If the optional nesting_key was defined, the merged result will be nested below.
+There is currently no way to nest multiple results under different keys.
+
+You can override the merging behavior per defined ext_pillar:
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - vault:
+           conf: path=secret/roles/{pillar[roles]}
+           merge_strategy: smart
+           merge_lists: false
 """
 
 
 import logging
 
 from requests.exceptions import HTTPError
 
+import salt.utils.dictupdate
+
 log = logging.getLogger(__name__)
 
 
@@ -140,36 +168,74 @@ def ext_pillar(
     pillar,  # pylint: disable=W0613
     conf,
     nesting_key=None,
+    merge_strategy=None,
+    merge_lists=None,
+    extra_minion_data=None,
 ):
     """
     Get pillar data from Vault for the configuration ``conf``.
     """
+    extra_minion_data = extra_minion_data or {}
+    if extra_minion_data.get("_vault_runner_is_compiling_pillar_templates"):
+        # Disable vault ext_pillar while compiling pillar for vault policy templates
+        return {}
+
     comps = conf.split()
 
     paths = [comp for comp in comps if comp.startswith("path=")]
     if not paths:
         log.error('"%s" is not a valid Vault ext_pillar config', conf)
         return {}
 
+    merge_strategy = merge_strategy or __opts__.get(
+        "pillar_source_merging_strategy", "smart"
+    )
+    merge_lists = merge_lists or __opts__.get("pillar_merge_lists", False)
     vault_pillar = {}
 
-    try:
-        path = paths[0].replace("path=", "")
-        path = path.format(**{"minion": minion_id})
-        version2 = __utils__["vault.is_v2"](path)
-        if version2["v2"]:
-            path = version2["data"]
-
-        url = "v1/{}".format(path)
-        response = __utils__["vault.make_request"]("GET", url)
-        response.raise_for_status()
-        vault_pillar = response.json().get("data", {})
-
-        if vault_pillar and version2["v2"]:
-            vault_pillar = vault_pillar["data"]
-    except HTTPError:
-        log.info("Vault secret not found for: %s", path)
+    path_pattern = paths[0].replace("path=", "")
+    for path in _get_paths(path_pattern, minion_id, pillar):
+        try:
+            version2 = __utils__["vault.is_v2"](path)
+            if version2["v2"]:
+                path = version2["data"]
+
+            url = "v1/{}".format(path)
+            response = __utils__["vault.make_request"]("GET", url)
+            response.raise_for_status()
+            vault_pillar_single = response.json().get("data", {})
+
+            if vault_pillar_single and version2["v2"]:
+                vault_pillar_single = vault_pillar_single["data"]
+
+            vault_pillar = salt.utils.dictupdate.merge(
+                vault_pillar,
+                vault_pillar_single,
+                strategy=merge_strategy,
+                merge_lists=merge_lists,
+            )
+        except HTTPError:
+            log.info("Vault secret not found for: %s", path)
 
     if nesting_key:
         vault_pillar = {nesting_key: vault_pillar}
     return vault_pillar
+
+
+def _get_paths(path_pattern, minion_id, pillar):
+    """
+    Get the paths that should be merged into the pillar dict
+    """
+    mappings = {"minion": minion_id, "pillar": pillar}
+
+    paths = []
+    try:
+        for expanded_pattern in __utils__["vault.expand_pattern_lists"](
+            path_pattern, **mappings
+        ):
+            paths.append(expanded_pattern.format(**mappings))
+    except KeyError:
+        log.warning("Could not resolve pillar path pattern %s", path_pattern)
+
+    log.debug(f"{minion_id} vault pillar paths: {paths}")
+    return paths