Fixes #25970: Make compilation resolved loops over Blocks/Methods in …

…techniques possible

policies/rudderc/docs/src/syntax.md

@@ -121,6 +121,8 @@ Blocks contains:
       * `worst-case-weighted-sum`: Select the worst status from all methods, and use it as the block compliance, with a weight equal to the total number of methods below the block.
       * `worst-case-weighted-one`: Select the worst status from all methods, and use it as the block compliance, with a weight of 1.
   * `id` (required with `focus` mode): ID of the method to focus reporting on.
+* `foreach` (optional): List of dictionnary, repeat the block for each items in the list and each time, replace every occurrence of `${item.x}` in the subitems, condition by the corresponding value in the dictionnary.
+* `foreach_name` (optional): Name of the local iterator variable to use if a `foreach` loop is used. Default to `item`.
 
 <div class="warning">
 Setting <code class="hljs">policy_mode_override</code> to <code class="hljs">enforce</code> will <strong>bypass the audit mode</strong>, so it must only be used
@@ -186,10 +188,121 @@ items:
       mode: disabled
 ```
 
+## Foreach
+
+Blocks and methods can be executed multiple times using the `foreach` field. Each iteration can be parameterized using a simple templating to fill the `params`, `condition` or `items` fields of the block|method being executed.
+The templating is done by using a local variable, replaced at technique compilation using the syntax `${<iterator name>.<key>}`.
+The `<iterator name>` is by default named `item` and can be changed using the `foreach_name` field, which is useful when nesting looping blocks.
+
+* The `foreach_name` must follow the pattern `^[a-zA-Z0-9_]+$`.
+
+For example
+
+```yaml
+  - name: "Install the '${item.name}' package"
+    method: package_present
+    params:
+      name: "${item.name}"
+      version: "latest"
+    foreach:
+      - name: "vim"
+      - name: "htop"
+```
+
+and
+
+```yaml
+  - name: "Install the '${tools.pkg_name}' package"
+    method: package_present
+    params:
+      name: "${tools.pkg_name}"
+      version: "${tools.version}"
+    foreach:
+      - pkg_name: "vim"
+        version: "latest"
+      - pkg_name: "htop"
+        version: "latest"
+```
+
+would both be strictly equivalent to
+
+```yaml
+  - name: "Install the vim package"
+    method: package_present
+    params:
+      name: "vim"
+      version: "latest"
+
+  - name: "Install the 'htop' package"
+    method: package_present
+    params:
+      name: htop"
+      version: "latest"
+```
+
+Foreach loops can also be nested when used on blocks. If it is the case, the replacement is done from child to parent.
+Make sure to rename the foreach iterator variable to avoid any conflict.
+
+Example:
+
+```yaml
+items:
+  - name: "Deploy utility files for ${user.login}"
+    foreach_name: "user"
+    foreach:
+      - login: "bob"
+      - login: "alice"
+    items:
+      - name: "Deploy file ~/${file.path}"
+        method: file_from_shared_folder
+        params:
+          hash_type: sha256
+          source: "${file.path}"
+          path: "/home/${user.login}/${file.path}"
+        foreach_name: "file"
+        foreach:
+          - path: ".vimrc"
+          - path: ".bashrc"
+```
+
+would resolve to
+
+```yaml
+items:
+  - name: "Deploy utility files for bob"
+    items:
+      - name: "Deploy file ~/.vimrc"
+        method: file_from_shared_folder
+        params:
+          hash_type: sha256
+          source: ".vimrc"
+          path: "/home/bob/.vimrc"
+      - name: "Deploy file ~/.bashrc"
+        method: file_from_shared_folder
+        params:
+          hash_type: sha256
+          source: ".bashrc"
+          path: "/home/bob/.bashrc"
+  - name: "Deploy utility files for alice"
+    items:
+      - name: "Deploy file ~/.vimrc"
+        method: file_from_shared_folder
+        params:
+          hash_type: sha256
+          source: ".vimrc"
+          path: "/home/alice/.vimrc"
+      - name: "Deploy file ~/.bashrc"
+        method: file_from_shared_folder
+        params:
+          hash_type: sha256
+          source: ".bashrc"
+          path: "/home/alice/.bashrc"
+```
+
 ## Resources
 
 Files can be attached to a technique, they will automatically be deployed in the policies when used on a node.
-The absolute path of the folder containing the resource files is accessible from within a technique using the variable `${resources_dir}`.
+The absolute path of the folder containing the resource files is accessible from within a technique using the variable `${}`.
 
 To add resources to a YAML technique, put the files under a `resources` folder in the technique directory.
 In the example below, the `file1.txt` will be available from within the technique using `${resources_dir}/file1.txt`.

policies/rudderc/src/backends/unix.rs

@@ -144,7 +144,7 @@ impl Backend for Unix {
                             call_bundles.push(bundle)
                         }
                     }
-                    _ => call_bundles.push(bundle)
+                    _ => call_bundles.push(bundle),
                 }
             }
         }

policies/rudderc/src/compiler.rs

@@ -289,8 +289,10 @@ fn check_ids_unicity(technique: &Technique) -> Result<()> {
             ItemKind::Method(r) => {
                 if r.generate_method_call_bundle {
                     vec![r.id.clone()]
-                } else { vec![] }
-            },
+                } else {
+                    vec![]
+                }
+            }
             _ => todo!(),
         }
     }

policies/rudderc/src/ir/technique.rs

@@ -379,10 +379,10 @@ pub struct DeserItem {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub foreach_name: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub foreach:  Option<Vec<HashMap<String, String>>>,
+    pub foreach: Option<Vec<HashMap<String, String>>>,
     // If a DeserItem is a result of a loop, "is_virtual" is true
     #[serde(default)]
-    pub is_virtual: bool
+    pub is_virtual: bool,
 }
 
 // Variant of Technique for first level of deserialization
@@ -418,69 +418,91 @@ impl DeserTechnique {
             items: self
                 .items
                 .into_iter()
-                .map(|i| {
+                .flat_map(|i| {
                     i.resolve_loop()
-                    .iter()
-                    .map(|j| j.clone().into_kind().unwrap())
-                    .collect::<Vec<ItemKind>>()
-                }).flatten().collect(),
+                        .iter()
+                        .map(|j| j.clone().into_kind().unwrap())
+                        .collect::<Vec<ItemKind>>()
+                })
+                .collect(),
             params: self.params,
         })
     }
 }
 
 impl DeserItem {
     fn resolve_loop(self) -> Vec<DeserItem> {
-        fn replace_placeholders(s: &str, h: &HashMap<String, String>, vn: Option<String>) -> String {
+        fn replace_placeholders(
+            s: &str,
+            h: &HashMap<String, String>,
+            vn: Option<String>,
+        ) -> String {
             let variable_name = vn.unwrap_or("item".to_string());
             // Define the pattern to match `${variable_name.key}`.
             let pattern = format!(r"\$\{{{}\.(\w+)\}}", regex::escape(&variable_name));
-            let regex = regex::Regex::new(&pattern).expect(&format!("Invalid loop variable iterator {}", variable_name));
-
+            let regex = regex::Regex::new(&pattern)
+                .unwrap_or_else(|_| panic!("Invalid loop variable iterator {}", variable_name));
             // Replace matches with corresponding values from the hashmap.
             regex
                 .replace_all(s, |captures: &regex::Captures| {
                     let key = &captures[1];
                     // Replace with the value from the hashmap, or keep the placeholder if the key doesn't exist.
-                    h.get(key).cloned().unwrap_or_else(|| captures[0].to_string())
+                    h.get(key)
+                        .cloned()
+                        .unwrap_or_else(|| captures[0].to_string())
                 })
                 .to_string()
         }
 
         if let Some(iterators) = self.foreach {
-            let vn =  self.foreach_name.clone();
-            let mut r: Vec<DeserItem> = iterators.iter().map(|h|
-                DeserItem {
-                    condition: self.condition.clone(),
+            let vn = self.foreach_name.clone();
+            let mut r: Vec<DeserItem> = iterators
+                .iter()
+                .map(|h| DeserItem {
+                    condition: Condition::Expression(replace_placeholders(
+                        self.condition.as_ref(),
+                        &h.clone(),
+                        vn.clone(),
+                    )),
                     name: replace_placeholders(&self.name, &h.clone(), vn.clone()),
-                    description: if let Some(d) = &self.description {
-                        Some(replace_placeholders(&d, &h.clone(), vn.clone()))
-                    } else  { None },
-                    documentation: if let Some(d) = &self.documentation {
-                        Some(replace_placeholders(&d, &h.clone(), vn.clone()))
-                    } else { None },
+                    description: self
+                        .documentation
+                        .as_ref()
+                        .map(|d| replace_placeholders(d, &h.clone(), vn.clone())),
+                    documentation: self
+                        .documentation
+                        .as_ref()
+                        .map(|d| replace_placeholders(d, &h.clone(), vn.clone())),
                     id: self.id.clone(),
                     reporting: self.reporting.clone(),
                     policy_mode_override: self.policy_mode_override,
                     foreach_name: None,
                     foreach: None,
                     method: self.method.clone(),
                     tags: self.tags.clone(),
-                    params: self.params.iter().map(|(k, v)| (k.clone(), replace_placeholders(v, h, vn.clone()))).collect(),
-                    items: self.items.iter().map(|i| i.clone().resolve_loop()).flatten().collect(),
+                    params: self
+                        .params
+                        .iter()
+                        .map(|(k, v)| (k.clone(), replace_placeholders(v, h, vn.clone())))
+                        .collect(),
+                    items: self
+                        .items
+                        .iter()
+                        .flat_map(|i| i.clone().resolve_loop())
+                        .collect(),
                     module: self.module.clone(),
                     is_virtual: true,
-                }
-            ).collect();
+                })
+                .collect();
             // The first element is always the only "true" one
             let loop_master = DeserItem {
-             is_virtual: false,
-             ..r.first().unwrap().clone()
+                is_virtual: false,
+                ..r.first().unwrap().clone()
             };
             if let Some(first) = r.get_mut(0) {
-              *first = loop_master;
+                *first = loop_master;
             }
-            r 
+            r
         } else {
             vec![self]
         }
@@ -510,7 +532,7 @@ impl DeserItem {
                 ))?,
                 info: None,
                 policy_mode_override: self.policy_mode_override,
-                generate_method_call_bundle: !self.is_virtual
+                generate_method_call_bundle: !self.is_virtual,
             })),
             (true, false, _, false) => {
                 bail!("Method {} ({}) requires params", self.name, self.id)
@@ -529,7 +551,7 @@ impl DeserItem {
                     self.name, self.id
                 ))?,
                 policy_mode_override: self.policy_mode_override,
-                generate_method_call_bundle: !self.is_virtual
+                generate_method_call_bundle: !self.is_virtual,
             })),
             (false, true, _, false) => {
                 bail!("Module {} ({}) requires params", self.name, self.id)

policies/rudderc/tests/cases/general/ntp/technique.cf

@@ -39,7 +39,7 @@ bundle agent call_ntp_technique_d86ce2e5_d5b6_45cc_87e8_c11cca71d907(c_name, c_k
   methods:
     "index_${local_index}_0" usebundle => _method_reporting_context_v4("${c_name}", "${c_key}", "${report_id}");
     "index_${local_index}_1" usebundle => _classes_noop(canonify("${class_prefix}_package_present_${c_key}"));
-    "index_${local_index}_2" usebundle => log_rudder("Skipping method 'Package present' with key parameter 'htop' since condition 'false' is not reached", "htop", canonify("${class_prefix}_package_present_${c_key}"), canonify("${class_prefix}_package_present_${c_key}"), @{args});
+    "index_${local_index}_2" usebundle => log_rudder("Skipping method 'Package present' with key parameter '${c_key}' since condition 'false' is not reached", "htop", canonify("${class_prefix}_package_present_${c_key}"), canonify("${class_prefix}_package_present_${c_key}"), @{args});
 
 }
 bundle agent call_ntp_technique_cf06e919_02b7_41a7_a03f_4239592f3c12(c_name, c_key, report_id, args, class_prefix, method_call_condition, name) {