Merge pull request #199 from feelpp/166-add-unit-tests

166 add unit tests

.github/workflows/ci.yml

@@ -51,8 +51,15 @@ jobs:
       run: |
         python -m venv .venv
         source .venv/bin/activate
+        pip3 install --upgrade pip
         pip3 install -I -r requirements.txt
         pip3 install dist/*.whl
+    - name: Unit tests
+      run: |
+        source .venv/bin/activate
+        python3 -m pytest
+      env:
+        GIRDER_API_KEY: ${{secrets.GIRDER}}
     - name: Compile test applications
       run: |
         mpic++ -std=c++17 -O3 tests/data/parallelSum.cpp -o tests/data/parallelSum

docs/modules/tutorial/pages/configurationfiles/benchmark.adoc

@@ -292,7 +292,7 @@ Valid generators are :
 ----
 {
     "name": "my_linspace_generator",
-    "logspace":{
+    "geomspace":{
         "min":2,
         "max":10,
         "n_steps":5
@@ -301,16 +301,15 @@ Valid generators are :
 ----
 The example will yield `[2,4,6,8,10]`. Min and max are inclusive.
 
-- `logspace`:
+- `geomspace`:
 [source,json]
 ----
 {
-    "name": "my_logspace_generator",
-    "logspace":{
+    "name": "my_geomspace_generator",
+    "geomspace":{
         "min":1,
         "max":10,
-        "n_steps":4,
-        "base":2
+        "n_steps":4
     }
 }
 ----
@@ -321,7 +320,7 @@ The example will yield `[2,16,128,1024]`. Min and max are inclusive.
 ----
 {
     "name": "my_range_generator",
-    "logspace":{
+    "geomspace":{
         "min":1,
         "max":5,
         "step":1
@@ -336,7 +335,7 @@ The example will yield `[1,2,3,4,5]`. Min and max are inclusive.
 ----
 {
     "name": "my_geometric_generator",
-    "logspace":{
+    "geometric":{
         "start":1,
         "ratio":2,
         "n_steps":5

requirements.txt

@@ -5,7 +5,7 @@ ipykernel
 numpy
 scipy
 plotly
-setuptools
+setuptools==66.1.1
 nbformat
 Jinja2
 ReFrame-HPC==4.6.3

src/feelpp/benchmarking/reframe/__main__.py

@@ -1,74 +1,10 @@
 import os, json, subprocess
-from datetime import datetime
 from feelpp.benchmarking.reframe.parser import Parser
 from feelpp.benchmarking.reframe.config.configReader import ConfigReader
 from feelpp.benchmarking.reframe.config.configSchemas import ConfigFile
 from feelpp.benchmarking.reframe.config.configMachines import MachineConfig
-from pathlib import Path
 from feelpp.benchmarking.reframe.reporting import WebsiteConfig
-
-
-class CommandBuilder:
-    def __init__(self, machine_config, parser):
-        self.machine_config = machine_config
-        self.parser = parser
-        self.current_date = datetime.now().strftime("%Y_%m_%dT%H_%M_%S")
-        self.report_folder_path = None
-
-    @staticmethod
-    def getScriptRootDir():
-        return Path(__file__).resolve().parent
-
-    def buildConfigFilePath(self):
-        return f'{self.getScriptRootDir() / "config/machineConfigs" / self.machine_config.machine}.py'
-
-    def buildRegressionTestFilePath(self):
-        return f'{self.getScriptRootDir() / "regression.py"}'
-
-    def createReportFolder(self,executable,use_case):
-        folder_path = os.path.join(self.machine_config.reports_base_dir,executable,use_case,self.machine_config.machine,str(self.current_date))
-        if not os.path.exists(folder_path):
-            os.makedirs(folder_path)
-        self.report_folder_path = folder_path
-
-        return str(self.report_folder_path)
-
-    def buildExecutionMode(self):
-        """Write the ReFrame execution flag depending on the parser arguments.
-            Examples are --dry-run or -r
-        """
-        if self.parser.args.dry_run:
-            return "--dry-run"
-        else:
-            return "-r"
-
-    def buildJobOptions(self,timeout,memory):
-        #TODO: Generalize (only workf for slurm ?)
-        options = []
-        if timeout:
-            options.append(f"-J time={timeout}")
-        if memory:
-            options.append(f"-J mem={memory}")
-        return " ".join(options)
-
-
-    def buildCommand(self,timeout,memory):
-        assert self.report_folder_path is not None, "Report folder path not set"
-        cmd = [
-            'reframe',
-            f'-C {self.buildConfigFilePath()}',
-            f'-c {self.buildRegressionTestFilePath()}',
-            f'-S report_dir_path={str(self.report_folder_path)}',
-            f'--system={self.machine_config.machine}',
-            f'--exec-policy={self.machine_config.execution_policy}',
-            f'--prefix={self.machine_config.reframe_base_dir}',
-            f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}',
-            f"{self.buildJobOptions(timeout,memory)}",
-            f'--perflogdir={os.path.join(self.machine_config.reframe_base_dir,"logs")}',
-            f'{"-"+"v"*self.parser.args.verbose  if self.parser.args.verbose else ""}',
-            f'{self.buildExecutionMode()}'
-        ]
-        return ' '.join(cmd)
+from feelpp.benchmarking.reframe.commandBuilder import CommandBuilder
 
 
 def main_cli():

src/feelpp/benchmarking/reframe/commandBuilder.py

@@ -0,0 +1,66 @@
+import os
+from datetime import datetime
+from pathlib import Path
+
+
+class CommandBuilder:
+    def __init__(self, machine_config, parser):
+        self.machine_config = machine_config
+        self.parser = parser
+        self.current_date = datetime.now().strftime("%Y_%m_%dT%H_%M_%S")
+        self.report_folder_path = None
+
+    @staticmethod
+    def getScriptRootDir():
+        return Path(__file__).resolve().parent
+
+    def buildConfigFilePath(self):
+        return f'{self.getScriptRootDir() / "config/machineConfigs" / self.machine_config.machine}.py'
+
+    def buildRegressionTestFilePath(self):
+        return f'{self.getScriptRootDir() / "regression.py"}'
+
+    def createReportFolder(self,executable,use_case):
+        folder_path = os.path.join(self.machine_config.reports_base_dir,executable,use_case,self.machine_config.machine,str(self.current_date))
+        if not os.path.exists(folder_path):
+            os.makedirs(folder_path)
+        self.report_folder_path = folder_path
+
+        return str(self.report_folder_path)
+
+    def buildExecutionMode(self):
+        """Write the ReFrame execution flag depending on the parser arguments.
+            Examples are --dry-run or -r
+        """
+        if self.parser.args.dry_run:
+            return "--dry-run"
+        else:
+            return "-r"
+
+    def buildJobOptions(self,timeout,memory):
+        #TODO: Generalize (only workf for slurm ?)
+        options = []
+        if timeout:
+            options.append(f"-J time={timeout}")
+        if memory:
+            options.append(f"-J mem={memory}")
+        return " ".join(options)
+
+
+    def buildCommand(self,timeout,memory):
+        assert self.report_folder_path is not None, "Report folder path not set"
+        cmd = [
+            'reframe',
+            f'-C {self.buildConfigFilePath()}',
+            f'-c {self.buildRegressionTestFilePath()}',
+            f'-S report_dir_path={str(self.report_folder_path)}',
+            f'--system={self.machine_config.machine}',
+            f'--exec-policy={self.machine_config.execution_policy}',
+            f'--prefix={self.machine_config.reframe_base_dir}',
+            f'--report-file={str(os.path.join(self.report_folder_path,"reframe_report.json"))}',
+            f"{self.buildJobOptions(timeout,memory)}",
+            f'--perflogdir={os.path.join(self.machine_config.reframe_base_dir,"logs")}',
+            f'{"-"+"v"*self.parser.args.verbose  if self.parser.args.verbose else ""}',
+            f'{self.buildExecutionMode()}'
+        ]
+        return ' '.join(cmd)

src/feelpp/benchmarking/reframe/config/configParameters.py

@@ -1,22 +1,58 @@
 from __future__ import annotations
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, model_validator, field_validator
 from typing import  Union, Optional, List, Any, Dict
 
 
-class Linspace(BaseModel):
+class NSteps(BaseModel):
+    n_steps:int
+
+    @field_validator("n_steps",mode="after")
+    @classmethod
+    def checkPositiveSteps(cls,v):
+        if v <= 0:
+            raise ValueError(f"Number of steps should be strictly positive ({v})")
+        return v
+
+class Linspace(NSteps):
     min:Union[float,int]
     max:Union[float,int]
-    n_steps:int
 
-class Logspace(Linspace):
-    base:Union[int,float]
+
+class Geomspace(Linspace):
+
+    @field_validator("min","max",mode="after")
+    @classmethod
+    def checkForZero(cls,v):
+        if v == 0:
+            raise ValueError("Geomspace cannot contain 0")
+        return v
+
+    @model_validator(mode="after")
+    def checkZeroInRange(self):
+        if (self.min < 0 and self.max > 0) or (self.max < 0 and self.min>0):
+            raise ValueError("0 cannot be contained between min and max")
+        return self
 
 class Range(BaseModel):
     min:Union[float,int]
     max:Union[float,int]
     step:Union[float,int]
 
-class Geometric(BaseModel):
+    @field_validator("step",mode="after")
+    @classmethod
+    def checkForZero(cls,v):
+        if v == 0:
+            raise ValueError("Step cannot be 0")
+        return v
+
+    @model_validator(mode="after")
+    def checkForEmpty(self):
+        if (self.min < self.max and self.step < 0) or (self.min > self.max and self.step > 0) or (self.min==self.max):
+            raise ValueError("Range will result empty")
+        return self
+
+
+class Geometric(NSteps):
     start:Union[float,int]
     ratio:Union[float,int]
     n_steps:int
@@ -31,7 +67,7 @@ class Parameter(BaseModel):
     active:Optional[bool] = True
 
     linspace:Optional[Linspace] = None
-    logspace:Optional[Logspace] = None
+    geomspace:Optional[Geomspace] = None
     range:Optional[Range] = None
     sequence:Optional[List[Union[int,float,str,Dict]]] = None
     repeat:Optional[Repeat] = None
@@ -40,9 +76,9 @@ class Parameter(BaseModel):
 
     @model_validator(mode="after")
     def setMode(self):
-        if self.logspace is not None:
-            self.mode = "logspace"
-        if self.linspace is not None:
+        if self.geomspace is not None:
+            self.mode = "geomspace"
+        elif self.linspace is not None:
             self.mode = "linspace"
         elif self.range is not None:
             self.mode = "range"
@@ -57,6 +93,6 @@ def setMode(self):
         else:
             raise NotImplementedError("Parameters need an implemented generator")
 
-        assert len([mode for mode in [self.linspace,self.logspace,self.geometric,self.range,self.zip,self.sequence,self.repeat] if mode is not None]) == 1, "Parameter can only have one generator"
+        assert len([mode for mode in [self.linspace,self.geomspace,self.geometric,self.range,self.zip,self.sequence,self.repeat] if mode is not None]) == 1, "Parameter can only have one generator"
 
         return self

src/feelpp/benchmarking/reframe/config/configPlots.py

@@ -22,7 +22,7 @@ class Plot(BaseModel):
     transformation:Literal["performance","relative_performance","speedup"]
     aggregations:Optional[List[Aggregation]] = None
     variables:Optional[List[str]] = None
-    names:List[str]
+    names:Optional[List[str]] = []
     xaxis:PlotAxis
     secondary_axis:Optional[PlotAxis] = None
     yaxis:PlotAxis

src/feelpp/benchmarking/reframe/config/configReader.py

@@ -36,17 +36,30 @@ def flattenDict(nested_json, parent_key='', separator='.'):
                 items.append((new_key, value))
         return dict(items)
 
-    def replacePlaceholders(self,target,flattened_source):
+    def replacePlaceholders(self,target,flattened_source,processed_placeholders=None):
+        if processed_placeholders is None:
+            processed_placeholders = set()
+
         def replaceMatch(match):
+            placeholder = match.group(1).strip()
+
+            if placeholder in processed_placeholders:
+                return match.group(0)
+
+            processed_placeholders.add(placeholder)
             resolved = flattened_source.get(match.group(1).strip(),match.group(0))
+
             if match.group(1) in flattened_source:
                 if isinstance(resolved,str) and "{{" in resolved and "}}" in resolved:
-                    resolved = self.replacePlaceholders(resolved,flattened_source)
-            return resolved
+                    resolved = self.replacePlaceholders(resolved,flattened_source,processed_placeholders)
+            return str(resolved)
 
-        replaced = self.template_pattern.sub(replaceMatch,target)
+        previous_target = None
+        while target != previous_target:
+            previous_target = target
+            target = self.template_pattern.sub(replaceMatch, target)
 
-        return os.path.expandvars(replaced) if "$" in replaced else replaced
+        return os.path.expandvars(target) if "$" in target else target
 
 
     def recursiveReplace(self,target,flattened_source):