Specialized Plot sub-directory ignoring (Fix) (#2314)

* functions in samplers can be now used pointing to the same module

* updated doc

* test same file

* added input file

* added test

* trailing spaces

* sampler

* sampler fix

* sampler fix

* few fixes in ensemble model

* serpent

* fixed serpent

* fixed EOL in case of multiple crossings

* fixed directory for custom plots

* set overwrite to True in tests

* sub dir in specialized plots

* Apply suggestions from code review

removed jobhandler modification

* removed redondant lines of code

* test subdir for one specialized plot

* fixed typo from function call

* removed trailing spaces

* modified tests with overwrite flag

* externalized counter progression in plots

* fixed filename for general plot

* updated all tests that had conflicting overwrite statements

* fixed final plot

doc/workshop/optimizer/Inputs/1_grad_desc.xml

@@ -101,7 +101,7 @@
       <source>search_path</source>
       <clusterLabel>trajID</clusterLabel>
     </Print>
-    <Plot name="path" subType="OptPath">
+    <Plot name="path" subType="OptPath" overwrite="True">
       <source>search_path</source>
       <vars>v0,angle,ymax,r</vars>
     </Plot>

ravenframework/CodeInterfaceClasses/SERPENT/SerpentInterface.py

@@ -50,7 +50,7 @@ def __init__(self):
     self._fileTypesToRead = ['ResultsReader'] # container of file types to read
     # in case of burnup calc, the interface can compute the time at which FOMs (e.g. keff) crosses
     # a target. For example (default), we can compute the time (burnDays) at which absKeff crosses 1.0
-    self.eolTarget = {'absKeff':1.0}
+    self.eolTarget = {}
 
   def _findInputFile(self, inputFiles):
     """

ravenframework/CodeInterfaceClasses/SERPENT/serpentOutputParser.py

@@ -185,10 +185,30 @@ def _resultsReader(self):
         if target not in res.resdata:
           raise ValueError(f"Target {target} for EOL calcs is not in result data")
         targetValues = res.resdata[target][:,0]
-        sorting = np.argsort(targetValues)
-        endOfLife = np.interp(value,targetValues[sorting],res.resdata['burnDays'][:,0][sorting],left=min(res.resdata['burnDays'][:,0]),right=max(res.resdata['burnDays'][:,0]))
-        resultsResults[f'EOL_{target}'] = np.asarray([endOfLife]*targetValues.size)
-
+        minTarget, maxTarget = np.min(targetValues), np.max(targetValues)
+        if value >= minTarget and value <= maxTarget:
+          endOfLifes = []
+          timeIntervals = []
+          for idx in range(targetValues.size-1):
+            if value >= targetValues[idx] and value <=  targetValues[idx+1] or value >=  targetValues[idx+1] and value <=  targetValues[idx]:
+              sorting = np.argsort(targetValues[idx:idx+2])
+              endOfLifes.append(np.interp(value,targetValues[idx:idx+2][sorting],res.resdata['burnDays'][:,0][idx:idx+2][sorting]))
+              timeIntervals.append(res.resdata['burnDays'][:,0][idx:idx+2].tolist())
+          endOfLife = np.max(endOfLifes)
+          if len(endOfLifes) > 1:
+            # the target has been crossed multiple times
+            msg = f"The target ({target}) value ({value}) has been crossed multiple times in burnup calculation. "
+            ti = ", ".join([f'{t[0]}|{t[1]}' for t in timeIntervals])
+            msg += f"The computed EOL_{target}(s) are: {', '.join([str(eol) for eol in endOfLifes])}. The value crossing happens at the following time (days) intervals: {ti}. "
+            msg += f"The maximum EOL_{target} ({endOfLife}) will be stored in the results' container."
+            print(f"SERPENT Interface: {msg}")
+          resultsResults[f'EOL_{target}'] = np.asarray([endOfLife]*targetValues.size)
+        else:
+          if value >= maxTarget:
+            # if value is > maximum, the EOL_target == res.resdata['burnDays'][:,0]
+            resultsResults[f'EOL_{target}'] = np.asarray([min(res.resdata['burnDays'][:,0])]*targetValues.size)
+          elif value <= minTarget:
+            resultsResults[f'EOL_{target}'] = np.asarray([max(res.resdata['burnDays'][:,0])]*targetValues.size)
     return resultsResults, nSteps
 
   def _detectorReader(self, buSteps):

ravenframework/Models/EnsembleModel.py

@@ -742,7 +742,7 @@ def __advanceModel(self, identifier, modelToExecute, origInputList, inputKwargs,
       # we evaluate the model directly
       try:
         evaluation = modelToExecute['Instance'].evaluateSample.original_function(modelToExecute['Instance'], origInputList, samplerType, inputKwargs)
-      except Exception as e:
+      except Exception:
         excType, excValue, excTrace = sys.exc_info()
         evaluation = None
     else:
@@ -759,9 +759,12 @@ def __advanceModel(self, identifier, modelToExecute, origInputList, inputKwargs,
       finishedRun = jobHandler.getFinished(jobIdentifier = localIdentifier, uniqueHandler=self.name+identifier)
       evaluation = finishedRun[0].getEvaluation()
       if isinstance(evaluation, rerror):
+        if finishedRun[0].exceptionTrace is not None:
+          excType, excValue, excTrace = finishedRun[0].exceptionTrace
+        else:
+          # the failure happened at the input creation stage
+          excType, excValue, excTrace = IOError, IOError("Failure happened at the input creation stage. See trace above"), None
         evaluation = None
-        excType, excValue, excTrace = finishedRun[0].exceptionTrace
-        e = rerror
         # the model failed
         for modelToRemove in list(set(self.orderList) - set([modelToExecute['Instance'].name])):
           jobHandler.getFinished(jobIdentifier = modelToRemove + utils.returnIdSeparator() + identifier, uniqueHandler = self.name + identifier)

ravenframework/OutStreams/OutStreamInterface.py

@@ -54,7 +54,7 @@ def __init__(self):
     self.printTag = 'OutStreamInterface'
     self.overwrite = True       # overwrite existing creations?
     self.subDirectory = None    # directory to save generated files to
-    self.filename = ''          # target file name
+    self.filename = None        # target file name
     self.numberAggregatedOS = 1 # number of aggregated outstreams # no addl info from original OutStream
 
   def handleInput(self, spec):

ravenframework/OutStreams/PlotEntity.py

@@ -99,6 +99,7 @@ def addOutput(self):
       @ In, None
       @ Out, None
     """
+    self._plotter.increaseCounter()
     self._plotter.run()
 
   ################

ravenframework/OutStreams/PlotInterfaces/GeneralPlot.py

@@ -99,7 +99,6 @@ def __init__(self):
     # general attributes
     self.printTag = 'OUTSTREAM PLOT'
     self.options = {}        # outstreaming options # no addl info from original developer
-    self.counter = 0         # keeps track of how many times the same plot has been plotted
     self.dim = None          # default plot is 2D
     self.sourceName = []     # list of source names
     self.sourceData = None   # source of data
@@ -1085,8 +1084,6 @@ def run(self):
     if not self.__fillCoordinatesFromSource():
       self.raiseAWarning('Nothing to Plot Yet. Returning.')
       return
-
-    self.counter += 1
     if self.counter > 1:
       self.actcm = None
     clusterDict = deepcopy(self.outStreamTypes)
@@ -2237,20 +2234,10 @@ def handle_close(event):
       if fileType == 'screen':
         continue
 
-      if not self.overwrite:
-        prefix = str(self.counter) + '-'
-      else:
-        prefix = ''
-
-      if len(self.filename) > 0:
-        name = self.filename
-      else:
-        name = prefix + self.name + '_' + str(self.outStreamTypes).replace("'", "").replace("[", "").replace("]", "").replace(",", "-").replace(" ", "")
-
-      if self.subDirectory is not None:
-        name = os.path.join(self.subDirectory,name)
-
-      self.fig.savefig(name + '.' + fileType, format=fileType)
+      defaultName = self.name + '_' + str(self.outStreamTypes).replace("'", "").replace("[", "").replace("]", "").replace(",", "-").replace(" ", "")
+      filename = self._createFilename(defaultName=f'{defaultName}.{fileType}')
+      filename = filename if filename.endswith(fileType) else f'{filename}.{fileType}'
+      self.fig.savefig(filename, format=fileType)
 
     if 'screen' not in self.destinations:
       plt.close(fig=self.fig)

ravenframework/OutStreams/PlotInterfaces/OptParallelCoordinate.py

@@ -18,12 +18,7 @@
 """
 
 # External Imports
-from collections import defaultdict
-import matplotlib.pyplot as plt
-from matplotlib.path import Path
-import matplotlib.patches as patches
 import numpy as np
-import pandas as pd
 import imageio
 
 # Internal Imports
@@ -59,13 +54,12 @@ def __init__(self):
       @ Out, None
     """
     super().__init__()
-    self.printTag = 'OptParallelCoordinate Plot'
+    self.printTag = 'OptParallelCoordinatePlot'
     self.source = None      # reference to DataObject source
     self.sourceName = None  # name of DataObject source
     self.vars = None        # variables to plot
     self.index = None       # index ID for each batch
 
-
   def handleInput(self, spec):
     """
       Loads the input specs for this object.
@@ -91,6 +85,7 @@ def initialize(self, stepEntities):
                                 current step. The sources are searched into this.
       @ Out, None
     """
+    super().initialize(stepEntities)
     src = self.findSource(self.sourceName, stepEntities)
     if src is None:
       self.raiseAnError(IOError, f'No source named "{self.sourceName}" was found in the Step for SamplePlot "{self.name}"!')
@@ -129,8 +124,10 @@ def run(self):
       plotUtils.generateParallelPlot(ys,genID,yMin,yMax,self.vars,fileID)
       filesID.append(fileID)
 
-    fig = plt.figure()
-    with imageio.get_writer(f'{self.name}.gif', mode='I') as writer:
+    # create filename
+    giffilename = self._createFilename(defaultName=f'{self.name}.gif')
+
+    with imageio.get_writer(giffilename, mode='I') as writer:
       for filename in filesID:
         image = imageio.imread(filename)
         writer.append_data(image)

ravenframework/OutStreams/PlotInterfaces/OptPath.py

@@ -84,6 +84,7 @@ def initialize(self, stepEntities):
                                 current step. The sources are searched into this.
       @ Out, None
     """
+    super().initialize(stepEntities)
     src = self.findSource(self.sourceName, stepEntities)
     if src is None:
       self.raiseAnError(IOError, f'No source named "{self.sourceName}" was found in the Step for SamplePlot "{self.name}"!')
@@ -124,7 +125,11 @@ def run(self):
                loc='center right',
                borderaxespad=0.1,
                title='Legend')
-    plt.savefig(f'{self.name}.png')
+
+    # create filename
+    filename = self._createFilename(defaultName=f'{self.name}.png')
+
+    plt.savefig(filename)
 
   def addPoint(self, ax, i, value, accepted):
     """

ravenframework/OutStreams/PlotInterfaces/PlotInterface.py

@@ -16,7 +16,7 @@
 
 @author: talbpaul
 """
-
+import os
 from abc import abstractmethod
 from .. import OutStreamInterface, OutStreamEntity
 from ...utils.utils import displayAvailable
@@ -42,6 +42,8 @@ def __init__(self):
       @ Out, None
     """
     super().__init__()
+    # keeps track of how many times the same plot has been plotted
+    self.counter = 0
     self.printTag = 'PlotInterface'
 
   def handleInput(self, spec):
@@ -60,8 +62,17 @@ def initialize(self, stepEntities):
       current step. The sources are searched into this.
       @ Out, None
     """
+    self.counter = 0
     super().initialize(stepEntities)
 
+  def increaseCounter(self):
+    """
+      Function to increase the counter (number of times this plot instance has been called after the initialization)
+      @ In, None
+      @ Out, None
+    """
+    self.counter += 1
+
   @abstractmethod
   def run(self):
     """
@@ -129,4 +140,22 @@ def getInitParams(self):
       paramDict['Overwrite output everytime called'] = 'True'
     else:
       paramDict['Overwrite output everytime called'] = 'False'
+    if self.filename is not None:
+      paramDict['Inputted filename'] = self.filename
+    if self.subDirectory is not None:
+      paramDict['Sub-directory'] = self.subDirectory
     return paramDict
+
+  def _createFilename(self, defaultName):
+    """
+      Utility method to create output file names (it considers the overwrite and subDirectory flag)
+      @ In, defaultName, str, the default filename if self.filename is not set
+      @ Out, filename, str, the formatted filename
+    """
+    filename = self.filename if self.filename is not None else defaultName
+    prefix = str(self.counter) + '-' if not self.overwrite else ''
+    filename = f'{prefix}{filename}'
+    if self.subDirectory is not None:
+      filename = os.path.join(self.subDirectory,filename)
+
+    return filename

ravenframework/OutStreams/PlotInterfaces/PopulationPlot.py

@@ -18,10 +18,8 @@
 """
 
 # External Imports
-from collections import defaultdict
 import matplotlib.pyplot as plt
 import numpy as np
-from matplotlib.lines import Line2D
 
 # Internal Imports
 from ...utils import plotUtils
@@ -101,6 +99,7 @@ def initialize(self, stepEntities):
                                 current step. The sources are searched into this.
       @ Out, None
     """
+    super().initialize(stepEntities)
     src = self.findSource(self.sourceName, stepEntities)
     if src is None:
       self.raiseAnError(IOError, f'No source named "{self.sourceName}" was found in the Step for SamplePlot "{self.name}"!')
@@ -157,8 +156,9 @@ def run(self):
     fig.tight_layout()
 
     if self.how in ['png','pdf','svg','jpeg']:
-      fileName = self.name +'.%s'  % self.how
-      plt.savefig(fileName, format=self.how)
+      # create filename
+      filename = self._createFilename(defaultName=self.name +'.%s'  % self.how)
+      plt.savefig(filename, format=self.how)
     else:
       self.raiseAnError(IOError, f'Digital format of the plot "{self.name}" is not available!')
 

ravenframework/OutStreams/PlotInterfaces/SamplePlot.py

@@ -46,7 +46,7 @@ def __init__(self):
       @ Out, None
     """
     super().__init__()
-    self.printTag = 'PlotInterface'
+    self.printTag = 'SamplePlot'
     self.source = None      # reference to DataObject source
     self.sourceName = None  # name of DataObject source
     self.vars = None        # variables to plot
@@ -102,7 +102,10 @@ def run(self):
       ax.set_ylabel(var)
     axes[-1].set_xlabel('RAVEN Sample Number')
     fig.align_ylabels(axes[:])
-    plt.savefig(f'{self.name}.png')
+
+    # create filename
+    filename = self._createFilename(defaultName=f'{self.name}.png')
+    plt.savefig(filename)
 
   def plotScalar(self, ax, ids, vals):
     """

ravenframework/OutStreams/PlotInterfaces/SyntheticCloud.py

@@ -94,6 +94,7 @@ def initialize(self, stepEntities):
                                 current step. The sources are searched into this.
       @ Out, None
     """
+    super().initialize(stepEntities)
     train = self.findSource(self.trainingName, stepEntities)
     if train is None:
       self.raiseAnError(IOError, f'No input named "{self.trainingName}" was found in the Step for Plotter "{self.name}"!')
@@ -115,7 +116,6 @@ def run(self):
       @ In, None
       @ Out, None
     """
-    tTag = self.training.sampleTag
     sTag = self.samples.sampleTag
     training = self.training.asDataset()
     samples = self.samples.asDataset()
@@ -151,7 +151,19 @@ def run(self):
         if mTraining is not None:
           ax.plot(mTraining[self.microName].values, mTraining[var].values, 'k-.')
 
-      filename =  f'{self.name}_{m}.png'
+      # create filename
+      originalFilename = None
+      if  self.filename is not None:
+        originalFilename = self.filename
+        rootname = originalFilename.split(".")[0]
+        self.filename = None
+      else:
+        rootname = self.name
+
+      filename = self._createFilename(defaultName= f'{rootname}_{m}.png')
+      if originalFilename is not None:
+        self.filename = originalFilename
+
       plt.savefig(filename)
       self.raiseAMessage(f'Wrote "{filename}".')
 

ravenframework/OutStreams/PrintInterfaces/FilePrint.py

@@ -133,7 +133,7 @@ def run(self):
     super().run()
     dictOptions = {}
     dictOptions['filenameroot'] = self.name
-    if len(self.filename) > 0:
+    if self.filename is not None:
       dictOptions['filenameroot'] = self.filename
     if self.subDirectory is not None:
       dictOptions['filenameroot'] = os.path.join(self.subDirectory,dictOptions['filenameroot'])

tests/framework/Optimizers/GeneticAlgorithms/discrete/unconstrained/testGAMaxwRep.xml

@@ -147,7 +147,7 @@
         <source>opt_export</source>
         <clusterLabel>trajID</clusterLabel>
       </Print>
-      <Plot name="opt_path" subType="OptPath">
+      <Plot name="opt_path" subType="OptPath" overwrite="True">
         <source>opt_export</source>
         <vars>x1,x2,x3,ans</vars>
       </Plot>

tests/framework/Optimizers/GradientDescent/basic.xml

@@ -118,7 +118,7 @@
       <source>opt_export</source>
       <clusterLabel>trajID</clusterLabel>
     </Print>
-    <Plot name="opt_path" subType="OptPath">
+    <Plot name="opt_path" subType="OptPath" overwrite="True">
       <source>opt_export</source>
       <vars>x,y,func,const,ans</vars>
     </Plot>

tests/framework/Optimizers/SimulatedAnnealing/CauchySA/test_simulatedAnnealing_OptPath.xml

@@ -107,7 +107,7 @@
       <source>opt_export</source>
       <clusterLabel>trajID</clusterLabel>
     </Print>
-    <Plot name="opt_path" subType="OptPath">
+    <Plot name="opt_path" subType="OptPath" overwrite="True">
       <source>opt_export</source>
       <vars>x,y,ans</vars>
     </Plot>

tests/framework/OutStreams/ga_population_plot.xml

@@ -39,7 +39,7 @@
   </DataObjects>
 
   <OutStreams>
-    <Plot name="plotter" subType="PopulationPlot">
+    <Plot name="plotter" subType="PopulationPlot"  overwrite="True" dir="OptimizationPlots">
       <source>samples</source>
       <vars>a,b,c,d,ans</vars>
       <logVars>ans</logVars>