[WIP] ND Sampled Variables

doc/workshop/ExternalModels/projectile.py

@@ -83,12 +83,12 @@ def current_angle(v0, ang, vel):
   return np.arccos(v0 * np.cos(ang) / vel)
 
 def run(raven, inputs):
-  vars = {'x0': get_from_raven('x0', raven, 0),
+  inps = {'x0': get_from_raven('x0', raven, 0),
           'y0': get_from_raven('y0', raven, 0),
           'v0': get_from_raven('v0', raven, 1),
           'angle': get_from_raven('angle', raven, 45),
           'timeOption': get_from_raven('timeOption', raven, 0)}
-  res = main(vars)
+  res = main(inps)
   raven.x = res['x']
   raven.y = res['y']
   raven.t = res['t']

doc/workshop/ensembleModels/inputs/ensembleExample/1_ensemble_model.xml

@@ -52,6 +52,7 @@
       </variable>
       <constant name="x0">0</constant>
       <constant name="y0">0</constant>
+      <constant name="v0">1</constant>
     </MonteCarlo>
   </Samplers>
 

ravenframework/BaseClasses/BaseEntity.py

@@ -209,14 +209,16 @@ def provideExpectedMetaKeys(self):
     """
     return self.metadataKeys, self.metadataParams
 
-  def addMetaKeys(self,args, params={}):
+  def addMetaKeys(self,args, params=None):
     """
       Adds keywords to a list of expected metadata keys.
       @ In, args, list(str), keywords to register
       @ In, params, dict, optional, {key:[indexes]}, keys of the dictionary are the variable names,
         values of the dictionary are lists of the corresponding indexes/coordinates of given variable
       @ Out, None
     """
+    if params is None:
+      params = {}
     if any(not mathUtils.isAString(a) for a in args):
       self.raiseAnError('Arguments to addMetaKeys were not all strings:',args)
     self.metadataKeys = self.metadataKeys.union(set(args))

ravenframework/DataObjects/DataSet.py

@@ -218,7 +218,8 @@ def addRealization(self, rlz):
     #         rlz = {'H': [[0.1, 0.2, 0.3], [0.2, 0.3, 0.4]],
     #                'X': [0, 1, 2],
     #                'Y': [0.5, 1.5],
-    #                '_indexMap': ['Y', 'X']}
+    #                '_indexMap': [{'H': ['Y', 'X']}]
+    #         }
     #         Note the order, H has shape (2, 3) so the first index is Y and the second is X.
     #         A sanity check is that H.shape == tuple(var.size for var in rlz['_indexMap'])
     #
@@ -638,7 +639,7 @@ def realization(self, index=None, matchDict=None, noMatchDict=None, tol=1e-15, u
           rlzs = [self._addIndexMapToRlz(rl) for rl in rlzs]
           dims = self.getDimensions()
           for index, rl in enumerate(rlzs):
-            d = {k:{'dims':tuple(dims[k]) ,'data': v} for (k,v) in rl.items()}
+            d = {k:{'dims':tuple(dims[k]) ,'data': v} for (k,v) in rl.items() if k not in ['_indexMap']}
             rlz[index] =  xr.Dataset.from_dict(d)
           if len(rlzs) > 1:
             # concatenate just in case there are multiple realizations
@@ -860,6 +861,7 @@ def vars(self):
       @ In, None
       @ Out, vars, list(str), variable names list
     """
+    # TODO should this include indexes as well??
     return self._inputs + self._outputs + self._metavars
 
   @property

ravenframework/DataObjects/HistorySet.py

@@ -15,23 +15,12 @@
   Specialized implementation of DataSet to accomodate outputs that share a pivot parameter (e.g. time)
 """
 import os
-import sys
-import copy
-import functools
 import itertools
-try:
-  import cPickle as pk
-except ImportError:
-  import pickle as pk
 import xml.etree.ElementTree as ET
 
-import abc
 import numpy as np
-import pandas as pd
-import xarray as xr
 
-from ..BaseClasses import BaseType
-from ..utils import utils, cached_ndarray, InputData, xmlUtils, mathUtils
+from ..utils import mathUtils
 try:
   from .DataSet import DataSet
 except ValueError: #attempted relative import in non-package

ravenframework/Functions.py

@@ -18,20 +18,11 @@
 
 This module contains interfaces to import external functions
 """
-#for future compatibility with Python 3--------------------------------------------------------------
-from __future__ import division, print_function, absolute_import
-# WARNING if you import unicode_literals here, we fail tests (e.g. framework.testFactorials).  This may be a future-proofing problem. 2015-04.
-#End compatibility block for Python 3----------------------------------------------------------------
-
-#External Modules------------------------------------------------------------------------------------
-#External Modules End--------------------------------------------------------------------------------
-
-#Internal Modules------------------------------------------------------------------------------------
 from .EntityFactoryBase import EntityFactory
 from .BaseClasses import BaseEntity, InputDataUser
 from .utils import utils, InputData, InputTypes
 from .CustomCommandExecuter import execCommand
-#Internal Modules End--------------------------------------------------------------------------------
+from .Realizations import Realization
 
 class FunctionCollection(InputData.ParameterInput):
   """
@@ -173,28 +164,24 @@ def __importValues(self,myInput):
       @ In, myInput, object (dataObjects,dict), object from which the data need to be imported
       @ Out, None
     """
-    if isinstance(myInput,dict):
+    if isinstance(myInput, (dict, Realization)):
       self.__inputFromWhat['dict'](myInput)
     else:
       self.raiseAnError(IOError,'Unknown type of input provided to the function '+str(self.name))
 
-  def __inputFromDict(self,myInputDict):
+  def __inputFromDict(self, inDict):
     """
       This is meant to be used to collect the input directly from a sampler generated input or simply from a generic dictionary
-      In case the input comes from a sampler the expected structure is myInputDict['SampledVars'][variable name] = value
-      In case it is a generic dictionary the expected structure is myInputDict[variable name] = value
-      @ In, myInputDict, dict, dict from which the data need to be imported
+      The expected structure is inDict[variable name] = value
+      @ In, inDict, dict, dict from which the data need to be imported
       @ Out, None
     """
-    if 'SampledVars' in myInputDict.keys():
-      inDict = myInputDict['SampledVars']
-    else:
-      inDict = myInputDict
     for name in self.__inputVariables:
-      if name in inDict.keys():
+      if name in inDict:
+        # FIXME this doesn't seem secure, and it's not even clear how it works ...
         execCommand('self.'+name+'=object["'+name+'"]',self=self,object=inDict)
       else:
-        self.raiseAnError(IOError,'The input variable '+name+' in external function seems not to be passed in')
+        self.raiseAnError(IOError,f'The input variable "{name}" in external function missing!')
 
   def evaluate(self,what,myInput):
     """

ravenframework/JobHandler.py

@@ -686,7 +686,36 @@ def startLoop(self):
       # probably when we move to Python 3.
       time.sleep(self.sleepTime)
 
-  def addJob(self, args, functionToRun, identifier, metadata=None, forceUseThreads = False, uniqueHandler="any", clientQueue = False, groupInfo = None):
+  def addJobBatch(self, batch, model, modelInput, samplerType, evalFunc):
+    """
+      Adds a batch of jobs to the internal queue.
+      @ In, batch, RealizationBatch, set of realizations to add
+      @ In, model, Model, model instance to run
+      @ In, modelInput, list, inputs for the Model
+      @ In, samplerType, str, sampler that generated this request
+      @ In, evalFunc, callable, method to be executed
+      @ Out, None
+    """
+    # TODO register batch to fill later?
+    for rlz in batch:
+      if rlz.isRestart:
+        self.addFinishedJob(rlz, metadata=rlz.inputInfo)
+      else:
+        # assure the realization knows about the batch it belongs to
+        rlz.inputInfo['batchID'] = batch.ID
+        self.addSingleJob(
+            (model, modelInput, samplerType, rlz),
+            evalFunc,
+            rlz.inputInfo['prefix'],
+            metadata = rlz.inputInfo,
+            uniqueHandler=rlz.inputInfo.get('uniqueHandler', 'any'),
+            forceUseThreads=rlz.inputInfo.get('forceThreads', False),
+            groupInfo={'id': batch.ID, 'size': len(batch)}
+        )
+
+  def addSingleJob(self, args, functionToRun, identifier, metadata=None,
+             forceUseThreads=False, uniqueHandler="any", clientQueue=False,
+             groupInfo=None):
     """
       Method to add an internal run (function execution)
       @ In, args, dict, this is a list of arguments that will be passed as
@@ -776,7 +805,8 @@ def reAddJob(self, runner):
         runner.trackTime('queue')
       self.__submittedJobs.append(runner.identifier)
 
-  def addClientJob(self, args, functionToRun, identifier, metadata=None, uniqueHandler="any", groupInfo = None):
+  def addClientJob(self, args, functionToRun, identifier,
+                   metadata=None, uniqueHandler="any", groupInfo=None):
     """
       Method to add an internal run (function execution), without consuming
       resources (free spots). This can be used for client handling (see
@@ -802,10 +832,9 @@ def addClientJob(self, args, functionToRun, identifier, metadata=None, uniqueHan
               Consequentially the size is immutable
       @ Out, None
     """
-    self.addJob(args, functionToRun, identifier, metadata,
-                forceUseThreads = True, uniqueHandler = uniqueHandler,
-                clientQueue = True, groupInfo = groupInfo)
-
+    self.addSingleJob(args, functionToRun, identifier, metadata,
+                forceUseThreads=True, uniqueHandler=uniqueHandler,
+                clientQueue=True, groupInfo=groupInfo)
 
   def addFinishedJob(self, data, metadata=None, uniqueHandler="any", profile=False):
     """