Make Xmile work with new translator

pysd/building/python/python_expressions_builder.py

@@ -893,6 +893,7 @@ def __init__(self, lookups_str,  component):
         self.arguments = {}
         self.x = lookups_str.x
         self.y = lookups_str.y
+        self.keyword = lookups_str.type
 
     def build(self, arguments):
         self.component.type = "Lookup"
@@ -908,6 +909,7 @@ def build(self, arguments):
             threshold=len(self.y)
         )
         arguments["subscripts"] = self.def_subs
+        arguments["interp"] = self.keyword
 
         if "hardcoded_lookups" in self.element.objects:
             # object already exists
@@ -926,7 +928,8 @@ def build(self, arguments):
             self.element.objects["hardcoded_lookups"] = {
                 "name": arguments["name"],
                 "expression": "%(name)s = HardcodedLookups(%(x)s, %(y)s, "
-                              "%(subscripts)s, '%(name)s')" % arguments
+                              "%(subscripts)s, '%(interp)s', '%(name)s')"
+                              % arguments
             }
 
             return BuildAST(

pysd/building/python/python_model_builder.py

@@ -505,13 +505,11 @@ def build_element_out(self):
 
         indent = 12
 
+        # convert newline indicator and add expected level of indentation
         self.contents = contents.replace("\n", "\n" + " " * (indent+4))
         self.objects = objects.replace("\n", "\n" + " " * indent)
-
-        # convert newline indicator and add expected level of indentation
-        # TODO check if this is neccessary
         self.documentation = self.documentation.replace(
-            "\\", "\n").replace("\n", "\n" + "" * indent)
+            "\\", "\n").replace("\n", "\n" + " " * indent)
 
         return textwrap.dedent('''
             %(subs_dec)s

pysd/py_backend/lookups.py

@@ -19,12 +19,14 @@ def _call(self, data, x):
             if not x.dims:
                 # shape 0 xarrays
                 return self._call(data, float(x))
-            if np.all(x > data['lookup_dim'].values[-1]):
+            if self.interp != "extrapolate" and\
+               np.all(x > data['lookup_dim'].values[-1]):
                 outdata, _ = xr.broadcast(data[-1], x)
                 warnings.warn(
                   self.py_name + "\n"
                   + "extrapolating data above the maximum value of the series")
-            elif np.all(x < data['lookup_dim'].values[0]):
+            elif self.interp != "extrapolate" and\
+              np.all(x < data['lookup_dim'].values[0]):
                 outdata, _ = xr.broadcast(data[0], x)
                 warnings.warn(
                   self.py_name + "\n"
@@ -43,15 +45,31 @@ def _call(self, data, x):
             if x in data['lookup_dim'].values:
                 outdata = data.sel(lookup_dim=x)
             elif x > data['lookup_dim'].values[-1]:
-                outdata = data[-1]
+                if self.interp == "extrapolate":
+                    # extrapolate method for xmile models
+                    k = (data[-1]-data[-2])\
+                        / (data['lookup_dim'].values[-1]
+                           - data['lookup_dim'].values[-2])
+                    outdata = data[-1] + k*(x - data['lookup_dim'].values[-1])
+                else:
+                    outdata = data[-1]
                 warnings.warn(
                   self.py_name + "\n"
                   + "extrapolating data above the maximum value of the series")
             elif x < data['lookup_dim'].values[0]:
-                outdata = data[0]
+                if self.interp == "extrapolate":
+                    # extrapolate method for xmile models
+                    k = (data[1]-data[0])\
+                        / (data['lookup_dim'].values[1]
+                           - data['lookup_dim'].values[0])
+                    outdata = data[0] + k*(x - data['lookup_dim'].values[0])
+                else:
+                    outdata = data[0]
                 warnings.warn(
                   self.py_name + "\n"
                   + "extrapolating data below the minimum value of the series")
+            elif self.interp == 'hold_backward':
+                outdata = data.sel(lookup_dim=x, method="pad")
             else:
                 outdata = data.interp(lookup_dim=x)
 
@@ -67,7 +85,7 @@ def _call(self, data, x):
 class HardcodedLookups(Lookups):
     """Class for lookups defined in the file"""
 
-    def __init__(self, x, y, coords, py_name):
+    def __init__(self, x, y, coords, interp, py_name):
         # TODO: avoid add and merge all declarations in one definition
         self.is_float = not bool(coords)
         self.py_name = py_name
@@ -78,6 +96,7 @@ def __init__(self, x, y, coords, py_name):
             ["lookup_dim"] + list(coords)
         )
         self.x = set(x)
+        self.interp = interp
 
     def add(self, x, y, coords):
         y = np.array(y).reshape((len(x),) + (1,)*len(coords))

pysd/pysd.py

@@ -71,7 +71,9 @@ def read_xmile(xmile_file, data_files=None, initialize=True, old=False,
         xmile_file_obj.parse()
 
         abs_model = xmile_file_obj.get_abstract_model()
+        #print(abs_model.dump(indent=" "))
         py_model_file = ModelBuilder(abs_model).build_model()
+
     model = load(py_model_file, data_files, initialize, missing_values)
     model.xmile_file = str(xmile_file)
     return model

pysd/translation/structures/abstract_expressions.py

@@ -207,10 +207,11 @@ class LookupsStructure:
     y: tuple
     x_range: tuple
     y_range: tuple
+    type: str
 
     def __str__(self) -> str:
-        return "LookupStructure:\n\tx %s = %s\n\ty %s = %s\n" % (
-            self.x_range, self.x, self.y_range, self.y
+        return "LookupStructure (%s):\n\tx %s = %s\n\ty %s = %s\n" % (
+            self.type, self.x_range, self.x, self.y_range, self.y
         )
 
 

pysd/translation/vensim/vensim_element.py

@@ -323,7 +323,8 @@ def visit_regularLookup(self, n, vc):
             x=tuple(values[:, 0]),
             y=tuple(values[:, 1]),
             x_range=tuple(xy_range[:, 0]),
-            y_range=tuple(xy_range[:, 1])
+            y_range=tuple(xy_range[:, 1]),
+            type="interpolate"
         )
 
     def visit_excelLookup(self, n, vc):
@@ -424,7 +425,8 @@ def visit_lookup_with_def(self, n, vc):
             x=tuple(values[:, 0]),
             y=tuple(values[:, 1]),
             x_range=tuple(xy_range[:, 0]),
-            y_range=tuple(xy_range[:, 1])
+            y_range=tuple(xy_range[:, 1]),
+            type="interpolate"
         )
 
         return self.add_element(structures["with_lookup"](

pysd/translation/xmile/parsing_grammars/equations.peg

@@ -0,0 +1,53 @@
+# Parsing Expression Grammar: components
+
+expr_type = array / final_expr / empty
+
+final_expr = conditional_statement / logic2_expr
+
+logic2_expr = logic_expr (_ logic_oper _ logic_expr)*  # logic operators (:and:, :or:)
+logic_expr = not_oper? _ comp_expr  # :not: operator
+comp_expr = add_expr (_ comp_oper _ add_expr)?  # comparison (e.g. '<', '=>')
+add_expr = prod_expr (_ add_oper _ prod_expr)*  # addition and substraction
+prod_expr = exp_expr (_ prod_oper _ exp_expr)*  # product and division
+exp_expr = neg_expr (_ exp_oper _ neg_expr)*  # exponential
+neg_expr = pre_oper? _ expr  # pre operators (-, +)
+expr = call / parens / number / reference
+
+arguments = ((string / final_expr) _ ","? _)*
+parens   = "(" _ final_expr _ ")"
+
+call = reference _ "(" _ arguments _ ")"
+conditional_statement = "IF" _ logic2_expr _ "THEN"  _ logic2_expr _ "ELSE" _ logic2_expr
+
+reference = (name _ subscript_list) / name  # check first for subscript
+subscript_list = "[" _ (name _ "!"? _ ","? _)+ _ "]"
+
+array = (raw_number _ ("," / ";")? _)+ !~r"."  # negative lookahead for
+
+logic_oper = ~r"(%(logic_ops)s)"IU
+not_oper = ~r"(%(not_ops)s)"IU
+comp_oper = ~r"(%(comp_ops)s)"IU
+add_oper = ~r"(%(add_ops)s)"IU
+prod_oper = ~r"(%(prod_ops)s)"IU
+exp_oper = ~r"(%(exp_ops)s)"IU
+pre_oper = ~r"(%(pre_ops)s)"IU
+
+empty = "" # empty string
+
+_ = spacechar*
+spacechar = " "* ~"\t"*
+
+name = basic_id / escape_group
+
+# This takes care of models with Unicode variable names
+basic_id = id_start id_continue*
+
+id_start = ~r"[\w]"IU
+id_continue = id_start / ~r"[0-9\'\$\_]"
+
+# between quotes, either escaped quote or character that is not a quote
+escape_group = "\"" ( "\\\"" / ~r"[^\"]" )* "\""
+
+number = raw_number
+raw_number = ("+"/"-")? ~r"\d+\.?\d*([eE][+-]?\d+)?"
+string = "\'" (~r"[^\']"IU)* "\'"