Merge pull request #304 from enekomartinmartinez/encoding-files

Solve Windows bugs and CI

.github/workflows/ci.yml

@@ -6,11 +6,10 @@ on: [push, pull_request]
 
 jobs:
   test:
-    #runs-on: ${{ matrix.os }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        #os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, windows-latest]
         python-version: [3.7, 3.9]
 
     steps:
@@ -26,7 +25,7 @@ jobs:
       - name: Test and coverage
         run: pytest tests/ --cov=pysd
       - name: Coveralls
-        if: matrix.python-version == 3.7
+        if: ${{ matrix.python-version == 3.7 && matrix.os == 'ubuntu-latest' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: coveralls --service=github

pysd/_version.py

@@ -1 +1 @@
-__version__ = "2.1.1"
+__version__ = "2.2.0"

pysd/py_backend/external.py

@@ -5,8 +5,8 @@
 """
 
 import re
-import os
 import warnings
+from pathlib import Path
 import pandas as pd  # TODO move to openpyxl
 import numpy as np
 import xarray as xr
@@ -26,15 +26,15 @@ def read(cls, file_name, sheet_name):
         """
         Read the Excel file or return the previously read one
         """
-        if file_name + sheet_name in cls._Excels:
-            return cls._Excels[file_name + sheet_name]
+        if file_name.joinpath(sheet_name) in cls._Excels:
+            return cls._Excels[file_name.joinpath(sheet_name)]
         else:
             excel = np.array([
                 pd.to_numeric(ex, errors='coerce')
                 for ex in
                 pd.read_excel(file_name, sheet_name, header=None).values
                 ])
-            cls._Excels[file_name + sheet_name] = excel
+            cls._Excels[file_name.joinpath(sheet_name)] = excel
             return excel
 
     @classmethod
@@ -110,7 +110,7 @@ def _get_data_from_file(self, rows, cols):
 
         """
         # TODO move to openpyxl to avoid pandas dependency in this file.
-        ext = os.path.splitext(self.file)[1].lower()
+        ext = self.file.suffix.lower()
         if ext in ['.xls', '.xlsx']:
             # read data
             data = Excels.read(
@@ -314,7 +314,7 @@ def _resolve_file(self, root):
 
         Parameters
         ----------
-        root: str
+        root: pathlib.Path or str
             The root path to the model file.
 
         Returns
@@ -328,12 +328,17 @@ def _resolve_file(self, root):
                 self.py_name + "\n"
                 + f"Indirect reference to file: {self.file}")
 
-        self.file = os.path.join(root, self.file)
+        if isinstance(root, str):  # pragma: no cover
+            # backwards compatibility
+            # TODO: remove with PySD 3.0.0
+            root = Path(root)
 
-        if not os.path.isfile(self.file):
+        self.file = root.joinpath(self.file)
+
+        if not self.file.is_file():
             raise FileNotFoundError(
                 self.py_name + "\n"
-                + f"File '{self.file}' not found.")
+                + "File '%s' not found." % self.file)
 
     def _initialize_data(self, element_type):
         """

pysd/py_backend/statefuls.py

@@ -609,7 +609,7 @@ def __init__(self, py_model_file, params=None, return_func=None,
         self.cache = Cache()
         self.py_name = py_name
         self.external_loaded = False
-        self.components = Components(py_model_file, self.set_components)
+        self.components = Components(str(py_model_file), self.set_components)
 
         if __version__.split(".")[0]\
            != self.get_pysd_compiler_version().split(".")[0]:
@@ -664,7 +664,7 @@ def __init__(self, py_model_file, params=None, return_func=None,
         else:
             self.return_func = lambda: 0
 
-        self.py_model_file = py_model_file
+        self.py_model_file = str(py_model_file)
 
     def __call__(self):
         return self.return_func()
@@ -702,7 +702,7 @@ def _get_initialize_order(self):
         self.stateful_initial_dependencies = {
             ext: set()
             for ext in self.components._dependencies
-            if ext.startswith("_")
+            if (ext.startswith("_") and not ext.startswith("_active_initial_"))
         }
         for element in self.stateful_initial_dependencies:
             self._get_full_dependencies(
@@ -712,8 +712,10 @@ def _get_initialize_order(self):
         # get the full dependencies of stateful objects taking into account
         # only other objects
         current_deps = {
-            element: [dep for dep in deps if dep.startswith("_")]
-            for element, deps in self.stateful_initial_dependencies.items()
+            element: [
+                dep for dep in deps
+                if dep in self.stateful_initial_dependencies
+            ] for element, deps in self.stateful_initial_dependencies.items()
         }
 
         # get initialization order of the stateful elements
@@ -887,6 +889,7 @@ def _isdynamic(self, dependencies):
             return True
         for dep in dependencies:
             if dep.startswith("_") and not dep.startswith("_initial_")\
+               and not dep.startswith("_active_initial_")\
                and not dep.startswith("__"):
                 return True
         return False

pysd/py_backend/utils.py

@@ -4,7 +4,6 @@
 functions.py
 """
 
-import os
 import json
 from pathlib import Path
 from chardet.universaldetector import UniversalDetector
@@ -326,15 +325,15 @@ def rearrange(data, dims, coords):
     return None
 
 
-def load_model_data(root_dir, model_name):
+def load_model_data(root, model_name):
 
     """
     Used for models split in several files.
     Loads subscripts_dic, namespace and modules dictionaries
 
     Parameters
     ----------
-    root_dir: str
+    root: pathlib.Path or str
         Path to the model file.
 
     model_name: str
@@ -355,22 +354,22 @@ def load_model_data(root_dir, model_name):
         corresponding variables as values.
 
     """
+    if isinstance(root, str):  # pragma: no cover
+        # backwards compatibility
+        # TODO: remove with PySD 3.0.0
+        root = Path(root)
 
-    with open(os.path.join(root_dir, "_subscripts_" + model_name + ".json")
-              ) as subs:
+    with open(root.joinpath("_subscripts_" + model_name + ".json")) as subs:
         subscripts = json.load(subs)
 
-    with open(os.path.join(root_dir, "_namespace_" + model_name + ".json")
-              ) as names:
+    with open(root.joinpath("_namespace_" + model_name + ".json")) as names:
         namespace = json.load(names)
-    with open(os.path.join(root_dir, "_dependencies_" + model_name + ".json")
-              ) as deps:
+    with open(root.joinpath("_dependencies_" + model_name + ".json")) as deps:
         dependencies = json.load(deps)
 
     # the _modules.json in the sketch_var folder shows to which module each
     # variable belongs
-    with open(os.path.join(root_dir, "modules_" + model_name, "_modules.json")
-              ) as mods:
+    with open(root.joinpath("modules_" + model_name, "_modules.json")) as mods:
         modules = json.load(mods)
 
     return namespace, subscripts, dependencies, modules
@@ -408,14 +407,20 @@ def load_modules(module_name, module_content, work_dir, submodules):
         model file.
 
     """
+    if isinstance(work_dir, str):  # pragma: no cover
+        # backwards compatibility
+        # TODO: remove with PySD 3.0.0
+        work_dir = Path(work_dir)
+
     if isinstance(module_content, list):
-        with open(os.path.join(work_dir, module_name + ".py"), "r") as mod:
+        with open(work_dir.joinpath(module_name + ".py"), "r",
+                  encoding="UTF-8") as mod:
             submodules.append(mod.read())
     else:
         for submod_name, submod_content in module_content.items():
             load_modules(
                 submod_name, submod_content,
-                os.path.join(work_dir, module_name),
+                work_dir.joinpath(module_name),
                 submodules)
 
     return "\n\n".join(submodules)

pysd/pysd.py

@@ -68,7 +68,8 @@ def read_xmile(xmile_file, data_files=None, initialize=True,
 
 
 def read_vensim(mdl_file, data_files=None, initialize=True,
-                missing_values="warning", split_views=False, **kwargs):
+                missing_values="warning", split_views=False,
+                encoding=None, **kwargs):
     """
     Construct a model from Vensim `.mdl` file.
 
@@ -99,6 +100,11 @@ def read_vensim(mdl_file, data_files=None, initialize=True,
         file. Setting this argument to True is recommended for large
         models split in many different views. Default is False.
 
+    encoding: str or None (optional)
+        Encoding of the source model file. If None, the encoding will be
+        read from the model, if the encoding is not defined in the model
+        file it will be set to 'UTF-8'. Default is None.
+
     **kwargs: (optional)
         Additional keyword arguments for translation.
         subview_sep: list
@@ -120,9 +126,9 @@ def read_vensim(mdl_file, data_files=None, initialize=True,
     """
     from .translation.vensim.vensim2py import translate_vensim
 
-    py_model_file = translate_vensim(mdl_file, split_views, **kwargs)
+    py_model_file = translate_vensim(mdl_file, split_views, encoding, **kwargs)
     model = load(py_model_file, data_files, initialize, missing_values)
-    model.mdl_file = mdl_file
+    model.mdl_file = str(mdl_file)
     return model
 
 
@@ -158,4 +164,4 @@ def load(py_model_file, data_files=None, initialize=True,
     >>> model = load('../tests/test-models/samples/teacup/teacup.py')
 
     """
-    return Model(str(py_model_file), data_files, initialize, missing_values)
+    return Model(py_model_file, data_files, initialize, missing_values)