From 85783cf811568dae931d06d3f434a34424599817 Mon Sep 17 00:00:00 2001 From: Paulo Meira <10246101+PMeira@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:40:56 -0300 Subject: [PATCH] Initial Oddie integration, including updates to tests --- dss/IBus.py | 7 ++ dss/ICircuit.py | 14 +-- dss/IDSS.py | 10 +- dss/Oddie.py | 84 ++++++++++++++++ dss/__init__.py | 4 +- dss/_cffi_api_util.py | 11 ++- pyproject.toml | 2 +- tests/_settings.py | 45 +++++++-- tests/compare_outputs.py | 88 ++++++++++++++++- tests/save_outputs.py | 139 +++++++++++++++++++++------ tests/test_ctrlqueue.py | 7 +- tests/test_general.py | 196 +++++++++++++++++++++++++++++--------- tests/test_past_issues.py | 25 ++--- 13 files changed, 514 insertions(+), 118 deletions(-) create mode 100644 dss/Oddie.py diff --git a/dss/IBus.py b/dss/IBus.py index bfe36a6c..3f1b6104 100644 --- a/dss/IBus.py +++ b/dss/IBus.py @@ -463,6 +463,13 @@ def __call__(self, index: Union[int, str]) -> IBus: return self.__getitem__(index) def __iter__(self) -> Iterator[IBus]: + if self._api_util._is_odd: + for i in range(self._lib.Circuit_Get_NumBuses()): + self._check_for_error(self._lib.Circuit_SetActiveBusi(i)) + yield self + + return + n = self._check_for_error(self._lib.Circuit_SetActiveBusi(0)) while n == 0: yield self diff --git a/dss/ICircuit.py b/dss/ICircuit.py index f03479ad..0f2e1811 100644 --- a/dss/ICircuit.py +++ b/dss/ICircuit.py @@ -206,14 +206,14 @@ def __init__(self, api_util): self.PVSystems = IPVSystems(api_util) self.Vsources = IVsources(api_util) self.LineCodes = ILineCodes(api_util) - self.LineGeometries = ILineGeometries(api_util) - self.LineSpacings = ILineSpacings(api_util) - self.WireData = IWireData(api_util) - self.CNData = ICNData(api_util) - self.TSData = ITSData(api_util) - self.Reactors = IReactors(api_util) + self.LineGeometries = ILineGeometries(api_util) if not api_util._is_odd else None + self.LineSpacings = ILineSpacings(api_util) if not api_util._is_odd else None + self.WireData = IWireData(api_util) if not api_util._is_odd else None + self.CNData = ICNData(api_util) if not api_util._is_odd else None + self.TSData = ITSData(api_util) if not api_util._is_odd else None + self.Reactors = IReactors(api_util) if not api_util._is_odd else None self.ReduceCkt = IReduceCkt(api_util) #: Circuit Reduction Interface - self.Storages = IStorages(api_util) + self.Storages = IStorages(api_util) if not api_util._is_odd else None self.GICSources = IGICSources(api_util) if hasattr(api_util.lib, 'Parallel_CreateActor'): diff --git a/dss/IDSS.py b/dss/IDSS.py index e4984046..201a4826 100644 --- a/dss/IDSS.py +++ b/dss/IDSS.py @@ -134,14 +134,14 @@ def __init__(self, api_util): self.Executive = IDSS_Executive(api_util) #: Kept for compatibility. - self.Events = IDSSEvents(api_util) + self.Events = IDSSEvents(api_util) if not api_util._is_odd else None #: Kept for compatibility. self.Parser = IParser(api_util) #: Kept for compatibility. Apparently was used for DSSim-PC (now OpenDSS-G), a #: closed-source software developed by EPRI using LabView. - self.DSSim_Coms = IDSSimComs(api_util) + self.DSSim_Coms = IDSSimComs(api_util) if not api_util._is_odd else None #: The YMatrix interface provides advanced access to the internals of #: the DSS engine. The sparse admittance matrix of the system is also @@ -157,7 +157,7 @@ def __init__(self, api_util): #: and run scripts inside the ZIP, without creating extra files on disk. #: #: **(API Extension)** - self.ZIP = IZIP(api_util) + self.ZIP = IZIP(api_util) if not api_util._is_odd else None Base.__init__(self, api_util) @@ -444,6 +444,10 @@ def NewContext(self) -> IDSS: **(API Extension)** ''' + + if self._api_util._is_odd: + raise NotImplementedError("NewContext is not supported for the official OpenDSS engine.") + ffi = self._api_util.ffi lib = self._api_util.lib_unpatched new_ctx = ffi.gc(lib.ctx_New(), lib.ctx_Dispose) diff --git a/dss/Oddie.py b/dss/Oddie.py new file mode 100644 index 00000000..af6d2c93 --- /dev/null +++ b/dss/Oddie.py @@ -0,0 +1,84 @@ +from __future__ import annotations +from typing import Optional +from ._cffi_api_util import CffiApiUtil +from .IDSS import IDSS +from enum import Flag + +class OddieOptions(Flag): + MapErrors = 0x01 + + +class IOddieDSS(IDSS): + r''' + The OddieDSS class exposes the official OpenDSSDirect.DLL binary, + as distributed by EPRI, with the same API as the DSS-Python and + the official COM interface object on Windows. It uses AltDSS Oddie + to achieve this. + + **Note:** This class requires the backend for Oddie to be included in + the `dss_python_backend` package. If it is not available, an import + error should occur when trying to use this. + + AltDSS Oddie wraps OpenDSSDirect.DLL, providing a minimal compatiliby layer + to expose it with the same API as AltDSS/DSS C-API. With it, we can + just reuse most of the tools from the other projects on DSS-Extensions + without too much extra work. + + Note that many functions from DSS-Extensions will not be available and/or + will return errors, and this is expected. There are some issues and/or + limitations from OpenDSSDirect.DLL that may or may not affect specific + use cases; check the documentation on https://dss-extensions.org for + more information. + + :param library_path: The name or full path of the target dynamic library to + load. Defaults to trying to load "OpenDSSDirect" from `c:\Program Files\OpenDSS\x64`, + followed by trying to load it from the current path. + + :param load_flags: Optional, flags to feed the [`LoadLibrary`](https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexa) + (on Windows) or `dlopen` (on Linux and macOS, whenever EPRI supports it). If not provided, a default set will be used. + + :param oddie_options: Optional, Oddie configuration flags. If none passed, + the default settings (recommended) will be used. For advanced users. + ''' + + def __init__(self, library_path: str = '', load_flags: Optional[int] = None, oddie_options: Optional[OddieOptions] = None): + from dss_python_backend import _altdss_oddie_capi + lib = _altdss_oddie_capi.lib + ffi = _altdss_oddie_capi.ffi + NULL = ffi.NULL + + c_load_flags = NULL + if load_flags is not None: + c_load_flags = ffi.new('uint32_t*', load_flags) + + if library_path: + library_path = library_path.encode() + lib.Oddie_SetLibOptions(library_path, c_load_flags) + ctx = lib.ctx_New() + else: + # Try the default install folder + library_path = rb'C:\Program Files\OpenDSS\x64\OpenDSSDirect.dll' + lib.Oddie_SetLibOptions(library_path, c_load_flags) + ctx = lib.ctx_New() + if ctx == NULL: + # Try from the general path, let the system resolve it + library_path = rb'OpenDSSDirect.dll' + lib.Oddie_SetLibOptions(library_path, c_load_flags) + ctx = lib.ctx_New() + + if ctx == NULL: + raise RuntimeError("Could not load the target library.") + + if lib.ctx_DSS_Start(ctx, 0) != 1: + raise RuntimeError("DSS_Start call was not successful.") + + if oddie_options is not None: + lib.Oddie_SetOptions(oddie_options) + + ctx = ffi.gc(ctx, lib.ctx_Dispose) + api_util = CffiApiUtil(ffi, lib, ctx, is_odd=True) + api_util._library_path = library_path + IDSS.__init__(self, api_util) + + +__all__ = ['IOddieDSS', 'OddieOptions'] \ No newline at end of file diff --git a/dss/__init__.py b/dss/__init__.py index 8379e5e5..07934c52 100644 --- a/dss/__init__.py +++ b/dss/__init__.py @@ -17,8 +17,8 @@ lib.DSS_SetPropertiesMO(_properties_mo.encode()) from ._cffi_api_util import CffiApiUtil, DSSException, set_case_insensitive_attributes -# from .altdss import Edit from .IDSS import IDSS +from .Oddie import IOddieDSS, OddieOptions from .enums import * DssException = DSSException @@ -44,4 +44,4 @@ except: __version__ = '0.0dev' -__all__ = ['dss', 'DSS', 'DSS_GR', 'prime_api_util', 'api_util', 'DSSException', 'patch_dss_com', 'set_case_insensitive_attributes', 'enums', 'Edit'] \ No newline at end of file +__all__ = ['dss', 'DSS', 'DSS_GR', 'prime_api_util', 'api_util', 'DSSException', 'patch_dss_com', 'set_case_insensitive_attributes', 'enums', 'IOddieDSS', 'OddieOptions'] \ No newline at end of file diff --git a/dss/_cffi_api_util.py b/dss/_cffi_api_util.py index c12f8fcc..2e906aa1 100644 --- a/dss/_cffi_api_util.py +++ b/dss/_cffi_api_util.py @@ -334,7 +334,8 @@ class CffiApiUtil(object): ''' _ctx_to_util = WeakKeyDictionary() - def __init__(self, ffi, lib, ctx=None): + def __init__(self, ffi, lib, ctx=None, is_odd=False): + self._is_odd = is_odd self.owns_ctx = True self.codec = codec self.ctx = ctx @@ -466,18 +467,26 @@ def clear_callback(self, step: int): def register_callbacks(self): + if self._is_odd: + return + mgr = get_manager_for_ctx(self.ctx) # if multiple calls, the extras are ignored mgr.register_func(AltDSSEvent.Clear, altdss_python_util_callback) mgr.register_func(AltDSSEvent.ReprocessBuses, altdss_python_util_callback) def unregister_callbacks(self): + if self._is_odd: + return mgr = get_manager_for_ctx(self.ctx) mgr.unregister_func(AltDSSEvent.Clear, altdss_python_util_callback) mgr.unregister_func(AltDSSEvent.ReprocessBuses, altdss_python_util_callback) # The context will die, no need to do anything else currently. def __del__(self): + if self._is_odd: + return + self.clear_callback(0) self.clear_callback(1) self.unregister_callbacks() diff --git a/pyproject.toml b/pyproject.toml index 5f6bb229..6a38eb75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ packages = ["dss"] name = "dss-python" dynamic = ["version"] dependencies = [ - "dss_python_backend==0.14.5", + "dss_python_backend==0.14.6a1", "numpy>=1.21.0", "typing_extensions>=4.5,<5", ] diff --git a/tests/_settings.py b/tests/_settings.py index 9e58cb77..f448eec7 100644 --- a/tests/_settings.py +++ b/tests/_settings.py @@ -1,5 +1,24 @@ + import sys, os +import faulthandler +faulthandler.disable() +from dss import DSS, IOddieDSS +faulthandler.enable() + +org_dir = os.getcwd() + +USE_ODDIE = os.getenv('DSS_PYTHON_ODDIE', None) +if USE_ODDIE: + # print("Using Oddie:", USE_ODDIE) + if USE_ODDIE != '1': + DSS = IOddieDSS(USE_ODDIE) + else: + DSS = IOddieDSS() + + os.chdir(org_dir) + + WIN32 = (sys.platform == 'win32') if os.path.exists('../../electricdss-tst/'): BASE_DIR = os.path.abspath('../../electricdss-tst/') @@ -25,6 +44,21 @@ #"L!Distrib/IEEETestCases/4wire-Delta/Kersting4wireIndMotor.dss", test_filenames = ''' +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-csv-p.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-csv-pq.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-dbl-p.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-sng-p.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-txt-p.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-mm-txt-pq.dss +Version8/Distrib/Examples/MemoryMappingLoadShapes/ckt24/master_ckt24-nomm.dss +Version8/Distrib/IEEETestCases/4Bus-DY-Bal/4Bus-DY-Bal.DSS +Version8/Distrib/IEEETestCases/4Bus-GrdYD-Bal/4Bus-GrdYD-Bal.DSS +Version8/Distrib/IEEETestCases/4Bus-OYOD-Bal/4Bus-OYOD-Bal.DSS +Version8/Distrib/IEEETestCases/4Bus-OYOD-UnBal/4bus-OYOD-UnBal.dss +Version8/Distrib/IEEETestCases/4Bus-YD-Bal/4Bus-YD-Bal.DSS +Version8/Distrib/IEEETestCases/4Bus-YY-Bal/4Bus-YY-Bal.DSS +Version8/Distrib/IEEETestCases/4Bus-YYD/YYD-Master.DSS +Version8/Distrib/IEEETestCases/13Bus/IEEE13Nodeckt.dss Test/CapControlFollow.dss Test/CapacitorConfigs.dss Test/ReactorConfigs.dss @@ -47,14 +81,6 @@ L!Version8/Distrib/IEEETestCases/123Bus/IEEE123Master.dss Version8/Distrib/IEEETestCases/37Bus/ieee37.dss Version8/Distrib/IEEETestCases/IEEE 30 Bus/Master.dss -Version8/Distrib/IEEETestCases/4Bus-DY-Bal/4Bus-DY-Bal.DSS -Version8/Distrib/IEEETestCases/4Bus-GrdYD-Bal/4Bus-GrdYD-Bal.DSS -Version8/Distrib/IEEETestCases/4Bus-OYOD-Bal/4Bus-OYOD-Bal.DSS -Version8/Distrib/IEEETestCases/4Bus-OYOD-UnBal/4bus-OYOD-UnBal.dss -Version8/Distrib/IEEETestCases/4Bus-YD-Bal/4Bus-YD-Bal.DSS -Version8/Distrib/IEEETestCases/4Bus-YY-Bal/4Bus-YY-Bal.DSS -Version8/Distrib/IEEETestCases/4Bus-YYD/YYD-Master.DSS -Version8/Distrib/IEEETestCases/13Bus/IEEE13Nodeckt.dss Test/IEEE13_LineSpacing.dss Test/IEEE13_LineGeometry.dss Test/IEEE13_Assets.dss @@ -105,6 +131,9 @@ L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_IEEE8500/Run_8500Node_GFMDailySmallerPV.dss L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_IEEE8500/Run_8500Node_GFMSnap.dss L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_IEEE8500/Run_8500Node_Unbal.dss +L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_AmpsLimit_123/Run_IEEE123Bus_GFMDaily.DSS +L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_AmpsLimit_123/Run_IEEE123Bus_GFMDailySwapRef.DSS +L!Version8/Distrib/Examples/Microgrid/GridFormingInverter/GFM_AmpsLimit_123/Run_IEEE123Bus_GFMSnap.DSS L!Version8/Distrib/IEEETestCases/123Bus/RevRegTest.dss L!Version8/Distrib/Examples/IBRDynamics_Cases/GFM_IEEE123/Run_IEEE123Bus_GFMDaily.DSS L!Version8/Distrib/Examples/IBRDynamics_Cases/GFM_IEEE123/Run_IEEE123Bus_GFMDaily_CannotPickUpLoad.DSS diff --git a/tests/compare_outputs.py b/tests/compare_outputs.py index cd8ed383..7600644e 100644 --- a/tests/compare_outputs.py +++ b/tests/compare_outputs.py @@ -8,6 +8,12 @@ import xmldiff.main, xmldiff.actions import pandas as pd + +try: + import colored_traceback.auto +except: + pass + np.set_printoptions(linewidth=300) BAD_PI = 3.14159265359 # from OpenDSS source code @@ -167,6 +173,27 @@ def element_compare(self, va, vb, path): def compare(self, a, b, org_path=None): for k in a.keys(): path = org_path + [k] + + if k in ('Monitors', 'IrradianceNow', ): + continue + + if len(path) > 2 and tuple(path[-2:]) in [('Topology', 'AllIsolatedBranches'), ('Topology', 'AllIsolatedLoads')]: + continue + + if len(path) > 3 and tuple(path[-3:]) in [ + ('PVSystems', 'metadata', 'AllPropertyNames'), + ('Relays', 'metadata', 'AllPropertyNames'), + ('Reclosers', 'metadata', 'AllPropertyNames'), + ]: + continue + + if len(path) > 5 and (path[-5], path[-4], path[-2], path[-1]) in [ + ('PVSystems', 'records', 'ActiveCktElement', 'NumProperties'), + ('Relays', 'records', 'ActiveCktElement', 'NumProperties'), + ('Generators', 'records', 'ActiveCktElement', 'NumProperties'), + ]: + continue + if tuple(path) in KNOWN_COM_DIFF: continue @@ -182,6 +209,9 @@ def compare(self, a, b, org_path=None): if k in ('TotalMiles', 'FaultRate', 'pctPermanent'): continue # TODO: investigate + if k == 'Sensor': + continue # TODO: Seems broken in COM, investigate + if k == 'AutoBusList': continue # buggy? @@ -196,6 +226,10 @@ def compare(self, a, b, org_path=None): ): continue + if tuple(path[-2:]) == ('ActiveCktElement', 'EnergyMeter'): + if (va, vb) == ('', '0') or (vb, va) == ('', '0'): + continue + if isinstance(va, dict): # recursive compare self.compare(va, vb, path) @@ -223,7 +257,11 @@ def compare(self, a, b, org_path=None): vb.remove('Rneut') vb.remove('Xneut') + if len(va) != len(vb) and k != 'ZIPV': + if k in ('Yprim', 'ZscMatrix', 'YscMatrix') and len(vb) in [0, 1, 2] and len(va) in [0, 1, 2] and (va == [0, 0] or va == [0] or va == []) and (vb == [0, 0] or vb == [0] or vb == []): + continue + if k == 'dblFreq': if va == [0.0]: #TODO: may be worth adjusting in DSS C-API assert all(x == 0 for x in vb), (path, va, vb) @@ -275,7 +313,15 @@ def compare(self, a, b, org_path=None): elif (k == 'YCurrents' or k.startswith('Cplx') or is_complex(path)) and len(va) % 2 == 0: va = va.view(dtype=complex) - vb = vb.view(dtype=complex) + try: + vb = vb.view(dtype=complex) + except: + print('[ISSUE: COMPLEX SIZE MISMATCH]', k, vb) + if vb == [0.0]: + vb = np.array([0.0], dtype=complex) + else: + raise + rtol = 1e-3 if 'Seq' in k: @@ -287,6 +333,9 @@ def compare(self, a, b, org_path=None): # abs(b - a) <= (atol + rtol * abs(a)) if len(vb) != len(va): + if k == 'ZIPV' and (len(va), len(vb)) in ((1, 7), (7, 1)): + continue + self.printe('ERROR (vector, shapes):', path, f'a: {len(va)}, b: {len(vb)}') continue @@ -346,7 +395,8 @@ def compare_all(self): fA = zipA.open(fn, 'r') except KeyError: if not fn.endswith('GISCoords.dss'): - print('MISSING:', fn) + # print('MISSING:', fn) + pass continue except BadZipFile: @@ -369,6 +419,8 @@ def compare_all(self): self.B_IS_COM = 'C-API' not in dataB['DSS']['Version'] try: self.compare(dataA, dataB, [fn]) + if not self.per_file[fn]: + print('FILE OK:', fn) except: print("COMPARE ERROR:", fn) raise @@ -406,6 +458,9 @@ def compare_all(self): if not ENABLE_CSV: continue + if 'yprim' in fn.lower(): + continue + print(fn) # The CSVs from OpenDSS can havbe some weird header, and we need to compare @@ -413,18 +468,41 @@ def compare_all(self): textA = fA.read().decode().lower() textB = fB.read().decode().lower() with io.StringIO(textA) as sfA, io.StringIO(textB) as sfB: - df_a = pd.read_csv(sfA) + try: + df_a = pd.read_csv(sfA) + except pd.errors.EmptyDataError: + continue + df_b = pd.read_csv(sfB) df_a.columns = [x.strip() for x in df_a.columns] df_b.columns = [x.strip() for x in df_b.columns] + if ('freq (hz)' in df_a.columns) != ('freq (hz)' in df_b.columns): + print("INFO: CSV: ignoring extra column 'freq (hz)'") + if 'freq (hz)' in df_a.columns: + del df_a['freq (hz)'] + if 'freq (hz)' in df_b.columns: + del df_b['freq (hz)'] + + for df in (df_a, df_b): + if len(df.columns): + col = df.columns[-1] + if col.startswith('Unnamed: '): + del df[col] + try: - pd.testing.assert_frame_equal(df_a, df_b, atol=tol, rtol=tol) + pd.testing.assert_frame_equal(df_a, df_b, atol=tol, rtol=tol, check_dtype=False) except: print("COMPARE CSV ERROR:", fn) + print(df_a.columns) + print(df_b.columns) + from traceback import print_exc + from sys import stdout + print_exc(file=stdout) + print() - raise + # raise self.total += 1 diff --git a/tests/save_outputs.py b/tests/save_outputs.py index 626c5e10..330c8aaf 100644 --- a/tests/save_outputs.py +++ b/tests/save_outputs.py @@ -2,19 +2,27 @@ from glob import glob from inspect import ismethod from math import isfinite -from time import perf_counter +from time import perf_counter, sleep from tempfile import TemporaryDirectory import numpy as np from zipfile import ZipFile, ZIP_DEFLATED from dss import enums, DSSException import dss +from pathlib import Path +from hashlib import sha1 +try: + from ._settings import test_filenames, cimxml_test_filenames, USE_ODDIE +except ImportError: + from _settings import test_filenames, cimxml_test_filenames, USE_ODDIE original_working_dir = os.getcwd() NO_PROPERTIES = os.getenv('DSS_PYTHON_VALIDATE') == 'NOPROP' WIN32 = (sys.platform == 'win32') COM_VLL_BROKEN = True -SAVE_DSSX_OUTPUT = ('dss-extensions' in sys.argv) or not WIN32 +SAVE_DSSX_OUTPUT_ODD = ('dss-extensions-odd' in sys.argv) or USE_ODDIE +SAVE_DSSX_OUTPUT = SAVE_DSSX_OUTPUT_ODD or ('dss-extensions' in sys.argv) or not WIN32 + VERBOSE = ('-v' in sys.argv) suffix = '' @@ -38,7 +46,7 @@ def printv(*args): def run(dss: dss.IDSS, fn: str, line_by_line: bool): os.chdir(original_working_dir) dss.Text.Command = f'cd "{original_working_dir}"' - dss.Start(0) + # dss.Start(0) -- move this outside the run loop dss.Text.Command = 'Clear' dss.Text.Command = 'new circuit.RESET' @@ -95,6 +103,17 @@ def run(dss: dss.IDSS, fn: str, line_by_line: bool): else: dss.Text.Command = 'Makebuslist' + if ('epri_dpv/M1/Master_NoPV.dss' not in fn and + 'epri_dpv/K1/Master_NoPV.dss' not in fn and + 'epri_dpv/J1/Master_withPV.dss' not in fn and + 'LVTestCase/Master.dss' not in fn + and 'ckt5/Master_ckt5.dss' not in fn + and 'IEEE-TIA-LV Model' not in fn + and 'GFM_IEEE8500/Run_8500Node_Unbal' not in fn + and 'Storage-Quasi-Static-Example/Run_Demo1' not in fn + ): + dss.Text.Command = 'export profile phases=all' + reliabity_ran = True try: dss.ActiveCircuit.Meters.DoReliabilityCalc(False) @@ -107,7 +126,12 @@ def run(dss: dss.IDSS, fn: str, line_by_line: bool): return reliabity_ran, has_closedi +reported_field_issues = set() + def adjust_to_json(cls, field): + if cls.__class__.__name__ in ['ICapControls'] and field in ['idx']: + return + try: data = getattr(cls, field) if ismethod(data): @@ -122,8 +146,11 @@ def adjust_to_json(cls, field): return data except StopIteration: return - except: - print(cls, field) + except Exception as ex: + key = (type(ex).__qualname__, cls, field) + if key not in reported_field_issues: + print('[FIELD ISSUE]', *key) + reported_field_issues.add(key) raise ckt_elem_columns_meta = {'AllPropertyNames'} @@ -135,8 +162,11 @@ def export_dss_api_cls(dss: dss.IDSS, dss_cls): has_iter = hasattr(type(dss_cls), '__iter__') is_ckt_element = getattr(type(dss_cls), '_is_circuit_element', False) ckt_elem = dss.ActiveCircuit.ActiveCktElement - ckt_elem_columns = set(type(ckt_elem)._columns) - ckt_elem_columns_meta - pc_elem_columns + ckt_elem_columns = set(type(ckt_elem)._columns) - ckt_elem_columns_meta - pc_elem_columns - {'Handle', 'IsIsolated', 'HasOCPDevice'} fields = list(type(dss_cls)._columns) + + if 'UserClasses' in fields: + fields.remove('UserClasses') if 'SAIFIKW' in fields: meter_section_fields = fields[fields.index('NumSections'):] @@ -144,18 +174,18 @@ def export_dss_api_cls(dss: dss.IDSS, dss_cls): else: meter_section_fields = None - if not SAVE_DSSX_OUTPUT: + if (not SAVE_DSSX_OUTPUT) or SAVE_DSSX_OUTPUT_ODD: if 'TotalPowers' in ckt_elem_columns: ckt_elem_columns.remove('TotalPowers') if 'IsIsolated' in ckt_elem_columns: ckt_elem_columns.remove('IsIsolated') - if COM_VLL_BROKEN and 'Coorddefined' in fields: - fields.remove('puVLL') - fields.remove('VLL') - fields.remove('AllPCEatBus') - fields.remove('AllPDEatBus') + if COM_VLL_BROKEN:### and 'Coorddefined' in fields: + if 'puVLL' in fields: fields.remove('puVLL') + if 'VLL' in fields: fields.remove('VLL') + if 'AllPCEatBus' in fields: fields.remove('AllPCEatBus') + if 'AllPDEatBus' in fields: fields.remove('AllPDEatBus') # if 'Sensor' in fields: # Both Loads and PVSystems if 'ipvsystems' in type(dss_cls).__name__.lower(): @@ -192,6 +222,12 @@ def export_dss_api_cls(dss: dss.IDSS, dss_cls): # printv('>', field) try: record[field] = adjust_to_json(dss_cls, field) + except DSSException as e: + # Check for methods not implemented + if 'not implemented' in e.args[1].lower(): + #print(e.args) + continue + raise except StopIteration: # Some fields are functions, skip those continue @@ -235,7 +271,16 @@ def export_dss_api_cls(dss: dss.IDSS, dss_cls): for field in ckt_iter_columns_meta: # printv('>', field) - metadata_record[field] = adjust_to_json(dss_cls, field) + try: + metadata_record[field] = adjust_to_json(dss_cls, field) + except DSSException as e: + if 'not implemented' in e.args[1].lower(): + # print(e.args) + continue + + raise + + if 'Meters' in type(dss_cls).__name__: # This breaks the iteration @@ -300,6 +345,9 @@ def save_state(dss: dss.IDSS, runtime: float = 0.0) -> str: #TODO: machine info? } for key, dss_api_cls in dss_classes.items(): + if dss_api_cls is None: + printv(f'{key}: Not implemented, skipping.') + continue # print(key) document[key] = export_dss_api_cls(dss, dss_api_cls) @@ -308,13 +356,18 @@ def save_state(dss: dss.IDSS, runtime: float = 0.0) -> str: return json.dumps(document) -def get_archive_fn(live_fn): +def get_archive_fn(live_fn, fn_prefix=None): actual_fn = os.path.normpath(live_fn) common_prefix = os.path.commonprefix([ROOT_DIR, actual_fn]) archive_fn = actual_fn[len(common_prefix) + 1:] if WIN32: archive_fn = archive_fn.replace('\\', '/') + if fn_prefix is not None: + archive_fn_parts = archive_fn.split('/') + archive_fn_parts[-1] = fn_prefix + '_' + archive_fn_parts[-1] + archive_fn = '/'.join(archive_fn_parts) + return archive_fn if __name__ == '__main__': @@ -323,11 +376,6 @@ def get_archive_fn(live_fn): else: ROOT_DIR = os.path.abspath('../electricdss-tst/') - try: - from ._settings import test_filenames, cimxml_test_filenames - except ImportError: - from _settings import test_filenames, cimxml_test_filenames - # test_filenames = [] # cimxml_test_filenames = [] try: @@ -337,8 +385,27 @@ def get_archive_fn(live_fn): except: colorizer = None + if SAVE_DSSX_OUTPUT_ODD: + try: + from ._settings import DSS + except ImportError: + from _settings import DSS - if SAVE_DSSX_OUTPUT: + oddd_ver = DSS.Version.split(' ')[1] + print("Using official OpenDSS through ODDIE:", DSS.Version) + if USE_ODDIE != '1': + print("User-provided library path:", USE_ODDIE) + + debug_suffix = '-debug' if 'debug' in DSS.Version.lower() else '' + suffix = f'-dssx_oddd-{sys.platform}-{platform.machine()}-{oddd_ver}{debug_suffix}' + #test_idx = test_filenames.index('L!Version8/Distrib/IEEETestCases/123Bus/RevRegTest.dss') + 50 + # test_filenames = [fn for fn in test_filenames if 'DOCTechNote' not in fn] # DOC not implemented + # test_filenames = ['L!Version8/Distrib/IEEETestCases/123Bus/Run_YearlySim.dss'] + # test_filenames = ['L!Version8/Distrib/IEEETestCases/123Bus/SolarRamp.DSS'] + cimxml_test_filenames = [] # Cannot run these now + DSS.AllowForms = False + + elif SAVE_DSSX_OUTPUT: from dss import DSS, DSSCompatFlags DSS.CompatFlags = 0 # DSSCompatFlags.InvControl9611 print("Using DSS-Extensions:", DSS.Version) @@ -364,8 +431,14 @@ def get_archive_fn(live_fn): try: DSS.Text.Command = 'new circuit.dummy' check_error() + if not WIN32: + DSS.Text.Command = 'set Editor=/bin/true' + else: + DSS.Text.Command = r'set Editor="C:\Program Files\Git\usr\bin\true.exe"' + DSS.Text.Command = 'set ShowExport=NO' check_error() + sleep(0.1) DSS.Text.Command = 'clear' check_error() except: @@ -376,6 +449,10 @@ def get_archive_fn(live_fn): zip_fn = f'results{suffix}.zip' with ZipFile(os.path.join(original_working_dir, zip_fn), mode='a', compression=ZIP_DEFLATED) as zip_out: for fn in test_filenames + cimxml_test_filenames: + if not fn.strip(): + break + + fn_hash = sha1(fn.encode()).hexdigest() org_fn = fn fixed_fn = fn if not fn.startswith('L!') else fn[2:] line_by_line = fn.startswith('L!') @@ -390,6 +467,7 @@ def get_archive_fn(live_fn): try: has_closedi = False tstart_run = perf_counter() + # print(fn) reliabity_ran, has_closedi = run(DSS, fn, line_by_line) runtime = perf_counter() - tstart_run total_runtime += runtime @@ -418,13 +496,6 @@ def get_archive_fn(live_fn): for xml_live_fn in xml_live_fns: zip_out.write(xml_live_fn, get_archive_fn(xml_live_fn).replace('.XML', '.xml')) - if has_closedi: - DSS.Text.Command = 'get CaseName' - res_dir = os.path.join(DSS.DataPath, DSS.Text.Result) - for csv_live_fn in glob(f'{res_dir}/*/*.csv'): - print(csv_live_fn) - zip_out.write(csv_live_fn, get_archive_fn(csv_live_fn).replace('.CSV', '.csv')) - with TemporaryDirectory() as tmp_dir: DSS.Text.Command = f'save circuit dir="{tmp_dir}"' base_zip_dir = get_archive_fn(fn) + '.saved/' @@ -436,6 +507,20 @@ def get_archive_fn(live_fn): # print((json_fn, base_zip_dir, saved_fn, rel_saved_fn.replace('.DSS', '.dss'))) zip_out.write(saved_fn, base_zip_dir + rel_saved_fn.replace('.DSS', '.dss')) + if True: # has_closedi: + # DSS.Text.Command = 'get CaseName' + # res_dir = os.path.join(DSS.DataPath, DSS.Text.Result) + res_path = Path(DSS.DataPath) + + for csv_live_fn in res_path.rglob('*.csv', case_sensitive=False): + csv_live_fn = str(csv_live_fn.absolute()) + print(csv_live_fn) + try: + zip_out.write(csv_live_fn, get_archive_fn(csv_live_fn, fn_hash).replace('.CSV', '.csv')) + except: + DSS.Text.Command = 'clear' + zip_out.write(csv_live_fn, get_archive_fn(csv_live_fn, fn_hash).replace('.CSV', '.csv')) + print(perf_counter() - t0_global, 'seconds') print(total_runtime, 'seconds (runtime only)') diff --git a/tests/test_ctrlqueue.py b/tests/test_ctrlqueue.py index e3339ec7..b8a0a6fb 100644 --- a/tests/test_ctrlqueue.py +++ b/tests/test_ctrlqueue.py @@ -6,9 +6,9 @@ import numpy as np try: - from ._settings import BASE_DIR, WIN32 + from ._settings import BASE_DIR, WIN32, DSS as DSSRef except ImportError: - from _settings import BASE_DIR, WIN32 + from _settings import BASE_DIR, WIN32, DSS as DSSRef USE_COM = False # Change to True to test with the COM DLL @@ -21,7 +21,7 @@ def test_ctrlqueue(): # When running pytest, the faulthandler seems too eager to grab FPC's exceptions, even when handled import faulthandler faulthandler.disable() - from dss import DSS as DSSobj + DSSobj = DSSRef faulthandler.enable() else: from dss import DSS as DSSobj @@ -33,6 +33,7 @@ def test_ctrlqueue(): DSSobj = win32com.client.gencache.EnsureDispatch('OpenDSSengine.DSS') os.chdir(old_cd) + DSSobj.AllowForms = False DSSText = DSSobj.Text DSSCircuit = DSSobj.ActiveCircuit DSSSolution = DSSCircuit.Solution diff --git a/tests/test_general.py b/tests/test_general.py index 7e76a437..1c84b80b 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -9,30 +9,26 @@ import pytest try: - from ._settings import BASE_DIR, ZIP_FN, WIN32 + from ._settings import BASE_DIR, ZIP_FN, WIN32, DSS except ImportError: - from _settings import BASE_DIR, ZIP_FN, WIN32 + from _settings import BASE_DIR, ZIP_FN, WIN32, DSS -if WIN32: - # When running pytest, the faulthandler seems too eager to grab FPC's exceptions, even when handled - import faulthandler - faulthandler.disable() - import dss - faulthandler.enable() -else: - import dss - -from dss import DSS, IDSS, DSSException, SparseSolverOptions, SolveModes, set_case_insensitive_attributes, DSSCompatFlags, LoadModels, DSSPropertyNameStyle +import dss +from dss import IDSS, DSSException, SparseSolverOptions, SolveModes, set_case_insensitive_attributes, DSSCompatFlags, LoadModels, DSSPropertyNameStyle, IOddieDSS org_dir = os.getcwd() def setup_function(): DSS.ClearAll() - DSS.AllowEditor = False - DSS.AdvancedTypes = False - DSS.AllowChangeDir = True - DSS.COMErrorResults = True # TODO: change to False - DSS.CompatFlags = 0 + + if not DSS._api_util._is_odd: + DSS.AllowEditor = False + DSS.AdvancedTypes = False + DSS.AllowChangeDir = True + DSS.COMErrorResults = True # TODO: change to False + DSS.CompatFlags = 0 + + DSS.AllowForms = False DSS.Error.UseExceptions = True DSS.Text.Command = 'set DefaultBaseFreq=60' @@ -271,13 +267,15 @@ def test_set_mode(): def test_pm_threads(): - DSS.AllowChangeDir = False + if not isinstance(DSS, IOddieDSS): + DSS.AllowChangeDir = False Parallel = DSS.ActiveCircuit.Parallel if Parallel.NumCPUs < 4: return # Cannot run in this machine, e.g. won't run on GitHub Actions - DSS.AdvancedTypes = True + if not isinstance(DSS, IOddieDSS): + DSS.AdvancedTypes = True DSS.Text.Command = 'set parallel=No' fn = os.path.abspath(f'{BASE_DIR}/Version8/Distrib/EPRITestCircuits/ckt5/Master_ckt5.dss') @@ -352,31 +350,34 @@ def test_pm_threads(): assert max(abs(v_pm[3] - v_pm[0])) > 1e-1 assert dt_pm < dt_seq - # Let's run with threads, using DSSContexts too - v_ctx = [None] * 4 - - def _run(ctx, i): - ctx.Text.Command = f'compile "{fn}"' - ctx.ActiveCircuit.Solution.Solve() - ctx.Text.Command = f'set mode=yearly number=144 hour={i * 24} controlmode=off stepsize=600' - ctx.ActiveCircuit.Solution.Solve() - v_ctx[i] = ctx.ActiveCircuit.AllBusVolts - - ctxs = [DSS.NewContext() for _ in range(4)] - t0 = perf_counter() - threads = [] - for i, ctx in enumerate(ctxs): - t = threading.Thread(target=_run, args=(ctx, i)) - threads.append(t) - - for t in threads: - t.start() - - for t in threads: - t.join() - - t1 = perf_counter() - dt_ctx = t1 - t0 + if not isinstance(DSS, IOddieDSS): + # Let's run with threads, using DSSContexts too + v_ctx = [None] * 4 + + def _run(ctx, i): + ctx.Text.Command = f'compile "{fn}"' + ctx.ActiveCircuit.Solution.Solve() + ctx.Text.Command = f'set mode=yearly number=144 hour={i * 24} controlmode=off stepsize=600' + ctx.ActiveCircuit.Solution.Solve() + v_ctx[i] = ctx.ActiveCircuit.AllBusVolts + + ctxs = [DSS.NewContext() for _ in range(4)] + t0 = perf_counter() + threads = [] + for i, ctx in enumerate(ctxs): + t = threading.Thread(target=_run, args=(ctx, i)) + threads.append(t) + + for t in threads: + t.start() + + for t in threads: + t.join() + + t1 = perf_counter() + dt_ctx = t1 - t0 + else: + dt_ctx = np.NaN np.testing.assert_allclose(v_ctx[0], v_seq[0]) np.testing.assert_allclose(v_ctx[1], v_seq[1]) @@ -474,6 +475,100 @@ def _run(ctx: IDSS, case_list, converged, results): assert dt_thread < dt_seq +def test_basic_input_errors(): + with pytest.raises(DSSException): + DSS('redirect this_file_does_not_exist_0293093022.dss') + + with pytest.raises(DSSException): + DSS('redirect ../this_file_does_not_exist_0293093022.dss') + + # LoadShape + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 CSVFile=this_file_does_not_exist_0293093022.csv') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 SngFile=this_file_does_not_exist_0293093022.sng') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 DblFile=this_file_does_not_exist_0293093022.dbl') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 PQCSVFile=this_file_does_not_exist_0293093022pq.csv') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 PMult=(File=this_file_does_not_exist_0293093022.txt)') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new loadshape.shape1 PMult=(SngFile=this_file_does_not_exist_0293093022.sng)') + + # PriceShape + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new priceshape.shape1 CSVFile=this_file_does_not_exist_0293093022.csv') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new priceshape.shape1 SngFile=this_file_does_not_exist_0293093022.sng') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new priceshape.shape1 DblFile=this_file_does_not_exist_0293093022.dbl') + + # TShape + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new tshape.shape1 CSVFile=this_file_does_not_exist_0293093022.csv') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new tshape.shape1 SngFile=this_file_does_not_exist_0293093022.sng') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new tshape.shape1 DblFile=this_file_does_not_exist_0293093022.dbl') + + # XYCurve + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new xycurve.curve1 CSVFile=this_file_does_not_exist_0293093022.csv') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new xycurve.curve1 SngFile=this_file_does_not_exist_0293093022.sng') + + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new xycurve.curve1 DblFile=this_file_does_not_exist_0293093022.dbl') + + # Spectrum + with pytest.raises(DSSException): + DSS('clear') + DSS('new circuit.test') + DSS('new spectrum.spec1 CSVFile=this_file_does_not_exist_0293093022.csv') + + + #TODO: finish/fix this for the current framework # DSS C-API change: to avoid issues with read-only folders, we # changed this from GetCurrentDir to OutputDirectory. This is @@ -836,7 +931,18 @@ def test_line_parent_compat(): assert res_compat[3:2:] == res_no_compat[3:2:] +def test_path_sideeffects(): + test_loadshape_save() + test_basic_input_errors() + test_loadshape_save() + + if __name__ == '__main__': + DSS.AllowForms = False + print(DSS.Version) # for _ in range(250): # test_pm_threads() - test_capacitor_reactor() \ No newline at end of file + + test_path_sideeffects() + test_capacitor_reactor() + print('DONE!') \ No newline at end of file diff --git a/tests/test_past_issues.py b/tests/test_past_issues.py index dce19270..9d8d5587 100644 --- a/tests/test_past_issues.py +++ b/tests/test_past_issues.py @@ -7,26 +7,19 @@ import scipy.sparse as sp try: - from ._settings import BASE_DIR, WIN32, ZIP_FN + from ._settings import BASE_DIR, WIN32, ZIP_FN, DSS except ImportError: - from _settings import BASE_DIR, WIN32, ZIP_FN - -if WIN32: - # When running pytest, the faulthandler seems too eager to grab FPC's exceptions, even when handled - import faulthandler - faulthandler.disable() - import dss - faulthandler.enable() -else: - import dss + from _settings import BASE_DIR, WIN32, ZIP_FN, DSS def setup_function(): DSS.ClearAll() - DSS.AllowEditor = False - DSS.AdvancedTypes = False - DSS.AllowChangeDir = True - DSS.COMErrorResults = True # TODO: change to False - DSS.CompatFlags = 0 + DSS.AllowForms = False + if not DSS._api_util._is_odd: + DSS.AllowEditor = False + DSS.AdvancedTypes = False + DSS.AllowChangeDir = True + DSS.COMErrorResults = True # TODO: change to False + DSS.CompatFlags = 0 def test_rxmatrix():