Merge pull request #87 from BDonnot/bd-dev

Ready for release 0.9.0

.github/workflows/main.yml

@@ -12,6 +12,8 @@ jobs:
     # build wheels for some linux
     name: Build linux ${{ matrix.python.name }} wheel on ${{ matrix.cont.name}}
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     strategy:
       matrix:
         python:
@@ -70,38 +72,43 @@ jobs:
       #     platforms: arm64
 
       - name: Checkout sources
-        uses: actions/checkout@v3 # v4 not working...
+        uses: actions/checkout@v1
         with:
           submodules: true
 
       - name: Setup path
         run: echo "/opt/python/${{ matrix.python.name }}-${{ matrix.python.abi }}/bin/" >> $GITHUB_PATH
 
+      - name: get python version
+        run: |
+          python3 --version
+
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements_compile.txt
+          python3 -m pip freeze
 
       - name: Compile SuiteSparse
         run: 
           make
 
-      - name: Build wheel
+      - name: Build lightsim2grid wheel
         run: |
           # export __O3_OPTIM=1
           python3 setup.py bdist_wheel
           auditwheel repair dist/*.whl
 
       - name: Build source archive
         if: matrix.python.name == 'cp311'
-        run: python setup.py sdist
+        run: python3 setup.py sdist
 
-      - name: Install wheel
+      - name: Install lightsim2grid wheel
         run: |
-          pip3 install wheelhouse/*.whl --user
-          pip freeze
+          python3 -m pip install wheelhouse/*.whl --user
+          python3 -m pip freeze
 
-      - name: Check package can be imported (bare install)
+      - name: Check package can be imported (bare install, default numpy)
         run: |
           mkdir tmp_for_import_checking
           cd tmp_for_import_checking
@@ -111,7 +118,20 @@ jobs:
           python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
           python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
           python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
-          python3 -c "from lightsim2grid.gridmodel import init, GridModel"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
+
+      - name: Check package can be imported (bare install, numpy < 2)
+        if: matrix.python.name != 'cp37' && matrix.python.name != 'cp38'
+        run: |
+          python3 -m pip install "numpy<2"
+          cd tmp_for_import_checking
+          python3 -c "import lightsim2grid"
+          python3 -c "from lightsim2grid import *"
+          python3 -c "from lightsim2grid.newtonpf import newtonpf"
+          python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
+          python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
+          python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
       
       - name: Fix urllib3 (python 3.7)
         if: matrix.python.name == 'cp37'
@@ -121,7 +141,7 @@ jobs:
 
       - name: Install grid2op
         run: | 
-          python3 -m pip install grid2op
+          python3 -m pip install grid2op pandapower
           python3 -m pip freeze
 
       - name: Check extra can be imported can be imported (with grid2op)
@@ -132,6 +152,7 @@ jobs:
           python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysis"
           python3 -c "from lightsim2grid.physical_law_checker import PhysicalLawChecker"
           python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysis"
+          python3 -c "from lightsim2grid.gridmodel import init_from_pandapower"
 
       - name: Check LightSimBackend can be used to create env
         run:
@@ -217,7 +238,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools
-          python -m pip install -r requirements.txt
+          python -m pip install -r requirements_compile.txt
 
       - name: Compile SuiteSparse cmake
         run: |
@@ -240,21 +261,46 @@ jobs:
         shell: bash
         run: python -m pip install dist/*.whl --user
 
-      - name: Check package can be imported
+      - name: Check package can be imported (bare install, default numpy)
         run: |
           mkdir tmp_for_import_checking
           cd tmp_for_import_checking
-          python -c "import lightsim2grid"
-          python -c "from lightsim2grid import *"
-          python -c "from lightsim2grid.newtonpf import newtonpf"
-          cd ..
-      
-      - name: Check LightSimBackend can be imported
+          python3 -c "import lightsim2grid"
+          python3 -c "from lightsim2grid import *"
+          python3 -c "from lightsim2grid.newtonpf import newtonpf"
+          python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
+          python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
+          python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
+
+      - name: Check package can be imported (bare install, numpy < 2)
+        if: matrix.python.name != 'cp37' && matrix.python.name != 'cp38'
         run: |
+          python3 -m pip install "numpy<2"
           cd tmp_for_import_checking
+          python3 -c "import lightsim2grid"
+          python3 -c "from lightsim2grid import *"
+          python3 -c "from lightsim2grid.newtonpf import newtonpf"
+          python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
+          python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
+          python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
+
+      - name: Check LightSimBackend can be imported (with grid2op)
+        run: |
           python -m pip install grid2op
+          cd tmp_for_import_checking
           python -c "from lightsim2grid import LightSimBackend"
           python -c "from lightsim2grid import LightSimBackend; import grid2op; env = grid2op.make('l2rpn_case14_sandbox', test=True, backend=LightSimBackend())"
+          python -c "from lightsim2grid.timeSerie import TimeSerie"
+          python -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysis"
+          python -c "from lightsim2grid.physical_law_checker import PhysicalLawChecker"
+          python -c "from lightsim2grid.securityAnalysis import SecurityAnalysis"
+          python -c "from lightsim2grid.gridmodel import init_from_pandapower"
+          
+      # - name: Debug windows CI
+      #   run: |
+      #     python -m unittest lightsim2grid.tests.test_n1contingencyrewards.TestN1ContingencyReward_Base.test_do_nothing
 
       - name: Upload wheel
         uses: actions/upload-artifact@v3
@@ -336,7 +382,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools  # for python >= 3.12
-          python -m pip install -r requirements.txt
+          python -m pip install -r requirements_compile.txt
 
       - name: Compile SuiteSparse make
         run: |
@@ -358,17 +404,30 @@ jobs:
           python -m pip install dist/*.whl --user
           python -m pip freeze
 
-      - name: Check package can be imported (bare install)
+      - name: Check package can be imported (bare install, default numpy)
         run: |
           mkdir tmp_for_import_checking
           cd tmp_for_import_checking
-          python -c "import lightsim2grid"
-          python -c "from lightsim2grid import *"
-          python -c "from lightsim2grid.newtonpf import newtonpf"
-          python -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
-          python -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
-          python -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
-          python -c "from lightsim2grid.gridmodel import init, GridModel"
+          python3 -c "import lightsim2grid"
+          python3 -c "from lightsim2grid import *"
+          python3 -c "from lightsim2grid.newtonpf import newtonpf"
+          python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
+          python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
+          python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
+
+      - name: Check package can be imported (bare install, numpy < 2)
+        if: matrix.python.name != 'cp37' && matrix.python.name != 'cp38'
+        run: |
+          python3 -m pip install "numpy<2"
+          cd tmp_for_import_checking
+          python3 -c "import lightsim2grid"
+          python3 -c "from lightsim2grid import *"
+          python3 -c "from lightsim2grid.newtonpf import newtonpf"
+          python3 -c "from lightsim2grid.timeSerie import TimeSeriesCPP"
+          python3 -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP"
+          python3 -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP"
+          python3 -c "from lightsim2grid.gridmodel import GridModel"
       
       - name: Fix urllib3 (python 3.7)
         if: matrix.python.name == 'cp37'
@@ -389,6 +448,7 @@ jobs:
           python -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysis"
           python -c "from lightsim2grid.physical_law_checker import PhysicalLawChecker"
           python -c "from lightsim2grid.securityAnalysis import SecurityAnalysis"
+          python -c "from lightsim2grid.gridmodel import init_from_pandapower"
       
       - name: Check LightSimBackend can be used to create env
         run: |
@@ -487,7 +547,7 @@ jobs:
           CIBW_ARCHS_MACOS: arm64 
           CIBW_ARCHS_WINDOWS: ARM64 
           CIBW_ENVIRONMENT: __O3_OPTIM=1
-          CIBW_BEFORE_ALL: python -m pip install --upgrade setuptools wheel pybind11 pip
+          CIBW_BEFORE_ALL: python -m pip install --upgrade setuptools wheel pybind11 pip && python -m pip install "numpy>=2; python_version >= '3.9'"  "numpy; python_version < '3.9'"
           CIBW_BEFORE_BUILD_MACOS: make clean && make CC=clang CXX=clang++  # no need to cross compile this because host is already arm64 based
           CIBW_BEFORE_BUILD_LINUX: make clean && make  # maybe we need to cross compile this (host x64) but maybe not (qemu) ?
           CIBW_BEFORE_BUILD_WINDOWS: > 
@@ -498,16 +558,18 @@ jobs:
             cmake -DCMAKE_INSTALL_PREFIX=..\built -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=arm64 .. &&
             cmake --build . --config Release &&
             cmake --install . --prefix ..\built
-          CIBW_TEST_REQUIRES: grid2op pandapower
+          CIBW_TEST_REQUIRES: grid2op pandapower "numpy<2"
           CIBW_TEST_SKIP: "cp312-* *-macosx_arm64 *-win_arm64" # to silence warning "While arm64 wheels can be built on x86_64, they cannot be tested."
           CIBW_TEST_COMMAND: >
             python -c "import lightsim2grid" &&
             python -c "from lightsim2grid import *" &&
             python -c "from lightsim2grid.newtonpf import newtonpf" &&
-            python -c "from lightsim2grid.timeSerie import TimeSeriesCPP" &&
-            python -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP" &&
-            python -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP" &&
-            python -c "from lightsim2grid.gridmodel import init, GridModel"
+            python -c "from lightsim2grid.timeSerie import TimeSeriesCPP, TimeSerie" &&
+            python -c "from lightsim2grid.contingencyAnalysis import ContingencyAnalysisCPP, ContingencyAnalysis" &&
+            python -c "from lightsim2grid.securityAnalysis import SecurityAnalysisCPP, SecurityAnalysis" &&
+            python -c "from lightsim2grid.gridmodel import GridModel, init_from_pandapower" &&
+            python -c "from lightsim2grid import LightSimBackend" &&
+            python -c "from lightsim2grid.physical_law_checker import PhysicalLawChecker"
       - name: Upload wheel
         uses: actions/upload-artifact@v3
         with:
@@ -517,7 +579,6 @@ jobs:
   package:
     name: Package wheels
     runs-on: ubuntu-latest
-    # needs: [manylinux_build, windows_build, macos_build_37_]
     needs: [exotic_build, macos_build_37_, windows_build, manylinux_build]
 
     steps:

.gitignore

@@ -279,4 +279,12 @@ bug_sparselu
 bug_sparselu_eigen.cpp
 test_segfault.sh
 nohup.out
-test_rte/
+test_rte/
+venv_312/
+benchmarks/requirements_venv_bench_solvers.txt
+patch_*.patch
+lightsim2grid/tests/venv_test_pp/
+*_quantique.py
+venv_legacy_311/
+venv_test
+lightsim2grid/gridmodel/from_pypowsybl/pypowsybl.ipynb

CHANGELOG.rst

@@ -18,6 +18,87 @@ Change Log
 - maybe have a look at suitesparse "sliplu" tools ?
 - easier building (get rid of the "make" part)
 
+TODO: https://github.com/haranjackson/NewtonKrylov for another type of algorithm ?
+TODO HVDC in Jacobian (see pandapower)
+TODO: in ContingencyAnalysisCpp: add back the `if(!ac_solver_used)` inside the  `remove_from_Ybus`
+      in order to perform the "invertibility" check
+TODO: in `main.cpp` check the returned policy of pybind11 and also the `py::call_guard<py::gil_scoped_release>()` stuff
+
+[0.9.0] 2024-07-29
+--------------------------
+- [BREAKING] installing pandapower lightsim2grid does not require anymore to install
+  pandapower (you can initialize `GridModel` with pypowsybl or pandapower if you want). To make it both
+  cleaner and clearer the function `lightsim2grid.gridmodel.init` has been removed.
+  Please use `lightsim2grid.gridmodel.init_from_pandapower` or 
+  `lightsim2grid.gridmodel.init_from_pypowsybl` from now on
+- [BREAKING] the previous `gridmodel.get_ptdf()` function was wrongly labelled with the
+  "solver" bus id and not the `gridmodel` bus id which could cause issue when it was computed 
+  on some grid configuration. It has now been fixed (so the `gridmodel.get_ptdf` returns the
+  proper things). If you want the previous behaviour, you need to use `gridmodel.get_ptdf_solver()`
+- [BREAKING] similarly, `gridmodel.get_Ybus()`, `gridmodel.get_dcYbus()`, `gridmodel.get_Sbus()`
+  and `gridmodel.get_dcSbus()` now return things in the `gridmodel` bus ordering. For the previous
+  behaviour you can use `gridmodel.get_Ybus_solver()`, `gridmodel.get_dcYbus_solver()`,
+  `gridmodel.get_Sbus_solver()` and `gridmodel.get_dcSbus_solver()`
+- [BREAKING] the more rational logic above also extends to all the functions listed in the 
+  table below:
+
+===============================    ===================================================
+Function with behaviour change      Name of the new function having the same behaviour
+===============================    ===================================================
+gridmodel.get_ptdf()                gridmodel.get_ptdf_solver()
+gridmodel.get_Ybus()                gridmodel.get_Ybus_solver()
+gridmodel.get_dcYbus()              gridmodel.get_dcYbus_solver()
+gridmodel.get_Sbus()                gridmodel.get_Sbus_solver()
+gridmodel.get_dcSbus()              gridmodel.get_dcSbus_solver()
+gridmodel.get_pv()                  gridmodel.get_pv_solver()
+gridmodel.get_pq()                  gridmodel.get_pq_solver()
+gridmodel.get_slack_ids()           gridmodel.get_slack_ids_solver()
+gridmodel.get_slack_ids_dc()        gridmodel.get_slack_ids_dc_solver()
+gridmodel.get_slack_weights()       gridmodel.get_slack_weights_solver()
+gridmodel.get_V()                   gridmodel.get_V_solver()
+gridmodel.get_Va()                  gridmodel.get_Va_solver()
+gridmodel.get_Vm()                  gridmodel.get_Vm_solver()
+gridmodel.get_J()                   gridmodel.get_J_solver()
+gridmodel.get_Bf()                  gridmodel.get_Bf_solver()
+===============================    ==================================================
+
+- [FIXED] the `change_solver` in the `ContingencyAnalysis` did not work correctly.
+  More specifically the solver type used might not be correct if changed which could 
+  lead to wrong Ybus being passed to the solver.
+- [FIXED] some compatibility mode with python `3.7`
+- [FIXED] a bug when "turned off" generator were not PV (slack was 
+  "turned off" when its target P was 0. But still the slack so ends up producing something...)
+- [FIXED] (consistency with pandapower) when an intial powerflow is run
+  to initialized an AC powerflow, the initial voltages are 1 pu (and 
+  not `gridmodel.get_init_vm_pu()` as previously).
+- [FIXED] `gridmodel.get_ptdf()` now have the 
+  normal "gridmodel" bus id representation and not the "solver" bus ordering.
+- [FIXED] `gridmodel.get_lodf()` issue wrong results in case of some
+  topological modification
+- [FIXED] calls to methods such as `gridmodel.get_pv` or `gridmodel.get_V` 
+  or `gridmodel.get_Ybus` could lead to severe crashes (segmentation fault)
+  on some (rare) cases. Now an exceptions should be thrown in these cases.
+- [FIXED] basic backward compatibility is ensured and tested for legacy grid2op >= 0.9.1.post1
+  Not all features are tested and only 1.x versions are tested 
+  (ie 1.1 or 1.2 but not 1.2.1, 1.2.2, 1.2.3 etc.) and only for python 3.11
+- [FIXED] a bug when using `LightSimBackend` with some old (but not too old) grid2op
+  versions.
+- [FIXED] various compatibility bugs when using old grid2op versions.
+- [ADDED] it is now possible to deactivate the support for shunts by 
+  subclassing the LightSimBackend class and setting the `shunts_data_available`
+  to `False`
+- [IMPROVED] in the `ContingencyAnalysis` class, the underlying cpp model will now
+  perform an initial powerflow.
+- [IMPROVED] distributed wheels are now compiled (whenever possible) with numpy 2. 
+  This makes them compatible with both numpy 1.x.y and numpy 2.z.t versions.
+- [IMPROVED] tests are now performed when lightsim2grid is compiled with 
+  the latest clang (18) and gcc (14) versions on the CI using python 3.11
+
+[0.8.2.post1] 2024-04-xx
+--------------------------
+- [FIXED] a "forward compatibility" issue with grid2op 1.10.2
+  (due to wrong usage of some internal classes when loading a pandapower grid)
+
 [0.8.2] 2024-04-22
 --------------------
 - [FIXED] CI was broken after migration to artifact v4, set it back to v3 

README.md

@@ -66,7 +66,7 @@ from lightsim2grid.newtonpf import newtonpf
 V, converged, iterations, J = newtonpf(Ybus, V, Sbus, ref, weights, pv, pq, ppci, options)
 ```
 
-This function uses the KLU algorithm and a c++ implementation of a Newton solver for speed.
+This function uses the KLU algorithm (when available) and a c++ implementation of a Newton solver for speed.
 
 ## Installation (from pypi official repository, recommended)