add timeseries profiles, GettingStarted tutorial

.github/workflows/github_test_action.yml

@@ -32,6 +32,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install pytest jupyter
+        python -m pip install git+https://github.com/SteffenMeinecke/pandapower.git@feature/DERController
+        python -m pip install git+https://github.com/e2nIEE/simbench.git@develop
         python -m pip install $GITHUB_WORKSPACE["test"]
     - name: List of installed packages
       run: |
@@ -53,6 +55,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        python -m pip install git+https://github.com/SteffenMeinecke/pandapower.git@feature/DERController
+        python -m pip install git+https://github.com/e2nIEE/simbench.git@develop
         python -m pip install $GITHUB_WORKSPACE["test"]
     - name: List of installed packages
       run: |
@@ -73,7 +77,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install $GITHUB_WORKSPACE["all"]
+          python -m pip install git+https://github.com/SteffenMeinecke/pandapower.git@feature/DERController
+          python -m pip install git+https://github.com/e2nIEE/simbench.git@develop
+          python -m pip install $GITHUB_WORKSPACE["test","all"]
       - name: List of installed packages
         run: |
           python -m pip list
@@ -93,7 +99,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install $GITHUB_WORKSPACE["all"]
+          python -m pip install git+https://github.com/SteffenMeinecke/pandapower.git@feature/DERController
+          python -m pip install git+https://github.com/e2nIEE/simbench.git@develop
+          python -m pip install $GITHUB_WORKSPACE["test","all"]
       - name: List of installed packages
         run: |
           python -m pip list

MANIFEST.in

@@ -1,7 +1,7 @@
 include LICENSE AUTHORS README.rst CHANGELOG.rst CITATION.bib
 
 global-include *.json
-global-include *.h5
+global-include *.parquet
 
 prune .git*
 prune tutorials*

README.md

@@ -1,2 +1,4 @@
 # SimBench_EHV_HV_excerpt
-Grid data from SimBench used for the Dissertation 'TSO-DSO Coordinated Distributed Optimizations in Grid Operations Considering System Operator Sovereignty' and two more papers
+
+This repository provides the applied data, i.e. power system grid data and time series profiles, to reproduce or compare with simulations of the Dissertation 'TSO-DSO Coordinated Distributed Optimizations in Grid Operations Considering System Operator Sovereignty' and the two referenced papers ['New distributed optimization method for tsodso coordinated grid operation preserving power system operator sovereignty'](https://www.mdpi.com/1996-1073/16/12/4753) and ['Distributed optimization of smart grid assets for grid operation preserving power system operator sovereignty'](https://ieeexplore.ieee.org/document/10202774).
+The [Tutorial](https://github.com/SteffenMeinecke/SimBench_EHV_HV_excerpt/blob/master/tutorials/GettingStarted.ipynb) gives advise how to apply provided data and code.

SimBench_EHV_HV_excerpt/SimBench_for_phd.py

@@ -5,6 +5,7 @@
 import pandapower as pp
 import simbench as sb
 
+from SimBench_EHV_HV_excerpt import home, data_path
 from SimBench_EHV_HV_excerpt.toolbox import *
 
 try:
@@ -72,7 +73,7 @@ def SimBench_for_phd(
     # --- only read from json file -----------------------------------------------------------------
     if from_json:
         net = pp.from_json(os.path.join(data_path, "net.json"))
-        add_profiles_from_h5_to_net(net, profiles_file, time_steps, False)
+        add_profiles_from_parquet_to_net(net, time_steps, False, kwargs.get("profiles_folder", None))
 
     # --- create net -------------------------------------------------------------------------------
     else:
@@ -125,7 +126,7 @@ def SimBench_for_phd(
             del res
 
             # downcast profiles
-            downcast_profiles(net)
+            downcast_profiles(net.profiles)
 
             # -- reduce the net to the relevant part
             logger.info("reduce_ehv() starts.")
@@ -228,20 +229,20 @@ def _store_files_to_desktop(net) -> None:
     del net_wo_profiles
 
     # --- store profiles to h5
-    profiles_file = os.path.join(home, "Desktop", "profiles.h5")
-    file_exists = os.path.exists(profiles_file)
-    store_profiles_to_hdf5_file(net.profiles, profiles_file)
-    if file_exists:
-        logger.info("'profiles.h5' is replaced/updated on your Desktop.")
+    profiles_folder = os.path.join(home, "Desktop", "profiles")
+    dir_exists = os.path.exists(profiles_folder)
+    store_profiles_to_parquet_files(net.profiles, profiles_folder)
+    if dir_exists:
+        logger.info("The profile folders at the desktop were refilled by parquet files.")
     else:
-        logger.info("'profiles.h5' has been created on your Desktop.")
+        logger.info("profiles were written to parquet files in the profiles folder on your Desktop.")
 
 
 if __name__ == "__main__":
 
     net = SimBench_for_phd()  # from_json=True, time_steps=False, merged_same_bus_gens=False
 
     if False:
-        net = SimBench_for_phd(time_steps=True)
+        net = SimBench_for_phd(from_json=False, time_steps=True)
         downcast_profiles(net.profiles)
         store_profiles_to_hdf5_file(net.profiles, os.path.join(home, "Desktop", "profiles.h5"))

SimBench_EHV_HV_excerpt/__init__.py

@@ -1,6 +1,12 @@
 import os
+from pathlib import Path
+
+# --- define paths applied in the repo
 sb_excerpt_dir = os.path.dirname(os.path.realpath(__file__))
+home = str(Path.home())
+data_path = os.path.join(sb_excerpt_dir, "data")
 
+# --- import functionality
 import SimBench_EHV_HV_excerpt.toolbox
 
 from .SimBench_for_phd import SimBench_for_phd

tests/__init__.py → SimBench_EHV_HV_excerpt/test/__init__.py

@@ -1,4 +1,5 @@
 import os
-from SimBench_EHV_HV_excerpt import sb_excerpt_dir
 
 test_path = os.path.dirname(os.path.realpath(__file__))
+
+from .run_tests import run_all_tests

tests/test_SimBench_for_phd.py → SimBench_EHV_HV_excerpt/test/test_SimBench_for_phd.py

@@ -158,8 +158,12 @@ def test_powers():
     t_dep_losses1 = res["res_line.pl_mw"].sum(axis=1) + res["res_trafo.pl_mw"].sum(axis=1)
     t_dep_losses2 = -res["res_bus.p_mw"].sum(axis=1)
     assert np.allclose(t_dep_losses1, t_dep_losses2, atol=1e-2)
-    expected_losses = np.array([203.95, 299.30, 290.23, 249.35, 220.03, 219.03, 169.38, 220.23])
-    assert np.allclose(t_dep_losses1, expected_losses, atol=0.1)
+    expected_losses = np.array([203.86, 298.17, 290.11, 249.22, 219.90, 218.92, 169.21, 220.14])
+    try:
+        assert np.allclose(t_dep_losses1, expected_losses, atol=0.1)
+    except AssertionError:
+        data = pd.DataFrame({"expected_losses": expected_losses, "calculated_losses": t_dep_losses1})
+        raise AssertionError(data)
 
 
 if __name__ == "__main__":

tests/test_h5_profiles.py → SimBench_EHV_HV_excerpt/test/test_parquet_profiles.py

@@ -1,6 +1,6 @@
 import pytest
-import os
 import tempfile
+import shutil
 import pandas as pd
 import pandapower as pp
 
@@ -13,37 +13,25 @@ def test_combined_profile_fcts():
         "load.p_mw": pd.DataFrame([[1, 2], [3, 4], [5, 6]]),
         "sgen.q_mvar": pd.DataFrame([[1, 2], [3, 4], [5, 6]]),
     }
-    test_file1 = os.path.join(tempfile.gettempdir(), "test_profile1.h5")
-    test_file2 = os.path.join(tempfile.gettempdir(), "test_profile2.h5")
+    temp_dir = tempfile.mkdtemp()
 
     # --- store to file (from dict)
-    sbe.toolbox.store_profiles_to_hdf5_file(profiles, test_file1)
+    sbe.toolbox.store_profiles_to_parquet_files(profiles, temp_dir)
 
-    # load data from hdf5 file into net
+    # load data from parquet files into net
     net1 = pp.create_empty_network()
-    sbe.toolbox.add_profiles_from_h5_to_net(net1, test_file1, True, False)
+    sbe.toolbox.add_profiles_from_parquet_to_net(net1, True, False, profiles_folder=temp_dir)
 
     pd.testing.assert_frame_equal(net1.profiles["load.p_mw"], profiles["load.p_mw"])
     pd.testing.assert_frame_equal(net1.profiles["sgen.q_mvar"], profiles["sgen.q_mvar"])
 
-    # --- store to file (from HDFStore - probably seldom needed)
-    with pd.HDFStore(test_file1) as hdf_store:
-        sbe.toolbox.store_profiles_to_hdf5_file(hdf_store, test_file2)
-
-    # test that file2 equals file1
-    net2 = pp.create_empty_network()
-    sbe.toolbox.add_profiles_from_h5_to_net(net2, test_file2, True, False)
-    for key in net2.profiles.keys():
-        pd.testing.assert_frame_equal(net2.profiles[key], net1.profiles[key])
-
     # load data with time_steps
     net3 = pp.create_empty_network()
-    sbe.toolbox.add_profiles_from_h5_to_net(net3, test_file1, [2, 1], False)
+    sbe.toolbox.add_profiles_from_parquet_to_net(net3, [2, 1], False, profiles_folder=temp_dir)
     pd.testing.assert_frame_equal(net3.profiles["load.p_mw"], profiles["load.p_mw"].loc[[1, 2]])
     pd.testing.assert_frame_equal(net3.profiles["sgen.q_mvar"], profiles["sgen.q_mvar"].loc[[1, 2]])
 
-    os.remove(test_file1)
-    os.remove(test_file2)
+    shutil.rmtree(temp_dir)
 
 
 if __name__ == "__main__":

SimBench_EHV_HV_excerpt/toolbox/__init__.py

@@ -2,6 +2,6 @@
 from .downcasting import *
 from .json_io import *
 from .set_values_to_net import *
-from .h5_profiles import *
+from .parquet_profiles import *
 from .run_custom_timeseries import *
 from .grid_manipulation import *

SimBench_EHV_HV_excerpt/toolbox/controller_functions.py

@@ -4,7 +4,8 @@
 from pandapower.control import DiscreteTapControl
 
 try:
-    from pandapower.control import DERController, QModelQV, PQVArea4120V2, CosphiPCurve
+    from pandapower.control.controller.DERController import DERController, QModelQV, \
+        PQVArea4120V2, CosphiPCurve
     controllers_imported = True
 except ImportError:
     try: