From 99bcbf34807402f7bd50da12854e2b9dfd22639d Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 25 Mar 2026 18:10:28 +0100 Subject: [PATCH 1/5] add variable statistic for 'Final Energy [by Sector]|Industry' --- .../configs/mapping.default.yaml | 1 + .../statistics_functions.py | 102 +++++++++++------- tests/test_statistics_functions.py | 45 ++++++++ 3 files changed, 112 insertions(+), 36 deletions(-) diff --git a/pypsa_validation_processing/configs/mapping.default.yaml b/pypsa_validation_processing/configs/mapping.default.yaml index 754603e..98ba12f 100644 --- a/pypsa_validation_processing/configs/mapping.default.yaml +++ b/pypsa_validation_processing/configs/mapping.default.yaml @@ -7,3 +7,4 @@ Final Energy [by Carrier]|Electricity: Final_Energy_by_Carrier__Electricity Final Energy [by Sector]|Transportation: Final_Energy_by_Sector__Transportation +Final Energy [by Sector]|Industry: Final_Energy_by_Sector__Industry diff --git a/pypsa_validation_processing/statistics_functions.py b/pypsa_validation_processing/statistics_functions.py index 007cfe3..b729b08 100644 --- a/pypsa_validation_processing/statistics_functions.py +++ b/pypsa_validation_processing/statistics_functions.py @@ -22,34 +22,25 @@ def (network_collection: pypsa.Network) -> pd.Series: def Final_Energy_by_Carrier__Electricity( n: pypsa.Network, ) -> pd.DataFrame: - """Extract electricity final energy from a PyPSA NetworkCollection. + """Extract electricity final energy from a PyPSA Network. Returns the total electricity consumption (excluding transmission / - distribution losses) across all networks in *network_collection*. + distribution losses) Parameters ---------- - network_collection : pypsa.NetworkCollection - Collection of PyPSA networks to process. + n : pypsa.Network + PyPSA network to process. Returns ------- - pd.DataFrame - Long-format DataFrame with columns ``variable``, ``unit``, ``year``, - and ``value``. The ``variable`` column contains - ``"Final Energy [by Carrier]|Electricity"`` for every row. + pd.Series + Pandas Series with Multiindex of ``country`` and ``unit`` Notes ----- - The actual extraction of electricity final energy from the network - collection will be implemented by the user. A typical call would be:: - - network_collection.statistics.energy_balance( - comps=["Load"], bus_carrier="AC" - ) - - The current implementation returns a dummy value of ``0.0 MWh`` for the - year 2020 so that the end-to-end workflow can be tested. + Extracts all withdrawals from elec network. low_voltage is included in AC withdrawal. + Remove discharger afterwards, as battery-connecting links have different carrier names. """ # withdrawal from electricity including low_voltage res = n.statistics.energy_balance( @@ -67,35 +58,25 @@ def Final_Energy_by_Carrier__Electricity( def Final_Energy_by_Sector__Transportation( n: pypsa.Network, ) -> pd.DataFrame: - """Extract transportation-sector final energy from a PyPSA NetworkCollection. + """Extract transportation-sector final energy from a PyPSA Network. Returns the total energy consumed by the transportation sector (excluding - transmission / distribution losses) across all networks in - *network_collection*. + transmission / distribution losses) Parameters ---------- - network_collection : pypsa.NetworkCollection - Collection of PyPSA networks to process. + n : pypsa.Network + PyPSA network to process. Returns ------- - pd.DataFrame - Long-format DataFrame with columns ``variable``, ``unit``, ``year``, - and ``value``. The ``variable`` column contains - ``"Final Energy [by Sector]|Transportation"`` for every row. + pd.Series + Pandas Series with Multiindex of ``country`` and ``unit`` Notes ----- - The actual extraction of transportation final energy from the network - collection will be implemented by the user. A typical call would be:: - - network_collection.statistics.energy_balance( - comps=["Load"], carrier="transport" - ) - - The current implementation returns a dummy value of ``0.0 MWh`` for the - year 2020 so that the end-to-end workflow can be tested. + Includes all carriers directly connected to loads in the transportation sector. + TODO: Needs futher clarification for bidirectional EV usage! """ # sum over all transportation-relevant sectors - 2 different units involved. result = ( @@ -103,12 +84,61 @@ def Final_Energy_by_Sector__Transportation( carrier=[ "land transport EV", "land transport fuel cell", + "land transport oil", "kerosene for aviation", "shipping methanol", + "shipping oil", ], components="Load", groupby=["carrier", "unit", "country"], - direction="withdrawal", + direction="withdrawal", # for positive values + ) + .groupby(["country", "unit"]) + .sum() + ) + return result + + +def Final_Energy_by_Sector__Industry( + n: pypsa.Network, +) -> pd.DataFrame: + """Extract transportation-sector final energy from a PyPSA Network. + + Returns the total energy consumed by the transportation sector (excluding + transmission / distribution losses) + + Parameters + ---------- + n : pypsa.Network + PyPSA network to process. + + Returns + ------- + pd.Series + Pandas Series with Multiindex of ``country`` and ``unit`` + + Notes + ----- + Includes all carriers directly connected to loads in the industry sector. Same Carrier + names are also attached to some links, so components-grouping is needed! + Values are exogenously set, so output values are round numbers! + """ + carriers = [ + "coal for industry", + "industry electricity", + "gas for industry", + "H2 for industry", + "solid biomass for industry", + "industry methanol", + "naphtha for industry", + "low-temperature heat for industry", + ] + result = ( + n.statistics.energy_balance( + carrier=carriers, + groupby=["carrier", "unit", "country"], + components="Load", + direction="withdrawal", # for positive values ) .groupby(["country", "unit"]) .sum() diff --git a/tests/test_statistics_functions.py b/tests/test_statistics_functions.py index bf16c45..d58eeea 100644 --- a/tests/test_statistics_functions.py +++ b/tests/test_statistics_functions.py @@ -7,6 +7,7 @@ from pypsa_validation_processing.statistics_functions import ( Final_Energy_by_Carrier__Electricity, + Final_Energy_by_Sector__Industry, Final_Energy_by_Sector__Transportation, ) @@ -99,3 +100,47 @@ def test_multiple_networks(self, mock_network_collection: MockNetworkCollection) result = Final_Energy_by_Sector__Transportation(network) assert isinstance(result, (pd.DataFrame, pd.Series)) assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# Tests for Final_Energy_by_Sector__Industry +# --------------------------------------------------------------------------- + + +class TestFinalEnergyBySectorIndustry: + """Test suite for Final_Energy_by_Sector__Industry function.""" + + def test_returns_dataframe(self, mock_network: MockPyPSANetwork): + """Test that the function returns a pandas DataFrame or Series.""" + result = Final_Energy_by_Sector__Industry(mock_network) + assert isinstance(result, (pd.DataFrame, pd.Series)) + + def test_has_country_and_unit_index(self, mock_network: MockPyPSANetwork): + """Test that result has country and unit in the index.""" + result = Final_Energy_by_Sector__Industry(mock_network) + assert "country" in result.index.names + assert "unit" in result.index.names + + def test_not_empty(self, mock_network: MockPyPSANetwork): + """Test that result is not empty.""" + result = Final_Energy_by_Sector__Industry(mock_network) + assert len(result) > 0 + + def test_numeric_values(self, mock_network: MockPyPSANetwork): + """Test that result values are numeric.""" + result = Final_Energy_by_Sector__Industry(mock_network) + assert result.dtype in [float, int] or pd.api.types.is_numeric_dtype( + result.dtype + ) + + def test_contains_austria(self, mock_network: MockPyPSANetwork): + """Test that result contains Austria (AT) data.""" + result = Final_Energy_by_Sector__Industry(mock_network) + assert "AT" in result.index.get_level_values("country") + + def test_multiple_networks(self, mock_network_collection: MockNetworkCollection): + """Test processing multiple networks from collection.""" + for network in mock_network_collection: + result = Final_Energy_by_Sector__Industry(network) + assert isinstance(result, (pd.DataFrame, pd.Series)) + assert len(result) > 0 From 8a268d5fa997f86760777f928efaa3e8b40fdd72 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 25 Mar 2026 18:27:01 +0100 Subject: [PATCH 2/5] PR Review: adapt function docstrings and type hints --- pypsa_validation_processing/statistics_functions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pypsa_validation_processing/statistics_functions.py b/pypsa_validation_processing/statistics_functions.py index b729b08..97d6433 100644 --- a/pypsa_validation_processing/statistics_functions.py +++ b/pypsa_validation_processing/statistics_functions.py @@ -21,7 +21,7 @@ def (network_collection: pypsa.Network) -> pd.Series: def Final_Energy_by_Carrier__Electricity( n: pypsa.Network, -) -> pd.DataFrame: +) -> pd.Series: """Extract electricity final energy from a PyPSA Network. Returns the total electricity consumption (excluding transmission / @@ -57,7 +57,7 @@ def Final_Energy_by_Carrier__Electricity( def Final_Energy_by_Sector__Transportation( n: pypsa.Network, -) -> pd.DataFrame: +) -> pd.Series: """Extract transportation-sector final energy from a PyPSA Network. Returns the total energy consumed by the transportation sector (excluding @@ -102,9 +102,9 @@ def Final_Energy_by_Sector__Transportation( def Final_Energy_by_Sector__Industry( n: pypsa.Network, ) -> pd.DataFrame: - """Extract transportation-sector final energy from a PyPSA Network. + """Extract Industry-sector final energy from a PyPSA Network. - Returns the total energy consumed by the transportation sector (excluding + Returns the total energy consumed by the Industry sector (excluding transmission / distribution losses) Parameters From 40391c3460269edb0f6efd1bdd59d11602fc02b0 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 25 Mar 2026 19:26:45 +0100 Subject: [PATCH 3/5] adapt testing function to test output-format to Series. --- tests/test_statistics_functions.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_statistics_functions.py b/tests/test_statistics_functions.py index d58eeea..2857506 100644 --- a/tests/test_statistics_functions.py +++ b/tests/test_statistics_functions.py @@ -110,16 +110,16 @@ def test_multiple_networks(self, mock_network_collection: MockNetworkCollection) class TestFinalEnergyBySectorIndustry: """Test suite for Final_Energy_by_Sector__Industry function.""" - def test_returns_dataframe(self, mock_network: MockPyPSANetwork): - """Test that the function returns a pandas DataFrame or Series.""" + def test_returns_series(self, mock_network: MockPyPSANetwork): + """Test that the function returns a pandas Series.""" result = Final_Energy_by_Sector__Industry(mock_network) - assert isinstance(result, (pd.DataFrame, pd.Series)) + assert isinstance(result, pd.Series) - def test_has_country_and_unit_index(self, mock_network: MockPyPSANetwork): - """Test that result has country and unit in the index.""" + def test_has_country_and_unit_multiindex(self, mock_network: MockPyPSANetwork): + """Test that result has MultiIndex with country and unit levels.""" result = Final_Energy_by_Sector__Industry(mock_network) - assert "country" in result.index.names - assert "unit" in result.index.names + assert isinstance(result.index, pd.MultiIndex) + assert result.index.names == ["country", "unit"] def test_not_empty(self, mock_network: MockPyPSANetwork): """Test that result is not empty.""" @@ -142,5 +142,7 @@ def test_multiple_networks(self, mock_network_collection: MockNetworkCollection) """Test processing multiple networks from collection.""" for network in mock_network_collection: result = Final_Energy_by_Sector__Industry(network) - assert isinstance(result, (pd.DataFrame, pd.Series)) + assert isinstance(result, pd.Series) + assert isinstance(result.index, pd.MultiIndex) + assert result.index.names == ["country", "unit"] assert len(result) > 0 From 396edb3149ddb740d150bf877a43497f26025e20 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 25 Mar 2026 19:27:43 +0100 Subject: [PATCH 4/5] adapt AGENTS-file to always test output-format on statistics-functions --- .github/copilot-instructions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a7514e8..0977a86 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -145,6 +145,7 @@ A task is complete when: - Add or update tests when behavior changes. - Tests belong only in `/tests`. - Prefer minimal unit tests over integration tests. +- all testing routines `test_statistics_functions.py` for functions in `statistics_functions.py` must test the output-format. The outputformat MUST be a pandas.Series with Multiindex of ``country`` and ``unit``. It CAN include more levels in the Multiindex. ## Background Information > [!WARNING] From bdbf55b97d6adb727bb2e677f459a1d510602748 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 25 Mar 2026 19:32:45 +0100 Subject: [PATCH 5/5] resolve merge effects --- pypsa_validation_processing/statistics_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypsa_validation_processing/statistics_functions.py b/pypsa_validation_processing/statistics_functions.py index 3875554..18a9331 100644 --- a/pypsa_validation_processing/statistics_functions.py +++ b/pypsa_validation_processing/statistics_functions.py @@ -96,7 +96,7 @@ def Final_Energy_by_Sector__Transportation( .groupby(["country", "unit"]) .sum() ) - return result + return res def Final_Energy_by_Sector__Industry( @@ -133,7 +133,7 @@ def Final_Energy_by_Sector__Industry( "naphtha for industry", "low-temperature heat for industry", ] - result = ( + res = ( n.statistics.energy_balance( carrier=carriers, groupby=["carrier", "unit", "country"],