From ed550380245306b568004383f629eab045babb6d Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 5 Sep 2022 10:21:27 +0200 Subject: [PATCH 001/119] Add first draft of heat pump desaggregation --- .../heat_supply/individual_heating.py | 325 ++++++++++++++++++ 1 file changed, 325 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 2aa442e07..2fb9d8018 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -3,7 +3,10 @@ """ import geopandas as gpd +import numpy as np import pandas as pd +import saio + from egon.data import config, db @@ -246,3 +249,325 @@ def plot_heat_supply(resulting_capacities): }, ) plt.savefig(f"plots/individual_heat_supply_{c}.png", dpi=300) + + +def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): + """ + Returns building IDs of buildings with decentral heat demand in given MV grid. + + As cells with district heating differ between scenarios, this is also depending + on the scenario. + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + mv_grid_id : int + ID of MV grid. + + Returns + -------- + pd.Index(int) + Building IDs (as int) of buildings with decentral heat demand in given MV grid. + Type is pandas Index to avoid errors later on when it is used in a query. + + """ + + # get zensus cells in grid + zensus_population_ids = db.select_dataframe( + f""" + SELECT zensus_population_id + FROM boundaries.egon_map_zensus_grid_districts + WHERE bus_id = {mv_grid_id} + """, + index_col=None + ).zensus_population_id.values + + # convert to pd.Index (otherwise type is np.int64, which will for some reason + # throw an error when used in a query) + zensus_population_ids = pd.Index(zensus_population_ids) + + # get zensus cells with district heating + from egon.data.datasets.district_heating_areas import MapZensusDistrictHeatingAreas + + with db.session_scope() as session: + query = session.query( + MapZensusDistrictHeatingAreas.zensus_population_id, + ).filter( + MapZensusDistrictHeatingAreas.scenario == scenario, + MapZensusDistrictHeatingAreas.zensus_population_id.in_( + zensus_population_ids) + ) + + cells_with_dh = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).zensus_population_id.values + + # remove zensus cells with district heating + zensus_population_ids = zensus_population_ids.drop(cells_with_dh, errors="ignore") + + # get buildings with decentral heat demand + engine = db.engine() + saio.register_schema("demand", engine) + from saio.demand import heat_timeseries_selected_profiles + + with db.session_scope() as session: + query = session.query( + heat_timeseries_selected_profiles.building_id, + ).filter( + heat_timeseries_selected_profiles.zensus_population_id.in_( + zensus_population_ids) + ) + + buildings_with_heat_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values + + # convert to pd.Index (otherwise type is np.int64, which will for some reason + # throw an error when used in a query) + return pd.Index(buildings_with_heat_demand) + + +def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): + """ + Returns total heat pump capacity per grid that was previously defined (by NEP or + pypsa-eur-sec). + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + mv_grid_id : int + ID of MV grid. + + Returns + -------- + float + Total heat pump capacity in MW in given MV grid. + + """ + from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply + + with db.session_scope() as session: + query = session.query( + EgonIndividualHeatingSupply.mv_grid_id, + EgonIndividualHeatingSupply.capacity, + ).filter( + EgonIndividualHeatingSupply.scenario == scenario + ).filter( + EgonIndividualHeatingSupply.carrier == "heat_pump" + ).filter( + EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id + ) + + hp_cap_mv_grid = pd.read_sql( + query.statement, query.session.bind, index_col="mv_grid_id" + ).capacity.values[0] + + return hp_cap_mv_grid + + +def get_heat_demand_timeseries_per_building(scenario, building_ids): + """ + Gets heat demand time series for all given buildings. + + ToDo: CTS demand still missing!! Also maybe use other function to make it faster. + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + building_ids : pd.Index(int) + Building IDs (as int) of buildings to get heat demand time series for. + + Returns + -------- + pd.DataFrame + Dataframe with hourly heat demand in MW for entire year. Index of the dataframe + contains the time steps and columns the building ID. + + """ + from egon.data.datasets.heat_demand_timeseries import create_timeseries_for_building + heat_demand_ts = pd.DataFrame() + for building_id in building_ids: + tmp = create_timeseries_for_building(building_id, scenario) + heat_demand_ts = pd.concat([heat_demand_ts, tmp], axis=1) + return heat_demand_ts + + +def determine_minimum_hp_capacity_per_building( + peak_heat_demand, flexibility_factor=24/18, cop=1.7): + """ + Determines minimum required heat pump capacity + + Parameters + ---------- + peak_heat_demand : pd.Series + Series with peak heat demand per building in MW. Index contains the building ID. + flexibility_factor : float + Factor to overdimension the heat pump to allow for some flexible dispatch in + times of high heat demand. Per default, a factor of 24/18 is used, to take into + account + + Returns + ------- + pd.Series + Pandas series with minimum required heat pump capacity per building in MW. + + """ + return peak_heat_demand * flexibility_factor / cop + + +def determine_buildings_with_hp_in_mv_grid(hp_cap_mv_grid, min_hp_cap_per_building): + """ + Distributes given total heat pump capacity to buildings based on their peak + heat demand. + + Parameters + ----------- + hp_cap_mv_grid : float + Total heat pump capacity in MW in given MV grid. + min_hp_cap_per_building : pd.Series + Pandas series with minimum required heat pump capacity per building in MW. + + Returns + ------- + pd.Index(int) + Building IDs (as int) of buildings to get heat demand time series for. + + """ + building_ids = min_hp_cap_per_building.index + + # get buildings with PV to give them a higher priority when selecting buildings + # a heat pump will be allocated to + engine = db.engine() + saio.register_schema("supply", engine) + from saio.supply import egon_power_plants_pv_roof_building + + with db.session_scope() as session: + query = session.query( + egon_power_plants_pv_roof_building.building_id + ).filter( + egon_power_plants_pv_roof_building.building_id.in_( + building_ids) + ) + + buildings_with_pv = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values + + # set different weights for buildings with PV and without PV + weight_with_pv = 1.5 + weight_without_pv = 1. + weights = pd.concat( + [ + pd.DataFrame( + {"weight": weight_without_pv}, + index=building_ids.drop(buildings_with_pv, errors="ignore") + ), + pd.DataFrame( + {"weight": weight_with_pv}, + index=buildings_with_pv + ), + ] + ) + # normalise weights (probability needs to add up to 1) + weights.weight = weights.weight / weights.weight.sum() + + # get random order at which buildings are chosen + np.random.seed(db.credentials()["--random-seed"]) + buildings_with_hp_order = np.random.choice( + weights.index, size=len(weights), replace=False, p=weights.weight.values) + + # select buildings until HP capacity in MV grid is reached (some rest capacity + # will remain) + hp_cumsum = min_hp_cap_per_building.loc[buildings_with_hp_order].cumsum() + buildings_with_hp = hp_cumsum[hp_cumsum <= hp_cap_mv_grid].index + + return buildings_with_hp + + +def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): + """ + Desaggregates the required total heat pump capacity to buildings. + + All buildings are previously assigned a minimum required heat pump capacity. If + the total heat pump capacity exceeds this, larger heat pumps are assigned. + + Parameters + ------------ + min_hp_cap_per_building : pd.Series + Pandas series with minimum required heat pump capacity per building in MW. + hp_cap_mv_grid : float + Total heat pump capacity in MW in given MV grid. + + Returns + -------- + + + """ + # ToDo Add warning if minimum is larger than total hp capacity + # distribute remaining capacity to all buildings with HP depending on installed HP capacity + + allocated_cap = min_hp_cap_per_building.sum() + remaining_cap = hp_cap_mv_grid - allocated_cap + + if remaining_cap < 0: + # ToDo raise warning? + return + + fac = remaining_cap / allocated_cap + hp_cap_per_building = min_hp_cap_per_building * fac + min_hp_cap_per_building + return hp_cap_per_building + + +def determine_hp_capacity_per_building(scenario): + + # get all MV grid IDs + mv_grid_ids = db.select_dataframe( + f""" + SELECT bus_id + FROM grid.egon_mv_grid_district + """, + index_col=None + ).bus_id.values + + for mv_grid_id in mv_grid_ids: + + # determine minimum required heat pump capacity per building + building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id) + heat_demand_ts = get_heat_demand_timeseries_per_building(scenario, building_ids) + # ToDo Write peak heat demand to table? + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + heat_demand_ts.max()) + + # in case this function is called to create pypsa-eur-sec input, only the + # minimum required heat pump capacity per MV grid is needed + if scenario == "pypsa-eur-sec": + # ToDo Write minimum required capacity to table for pypsa-eur-sec input + return + + # in case this function is called to create data for 2035 scenario, the + # buildings with heat pumps are determined; for 2050 scenario all buildings + # with decentral heating system get a heat pump + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id) + if scenario == "eGon2035": + buildings_with_hp = determine_buildings_with_hp_in_mv_grid( + hp_cap_grid, min_hp_cap_buildings) + min_hp_cap_buildings = min_hp_cap_buildings.loc[buildings_with_hp] + + # distribute total heat pump capacity to all buildings with HP + hp_cap_per_building = desaggregate_hp_capacity( + min_hp_cap_buildings, hp_cap_grid) + + # ToDo Write desaggregated HP capacity to table + heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ + :, hp_cap_per_building.index].sum() + + # ToDo Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + + # ToDo Write other heat demand time series to database - gas voronoi + # (grid - egon_gas_voronoi mit carrier CH4) + # erstmal intermediate table From 42f79ef2a9cdd715f3db5a489360322cb30e90f0 Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 5 Sep 2022 11:06:36 +0200 Subject: [PATCH 002/119] Minor changes --- .../datasets/heat_supply/individual_heating.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 2fb9d8018..56d56e79e 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -503,11 +503,12 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): Returns -------- - + pd.Series + Pandas series with heat pump capacity per building in MW. """ - # ToDo Add warning if minimum is larger than total hp capacity - # distribute remaining capacity to all buildings with HP depending on installed HP capacity + # distribute remaining capacity to all buildings with HP depending on installed + # HP capacity allocated_cap = min_hp_cap_per_building.sum() remaining_cap = hp_cap_mv_grid - allocated_cap @@ -522,6 +523,13 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): def determine_hp_capacity_per_building(scenario): + """ + Parameters + ----------- + scenario : str + "pypsa-eur-sec", "eGon2035", "eGon100RE" + + """ # get all MV grid IDs mv_grid_ids = db.select_dataframe( @@ -545,6 +553,7 @@ def determine_hp_capacity_per_building(scenario): # in case this function is called to create pypsa-eur-sec input, only the # minimum required heat pump capacity per MV grid is needed if scenario == "pypsa-eur-sec": + min_hp_cap_buildings.sum() # ToDo Write minimum required capacity to table for pypsa-eur-sec input return From 8b6107be45159fc7a9b1e3978c962a83316bc080 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 5 Sep 2022 12:26:04 +0200 Subject: [PATCH 003/119] Add scenario specific wrapper functions --- .../heat_supply/individual_heating.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 56d56e79e..24a6af943 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -545,6 +545,7 @@ def determine_hp_capacity_per_building(scenario): # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( scenario, mv_grid_id) + # TODO alternative get peak demand from db? heat_demand_ts = get_heat_demand_timeseries_per_building(scenario, building_ids) # ToDo Write peak heat demand to table? min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( @@ -580,3 +581,27 @@ def determine_hp_capacity_per_building(scenario): # ToDo Write other heat demand time series to database - gas voronoi # (grid - egon_gas_voronoi mit carrier CH4) # erstmal intermediate table + + +def determine_hp_cap_pypsa_eur_sec(): + """Wrapper function to determine heat pump capacities for scenario + pypsa-eur-sec. Only the minimum required heat pump capacity per MV grid is + exported to db + """ + determine_hp_capacity_per_building(scenario="pypsa-eur-sec") + + +def determine_hp_cap_eGon2035(): + """Wrapper function to determine Heat Pump capacities + for scenario eGon2035. Only selected buildings get a heat pump capacity + assigned. Buildings with PV rooftop are more likely to be assigned. + """ + determine_hp_capacity_per_building(scenario="eGon2035") + + +def determine_hp_cap_eGon100RE(): + """Wrapper function to determine Heat Pump capacities + for scenario eGon100RE. All buildings without district heating get a heat + pump capacity assigned. + """ + determine_hp_capacity_per_building(scenario="eGon100RE") From 997dfcc88e161ed8c857b415ab557ec30a31acd9 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 5 Sep 2022 12:26:43 +0200 Subject: [PATCH 004/119] Black --- .../heat_supply/individual_heating.py | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 24a6af943..58740e014 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -18,7 +18,7 @@ def cascade_per_technology( max_size_individual_chp=0.05, ): - """ Add plants for individual heat. + """Add plants for individual heat. Currently only on mv grid district level. Parameters @@ -280,7 +280,7 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): FROM boundaries.egon_map_zensus_grid_districts WHERE bus_id = {mv_grid_id} """, - index_col=None + index_col=None, ).zensus_population_id.values # convert to pd.Index (otherwise type is np.int64, which will for some reason @@ -288,7 +288,9 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): zensus_population_ids = pd.Index(zensus_population_ids) # get zensus cells with district heating - from egon.data.datasets.district_heating_areas import MapZensusDistrictHeatingAreas + from egon.data.datasets.district_heating_areas import ( + MapZensusDistrictHeatingAreas, + ) with db.session_scope() as session: query = session.query( @@ -296,7 +298,8 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): ).filter( MapZensusDistrictHeatingAreas.scenario == scenario, MapZensusDistrictHeatingAreas.zensus_population_id.in_( - zensus_population_ids) + zensus_population_ids + ), ) cells_with_dh = pd.read_sql( @@ -304,7 +307,9 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): ).zensus_population_id.values # remove zensus cells with district heating - zensus_population_ids = zensus_population_ids.drop(cells_with_dh, errors="ignore") + zensus_population_ids = zensus_population_ids.drop( + cells_with_dh, errors="ignore" + ) # get buildings with decentral heat demand engine = db.engine() @@ -316,7 +321,8 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): heat_timeseries_selected_profiles.building_id, ).filter( heat_timeseries_selected_profiles.zensus_population_id.in_( - zensus_population_ids) + zensus_population_ids + ) ) buildings_with_heat_demand = pd.read_sql( @@ -349,15 +355,14 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply with db.session_scope() as session: - query = session.query( - EgonIndividualHeatingSupply.mv_grid_id, - EgonIndividualHeatingSupply.capacity, - ).filter( - EgonIndividualHeatingSupply.scenario == scenario - ).filter( - EgonIndividualHeatingSupply.carrier == "heat_pump" - ).filter( - EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id + query = ( + session.query( + EgonIndividualHeatingSupply.mv_grid_id, + EgonIndividualHeatingSupply.capacity, + ) + .filter(EgonIndividualHeatingSupply.scenario == scenario) + .filter(EgonIndividualHeatingSupply.carrier == "heat_pump") + .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) ) hp_cap_mv_grid = pd.read_sql( @@ -387,7 +392,10 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): contains the time steps and columns the building ID. """ - from egon.data.datasets.heat_demand_timeseries import create_timeseries_for_building + from egon.data.datasets.heat_demand_timeseries import ( + create_timeseries_for_building, + ) + heat_demand_ts = pd.DataFrame() for building_id in building_ids: tmp = create_timeseries_for_building(building_id, scenario) @@ -396,7 +404,8 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): def determine_minimum_hp_capacity_per_building( - peak_heat_demand, flexibility_factor=24/18, cop=1.7): + peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 +): """ Determines minimum required heat pump capacity @@ -418,7 +427,9 @@ def determine_minimum_hp_capacity_per_building( return peak_heat_demand * flexibility_factor / cop -def determine_buildings_with_hp_in_mv_grid(hp_cap_mv_grid, min_hp_cap_per_building): +def determine_buildings_with_hp_in_mv_grid( + hp_cap_mv_grid, min_hp_cap_per_building +): """ Distributes given total heat pump capacity to buildings based on their peak heat demand. @@ -448,8 +459,7 @@ def determine_buildings_with_hp_in_mv_grid(hp_cap_mv_grid, min_hp_cap_per_buildi query = session.query( egon_power_plants_pv_roof_building.building_id ).filter( - egon_power_plants_pv_roof_building.building_id.in_( - building_ids) + egon_power_plants_pv_roof_building.building_id.in_(building_ids) ) buildings_with_pv = pd.read_sql( @@ -458,17 +468,14 @@ def determine_buildings_with_hp_in_mv_grid(hp_cap_mv_grid, min_hp_cap_per_buildi # set different weights for buildings with PV and without PV weight_with_pv = 1.5 - weight_without_pv = 1. + weight_without_pv = 1.0 weights = pd.concat( [ pd.DataFrame( {"weight": weight_without_pv}, - index=building_ids.drop(buildings_with_pv, errors="ignore") - ), - pd.DataFrame( - {"weight": weight_with_pv}, - index=buildings_with_pv + index=building_ids.drop(buildings_with_pv, errors="ignore"), ), + pd.DataFrame({"weight": weight_with_pv}, index=buildings_with_pv), ] ) # normalise weights (probability needs to add up to 1) @@ -477,7 +484,11 @@ def determine_buildings_with_hp_in_mv_grid(hp_cap_mv_grid, min_hp_cap_per_buildi # get random order at which buildings are chosen np.random.seed(db.credentials()["--random-seed"]) buildings_with_hp_order = np.random.choice( - weights.index, size=len(weights), replace=False, p=weights.weight.values) + weights.index, + size=len(weights), + replace=False, + p=weights.weight.values, + ) # select buildings until HP capacity in MV grid is reached (some rest capacity # will remain) @@ -518,7 +529,9 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): return fac = remaining_cap / allocated_cap - hp_cap_per_building = min_hp_cap_per_building * fac + min_hp_cap_per_building + hp_cap_per_building = ( + min_hp_cap_per_building * fac + min_hp_cap_per_building + ) return hp_cap_per_building @@ -537,19 +550,23 @@ def determine_hp_capacity_per_building(scenario): SELECT bus_id FROM grid.egon_mv_grid_district """, - index_col=None + index_col=None, ).bus_id.values for mv_grid_id in mv_grid_ids: # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id) + scenario, mv_grid_id + ) # TODO alternative get peak demand from db? - heat_demand_ts = get_heat_demand_timeseries_per_building(scenario, building_ids) + heat_demand_ts = get_heat_demand_timeseries_per_building( + scenario, building_ids + ) # ToDo Write peak heat demand to table? min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - heat_demand_ts.max()) + heat_demand_ts.max() + ) # in case this function is called to create pypsa-eur-sec input, only the # minimum required heat pump capacity per MV grid is needed @@ -561,19 +578,24 @@ def determine_hp_capacity_per_building(scenario): # in case this function is called to create data for 2035 scenario, the # buildings with heat pumps are determined; for 2050 scenario all buildings # with decentral heating system get a heat pump - hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id) + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( + scenario, mv_grid_id + ) if scenario == "eGon2035": buildings_with_hp = determine_buildings_with_hp_in_mv_grid( - hp_cap_grid, min_hp_cap_buildings) + hp_cap_grid, min_hp_cap_buildings + ) min_hp_cap_buildings = min_hp_cap_buildings.loc[buildings_with_hp] # distribute total heat pump capacity to all buildings with HP hp_cap_per_building = desaggregate_hp_capacity( - min_hp_cap_buildings, hp_cap_grid) + min_hp_cap_buildings, hp_cap_grid + ) # ToDo Write desaggregated HP capacity to table heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ - :, hp_cap_per_building.index].sum() + :, hp_cap_per_building.index + ].sum() # ToDo Write aggregated heat demand time series of buildings with HP to # table to be used in eTraGo - egon_etrago_timeseries_individual_heating From 4e17813f2a85f12e00da96b106725c37e0ba6945 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 5 Sep 2022 14:39:53 +0200 Subject: [PATCH 005/119] Add table for EgonEtragoTimeSeriesIndividualHeating --- .../datasets/heat_supply/individual_heating.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 58740e014..92487fdec 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,6 +2,8 @@ individual heat supply. """ +from sqlalchemy import ARRAY, REAL, Column, Integer, String +from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd import numpy as np import pandas as pd @@ -9,6 +11,19 @@ from egon.data import config, db +engine = db.engine() +Base = declarative_base() + + +class EgonEtragoTimeSeriesIndividualHeating(Base): + __tablename__ = "egon_etrago_timeseries_individual_heating" + __table_args__ = {"schema": "demand"} + + bus_id = Column(Integer, primary_key=True) + scn_name = Column(String, primary_key=True) + p_set = Column(ARRAY(REAL)) + q_set = Column(ARRAY(REAL)) + def cascade_per_technology( heat_per_mv, From 42d35135f2b23af02b7a63e94cc4a653d7cde2b1 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 5 Sep 2022 14:41:08 +0200 Subject: [PATCH 006/119] Add db export template --- .../heat_supply/individual_heating.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 92487fdec..703febc67 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -541,6 +541,8 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): if remaining_cap < 0: # ToDo raise warning? + # TODO alternative outsource sanity check + # only possible if peak_load is determined beforehand return fac = remaining_cap / allocated_cap @@ -614,7 +616,27 @@ def determine_hp_capacity_per_building(scenario): # ToDo Write aggregated heat demand time series of buildings with HP to # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - + # EgonEtragoTimeSeriesIndividualHeating + + # # Change format + # data = CTS_grid.drop(columns="scenario") + # df_etrago_cts_heat_profiles = pd.DataFrame( + # index=data.index, columns=["scn_name", "p_set"] + # ) + # df_etrago_cts_heat_profiles.p_set = data.values.tolist() + # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] + # df_etrago_cts_heat_profiles.reset_index(inplace=True) + # + # # Drop and recreate Table if exists + # EgonEtragoTimeSeriesIndividualHeating.__table__.drop(bind=db.engine(), checkfirst=True) + # EgonEtragoTimeSeriesIndividualHeating.__table__.create(bind=db.engine(), checkfirst=True) + # + # # Write heat ts into db + # with db.session_scope() as session: + # session.bulk_insert_mappings( + # EgonEtragoTimeSeriesIndividualHeating, + # df_etrago_cts_heat_profiles.to_dict(orient="records"), + # ) # ToDo Write other heat demand time series to database - gas voronoi # (grid - egon_gas_voronoi mit carrier CH4) # erstmal intermediate table From 67ca91e17aeb86ad6b18c101f28a06cafd852cef Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 17:00:06 +0200 Subject: [PATCH 007/119] Disable pv_rooftop import --- .../heat_supply/individual_heating.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 703febc67..3278991f2 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -468,19 +468,20 @@ def determine_buildings_with_hp_in_mv_grid( # a heat pump will be allocated to engine = db.engine() saio.register_schema("supply", engine) - from saio.supply import egon_power_plants_pv_roof_building - - with db.session_scope() as session: - query = session.query( - egon_power_plants_pv_roof_building.building_id - ).filter( - egon_power_plants_pv_roof_building.building_id.in_(building_ids) - ) - - buildings_with_pv = pd.read_sql( - query.statement, query.session.bind, index_col=None - ).building_id.values - + # TODO Adhoc Pv rooftop fix + # from saio.supply import egon_power_plants_pv_roof_building + # + # with db.session_scope() as session: + # query = session.query( + # egon_power_plants_pv_roof_building.building_id + # ).filter( + # egon_power_plants_pv_roof_building.building_id.in_(building_ids) + # ) + # + # buildings_with_pv = pd.read_sql( + # query.statement, query.session.bind, index_col=None + # ).building_id.values + buildings_with_pv = [] # set different weights for buildings with PV and without PV weight_with_pv = 1.5 weight_without_pv = 1.0 From a2a999ef4063a934d4d43669384895140cdc1b3d Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 17:00:48 +0200 Subject: [PATCH 008/119] Add seperate datasets for each scenario --- src/egon/data/airflow/dags/pipeline.py | 25 +++++++++++++++ .../heat_supply/individual_heating.py | 31 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index b69f11914..f0fa0d8dd 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -43,6 +43,11 @@ from egon.data.datasets.heat_etrago import HeatEtrago from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable from egon.data.datasets.heat_supply import HeatSupply +from egon.data.datasets.heat_supply.individual_heating import ( + HeatPumps2035, + HeatPumps2050, + HeatPumpsEtrago, +) from egon.data.datasets.hydrogen_etrago import ( HydrogenBusEtrago, HydrogenGridEtrago, @@ -572,6 +577,26 @@ ] ) + heat_pumps_etrago = HeatPumpsEtrago( + dependencies=[ + DistrictHeatingAreas, + ] + ) + + heat_pumps_2050 = HeatPumps2050( + dependencies=[ + DistrictHeatingAreas, + run_pypsaeursec, + ] + ) + + heat_pumps_2035 = HeatPumps2035( + dependencies=[ + DistrictHeatingAreas, + # TODO add PV rooftop + ] + ) + # ########## Keep this dataset at the end # Sanity Checks sanity_checks = SanityChecks( diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 3278991f2..c867d2c04 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -10,6 +10,7 @@ import saio from egon.data import config, db +from egon.data.datasets import Dataset engine = db.engine() Base = declarative_base() @@ -25,6 +26,36 @@ class EgonEtragoTimeSeriesIndividualHeating(Base): q_set = Column(ARRAY(REAL)) +class HeatPumpsEtrago(Dataset): + def __init__(self, dependencies): + super().__init__( + name="HeatPumpsEtrago", + version="0.0.0", + dependencies=dependencies, + tasks=(determine_hp_cap_pypsa_eur_sec,), + ) + + +class HeatPumps2035(Dataset): + def __init__(self, dependencies): + super().__init__( + name="HeatPumps2035", + version="0.0.0", + dependencies=dependencies, + tasks=(determine_hp_cap_eGon2035,), + ) + + +class HeatPumps2050(Dataset): + def __init__(self, dependencies): + super().__init__( + name="HeatPumps2050", + version="0.0.0", + dependencies=dependencies, + tasks=(determine_hp_cap_eGon100RE), + ) + + def cascade_per_technology( heat_per_mv, technologies, From a1804fd743218090b75c3b829def557fc7104bed Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:02:54 +0200 Subject: [PATCH 009/119] Add further dependencies --- src/egon/data/airflow/dags/pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index f0fa0d8dd..8f144c480 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -579,12 +579,14 @@ heat_pumps_etrago = HeatPumpsEtrago( dependencies=[ + cts_demand_buildings, DistrictHeatingAreas, ] ) heat_pumps_2050 = HeatPumps2050( dependencies=[ + cts_demand_buildings, DistrictHeatingAreas, run_pypsaeursec, ] @@ -592,6 +594,7 @@ heat_pumps_2035 = HeatPumps2035( dependencies=[ + cts_demand_buildings, DistrictHeatingAreas, # TODO add PV rooftop ] From 0f61ed930f77af62b190fd508ce77852ee2ebf94 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:07:02 +0200 Subject: [PATCH 010/119] Rename db table --- .../heat_supply/individual_heating.py | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index c867d2c04..310fec365 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -16,14 +16,13 @@ Base = declarative_base() -class EgonEtragoTimeSeriesIndividualHeating(Base): +class EgonEtragoTimeseriesIndividualHeating(Base): __tablename__ = "egon_etrago_timeseries_individual_heating" __table_args__ = {"schema": "demand"} - bus_id = Column(Integer, primary_key=True) - scn_name = Column(String, primary_key=True) - p_set = Column(ARRAY(REAL)) - q_set = Column(ARRAY(REAL)) + scenario = Column(Text, primary_key=True) + carrier = Column(String, primary_key=True) + dist_aggregated_mw = Column(ARRAY(Float(53))) class HeatPumpsEtrago(Dataset): @@ -422,8 +421,6 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): """ Gets heat demand time series for all given buildings. - ToDo: CTS demand still missing!! Also maybe use other function to make it faster. - Parameters ----------- scenario : str @@ -646,32 +643,38 @@ def determine_hp_capacity_per_building(scenario): :, hp_cap_per_building.index ].sum() - # ToDo Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - # EgonEtragoTimeSeriesIndividualHeating - - # # Change format - # data = CTS_grid.drop(columns="scenario") - # df_etrago_cts_heat_profiles = pd.DataFrame( - # index=data.index, columns=["scn_name", "p_set"] - # ) - # df_etrago_cts_heat_profiles.p_set = data.values.tolist() - # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] - # df_etrago_cts_heat_profiles.reset_index(inplace=True) - # - # # Drop and recreate Table if exists - # EgonEtragoTimeSeriesIndividualHeating.__table__.drop(bind=db.engine(), checkfirst=True) - # EgonEtragoTimeSeriesIndividualHeating.__table__.create(bind=db.engine(), checkfirst=True) - # - # # Write heat ts into db - # with db.session_scope() as session: - # session.bulk_insert_mappings( - # EgonEtragoTimeSeriesIndividualHeating, - # df_etrago_cts_heat_profiles.to_dict(orient="records"), - # ) - # ToDo Write other heat demand time series to database - gas voronoi - # (grid - egon_gas_voronoi mit carrier CH4) - # erstmal intermediate table + df_etrago_timeseries_heat_pumps[mv_grid_id] = heat_timeseries_hp_buildings_mv_grid.values + + + # ToDo Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # TODO Clara uses this table already + # but will not need it anymore for pypsa eur sec + # EgonEtragoTimeseriesIndividualHeating + + # # Change format + # data = CTS_grid.drop(columns="scenario") + # df_etrago_cts_heat_profiles = pd.DataFrame( + # index=data.index, columns=["scn_name", "p_set"] + # ) + # df_etrago_cts_heat_profiles.p_set = data.values.tolist() + # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] + # df_etrago_cts_heat_profiles.reset_index(inplace=True) + # + # # Drop and recreate Table if exists + # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), checkfirst=True) + # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), checkfirst=True) + # + # # Write heat ts into db + # with db.session_scope() as session: + # session.bulk_insert_mappings( + # EgonEtragoTimeseriesIndividualHeating, + # df_etrago_cts_heat_profiles.to_dict(orient="records"), + # ) + # ToDo Write other heat demand time series to database - gas voronoi + # (grid - egon_gas_voronoi mit carrier CH4) + # erstmal intermediate table + # TODO Gas aggregiert pro MV Grid def determine_hp_cap_pypsa_eur_sec(): From dc5d39cfbbf34fb1d684bc41d7f00774009f821a Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:07:48 +0200 Subject: [PATCH 011/119] Add cts heat demand --- .../heat_supply/individual_heating.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 310fec365..9f47f2ca9 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -11,7 +11,9 @@ from egon.data import config, db from egon.data.datasets import Dataset - +from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( + calc_building_profiles +) engine = db.engine() Base = declarative_base() @@ -439,10 +441,27 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): create_timeseries_for_building, ) - heat_demand_ts = pd.DataFrame() - for building_id in building_ids: + heat_demand_residential_ts = pd.DataFrame() + # TODO remove testmode + # for building_id in building_ids: + for building_id in building_ids[:3]: + # ToDo: maybe use other function to make it faster. tmp = create_timeseries_for_building(building_id, scenario) - heat_demand_ts = pd.concat([heat_demand_ts, tmp], axis=1) + # # TODO check what happens if tmp emtpy + # tmp = pd.Series() if tmp.empty else tmp + heat_demand_residential_ts = pd.concat([heat_demand_residential_ts, tmp], axis=1) + + # TODO add cts profiles per building_id + # extend calc_building_profiles for heat + # extend calc_building_profiles for list of ids + heat_demand_cts_ts = calc_building_profiles(building_id = building_ids, + scenario=scenario, + sector="heat", + ) + + heat_demand_ts = pd.concat([heat_demand_residential_ts, heat_demand_cts_ts]) + # sum residential and heat if same building id in header + heat_demand_ts = heat_demand_ts.groupby(axis=1, level=0).sum() return heat_demand_ts From f57531636dbc96d2d7762cc33f042ad8b7f2130e Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:08:10 +0200 Subject: [PATCH 012/119] Remove sanity check --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 9f47f2ca9..32e39d808 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -587,12 +587,6 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): allocated_cap = min_hp_cap_per_building.sum() remaining_cap = hp_cap_mv_grid - allocated_cap - if remaining_cap < 0: - # ToDo raise warning? - # TODO alternative outsource sanity check - # only possible if peak_load is determined beforehand - return - fac = remaining_cap / allocated_cap hp_cap_per_building = ( min_hp_cap_per_building * fac + min_hp_cap_per_building From 5b5304ad94794435f412e0305a6bf08670014402 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:08:59 +0200 Subject: [PATCH 013/119] Use different peak load for each scenario --- .../heat_supply/individual_heating.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 32e39d808..b1f352ba8 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -612,26 +612,40 @@ def determine_hp_capacity_per_building(scenario): index_col=None, ).bus_id.values + df_etrago_timeseries_heat_pumps = pd.DataFrame() + for mv_grid_id in mv_grid_ids: # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( scenario, mv_grid_id ) - # TODO alternative get peak demand from db? - heat_demand_ts = get_heat_demand_timeseries_per_building( - scenario, building_ids - ) - # ToDo Write peak heat demand to table? - min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - heat_demand_ts.max() - ) + + if scenario == "eGon100RE": + # TODO alternative get peak demand from db? + # residential and cts peak sum + else: + # iterates for residential heat over building id > slow + heat_demand_ts = get_heat_demand_timeseries_per_building( + scenario, building_ids + ) + # ToDo Write peak heat demand to table? + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + heat_demand_ts.max() + ) # in case this function is called to create pypsa-eur-sec input, only the # minimum required heat pump capacity per MV grid is needed if scenario == "pypsa-eur-sec": min_hp_cap_buildings.sum() # ToDo Write minimum required capacity to table for pypsa-eur-sec input + + # ToDo Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # TODO Clara uses this table already + # but will not need it anymore for pypsa eur sec + # EgonEtragoTimeseriesIndividualHeating + return # in case this function is called to create data for 2035 scenario, the From 7203cb4afd0fbfdddaf12690bfd280e0924064e7 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 7 Sep 2022 18:11:41 +0200 Subject: [PATCH 014/119] Add pass --- src/egon/data/datasets/heat_supply/individual_heating.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index b1f352ba8..dc54a340c 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -622,6 +622,7 @@ def determine_hp_capacity_per_building(scenario): ) if scenario == "eGon100RE": + pass # TODO alternative get peak demand from db? # residential and cts peak sum else: From 8fc36b8a7618b5640122ab5a43b212dd4be42d46 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 7 Sep 2022 18:37:22 +0200 Subject: [PATCH 015/119] Devide disaggregation function --- .../heat_supply/individual_heating.py | 196 ++++++++++++------ 1 file changed, 133 insertions(+), 63 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index dc54a340c..b826a07f0 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -465,6 +465,28 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): return heat_demand_ts +def get_peak_demand_per_building(scenario, building_ids): + """ + Gets peak heat demand for all given buildings. + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + building_ids : pd.Index(int) + Building IDs (as int) of buildings to get heat demand time series for. + + Returns + -------- + pd.Series + Series with peak heat demand per building in MW. Index contains the building ID. + + """ + # TODO Implement + + return peak_heat_demand + + def determine_minimum_hp_capacity_per_building( peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 ): @@ -594,13 +616,10 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): return hp_cap_per_building -def determine_hp_capacity_per_building(scenario): - """ - Parameters - ----------- - scenario : str - "pypsa-eur-sec", "eGon2035", "eGon100RE" - +def determine_hp_cap_pypsa_eur_sec(): + """Wrapper function to determine heat pump capacities for scenario + pypsa-eur-sec. Only the minimum required heat pump capacity per MV grid is + exported to db """ # get all MV grid IDs @@ -618,48 +637,81 @@ def determine_hp_capacity_per_building(scenario): # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id + "eGon100RE", mv_grid_id ) - if scenario == "eGon100RE": - pass - # TODO alternative get peak demand from db? - # residential and cts peak sum - else: - # iterates for residential heat over building id > slow - heat_demand_ts = get_heat_demand_timeseries_per_building( - scenario, building_ids - ) - # ToDo Write peak heat demand to table? - min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - heat_demand_ts.max() - ) + # get heat demand time series per building + # iterates for residential heat over building id > slow + heat_demand_ts = get_heat_demand_timeseries_per_building( + "eGon100RE", building_ids + ) + + # ToDo Write peak heat demand to table + + # write aggregated heat time series to dataframe to write it to table later on + df_etrago_timeseries_heat_pumps[ + mv_grid_id] = heat_demand_ts.sum(axis=1).values + + # determine minimum required heat pump capacity per building + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + heat_demand_ts.max() + ) + # ToDo Write minimum required capacity to table for pypsa-eur-sec input + # min_hp_cap_buildings.sum() + + # ToDo Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # TODO Clara uses this table already + # but will not need it anymore for pypsa eur sec + # EgonEtragoTimeseriesIndividualHeating + + return + + +def determine_hp_cap_eGon2035(): + """Wrapper function to determine Heat Pump capacities + for scenario eGon2035. Only selected buildings get a heat pump capacity + assigned. Buildings with PV rooftop are more likely to be assigned. + """ + # get all MV grid IDs + mv_grid_ids = db.select_dataframe( + f""" + SELECT bus_id + FROM grid.egon_mv_grid_district + """, + index_col=None, + ).bus_id.values - # in case this function is called to create pypsa-eur-sec input, only the - # minimum required heat pump capacity per MV grid is needed - if scenario == "pypsa-eur-sec": - min_hp_cap_buildings.sum() - # ToDo Write minimum required capacity to table for pypsa-eur-sec input + df_etrago_timeseries_heat_pumps = pd.DataFrame() + + for mv_grid_id in mv_grid_ids: + + # determine minimum required heat pump capacity per building + building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon2035", mv_grid_id + ) - # ToDo Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - # TODO Clara uses this table already - # but will not need it anymore for pypsa eur sec - # EgonEtragoTimeseriesIndividualHeating + # get heat demand time series per building + # iterates for residential heat over building id > slow + heat_demand_ts = get_heat_demand_timeseries_per_building( + "eGon2035", building_ids + ) - return + # ToDo Write peak heat demand to table - # in case this function is called to create data for 2035 scenario, the - # buildings with heat pumps are determined; for 2050 scenario all buildings - # with decentral heating system get a heat pump + # determine minimum required heat pump capacity per building + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + heat_demand_ts.max() + ) + + # select buildings that will have a heat pump hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( - scenario, mv_grid_id + "eGon2035", mv_grid_id ) - if scenario == "eGon2035": - buildings_with_hp = determine_buildings_with_hp_in_mv_grid( - hp_cap_grid, min_hp_cap_buildings - ) - min_hp_cap_buildings = min_hp_cap_buildings.loc[buildings_with_hp] + buildings_with_hp = determine_buildings_with_hp_in_mv_grid( + hp_cap_grid, min_hp_cap_buildings + ) + min_hp_cap_buildings = min_hp_cap_buildings.loc[buildings_with_hp] # distribute total heat pump capacity to all buildings with HP hp_cap_per_building = desaggregate_hp_capacity( @@ -667,12 +719,13 @@ def determine_hp_capacity_per_building(scenario): ) # ToDo Write desaggregated HP capacity to table - heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ - :, hp_cap_per_building.index - ].sum() - - df_etrago_timeseries_heat_pumps[mv_grid_id] = heat_timeseries_hp_buildings_mv_grid.values + # write aggregated heat time series to dataframe to write it to table later on + heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ + :, hp_cap_per_building.index + ].sum(axis=1) + df_etrago_timeseries_heat_pumps[ + mv_grid_id] = heat_timeseries_hp_buildings_mv_grid.values # ToDo Write aggregated heat demand time series of buildings with HP to # table to be used in eTraGo - egon_etrago_timeseries_individual_heating @@ -705,25 +758,42 @@ def determine_hp_capacity_per_building(scenario): # TODO Gas aggregiert pro MV Grid -def determine_hp_cap_pypsa_eur_sec(): - """Wrapper function to determine heat pump capacities for scenario - pypsa-eur-sec. Only the minimum required heat pump capacity per MV grid is - exported to db - """ - determine_hp_capacity_per_building(scenario="pypsa-eur-sec") - - -def determine_hp_cap_eGon2035(): - """Wrapper function to determine Heat Pump capacities - for scenario eGon2035. Only selected buildings get a heat pump capacity - assigned. Buildings with PV rooftop are more likely to be assigned. - """ - determine_hp_capacity_per_building(scenario="eGon2035") - - def determine_hp_cap_eGon100RE(): """Wrapper function to determine Heat Pump capacities for scenario eGon100RE. All buildings without district heating get a heat pump capacity assigned. """ - determine_hp_capacity_per_building(scenario="eGon100RE") + + # get all MV grid IDs + mv_grid_ids = db.select_dataframe( + f""" + SELECT bus_id + FROM grid.egon_mv_grid_district + """, + index_col=None, + ).bus_id.values + + for mv_grid_id in mv_grid_ids: + + # determine minimum required heat pump capacity per building + building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mv_grid_id + ) + + # TODO get peak demand from db + peak_heat_demand = get_peak_demand_per_building("eGon100RE", building_ids) + + # determine minimum required heat pump capacity per building + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 + ) + + # distribute total heat pump capacity to all buildings with HP + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( + "eGon100RE", mv_grid_id + ) + hp_cap_per_building = desaggregate_hp_capacity( + min_hp_cap_buildings, hp_cap_grid + ) + + # ToDo Write desaggregated HP capacity to table From c784bb0ee92bb1f4c195489349940857da7aca36 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 9 Sep 2022 17:08:28 +0200 Subject: [PATCH 016/119] Update function name --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index b826a07f0..20820e676 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -12,7 +12,7 @@ from egon.data import config, db from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( - calc_building_profiles + calc_cts_building_profiles ) engine = db.engine() Base = declarative_base() @@ -452,9 +452,7 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): heat_demand_residential_ts = pd.concat([heat_demand_residential_ts, tmp], axis=1) # TODO add cts profiles per building_id - # extend calc_building_profiles for heat - # extend calc_building_profiles for list of ids - heat_demand_cts_ts = calc_building_profiles(building_id = building_ids, + heat_demand_cts_ts = calc_cts_building_profiles(building_id = building_ids, scenario=scenario, sector="heat", ) From 5f28eb54ef91d8360a2b7f72ea50b55272e5eb52 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 9 Sep 2022 17:08:42 +0200 Subject: [PATCH 017/119] Black --- .../heat_supply/individual_heating.py | 112 +++++++++++------- 1 file changed, 66 insertions(+), 46 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 20820e676..452714360 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -12,8 +12,9 @@ from egon.data import config, db from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( - calc_cts_building_profiles + calc_cts_building_profiles, ) + engine = db.engine() Base = declarative_base() @@ -22,9 +23,9 @@ class EgonEtragoTimeseriesIndividualHeating(Base): __tablename__ = "egon_etrago_timeseries_individual_heating" __table_args__ = {"schema": "demand"} bus_id = Column(Integer, primary_key=True) - scenario = Column(Text, primary_key=True) + scenario = Column(String, primary_key=True) carrier = Column(String, primary_key=True) - dist_aggregated_mw = Column(ARRAY(Float(53))) + dist_aggregated_mw = Column(ARRAY(REAL)) class HeatPumpsEtrago(Dataset): @@ -300,10 +301,11 @@ def plot_heat_supply(resulting_capacities): def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): """ - Returns building IDs of buildings with decentral heat demand in given MV grid. + Returns building IDs of buildings with decentral heat demand in given MV + grid. - As cells with district heating differ between scenarios, this is also depending - on the scenario. + As cells with district heating differ between scenarios, this is also + depending on the scenario. Parameters ----------- @@ -315,8 +317,9 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heat demand in given MV grid. - Type is pandas Index to avoid errors later on when it is used in a query. + Building IDs (as int) of buildings with decentral heat demand in given + MV grid. Type is pandas Index to avoid errors later on when it is + used in a query. """ @@ -330,8 +333,8 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): index_col=None, ).zensus_population_id.values - # convert to pd.Index (otherwise type is np.int64, which will for some reason - # throw an error when used in a query) + # convert to pd.Index (otherwise type is np.int64, which will for some + # reason throw an error when used in a query) zensus_population_ids = pd.Index(zensus_population_ids) # get zensus cells with district heating @@ -376,15 +379,15 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): query.statement, query.session.bind, index_col=None ).building_id.values - # convert to pd.Index (otherwise type is np.int64, which will for some reason - # throw an error when used in a query) + # convert to pd.Index (otherwise type is np.int64, which will for some + # reason throw an error when used in a query) return pd.Index(buildings_with_heat_demand) def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): """ - Returns total heat pump capacity per grid that was previously defined (by NEP or - pypsa-eur-sec). + Returns total heat pump capacity per grid that was previously defined + (by NEP or pypsa-eur-sec). Parameters ----------- @@ -433,8 +436,8 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): Returns -------- pd.DataFrame - Dataframe with hourly heat demand in MW for entire year. Index of the dataframe - contains the time steps and columns the building ID. + Dataframe with hourly heat demand in MW for entire year. Index of the + dataframe contains the time steps and columns the building ID. """ from egon.data.datasets.heat_demand_timeseries import ( @@ -449,15 +452,20 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): tmp = create_timeseries_for_building(building_id, scenario) # # TODO check what happens if tmp emtpy # tmp = pd.Series() if tmp.empty else tmp - heat_demand_residential_ts = pd.concat([heat_demand_residential_ts, tmp], axis=1) + heat_demand_residential_ts = pd.concat( + [heat_demand_residential_ts, tmp], axis=1 + ) # TODO add cts profiles per building_id - heat_demand_cts_ts = calc_cts_building_profiles(building_id = building_ids, - scenario=scenario, - sector="heat", - ) + heat_demand_cts_ts = calc_cts_building_profiles( + building_id=building_ids, + scenario=scenario, + sector="heat", + ) - heat_demand_ts = pd.concat([heat_demand_residential_ts, heat_demand_cts_ts]) + heat_demand_ts = pd.concat( + [heat_demand_residential_ts, heat_demand_cts_ts] + ) # sum residential and heat if same building id in header heat_demand_ts = heat_demand_ts.groupby(axis=1, level=0).sum() return heat_demand_ts @@ -477,7 +485,8 @@ def get_peak_demand_per_building(scenario, building_ids): Returns -------- pd.Series - Series with peak heat demand per building in MW. Index contains the building ID. + Series with peak heat demand per building in MW. Index contains the + building ID. """ # TODO Implement @@ -494,16 +503,18 @@ def determine_minimum_hp_capacity_per_building( Parameters ---------- peak_heat_demand : pd.Series - Series with peak heat demand per building in MW. Index contains the building ID. + Series with peak heat demand per building in MW. Index contains the + building ID. flexibility_factor : float - Factor to overdimension the heat pump to allow for some flexible dispatch in - times of high heat demand. Per default, a factor of 24/18 is used, to take into - account + Factor to overdimension the heat pump to allow for some flexible + dispatch in times of high heat demand. Per default, a factor of 24/18 + is used, to take into account Returns ------- pd.Series - Pandas series with minimum required heat pump capacity per building in MW. + Pandas series with minimum required heat pump capacity per building in + MW. """ return peak_heat_demand * flexibility_factor / cop @@ -521,7 +532,8 @@ def determine_buildings_with_hp_in_mv_grid( hp_cap_mv_grid : float Total heat pump capacity in MW in given MV grid. min_hp_cap_per_building : pd.Series - Pandas series with minimum required heat pump capacity per building in MW. + Pandas series with minimum required heat pump capacity per building + in MW. Returns ------- @@ -531,8 +543,8 @@ def determine_buildings_with_hp_in_mv_grid( """ building_ids = min_hp_cap_per_building.index - # get buildings with PV to give them a higher priority when selecting buildings - # a heat pump will be allocated to + # get buildings with PV to give them a higher priority when selecting + # buildings a heat pump will be allocated to engine = db.engine() saio.register_schema("supply", engine) # TODO Adhoc Pv rooftop fix @@ -573,8 +585,8 @@ def determine_buildings_with_hp_in_mv_grid( p=weights.weight.values, ) - # select buildings until HP capacity in MV grid is reached (some rest capacity - # will remain) + # select buildings until HP capacity in MV grid is reached (some rest + # capacity will remain) hp_cumsum = min_hp_cap_per_building.loc[buildings_with_hp_order].cumsum() buildings_with_hp = hp_cumsum[hp_cumsum <= hp_cap_mv_grid].index @@ -585,13 +597,15 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): """ Desaggregates the required total heat pump capacity to buildings. - All buildings are previously assigned a minimum required heat pump capacity. If - the total heat pump capacity exceeds this, larger heat pumps are assigned. + All buildings are previously assigned a minimum required heat pump + capacity. If the total heat pump capacity exceeds this, larger heat pumps + are assigned. Parameters ------------ min_hp_cap_per_building : pd.Series - Pandas series with minimum required heat pump capacity per building in MW. + Pandas series with minimum required heat pump capacity per building + in MW. hp_cap_mv_grid : float Total heat pump capacity in MW in given MV grid. @@ -601,8 +615,8 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): Pandas series with heat pump capacity per building in MW. """ - # distribute remaining capacity to all buildings with HP depending on installed - # HP capacity + # distribute remaining capacity to all buildings with HP depending on + # installed HP capacity allocated_cap = min_hp_cap_per_building.sum() remaining_cap = hp_cap_mv_grid - allocated_cap @@ -646,9 +660,11 @@ def determine_hp_cap_pypsa_eur_sec(): # ToDo Write peak heat demand to table - # write aggregated heat time series to dataframe to write it to table later on - df_etrago_timeseries_heat_pumps[ - mv_grid_id] = heat_demand_ts.sum(axis=1).values + # write aggregated heat time series to dataframe to write it to table + # later on + df_etrago_timeseries_heat_pumps[mv_grid_id] = heat_demand_ts.sum( + axis=1 + ).values # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( @@ -718,12 +734,14 @@ def determine_hp_cap_eGon2035(): # ToDo Write desaggregated HP capacity to table - # write aggregated heat time series to dataframe to write it to table later on + # write aggregated heat time series to dataframe to write it to table + # later on heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ - :, hp_cap_per_building.index - ].sum(axis=1) + :, hp_cap_per_building.index + ].sum(axis=1) df_etrago_timeseries_heat_pumps[ - mv_grid_id] = heat_timeseries_hp_buildings_mv_grid.values + mv_grid_id + ] = heat_timeseries_hp_buildings_mv_grid.values # ToDo Write aggregated heat demand time series of buildings with HP to # table to be used in eTraGo - egon_etrago_timeseries_individual_heating @@ -779,7 +797,9 @@ def determine_hp_cap_eGon100RE(): ) # TODO get peak demand from db - peak_heat_demand = get_peak_demand_per_building("eGon100RE", building_ids) + peak_heat_demand = get_peak_demand_per_building( + "eGon100RE", building_ids + ) # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( From b4ac9ff7b51463e744866242b49e152774b9cf1c Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 9 Sep 2022 17:51:23 +0200 Subject: [PATCH 018/119] Use sqla adapter for int --- .../datasets/heat_supply/individual_heating.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 452714360..13cc873b9 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,6 +2,7 @@ individual heat supply. """ +from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -58,6 +59,20 @@ def __init__(self, dependencies): ) +# ========== Register np datatypes with SQLA ========== +def adapt_numpy_float64(numpy_float64): + return AsIs(numpy_float64) + + +def adapt_numpy_int64(numpy_int64): + return AsIs(numpy_int64) + + +register_adapter(np.float64, adapt_numpy_float64) +register_adapter(np.int64, adapt_numpy_int64) +# ===================================================== + + def cascade_per_technology( heat_per_mv, technologies, From ec13661b661e1b1c831397c3e7808d8ba92cf312 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 9 Sep 2022 17:52:13 +0200 Subject: [PATCH 019/119] Update function name --- .../heat_supply/individual_heating.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 13cc873b9..73ef3dcce 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -379,13 +379,13 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): # get buildings with decentral heat demand engine = db.engine() saio.register_schema("demand", engine) - from saio.demand import heat_timeseries_selected_profiles + from saio.demand import egon_heat_timeseries_selected_profiles with db.session_scope() as session: query = session.query( - heat_timeseries_selected_profiles.building_id, + egon_heat_timeseries_selected_profiles.building_id, ).filter( - heat_timeseries_selected_profiles.zensus_population_id.in_( + egon_heat_timeseries_selected_profiles.zensus_population_id.in_( zensus_population_ids ) ) @@ -437,7 +437,9 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): return hp_cap_mv_grid -def get_heat_demand_timeseries_per_building(scenario, building_ids): +def get_heat_demand_timeseries_per_building( + scenario, building_ids, mv_grid_id +): """ Gets heat demand time series for all given buildings. @@ -447,6 +449,8 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): Name of scenario. Can be either "eGon2035" or "eGon100RE". building_ids : pd.Index(int) Building IDs (as int) of buildings to get heat demand time series for. + mv_grid_id : int + MV grid of the buildings Returns -------- @@ -461,19 +465,25 @@ def get_heat_demand_timeseries_per_building(scenario, building_ids): heat_demand_residential_ts = pd.DataFrame() # TODO remove testmode - # for building_id in building_ids: - for building_id in building_ids[:3]: + for building_id in building_ids: + # for building_id in building_ids[:3]: # ToDo: maybe use other function to make it faster. + # in MW tmp = create_timeseries_for_building(building_id, scenario) # # TODO check what happens if tmp emtpy # tmp = pd.Series() if tmp.empty else tmp heat_demand_residential_ts = pd.concat( - [heat_demand_residential_ts, tmp], axis=1 + [ + heat_demand_residential_ts, + tmp.rename(columns={"demand": building_id}), + ], + axis=1, ) # TODO add cts profiles per building_id heat_demand_cts_ts = calc_cts_building_profiles( - building_id=building_ids, + egon_building_ids=building_ids, + bus_ids=[mv_grid_id], scenario=scenario, sector="heat", ) @@ -670,7 +680,9 @@ def determine_hp_cap_pypsa_eur_sec(): # get heat demand time series per building # iterates for residential heat over building id > slow heat_demand_ts = get_heat_demand_timeseries_per_building( - "eGon100RE", building_ids + scenario="eGon100RE", + building_ids=building_ids, + mv_grid_id=mv_grid_id, ) # ToDo Write peak heat demand to table From d8319a67022b988e7d6e3dc2cad458064d17695e Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 19 Sep 2022 18:40:15 +0200 Subject: [PATCH 020/119] Move and fix adapter bug --- .../datasets/heat_supply/individual_heating.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 73ef3dcce..0b080eac4 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -59,7 +59,6 @@ def __init__(self, dependencies): ) -# ========== Register np datatypes with SQLA ========== def adapt_numpy_float64(numpy_float64): return AsIs(numpy_float64) @@ -68,11 +67,6 @@ def adapt_numpy_int64(numpy_int64): return AsIs(numpy_int64) -register_adapter(np.float64, adapt_numpy_float64) -register_adapter(np.int64, adapt_numpy_int64) -# ===================================================== - - def cascade_per_technology( heat_per_mv, technologies, @@ -348,6 +342,11 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): index_col=None, ).zensus_population_id.values + # TODO replace with sql adapter? + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== # convert to pd.Index (otherwise type is np.int64, which will for some # reason throw an error when used in a query) zensus_population_ids = pd.Index(zensus_population_ids) @@ -394,9 +393,7 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): query.statement, query.session.bind, index_col=None ).building_id.values - # convert to pd.Index (otherwise type is np.int64, which will for some - # reason throw an error when used in a query) - return pd.Index(buildings_with_heat_demand) + return buildings_with_heat_demand def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): From c3ba42243e6b8a61b7c69b9295d049dbf259cb3a Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 19 Sep 2022 18:42:43 +0200 Subject: [PATCH 021/119] Add timeit decorator --- .../electricity_demand_timeseries/tools.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/tools.py b/src/egon/data/datasets/electricity_demand_timeseries/tools.py index 59a75fa4e..798616752 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/tools.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/tools.py @@ -1,5 +1,6 @@ from io import StringIO import csv +import time from shapely.geometry import Point import geopandas as gpd @@ -11,6 +12,23 @@ engine = db.engine() +def timeit(func): + """ + Decorator for measuring function's running time. + """ + + def measure_time(*args, **kw): + start_time = time.time() + result = func(*args, **kw) + print( + "Processing time of %s(): %.2f seconds." + % (func.__qualname__, time.time() - start_time) + ) + return result + + return measure_time + + def random_point_in_square(geom, tol): """ Generate a random point within a square From a7de35ec839f542035eec9c87bb6aea2584ed9ec Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 19 Sep 2022 18:45:17 +0200 Subject: [PATCH 022/119] Use timeit decorator --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 0b080eac4..be12e5d84 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -15,6 +15,7 @@ from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( calc_cts_building_profiles, ) +from egon.data.datasets.electricity_demand_timeseries.tools import timeit engine = db.engine() Base = declarative_base() @@ -308,6 +309,7 @@ def plot_heat_supply(resulting_capacities): plt.savefig(f"plots/individual_heat_supply_{c}.png", dpi=300) +@timeit def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): """ Returns building IDs of buildings with decentral heat demand in given MV From c5c8a44266763b46ab9d8497cce061ad59cb32d9 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 28 Sep 2022 21:16:01 +0200 Subject: [PATCH 023/119] Adapt column name --- src/egon/data/datasets/heat_demand_timeseries/daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_demand_timeseries/daily.py b/src/egon/data/datasets/heat_demand_timeseries/daily.py index bbbff15bc..9217bd9f8 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/daily.py +++ b/src/egon/data/datasets/heat_demand_timeseries/daily.py @@ -32,7 +32,7 @@ class EgonDailyHeatDemandPerClimateZone(Base): climate_zone = Column(Text, primary_key=True) day_of_year = Column(Integer, primary_key=True) temperature_class = Column(Integer) - heat_demand_share = Column(Float(53)) + daily_demand_share = Column(Float(53)) def temperature_classes(): From 58b54fbe3c0ff1148c07b3f69375170ef46c631e Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 28 Sep 2022 21:16:18 +0200 Subject: [PATCH 024/119] Add description of resulting dataframe --- .../datasets/electricity_demand_timeseries/cts_buildings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index 2917a420a..f58059b3f 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -936,7 +936,9 @@ def calc_cts_building_profiles( Returns ------- df_building_profiles: pd.DataFrame - Table of demand profile per building + Table of demand profile per building. Column names are building IDs and index + is hour of the year as int (0-8759). + """ if sector == "electricity": # Get cts building electricity demand share of selected buildings From 6b3256e9f493ae5bfbfc90c4ab8bb30773b397cf Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 28 Sep 2022 21:16:41 +0200 Subject: [PATCH 025/119] Adapt heat pump desaggregation --- .../heat_supply/individual_heating.py | 897 +++++++++++++----- 1 file changed, 660 insertions(+), 237 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index be12e5d84..dfeb614d1 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,20 +2,39 @@ individual heat supply. """ +from loguru import logger +import numpy as np +import pandas as pd +import random +import saio + +from pathlib import Path +import time + from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd -import numpy as np -import pandas as pd -import saio + from egon.data import config, db from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( calc_cts_building_profiles, ) -from egon.data.datasets.electricity_demand_timeseries.tools import timeit +from egon.data.datasets.electricity_demand_timeseries.tools import ( + write_table_to_postgres, +) +from egon.data.datasets.heat_demand import EgonPetaHeat +from egon.data.datasets.heat_demand_timeseries.daily import ( + EgonDailyHeatDemandPerClimateZone, + EgonMapZensusClimateZones, +) +from egon.data.datasets.heat_demand_timeseries.idp_pool import ( + EgonHeatTimeseries, +) +# get zensus cells with district heating +from egon.data.datasets.zensus_mv_grid_districts import MapZensusGridDistricts engine = db.engine() Base = declarative_base() @@ -30,6 +49,7 @@ class EgonEtragoTimeseriesIndividualHeating(Base): dist_aggregated_mw = Column(ARRAY(REAL)) +# ToDo @Julian muss angepasst werden? class HeatPumpsEtrago(Dataset): def __init__(self, dependencies): super().__init__( @@ -40,6 +60,7 @@ def __init__(self, dependencies): ) +# ToDo @Julian muss angepasst werden? class HeatPumps2035(Dataset): def __init__(self, dependencies): super().__init__( @@ -50,6 +71,7 @@ def __init__(self, dependencies): ) +# ToDo @Julian muss angepasst werden? class HeatPumps2050(Dataset): def __init__(self, dependencies): super().__init__( @@ -60,6 +82,16 @@ def __init__(self, dependencies): ) +class BuildingHeatPeakLoads(Base): + __tablename__ = "egon_building_heat_peak_loads" + __table_args__ = {"schema": "demand"} + + building_id = Column(Integer, primary_key=True) + scenario = Column(String, primary_key=True) + sector = Column(String, primary_key=True) + peak_load_in_w = Column(REAL) + + def adapt_numpy_float64(numpy_float64): return AsIs(numpy_float64) @@ -68,6 +100,61 @@ def adapt_numpy_int64(numpy_int64): return AsIs(numpy_int64) +def log_to_file(name): + """Simple only file logger""" + logger.remove() + logger.add( + Path(f"{name}.log"), + format="{time} {level} {message}", + # filter="my_module", + level="TRACE", + ) + logger.trace("Start trace logging") + return logger + + +def timeit(func): + """ + Decorator for measuring function's running time. + """ + + def measure_time(*args, **kw): + start_time = time.time() + result = func(*args, **kw) + print( + "Processing time of %s(): %.2f seconds." + % (func.__qualname__, time.time() - start_time) + ) + return result + + return measure_time + + +def timeitlog(func): + """ + Decorator for measuring running time of residential heat peak load and + logging it. + """ + + def measure_time(*args, **kw): + start_time = time.time() + result = func(*args, **kw) + process_time = time.time() - start_time + try: + mvgd = kw["mvgd"] + except KeyError: + mvgd = "bulk" + statement = ( + f"MVGD={mvgd} | Processing time of {func.__qualname__} | " + f"{time.strftime('%H h, %M min, %S s', time.gmtime(process_time))}" + ) + logger.trace(statement) + print(statement) + return result + + return measure_time + + def cascade_per_technology( heat_per_mv, technologies, @@ -276,6 +363,226 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): ) +# @timeit +def get_peta_demand(mvgd): + """only residential""" + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.zensus_population_id, + EgonPetaHeat.demand.label("peta_2035"), + ) + .filter(MapZensusGridDistricts.bus_id == mvgd) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .filter(EgonPetaHeat.scenario == "eGon2035") + .filter(EgonPetaHeat.sector == "residential") + ) + + df_peta_2035 = pd.read_sql( + query.statement, query.session.bind, index_col="zensus_population_id" + ) + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.zensus_population_id, + EgonPetaHeat.demand.label("peta_2050"), + ) + .filter(MapZensusGridDistricts.bus_id == mvgd) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .filter(EgonPetaHeat.scenario == "eGon100RE") + .filter(EgonPetaHeat.sector == "residential") + ) + + df_peta_100RE = pd.read_sql( + query.statement, query.session.bind, index_col="zensus_population_id" + ) + + df_peta_demand = pd.concat( + [df_peta_2035, df_peta_100RE], axis=1 + ).reset_index() + + return df_peta_demand + + +# @timeit +def get_profile_ids(mvgd): + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.zensus_population_id, + EgonHeatTimeseries.building_id, + EgonHeatTimeseries.selected_idp_profiles, + ) + .filter(MapZensusGridDistricts.bus_id == mvgd) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonHeatTimeseries.zensus_population_id + ) + ) + + df_profiles_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) + # Add building count per cell + df_profiles_ids = pd.merge( + left=df_profiles_ids, + right=df_profiles_ids.groupby("zensus_population_id")["building_id"] + .count() + .rename("buildings"), + left_on="zensus_population_id", + right_index=True, + ) + + df_profiles_ids = df_profiles_ids.explode("selected_idp_profiles") + df_profiles_ids["day_of_year"] = ( + df_profiles_ids.groupby("building_id").cumcount() + 1 + ) + return df_profiles_ids + + +# @timeit +def get_daily_profiles(profile_ids): + saio.register_schema("demand", db.engine()) + from saio.demand import egon_heat_idp_pool + + with db.session_scope() as session: + query = session.query(egon_heat_idp_pool).filter( + egon_heat_idp_pool.index.in_(profile_ids) + ) + + df_profiles = pd.read_sql( + query.statement, query.session.bind, index_col="index" + ) + + df_profiles = df_profiles.explode("idp") + df_profiles["hour"] = df_profiles.groupby(axis=0, level=0).cumcount() + 1 + + return df_profiles + + +# @timeit +def get_daily_demand_share(mvgd): + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.zensus_population_id, + EgonDailyHeatDemandPerClimateZone.day_of_year, + EgonDailyHeatDemandPerClimateZone.daily_demand_share, + ) + .filter( + EgonMapZensusClimateZones.climate_zone + == EgonDailyHeatDemandPerClimateZone.climate_zone + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonMapZensusClimateZones.zensus_population_id + ) + .filter(MapZensusGridDistricts.bus_id == mvgd) + ) + + df_daily_demand_share = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) + return df_daily_demand_share + + +@timeitlog +def calc_residential_heat_profiles_per_mvgd(mvgd): + """ + Gets residential heat profiles per building in MV grid for both eGon2035 and + eGon100RE scenario. + + Parameters + ---------- + mvgd : int + MV grid ID. + + Returns + -------- + pd.DataFrame + Heat demand profiles of buildings. Columns are: + * zensus_population_id : int + Zensus cell ID building is in. + * building_id : int + ID of building. + * day_of_year : int + Day of the year (1 - 365). + * hour : int + Hour of the day (1 - 24). + * eGon2035 : float + Building's residential heat demand in MW, for specified hour of the + year (specified through columns `day_of_year` and `hour`). + * eGon100RE : float + Building's residential heat demand in MW, for specified hour of the + year (specified through columns `day_of_year` and `hour`). + + """ + df_peta_demand = get_peta_demand(mvgd) + + if df_peta_demand.empty: + return None + + df_profiles_ids = get_profile_ids(mvgd) + + if df_profiles_ids.empty: + return None + + df_profiles = get_daily_profiles( + df_profiles_ids["selected_idp_profiles"].unique() + ) + + df_daily_demand_share = get_daily_demand_share(mvgd) + + # Merge profile ids to peta demand by zensus_population_id + df_profile_merge = pd.merge( + left=df_peta_demand, right=df_profiles_ids, on="zensus_population_id" + ) + + # Merge daily demand to daily profile ids by zensus_population_id and day + df_profile_merge = pd.merge( + left=df_profile_merge, + right=df_daily_demand_share, + on=["zensus_population_id", "day_of_year"], + ) + + # Merge daily profiles by profile id + df_profile_merge = pd.merge( + left=df_profile_merge, + right=df_profiles[["idp", "hour"]], + left_on="selected_idp_profiles", + right_index=True, + ) + + # Scale profiles + df_profile_merge["eGon2035"] = ( + df_profile_merge["idp"] + .mul(df_profile_merge["daily_demand_share"]) + .mul(df_profile_merge["peta_2035"]) + .div(df_profile_merge["buildings"]) + ) + + df_profile_merge["eGon100RE"] = ( + df_profile_merge["idp"] + .mul(df_profile_merge["daily_demand_share"]) + .mul(df_profile_merge["peta_2050"]) + .div(df_profile_merge["buildings"]) + ) + + columns = ["zensus_population_id", "building_id", "day_of_year", "hour", + "eGon2035", "eGon100RE"] + + return df_profile_merge.loc[:, columns] + + def plot_heat_supply(resulting_capacities): from matplotlib import pyplot as plt @@ -328,9 +635,9 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heat demand in given - MV grid. Type is pandas Index to avoid errors later on when it is - used in a query. + Building IDs (as int) of buildings with decentral heating system in given + MV grid. Type is pandas Index to avoid errors later on when it is + used in a query. """ @@ -436,93 +743,11 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): return hp_cap_mv_grid -def get_heat_demand_timeseries_per_building( - scenario, building_ids, mv_grid_id -): - """ - Gets heat demand time series for all given buildings. - - Parameters - ----------- - scenario : str - Name of scenario. Can be either "eGon2035" or "eGon100RE". - building_ids : pd.Index(int) - Building IDs (as int) of buildings to get heat demand time series for. - mv_grid_id : int - MV grid of the buildings - - Returns - -------- - pd.DataFrame - Dataframe with hourly heat demand in MW for entire year. Index of the - dataframe contains the time steps and columns the building ID. - - """ - from egon.data.datasets.heat_demand_timeseries import ( - create_timeseries_for_building, - ) - - heat_demand_residential_ts = pd.DataFrame() - # TODO remove testmode - for building_id in building_ids: - # for building_id in building_ids[:3]: - # ToDo: maybe use other function to make it faster. - # in MW - tmp = create_timeseries_for_building(building_id, scenario) - # # TODO check what happens if tmp emtpy - # tmp = pd.Series() if tmp.empty else tmp - heat_demand_residential_ts = pd.concat( - [ - heat_demand_residential_ts, - tmp.rename(columns={"demand": building_id}), - ], - axis=1, - ) - - # TODO add cts profiles per building_id - heat_demand_cts_ts = calc_cts_building_profiles( - egon_building_ids=building_ids, - bus_ids=[mv_grid_id], - scenario=scenario, - sector="heat", - ) - - heat_demand_ts = pd.concat( - [heat_demand_residential_ts, heat_demand_cts_ts] - ) - # sum residential and heat if same building id in header - heat_demand_ts = heat_demand_ts.groupby(axis=1, level=0).sum() - return heat_demand_ts - - -def get_peak_demand_per_building(scenario, building_ids): - """ - Gets peak heat demand for all given buildings. - - Parameters - ----------- - scenario : str - Name of scenario. Can be either "eGon2035" or "eGon100RE". - building_ids : pd.Index(int) - Building IDs (as int) of buildings to get heat demand time series for. - - Returns - -------- - pd.Series - Series with peak heat demand per building in MW. Index contains the - building ID. - - """ - # TODO Implement - - return peak_heat_demand - - def determine_minimum_hp_capacity_per_building( peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 ): """ - Determines minimum required heat pump capacity + Determines minimum required heat pump capacity. Parameters ---------- @@ -538,7 +763,7 @@ def determine_minimum_hp_capacity_per_building( ------- pd.Series Pandas series with minimum required heat pump capacity per building in - MW. + MW. """ return peak_heat_demand * flexibility_factor / cop @@ -614,6 +839,27 @@ def determine_buildings_with_hp_in_mv_grid( hp_cumsum = min_hp_cap_per_building.loc[buildings_with_hp_order].cumsum() buildings_with_hp = hp_cumsum[hp_cumsum <= hp_cap_mv_grid].index + # choose random heat pumps until remaining heat pumps are larger than remaining + # heat pump capacity + remaining_hp_cap = ( + hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum()) + min_cap_buildings_wo_hp = min_hp_cap_per_building.loc[ + building_ids.drop(buildings_with_hp)] + possible_buildings = min_cap_buildings_wo_hp[ + min_cap_buildings_wo_hp <= remaining_hp_cap].index + while len(possible_buildings) > 0: + random.seed(db.credentials()["--random-seed"]) + new_hp_building = random.choice(possible_buildings) + # add new building to building with HP + buildings_with_hp = buildings_with_hp.append(pd.Index([new_hp_building])) + # determine if there are still possible buildings + remaining_hp_cap = ( + hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum()) + min_cap_buildings_wo_hp = min_hp_cap_per_building.loc[ + building_ids.drop(buildings_with_hp)] + possible_buildings = min_cap_buildings_wo_hp[ + min_cap_buildings_wo_hp <= remaining_hp_cap].index + return buildings_with_hp @@ -623,7 +869,7 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): All buildings are previously assigned a minimum required heat pump capacity. If the total heat pump capacity exceeds this, larger heat pumps - are assigned. + are assigned. Parameters ------------ @@ -652,96 +898,62 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): return hp_cap_per_building -def determine_hp_cap_pypsa_eur_sec(): - """Wrapper function to determine heat pump capacities for scenario - pypsa-eur-sec. Only the minimum required heat pump capacity per MV grid is - exported to db +def determine_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): """ + Determines minimum required HP capacity in MV grid in MW as input for + pypsa-eur-sec. - # get all MV grid IDs - mv_grid_ids = db.select_dataframe( - f""" - SELECT bus_id - FROM grid.egon_mv_grid_district - """, - index_col=None, - ).bus_id.values - - df_etrago_timeseries_heat_pumps = pd.DataFrame() - - for mv_grid_id in mv_grid_ids: - - # determine minimum required heat pump capacity per building - building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mv_grid_id - ) - - # get heat demand time series per building - # iterates for residential heat over building id > slow - heat_demand_ts = get_heat_demand_timeseries_per_building( - scenario="eGon100RE", - building_ids=building_ids, - mv_grid_id=mv_grid_id, - ) - - # ToDo Write peak heat demand to table + Parameters + ---------- + peak_heat_demand : pd.Series + Series with peak heat demand per building in MW. Index contains the + building ID. + building_ids : pd.Index(int) + Building IDs (as int) of buildings with decentral heating system in given + MV grid. - # write aggregated heat time series to dataframe to write it to table - # later on - df_etrago_timeseries_heat_pumps[mv_grid_id] = heat_demand_ts.sum( - axis=1 - ).values + Returns + -------- + float + Minimum required HP capacity in MV grid in MW. + """ + if len(building_ids) > 0: + peak_heat_demand = peak_heat_demand.loc[building_ids] # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - heat_demand_ts.max() + peak_heat_demand ) - # ToDo Write minimum required capacity to table for pypsa-eur-sec input - # min_hp_cap_buildings.sum() - - # ToDo Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - # TODO Clara uses this table already - # but will not need it anymore for pypsa eur sec - # EgonEtragoTimeseriesIndividualHeating - - return + return min_hp_cap_buildings.sum() + else: + return 0.0 -def determine_hp_cap_eGon2035(): - """Wrapper function to determine Heat Pump capacities - for scenario eGon2035. Only selected buildings get a heat pump capacity - assigned. Buildings with PV rooftop are more likely to be assigned. +def determine_hp_cap_eGon2035(mv_grid_id, peak_heat_demand, building_ids): """ - # get all MV grid IDs - mv_grid_ids = db.select_dataframe( - f""" - SELECT bus_id - FROM grid.egon_mv_grid_district - """, - index_col=None, - ).bus_id.values - - df_etrago_timeseries_heat_pumps = pd.DataFrame() + Determines which buildings in the MV grid will have a HP (buildings with PV + rooftop are more likely to be assigned) in the eGon2035 scenario, as well as + their respective HP capacity in MW. - for mv_grid_id in mv_grid_ids: - - # determine minimum required heat pump capacity per building - building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon2035", mv_grid_id - ) + Parameters + ----------- + mv_grid_id : int + ID of MV grid. + peak_heat_demand : pd.Series + Series with peak heat demand per building in MW. Index contains the + building ID. + building_ids : pd.Index(int) + Building IDs (as int) of buildings with decentral heating system in + given MV grid. - # get heat demand time series per building - # iterates for residential heat over building id > slow - heat_demand_ts = get_heat_demand_timeseries_per_building( - "eGon2035", building_ids - ) + """ - # ToDo Write peak heat demand to table + if len(building_ids) > 0: + peak_heat_demand = peak_heat_demand.loc[building_ids] # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - heat_demand_ts.max() + peak_heat_demand ) # select buildings that will have a heat pump @@ -751,93 +963,304 @@ def determine_hp_cap_eGon2035(): buildings_with_hp = determine_buildings_with_hp_in_mv_grid( hp_cap_grid, min_hp_cap_buildings ) - min_hp_cap_buildings = min_hp_cap_buildings.loc[buildings_with_hp] # distribute total heat pump capacity to all buildings with HP hp_cap_per_building = desaggregate_hp_capacity( - min_hp_cap_buildings, hp_cap_grid - ) - - # ToDo Write desaggregated HP capacity to table - - # write aggregated heat time series to dataframe to write it to table - # later on - heat_timeseries_hp_buildings_mv_grid = heat_demand_ts.loc[ - :, hp_cap_per_building.index - ].sum(axis=1) - df_etrago_timeseries_heat_pumps[ - mv_grid_id - ] = heat_timeseries_hp_buildings_mv_grid.values - - # ToDo Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - # TODO Clara uses this table already - # but will not need it anymore for pypsa eur sec - # EgonEtragoTimeseriesIndividualHeating - - # # Change format - # data = CTS_grid.drop(columns="scenario") - # df_etrago_cts_heat_profiles = pd.DataFrame( - # index=data.index, columns=["scn_name", "p_set"] - # ) - # df_etrago_cts_heat_profiles.p_set = data.values.tolist() - # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] - # df_etrago_cts_heat_profiles.reset_index(inplace=True) - # - # # Drop and recreate Table if exists - # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), checkfirst=True) - # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), checkfirst=True) - # - # # Write heat ts into db - # with db.session_scope() as session: - # session.bulk_insert_mappings( - # EgonEtragoTimeseriesIndividualHeating, - # df_etrago_cts_heat_profiles.to_dict(orient="records"), - # ) - # ToDo Write other heat demand time series to database - gas voronoi - # (grid - egon_gas_voronoi mit carrier CH4) - # erstmal intermediate table - # TODO Gas aggregiert pro MV Grid - - -def determine_hp_cap_eGon100RE(): + min_hp_cap_buildings.loc[buildings_with_hp], hp_cap_grid + ) + + return hp_cap_per_building + + else: + return pd.Series() + + +def determine_hp_cap_eGon100RE(mv_grid_id): """Wrapper function to determine Heat Pump capacities for scenario eGon100RE. All buildings without district heating get a heat pump capacity assigned. """ - # get all MV grid IDs - mv_grid_ids = db.select_dataframe( - f""" - SELECT bus_id - FROM grid.egon_mv_grid_district - """, - index_col=None, - ).bus_id.values + # determine minimum required heat pump capacity per building + building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mv_grid_id + ) - for mv_grid_id in mv_grid_ids: + # TODO get peak demand from db + peak_heat_demand = get_peak_demand_per_building( + "eGon100RE", building_ids + ) - # determine minimum required heat pump capacity per building - building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mv_grid_id + # determine minimum required heat pump capacity per building + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 + ) + + # distribute total heat pump capacity to all buildings with HP + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( + "eGon100RE", mv_grid_id + ) + hp_cap_per_building = desaggregate_hp_capacity( + min_hp_cap_buildings, hp_cap_grid + ) + + # ToDo Write desaggregated HP capacity to table + + +@timeitlog +def residential_heat_peak_load_export_bulk(n, max_n=5): + """n= [1;max_n]""" + + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + log_to_file(residential_heat_peak_load_export_bulk.__qualname__ + f"_{n}") + if n == 0: + raise KeyError("n >= 1") + + # ToDo @Julian warum ist Abfrage so umständlich? + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .filter(EgonPetaHeat.sector == "residential") + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) + + # TODO mvgd_ids = [kleines mvgd] + for mvgd in [1556]: #mvgd_ids[n - 1]: + + logger.trace(f"MVGD={mvgd} | Start") + + # ############### get residential heat demand profiles ############### + df_heat_ts = calc_residential_heat_profiles_per_mvgd( + mvgd=mvgd ) - # TODO get peak demand from db - peak_heat_demand = get_peak_demand_per_building( - "eGon100RE", building_ids + # pivot to allow aggregation with CTS profiles + df_heat_ts_2035 = df_heat_ts.loc[ + :, ["building_id", "day_of_year", "hour", "eGon2035"]] + df_heat_ts_2035 = df_heat_ts_2035.pivot( + index=["day_of_year", "hour"], + columns="building_id", + values="eGon2035", + ) + df_heat_ts_2035 = df_heat_ts_2035.sort_index().reset_index(drop=True) + + df_heat_ts_100RE = df_heat_ts.loc[ + :, ["building_id", "day_of_year", "hour", "eGon100RE"]] + df_heat_ts_100RE = df_heat_ts_100RE.pivot( + index=["day_of_year", "hour"], + columns="building_id", + values="eGon100RE", ) + df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) - # determine minimum required heat pump capacity per building - min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 + del df_heat_ts + + # ############### get CTS heat demand profiles ############### + heat_demand_cts_ts_2035 = calc_cts_building_profiles( + egon_building_ids=[644, 645], + bus_ids=[1366], + scenario="eGon2035", + sector="heat", + ) + heat_demand_cts_ts_2035.rename( + columns={644: 1225533, 645: 1225527}, inplace=True) + heat_demand_cts_ts_100RE = calc_cts_building_profiles( + egon_building_ids=[644, 645], + bus_ids=[1366], + scenario="eGon100RE", + sector="heat", ) + heat_demand_cts_ts_100RE.rename( + columns={644: 1225533, 645: 1225527}, inplace=True) + # ToDo change back + # heat_demand_cts_ts_2035 = calc_cts_building_profiles( + # egon_building_ids=df_heat_ts.building_id.unique(), + # bus_ids=[mvgd], + # scenario="eGon2035", + # sector="heat", + # ) + # heat_demand_cts_ts_100RE = calc_cts_building_profiles( + # egon_building_ids=df_heat_ts.building_id.unique(), + # bus_ids=[mvgd], + # scenario="eGon100RE", + # sector="heat", + # ) + + # ############# aggregate residential and CTS demand profiles ############# + df_heat_ts_2035 = pd.concat( + [df_heat_ts_2035, heat_demand_cts_ts_2035], axis=1 + ) + df_heat_ts_2035 = df_heat_ts_2035.groupby(axis=1, level=0).sum() - # distribute total heat pump capacity to all buildings with HP - hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( - "eGon100RE", mv_grid_id + df_heat_ts_100RE = pd.concat( + [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 ) - hp_cap_per_building = desaggregate_hp_capacity( - min_hp_cap_buildings, hp_cap_grid + df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() + + del heat_demand_cts_ts_2035, heat_demand_cts_ts_100RE + + # ##################### export peak loads to DB ################### + + # ToDo @Julian kombinierte peak load oder getrennt nach residential und CTS? + df_peak_loads_2035 = df_heat_ts_2035.max() + df_peak_loads_100RE = df_heat_ts_100RE.max() + + df_peak_loads_db_2035 = df_peak_loads_2035.reset_index().melt( + id_vars="building_id", + var_name="scenario", + value_name="peak_load_in_w", + ) + df_peak_loads_db_2035["scenario"] = "eGon2035" + df_peak_loads_db_100RE = df_peak_loads_100RE.reset_index().melt( + id_vars="building_id", + var_name="scenario", + value_name="peak_load_in_w", ) + df_peak_loads_db_100RE["scenario"] = "eGon100RE" + df_peak_loads_db = pd.concat( + [df_peak_loads_db_2035, df_peak_loads_db_100RE]) + + del df_peak_loads_db_2035, df_peak_loads_db_100RE + + df_peak_loads_db["sector"] = "residential+CTS" + # From MW to W + # ToDo @Julian warum in W? + df_peak_loads_db["peak_load_in_w"] = df_peak_loads_db["peak_load_in_w"] * 1e6 + + logger.trace(f"MVGD={mvgd} | Export to DB") + + # TODO export peak loads all buildings both scenarios to db + # write_table_to_postgres( + # df_peak_loads_db, BuildingHeatPeakLoads, engine=engine + # ) + # logger.trace(f"MVGD={mvgd} | Done") + + # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ########## + + # get buildings with decentral heating systems in both scenarios + buildings_decentral_heating_2035 = ( + get_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon2035", mvgd + ) + ) + buildings_decentral_heating_100RE = ( + get_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mvgd + ) + ) + + # determine HP capacity per building for NEP2035 scenario + hp_cap_per_building_2035 = determine_hp_cap_eGon2035( + mvgd, df_peak_loads_2035, buildings_decentral_heating_2035) + buildings_hp_2035 = hp_cap_per_building_2035.index + buildings_gas_2035 = pd.Index(buildings_decentral_heating_2035).drop( + buildings_hp_2035) + + # determine minimum HP capacity per building for pypsa-eur-sec + hp_min_cap_mv_grid_pypsa_eur_sec = determine_hp_cap_pypsa_eur_sec( + df_peak_loads_100RE, buildings_decentral_heating_100RE) + + # ######################## write HP capacities to DB ###################### + + # ToDo Write HP capacity per building in 2035 (hp_cap_per_building_2035) to + # db table + + # ToDo Write minimum required capacity in pypsa-eur-sec + # (hp_min_cap_mv_grid_pypsa_eur_sec) to + # db table for pypsa-eur-sec input + + # ################ write aggregated heat profiles to DB ################### + + # heat demand time series for buildings with heat pumps + + # ToDo Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # TODO Clara uses this table already + # but will not need it anymore for pypsa eur sec - @Julian? + # EgonEtragoTimeseriesIndividualHeating + df_heat_ts_2035.loc[:, buildings_hp_2035].sum(axis=1) + df_heat_ts_100RE.loc[:, buildings_decentral_heating_100RE].sum(axis=1) + + # Change format + # ToDo @Julian noch notwendig? + # data = CTS_grid.drop(columns="scenario") + # df_etrago_cts_heat_profiles = pd.DataFrame( + # index=data.index, columns=["scn_name", "p_set"] + # ) + # df_etrago_cts_heat_profiles.p_set = data.values.tolist() + # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] + # df_etrago_cts_heat_profiles.reset_index(inplace=True) + + # # Drop and recreate Table if exists + # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), + # checkfirst=True) + # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), + # checkfirst=True) + # + # # Write heat ts into db + # with db.session_scope() as session: + # session.bulk_insert_mappings( + # EgonEtragoTimeseriesIndividualHeating, + # df_etrago_cts_heat_profiles.to_dict(orient="records"), + # ) + + # heat demand time series for buildings with gas boilers (only 2035 scenario) + df_heat_ts_2035.loc[:, buildings_gas_2035].sum(axis=1) + # ToDo Write other heat demand time series to database - gas voronoi + # (grid - egon_gas_voronoi mit carrier CH4) + # erstmal intermediate table + + +def residential_heat_peak_load_export_bulk_1(): + residential_heat_peak_load_export_bulk(1, max_n=5) + + +def residential_heat_peak_load_export_bulk_2(): + residential_heat_peak_load_export_bulk(2, max_n=5) + + +def residential_heat_peak_load_export_bulk_3(): + residential_heat_peak_load_export_bulk(3, max_n=5) + + +def residential_heat_peak_load_export_bulk_4(): + residential_heat_peak_load_export_bulk(4, max_n=5) + + +def residential_heat_peak_load_export_bulk_5(): + residential_heat_peak_load_export_bulk(5, max_n=5) + + +def create_peak_load_table(): + + BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) + + +def delete_peak_loads_if_existing(): + """Remove all entries""" + + with db.session_scope() as session: + # Buses + session.query(BuildingHeatPeakLoads).filter( + BuildingHeatPeakLoads.sector == "residential" + ).delete(synchronize_session=False) + - # ToDo Write desaggregated HP capacity to table +if __name__ == "__main__": + #calc_residential_heat_profiles_per_mvgd(mvgd) + residential_heat_peak_load_export_bulk_1() From 622aa4580333753ef2f962b34ff5967f055d9ccf Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 29 Sep 2022 22:58:45 +0200 Subject: [PATCH 026/119] Take out building_ids as parameter for calc_cts_building_ids --- .../electricity_demand_timeseries/cts_buildings.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index f58059b3f..bd6704afb 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -912,7 +912,6 @@ def calc_building_amenity_share(df_cts_buildings): def calc_cts_building_profiles( - egon_building_ids, bus_ids, scenario, sector, @@ -923,8 +922,6 @@ def calc_cts_building_profiles( Parameters ---------- - egon_building_ids: list of int - Ids of the building for which the profile is calculated. bus_ids: list of int Ids of the substation for which selected building profiles are calculated. @@ -951,8 +948,8 @@ def calc_cts_building_profiles( EgonCtsElectricityDemandBuildingShare.scenario == scenario ) .filter( - EgonCtsElectricityDemandBuildingShare.building_id.in_( - egon_building_ids + EgonCtsElectricityDemandBuildingShare.bus_id.in_( + bus_ids ) ) ) @@ -988,8 +985,8 @@ def calc_cts_building_profiles( ) .filter(EgonCtsHeatDemandBuildingShare.scenario == scenario) .filter( - EgonCtsHeatDemandBuildingShare.building_id.in_( - egon_building_ids + EgonCtsHeatDemandBuildingShare.bus_id.in_( + bus_ids ) ) ) From 2457d50e8de3f7394da083d040b56087a9a7bc17 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 29 Sep 2022 22:59:16 +0200 Subject: [PATCH 027/119] Adapt heat pump desaggregation --- .../heat_supply/individual_heating.py | 276 ++++++++++++------ 1 file changed, 182 insertions(+), 94 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index dfeb614d1..d3066fd88 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -21,6 +21,7 @@ from egon.data.datasets import Dataset from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( calc_cts_building_profiles, + CtsBuildings, ) from egon.data.datasets.electricity_demand_timeseries.tools import ( write_table_to_postgres, @@ -49,36 +50,31 @@ class EgonEtragoTimeseriesIndividualHeating(Base): dist_aggregated_mw = Column(ARRAY(REAL)) -# ToDo @Julian muss angepasst werden? -class HeatPumpsEtrago(Dataset): +class HeatPumpsPypsaEurSecAnd2035(Dataset): def __init__(self, dependencies): super().__init__( - name="HeatPumpsEtrago", + name="HeatPumpsPypsaEurSecAnd2035", version="0.0.0", dependencies=dependencies, - tasks=(determine_hp_cap_pypsa_eur_sec,), + tasks=(create_peak_load_table, + delete_peak_loads_if_existing, + {determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5, + } + ), ) -# ToDo @Julian muss angepasst werden? -class HeatPumps2035(Dataset): - def __init__(self, dependencies): - super().__init__( - name="HeatPumps2035", - version="0.0.0", - dependencies=dependencies, - tasks=(determine_hp_cap_eGon2035,), - ) - - -# ToDo @Julian muss angepasst werden? class HeatPumps2050(Dataset): def __init__(self, dependencies): super().__init__( name="HeatPumps2050", version="0.0.0", dependencies=dependencies, - tasks=(determine_hp_cap_eGon100RE), + tasks=(determine_hp_cap_buildings_eGon100RE,), ) @@ -617,10 +613,10 @@ def plot_heat_supply(resulting_capacities): @timeit -def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): +def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id): """ - Returns building IDs of buildings with decentral heat demand in given MV - grid. + Returns zensus cell IDs with decentral heating systems in given MV grid. As cells with district heating differ between scenarios, this is also depending on the scenario. @@ -635,7 +631,7 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heating system in given + Zensus cell IDs (as int) of buildings with decentral heating systems in given MV grid. Type is pandas Index to avoid errors later on when it is used in a query. @@ -651,11 +647,6 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): index_col=None, ).zensus_population_id.values - # TODO replace with sql adapter? - # ========== Register np datatypes with SQLA ========== - register_adapter(np.float64, adapt_numpy_float64) - register_adapter(np.int64, adapt_numpy_int64) - # ===================================================== # convert to pd.Index (otherwise type is np.int64, which will for some # reason throw an error when used in a query) zensus_population_ids = pd.Index(zensus_population_ids) @@ -683,6 +674,37 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): zensus_population_ids = zensus_population_ids.drop( cells_with_dh, errors="ignore" ) + return zensus_population_ids + + +@timeit +def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id): + """ + Returns building IDs of buildings with decentral residential heat demand in + given MV grid. + + As cells with district heating differ between scenarios, this is also + depending on the scenario. + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + mv_grid_id : int + ID of MV grid. + + Returns + -------- + pd.Index(int) + Building IDs (as int) of buildings with decentral heating system in given + MV grid. Type is pandas Index to avoid errors later on when it is + used in a query. + + """ + # get zensus cells with decentral heating + zensus_population_ids = get_zensus_cells_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id) # get buildings with decentral heat demand engine = db.engine() @@ -702,7 +724,55 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mv_grid_id): query.statement, query.session.bind, index_col=None ).building_id.values - return buildings_with_heat_demand + return pd.Index(buildings_with_heat_demand) + + +@timeit +def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id): + """ + Returns building IDs of buildings with decentral CTS heat demand in + given MV grid. + + As cells with district heating differ between scenarios, this is also + depending on the scenario. + + Parameters + ----------- + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + mv_grid_id : int + ID of MV grid. + + Returns + -------- + pd.Index(int) + Building IDs (as int) of buildings with decentral heating system in given + MV grid. Type is pandas Index to avoid errors later on when it is + used in a query. + + """ + + # get zensus cells with decentral heating + zensus_population_ids = get_zensus_cells_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id) + + # get buildings with decentral heat demand + # ToDo @Julian, sind das alle CTS buildings in der Tabelle? + with db.session_scope() as session: + query = session.query( + CtsBuildings.id, + ).filter( + CtsBuildings.zensus_population_id.in_( + zensus_population_ids + ) + ) + + buildings_with_heat_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).id.values + + return pd.Index(buildings_with_heat_demand) def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): @@ -898,7 +968,7 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): return hp_cap_per_building -def determine_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): +def determine_min_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): """ Determines minimum required HP capacity in MV grid in MW as input for pypsa-eur-sec. @@ -929,7 +999,7 @@ def determine_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): return 0.0 -def determine_hp_cap_eGon2035(mv_grid_id, peak_heat_demand, building_ids): +def determine_hp_cap_buildings_eGon2035(mv_grid_id, peak_heat_demand, building_ids): """ Determines which buildings in the MV grid will have a HP (buildings with PV rooftop are more likely to be assigned) in the eGon2035 scenario, as well as @@ -975,10 +1045,12 @@ def determine_hp_cap_eGon2035(mv_grid_id, peak_heat_demand, building_ids): return pd.Series() -def determine_hp_cap_eGon100RE(mv_grid_id): - """Wrapper function to determine Heat Pump capacities - for scenario eGon100RE. All buildings without district heating get a heat - pump capacity assigned. +def determine_hp_cap_buildings_eGon100RE(mv_grid_id): + """ + Main function to determine HP capacity per building in eGon100RE scenario. + + In eGon100RE scenario all buildings without district heating get a heat pump. + """ # determine minimum required heat pump capacity per building @@ -1004,23 +1076,36 @@ def determine_hp_cap_eGon100RE(mv_grid_id): min_hp_cap_buildings, hp_cap_grid ) - # ToDo Write desaggregated HP capacity to table + # ToDo Julian Write desaggregated HP capacity to table (same as for 2035 scenario) @timeitlog -def residential_heat_peak_load_export_bulk(n, max_n=5): - """n= [1;max_n]""" +def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): + """ + Main function to determine HP capacity per building in eGon2035 scenario and + minimum required HP capacity in MV for pypsa-eur-sec. + Further, creates heat demand time series for all buildings with heat pumps + (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all buildings + with gas boilers (only in eGon2035scenario), used in eTraGo. + + Parameters + ----------- + n : int + Number between [1;max_n]. + max_n : int + Maximum number of bulks (MV grid sets run in parallel). + + """ # ========== Register np datatypes with SQLA ========== register_adapter(np.float64, adapt_numpy_float64) register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - log_to_file(residential_heat_peak_load_export_bulk.__qualname__ + f"_{n}") + log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + f"_{n}") if n == 0: raise KeyError("n >= 1") - # ToDo @Julian warum ist Abfrage so umständlich? with db.session_scope() as session: query = ( session.query( @@ -1030,7 +1115,6 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): MapZensusGridDistricts.zensus_population_id == EgonPetaHeat.zensus_population_id ) - .filter(EgonPetaHeat.sector == "residential") .distinct(MapZensusGridDistricts.bus_id) ) mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) @@ -1072,34 +1156,15 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): # ############### get CTS heat demand profiles ############### heat_demand_cts_ts_2035 = calc_cts_building_profiles( - egon_building_ids=[644, 645], - bus_ids=[1366], + bus_ids=[mvgd], scenario="eGon2035", sector="heat", ) - heat_demand_cts_ts_2035.rename( - columns={644: 1225533, 645: 1225527}, inplace=True) heat_demand_cts_ts_100RE = calc_cts_building_profiles( - egon_building_ids=[644, 645], - bus_ids=[1366], + bus_ids=[mvgd], scenario="eGon100RE", sector="heat", ) - heat_demand_cts_ts_100RE.rename( - columns={644: 1225533, 645: 1225527}, inplace=True) - # ToDo change back - # heat_demand_cts_ts_2035 = calc_cts_building_profiles( - # egon_building_ids=df_heat_ts.building_id.unique(), - # bus_ids=[mvgd], - # scenario="eGon2035", - # sector="heat", - # ) - # heat_demand_cts_ts_100RE = calc_cts_building_profiles( - # egon_building_ids=df_heat_ts.building_id.unique(), - # bus_ids=[mvgd], - # scenario="eGon100RE", - # sector="heat", - # ) # ############# aggregate residential and CTS demand profiles ############# df_heat_ts_2035 = pd.concat( @@ -1116,7 +1181,6 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): # ##################### export peak loads to DB ################### - # ToDo @Julian kombinierte peak load oder getrennt nach residential und CTS? df_peak_loads_2035 = df_heat_ts_2035.max() df_peak_loads_100RE = df_heat_ts_100RE.max() @@ -1139,7 +1203,6 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): df_peak_loads_db["sector"] = "residential+CTS" # From MW to W - # ToDo @Julian warum in W? df_peak_loads_db["peak_load_in_w"] = df_peak_loads_db["peak_load_in_w"] * 1e6 logger.trace(f"MVGD={mvgd} | Export to DB") @@ -1152,52 +1215,77 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ########## - # get buildings with decentral heating systems in both scenarios - buildings_decentral_heating_2035 = ( - get_buildings_with_decentral_heat_demand_in_mv_grid( + # get residential buildings with decentral heating systems in both scenarios + buildings_decentral_heating_2035_res = ( + get_residential_buildings_with_decentral_heat_demand_in_mv_grid( "eGon2035", mvgd ) ) - buildings_decentral_heating_100RE = ( - get_buildings_with_decentral_heat_demand_in_mv_grid( + buildings_decentral_heating_100RE_res = ( + get_residential_buildings_with_decentral_heat_demand_in_mv_grid( "eGon100RE", mvgd ) ) + # get CTS buildings with decentral heating systems in both scenarios + buildings_decentral_heating_2035_cts = ( + get_cts_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon2035", mvgd + ) + ) + buildings_decentral_heating_100RE_cts = ( + get_cts_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mvgd + ) + ) + + # merge residential and CTS buildings + buildings_decentral_heating_2035 = ( + buildings_decentral_heating_2035_res.append( + buildings_decentral_heating_2035_cts + ).unique() + ) + buildings_decentral_heating_100RE = ( + buildings_decentral_heating_100RE_res.append( + buildings_decentral_heating_100RE_cts + ).unique() + ) + # determine HP capacity per building for NEP2035 scenario - hp_cap_per_building_2035 = determine_hp_cap_eGon2035( + hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( mvgd, df_peak_loads_2035, buildings_decentral_heating_2035) buildings_hp_2035 = hp_cap_per_building_2035.index buildings_gas_2035 = pd.Index(buildings_decentral_heating_2035).drop( buildings_hp_2035) # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = determine_hp_cap_pypsa_eur_sec( + hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( df_peak_loads_100RE, buildings_decentral_heating_100RE) # ######################## write HP capacities to DB ###################### - # ToDo Write HP capacity per building in 2035 (hp_cap_per_building_2035) to - # db table + # ToDo Julian Write HP capacity per building in 2035 (hp_cap_per_building_2035) to + # db table - neue Tabelle egon_hp_capacity_buildings - # ToDo Write minimum required capacity in pypsa-eur-sec + # ToDo Julian Write minimum required capacity in pypsa-eur-sec # (hp_min_cap_mv_grid_pypsa_eur_sec) to - # db table for pypsa-eur-sec input + # csv for pypsa-eur-sec input - im working directory gibt es directory + # input_pypsa_eur_sec - minimum_hp_capacity_mv_grid.csv # ################ write aggregated heat profiles to DB ################### # heat demand time series for buildings with heat pumps - # ToDo Write aggregated heat demand time series of buildings with HP to + # ToDo Julian Write aggregated heat demand time series of buildings with HP to # table to be used in eTraGo - egon_etrago_timeseries_individual_heating # TODO Clara uses this table already - # but will not need it anymore for pypsa eur sec - @Julian? + # but will not need it anymore for eTraGo # EgonEtragoTimeseriesIndividualHeating - df_heat_ts_2035.loc[:, buildings_hp_2035].sum(axis=1) - df_heat_ts_100RE.loc[:, buildings_decentral_heating_100RE].sum(axis=1) + df_heat_ts_2035.loc[:, buildings_hp_2035].sum(axis=1) # carrier heat_pump + df_heat_ts_100RE.loc[:, buildings_decentral_heating_100RE].sum(axis=1) # carrier heat_pump # Change format - # ToDo @Julian noch notwendig? + # ToDo Julian # data = CTS_grid.drop(columns="scenario") # df_etrago_cts_heat_profiles = pd.DataFrame( # index=data.index, columns=["scn_name", "p_set"] @@ -1220,30 +1308,31 @@ def residential_heat_peak_load_export_bulk(n, max_n=5): # ) # heat demand time series for buildings with gas boilers (only 2035 scenario) - df_heat_ts_2035.loc[:, buildings_gas_2035].sum(axis=1) - # ToDo Write other heat demand time series to database - gas voronoi - # (grid - egon_gas_voronoi mit carrier CH4) - # erstmal intermediate table + df_heat_ts_2035.loc[:, buildings_gas_2035].sum(axis=1) # carrier gas_boilers + # ToDo Julian Write heat demand time series for buildings with gas boiler to + # database - in gleiche Tabelle wie Zeitreihen für WP Gebäude, falls Clara + # nichts anderes sagt; wird später weiter aggregiert nach gas voronoi + # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia -def residential_heat_peak_load_export_bulk_1(): - residential_heat_peak_load_export_bulk(1, max_n=5) +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(1, max_n=5) -def residential_heat_peak_load_export_bulk_2(): - residential_heat_peak_load_export_bulk(2, max_n=5) +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(2, max_n=5) -def residential_heat_peak_load_export_bulk_3(): - residential_heat_peak_load_export_bulk(3, max_n=5) +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(3, max_n=5) -def residential_heat_peak_load_export_bulk_4(): - residential_heat_peak_load_export_bulk(4, max_n=5) +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(4, max_n=5) -def residential_heat_peak_load_export_bulk_5(): - residential_heat_peak_load_export_bulk(5, max_n=5) +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(5, max_n=5) def create_peak_load_table(): @@ -1262,5 +1351,4 @@ def delete_peak_loads_if_existing(): if __name__ == "__main__": - #calc_residential_heat_profiles_per_mvgd(mvgd) - residential_heat_peak_load_export_bulk_1() + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1() From b93c0718d7f34bcaea8900c2ff60888b11fba656 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 7 Oct 2022 14:44:03 +0200 Subject: [PATCH 028/119] Implement dynamic parallelisation --- src/egon/data/datasets.yml | 3 + .../heat_supply/individual_heating.py | 124 ++++++++++-------- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index ada9faef0..de5659e9d 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1087,3 +1087,6 @@ emobility_mit: reduce_memory: True export_results_to_csv: True parallel_tasks: 10 + +demand_timeseries_mvgd: + parallel_tasks: 5 diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index d3066fd88..00bb286cf 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,6 +2,8 @@ individual heat supply. """ +import os + from loguru import logger import numpy as np import pandas as pd @@ -11,6 +13,7 @@ from pathlib import Path import time +from airflow.operators.python_operator import PythonOperator from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -52,17 +55,71 @@ class EgonEtragoTimeseriesIndividualHeating(Base): class HeatPumpsPypsaEurSecAnd2035(Dataset): def __init__(self, dependencies): + def dyn_parallel_tasks(): + """Dynamically generate tasks + + The goal is to speed up tasks by parallelising bulks of mvgds. + + The number of parallel tasks is defined via parameter + `parallel_tasks` in the dataset config `datasets.yml`. + + Returns + ------- + set of airflow.PythonOperators + The tasks. Each element is of + :func:`egon.data.datasets.heat_supply.individual_heating. + determine_hp_capacity_eGon2035_pypsa_eur_sec` + """ + parallel_tasks = egon.data.config.datasets()["demand_timeseries_mvgd"].get( + "parallel_tasks", 1 + ) + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, parallel_tasks) + + + # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] + tasks = set() + for i, bulk in enumerate(mvgd_ids): + tasks.add( + PythonOperator( + task_id=( + f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" + f"mvgd_{min(bulk)}-{max(bulk)}" + ), + python_callable=determine_hp_capacity_eGon2035_pypsa_eur_sec, + op_kwargs={ + "mvgd_ids": bulk, + }, + ) + ) + return tasks + super().__init__( name="HeatPumpsPypsaEurSecAnd2035", version="0.0.0", dependencies=dependencies, tasks=(create_peak_load_table, delete_peak_loads_if_existing, - {determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5, + { *dyn_parallel_tasks() } ), ) @@ -1080,7 +1137,7 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): @timeitlog -def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): +def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): """ Main function to determine HP capacity per building in eGon2035 scenario and minimum required HP capacity in MV for pypsa-eur-sec. @@ -1090,11 +1147,8 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): Parameters ----------- - n : int - Number between [1;max_n]. - max_n : int - Maximum number of bulks (MV grid sets run in parallel). - + mvgd_ids : list(int) + List of MVGD ids """ # ========== Register np datatypes with SQLA ========== @@ -1102,31 +1156,13 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + f"_{n}") - if n == 0: - raise KeyError("n >= 1") - - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) - - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - - mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) + log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + + f"_{min(mvgd_ids)}-{max(mvgd_ids)}") # TODO mvgd_ids = [kleines mvgd] - for mvgd in [1556]: #mvgd_ids[n - 1]: + for mvgd in mvgd_ids: - logger.trace(f"MVGD={mvgd} | Start") + logger.debug(f"MVGD={mvgd} | Start") # ############### get residential heat demand profiles ############### df_heat_ts = calc_residential_heat_profiles_per_mvgd( @@ -1205,7 +1241,7 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): # From MW to W df_peak_loads_db["peak_load_in_w"] = df_peak_loads_db["peak_load_in_w"] * 1e6 - logger.trace(f"MVGD={mvgd} | Export to DB") + logger.debug(f"MVGD={mvgd} | Export to DB") # TODO export peak loads all buildings both scenarios to db # write_table_to_postgres( @@ -1315,26 +1351,6 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(1, max_n=5) - - -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(2, max_n=5) - - -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(3, max_n=5) - - -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(4, max_n=5) - - -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(5, max_n=5) - - def create_peak_load_table(): BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) From 815326140b6d8b0803e920ffdd2c05b6ca3b42ea Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 10 Oct 2022 14:27:03 +0200 Subject: [PATCH 029/119] Implement dynamic parallelisation --- src/egon/data/datasets.yml | 3 - .../heat_supply/individual_heating.py | 124 ++++++++---------- 2 files changed, 54 insertions(+), 73 deletions(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index de5659e9d..ada9faef0 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1087,6 +1087,3 @@ emobility_mit: reduce_memory: True export_results_to_csv: True parallel_tasks: 10 - -demand_timeseries_mvgd: - parallel_tasks: 5 diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 00bb286cf..d3066fd88 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,8 +2,6 @@ individual heat supply. """ -import os - from loguru import logger import numpy as np import pandas as pd @@ -13,7 +11,6 @@ from pathlib import Path import time -from airflow.operators.python_operator import PythonOperator from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -55,71 +52,17 @@ class EgonEtragoTimeseriesIndividualHeating(Base): class HeatPumpsPypsaEurSecAnd2035(Dataset): def __init__(self, dependencies): - def dyn_parallel_tasks(): - """Dynamically generate tasks - - The goal is to speed up tasks by parallelising bulks of mvgds. - - The number of parallel tasks is defined via parameter - `parallel_tasks` in the dataset config `datasets.yml`. - - Returns - ------- - set of airflow.PythonOperators - The tasks. Each element is of - :func:`egon.data.datasets.heat_supply.individual_heating. - determine_hp_capacity_eGon2035_pypsa_eur_sec` - """ - parallel_tasks = egon.data.config.datasets()["demand_timeseries_mvgd"].get( - "parallel_tasks", 1 - ) - # ========== Register np datatypes with SQLA ========== - register_adapter(np.float64, adapt_numpy_float64) - register_adapter(np.int64, adapt_numpy_int64) - # ===================================================== - - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) - - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - - mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, parallel_tasks) - - - # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] - tasks = set() - for i, bulk in enumerate(mvgd_ids): - tasks.add( - PythonOperator( - task_id=( - f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" - f"mvgd_{min(bulk)}-{max(bulk)}" - ), - python_callable=determine_hp_capacity_eGon2035_pypsa_eur_sec, - op_kwargs={ - "mvgd_ids": bulk, - }, - ) - ) - return tasks - super().__init__( name="HeatPumpsPypsaEurSecAnd2035", version="0.0.0", dependencies=dependencies, tasks=(create_peak_load_table, delete_peak_loads_if_existing, - { *dyn_parallel_tasks() + {determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4, + determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5, } ), ) @@ -1137,7 +1080,7 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): @timeitlog -def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): +def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): """ Main function to determine HP capacity per building in eGon2035 scenario and minimum required HP capacity in MV for pypsa-eur-sec. @@ -1147,8 +1090,11 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): Parameters ----------- - mvgd_ids : list(int) - List of MVGD ids + n : int + Number between [1;max_n]. + max_n : int + Maximum number of bulks (MV grid sets run in parallel). + """ # ========== Register np datatypes with SQLA ========== @@ -1156,13 +1102,31 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + - f"_{min(mvgd_ids)}-{max(mvgd_ids)}") + log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + f"_{n}") + if n == 0: + raise KeyError("n >= 1") + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) # TODO mvgd_ids = [kleines mvgd] - for mvgd in mvgd_ids: + for mvgd in [1556]: #mvgd_ids[n - 1]: - logger.debug(f"MVGD={mvgd} | Start") + logger.trace(f"MVGD={mvgd} | Start") # ############### get residential heat demand profiles ############### df_heat_ts = calc_residential_heat_profiles_per_mvgd( @@ -1241,7 +1205,7 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): # From MW to W df_peak_loads_db["peak_load_in_w"] = df_peak_loads_db["peak_load_in_w"] * 1e6 - logger.debug(f"MVGD={mvgd} | Export to DB") + logger.trace(f"MVGD={mvgd} | Export to DB") # TODO export peak loads all buildings both scenarios to db # write_table_to_postgres( @@ -1351,6 +1315,26 @@ def determine_hp_capacity_eGon2035_pypsa_eur_sec(mvgd_ids): # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(1, max_n=5) + + +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(2, max_n=5) + + +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(3, max_n=5) + + +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(4, max_n=5) + + +def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5(): + determine_hp_capacity_eGon2035_pypsa_eur_sec(5, max_n=5) + + def create_peak_load_table(): BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) From 5db9383fc1a19b22ba5ab629f4449282b9edaa0b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:36:28 +0200 Subject: [PATCH 030/119] Implement export and debug --- .../electricity_demand_timeseries/tools.py | 2 + .../heat_supply/individual_heating.py | 1020 +++++++++++------ 2 files changed, 690 insertions(+), 332 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/tools.py b/src/egon/data/datasets/electricity_demand_timeseries/tools.py index 524ed7070..fe32d4bb0 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/tools.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/tools.py @@ -192,6 +192,8 @@ def write_table_to_postgres( if drop: db_table.__table__.drop(bind=engine, checkfirst=True) db_table.__table__.create(bind=engine) + else: + db_table.__table__.create(bind=engine, checkfirst=True) df.to_sql( name=db_table.__table__.name, diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index d3066fd88..26366fb8a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -2,26 +2,31 @@ individual heat supply. """ -from loguru import logger -import numpy as np -import pandas as pd -import random -import saio - from pathlib import Path +import os +import random import time +from loguru import logger from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd - +import numpy as np +import pandas as pd +import saio from egon.data import config, db from egon.data.datasets import Dataset +from egon.data.datasets.district_heating_areas import ( + MapZensusDistrictHeatingAreas, +) from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( - calc_cts_building_profiles, CtsBuildings, + calc_cts_building_profiles, +) +from egon.data.datasets.electricity_demand_timeseries.mapping import ( + EgonMapZensusMvgdBuildings, ) from egon.data.datasets.electricity_demand_timeseries.tools import ( write_table_to_postgres, @@ -34,13 +39,14 @@ from egon.data.datasets.heat_demand_timeseries.idp_pool import ( EgonHeatTimeseries, ) + # get zensus cells with district heating from egon.data.datasets.zensus_mv_grid_districts import MapZensusGridDistricts engine = db.engine() Base = declarative_base() - +# TODO check column names> class EgonEtragoTimeseriesIndividualHeating(Base): __tablename__ = "egon_etrago_timeseries_individual_heating" __table_args__ = {"schema": "demand"} @@ -50,21 +56,87 @@ class EgonEtragoTimeseriesIndividualHeating(Base): dist_aggregated_mw = Column(ARRAY(REAL)) +class EgonHpCapacityBuildings(Base): + __tablename__ = "egon_hp_capacity_buildings" + __table_args__ = {"schema": "demand"} + building_id = Column(Integer, primary_key=True) + scenario = Column(String, primary_key=True) + hp_capacity = Column(REAL) + + class HeatPumpsPypsaEurSecAnd2035(Dataset): def __init__(self, dependencies): + def dyn_parallel_tasks(): + """Dynamically generate tasks + + The goal is to speed up tasks by parallelising bulks of mvgds. + + The number of parallel tasks is defined via parameter + `parallel_tasks` in the dataset config `datasets.yml`. + + Returns + ------- + set of airflow.PythonOperators + The tasks. Each element is of + :func:`egon.data.datasets.heat_supply.individual_heating. + determine_hp_capacity_eGon2035_pypsa_eur_sec` + """ + parallel_tasks = egon.data.config.datasets()[ + "demand_timeseries_mvgd" + ].get("parallel_tasks", 1) + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split( + mvgd_ids["bus_id"].values, parallel_tasks + ) + + # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] + tasks = set() + for i, bulk in enumerate(mvgd_ids): + tasks.add( + PythonOperator( + task_id=( + f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" + f"mvgd_{min(bulk)}-{max(bulk)}" + ), + python_callable=determine_hp_cap_peak_load_mvgd_ts, + op_kwargs={ + "mvgd_ids": bulk, + }, + ) + ) + return tasks + super().__init__( name="HeatPumpsPypsaEurSecAnd2035", version="0.0.0", dependencies=dependencies, - tasks=(create_peak_load_table, - delete_peak_loads_if_existing, - {determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4, - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5, - } - ), + tasks=( + create_peak_load_table, + create_hp_capacity_table, + # delete_peak_loads_if_existing, + {*dyn_parallel_tasks()}, + ), ) @@ -98,14 +170,17 @@ def adapt_numpy_int64(numpy_int64): def log_to_file(name): """Simple only file logger""" + file = os.path.basename(__file__).rstrip(".py") + file_path = Path(f"./{file}_logs") + os.makedirs(file_path, exist_ok=True) logger.remove() logger.add( - Path(f"{name}.log"), + file_path / Path(f"{name}.log"), format="{time} {level} {message}", # filter="my_module", - level="TRACE", + level="DEBUG", ) - logger.trace("Start trace logging") + logger.trace(f"Start logging of: {name}") return logger @@ -144,7 +219,7 @@ def measure_time(*args, **kw): f"MVGD={mvgd} | Processing time of {func.__qualname__} | " f"{time.strftime('%H h, %M min, %S s', time.gmtime(process_time))}" ) - logger.trace(statement) + logger.debug(statement) print(statement) return result @@ -359,57 +434,64 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): ) -# @timeit +# @timeitlog def get_peta_demand(mvgd): - """only residential""" + """ + Retrieve annual peta heat demand for residential buildings and both + scenarios. - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.zensus_population_id, - EgonPetaHeat.demand.label("peta_2035"), - ) - .filter(MapZensusGridDistricts.bus_id == mvgd) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .filter(EgonPetaHeat.scenario == "eGon2035") - .filter(EgonPetaHeat.sector == "residential") - ) + Parameters + ---------- + mvgd : int + ID of MVGD - df_peta_2035 = pd.read_sql( - query.statement, query.session.bind, index_col="zensus_population_id" - ) + Returns + ------- + df_peta_demand : pd.DataFrame + Annual residential heat demand per building and scenario + """ with db.session_scope() as session: query = ( session.query( MapZensusGridDistricts.zensus_population_id, - EgonPetaHeat.demand.label("peta_2050"), + EgonPetaHeat.scenario, + EgonPetaHeat.demand, ) .filter(MapZensusGridDistricts.bus_id == mvgd) .filter( MapZensusGridDistricts.zensus_population_id == EgonPetaHeat.zensus_population_id ) - .filter(EgonPetaHeat.scenario == "eGon100RE") .filter(EgonPetaHeat.sector == "residential") ) - df_peta_100RE = pd.read_sql( - query.statement, query.session.bind, index_col="zensus_population_id" + df_peta_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None ) - - df_peta_demand = pd.concat( - [df_peta_2035, df_peta_100RE], axis=1 + df_peta_demand = df_peta_demand.pivot( + index="zensus_population_id", columns="scenario", values="demand" ).reset_index() return df_peta_demand -# @timeit -def get_profile_ids(mvgd): +# @timeitlog +def get_residential_heat_profile_ids(mvgd): + """ + Retrieve 365 daily heat profiles ids per residential building and selected + mvgd. + + Parameters + ---------- + mvgd : int + ID of MVGD + + Returns + ------- + df_profiles_ids : pd.DataFrame + Residential daily heat profile ID's per building + """ with db.session_scope() as session: query = ( session.query( @@ -437,15 +519,28 @@ def get_profile_ids(mvgd): right_index=True, ) + # unnest array of ids per building df_profiles_ids = df_profiles_ids.explode("selected_idp_profiles") + # add day of year column by order of list df_profiles_ids["day_of_year"] = ( df_profiles_ids.groupby("building_id").cumcount() + 1 ) return df_profiles_ids -# @timeit +# @timeitlog def get_daily_profiles(profile_ids): + """ + Parameters + ---------- + profile_ids : list(int) + daily heat profile ID's + + Returns + ------- + df_profiles : pd.DataFrame + Residential daily heat profiles + """ saio.register_schema("demand", db.engine()) from saio.demand import egon_heat_idp_pool @@ -458,31 +553,39 @@ def get_daily_profiles(profile_ids): query.statement, query.session.bind, index_col="index" ) + # unnest array of profile values per id df_profiles = df_profiles.explode("idp") + # Add column for hour of day df_profiles["hour"] = df_profiles.groupby(axis=0, level=0).cumcount() + 1 return df_profiles -# @timeit +# @timeitlog def get_daily_demand_share(mvgd): + """per census cell + Parameters + ---------- + mvgd : int + MVGD id + + Returns + ------- + df_daily_demand_share : pd.DataFrame + Daily annual demand share per cencus cell + """ with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.zensus_population_id, - EgonDailyHeatDemandPerClimateZone.day_of_year, - EgonDailyHeatDemandPerClimateZone.daily_demand_share, - ) - .filter( - EgonMapZensusClimateZones.climate_zone - == EgonDailyHeatDemandPerClimateZone.climate_zone - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonMapZensusClimateZones.zensus_population_id - ) - .filter(MapZensusGridDistricts.bus_id == mvgd) + query = session.query( + MapZensusGridDistricts.zensus_population_id, + EgonDailyHeatDemandPerClimateZone.day_of_year, + EgonDailyHeatDemandPerClimateZone.daily_demand_share, + ).filter( + EgonMapZensusClimateZones.climate_zone + == EgonDailyHeatDemandPerClimateZone.climate_zone, + MapZensusGridDistricts.zensus_population_id + == EgonMapZensusClimateZones.zensus_population_id, + MapZensusGridDistricts.bus_id == mvgd, ) df_daily_demand_share = pd.read_sql( @@ -494,8 +597,8 @@ def get_daily_demand_share(mvgd): @timeitlog def calc_residential_heat_profiles_per_mvgd(mvgd): """ - Gets residential heat profiles per building in MV grid for both eGon2035 and - eGon100RE scenario. + Gets residential heat profiles per building in MV grid for both eGon2035 + and eGon100RE scenario. Parameters ---------- @@ -515,21 +618,25 @@ def calc_residential_heat_profiles_per_mvgd(mvgd): * hour : int Hour of the day (1 - 24). * eGon2035 : float - Building's residential heat demand in MW, for specified hour of the - year (specified through columns `day_of_year` and `hour`). + Building's residential heat demand in MW, for specified hour + of the year (specified through columns `day_of_year` and + `hour`). * eGon100RE : float - Building's residential heat demand in MW, for specified hour of the - year (specified through columns `day_of_year` and `hour`). - + Building's residential heat demand in MW, for specified hour + of the year (specified through columns `day_of_year` and + `hour`). """ df_peta_demand = get_peta_demand(mvgd) + # TODO maybe return empty dataframe if df_peta_demand.empty: + logger.info(f"No demand for MVGD: {mvgd}") return None - df_profiles_ids = get_profile_ids(mvgd) + df_profiles_ids = get_residential_heat_profile_ids(mvgd) if df_profiles_ids.empty: + logger.info(f"No profiles for MVGD: {mvgd}") return None df_profiles = get_daily_profiles( @@ -562,19 +669,25 @@ def calc_residential_heat_profiles_per_mvgd(mvgd): df_profile_merge["eGon2035"] = ( df_profile_merge["idp"] .mul(df_profile_merge["daily_demand_share"]) - .mul(df_profile_merge["peta_2035"]) + .mul(df_profile_merge["eGon2035"]) .div(df_profile_merge["buildings"]) ) df_profile_merge["eGon100RE"] = ( df_profile_merge["idp"] .mul(df_profile_merge["daily_demand_share"]) - .mul(df_profile_merge["peta_2050"]) + .mul(df_profile_merge["eGon100RE"]) .div(df_profile_merge["buildings"]) ) - columns = ["zensus_population_id", "building_id", "day_of_year", "hour", - "eGon2035", "eGon100RE"] + columns = [ + "zensus_population_id", + "building_id", + "day_of_year", + "hour", + "eGon2035", + "eGon100RE", + ] return df_profile_merge.loc[:, columns] @@ -612,9 +725,10 @@ def plot_heat_supply(resulting_capacities): plt.savefig(f"plots/individual_heat_supply_{c}.png", dpi=300) -@timeit +@timeitlog def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id): + scenario, mv_grid_id +): """ Returns zensus cell IDs with decentral heating systems in given MV grid. @@ -631,8 +745,8 @@ def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( Returns -------- pd.Index(int) - Zensus cell IDs (as int) of buildings with decentral heating systems in given - MV grid. Type is pandas Index to avoid errors later on when it is + Zensus cell IDs (as int) of buildings with decentral heating systems in + given MV grid. Type is pandas Index to avoid errors later on when it is used in a query. """ @@ -647,15 +761,12 @@ def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( index_col=None, ).zensus_population_id.values + # maybe use adapter # convert to pd.Index (otherwise type is np.int64, which will for some # reason throw an error when used in a query) zensus_population_ids = pd.Index(zensus_population_ids) # get zensus cells with district heating - from egon.data.datasets.district_heating_areas import ( - MapZensusDistrictHeatingAreas, - ) - with db.session_scope() as session: query = session.query( MapZensusDistrictHeatingAreas.zensus_population_id, @@ -674,12 +785,13 @@ def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( zensus_population_ids = zensus_population_ids.drop( cells_with_dh, errors="ignore" ) - return zensus_population_ids + return pd.Index(zensus_population_ids) -@timeit +@timeitlog def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id): + scenario, mv_grid_id +): """ Returns building IDs of buildings with decentral residential heat demand in given MV grid. @@ -703,11 +815,13 @@ def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( """ # get zensus cells with decentral heating - zensus_population_ids = get_zensus_cells_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id) + zensus_population_ids = ( + get_zensus_cells_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id + ) + ) # get buildings with decentral heat demand - engine = db.engine() saio.register_schema("demand", engine) from saio.demand import egon_heat_timeseries_selected_profiles @@ -727,9 +841,10 @@ def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( return pd.Index(buildings_with_heat_demand) -@timeit +@timeitlog def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id): + scenario, mv_grid_id +): """ Returns building IDs of buildings with decentral CTS heat demand in given MV grid. @@ -754,27 +869,79 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( """ # get zensus cells with decentral heating - zensus_population_ids = get_zensus_cells_with_decentral_heat_demand_in_mv_grid( - scenario, mv_grid_id) + zensus_population_ids = ( + get_zensus_cells_with_decentral_heat_demand_in_mv_grid( + scenario, mv_grid_id + ) + ) # get buildings with decentral heat demand # ToDo @Julian, sind das alle CTS buildings in der Tabelle? + # ja aber die zensus_population_id stimmt nicht + # boundaries.egon_map_zensus_mvgd_buildings_used benutzen + # with db.session_scope() as session: - query = session.query( - CtsBuildings.id, - ).filter( - CtsBuildings.zensus_population_id.in_( + query = session.query(EgonMapZensusMvgdBuildings.building_id).filter( + EgonMapZensusMvgdBuildings.sector == "cts", + EgonMapZensusMvgdBuildings.zensus_population_id.in_( zensus_population_ids ) + # ).unique(EgonMapZensusMvgdBuildings.building_id) ) buildings_with_heat_demand = pd.read_sql( query.statement, query.session.bind, index_col=None - ).id.values + ).building_id.values return pd.Index(buildings_with_heat_demand) +def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd): + """""" + # get residential buildings with decentral heating systems in both scenarios + buildings_decentral_heating_2035_res = ( + get_residential_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon2035", mvgd + ) + ) + buildings_decentral_heating_100RE_res = ( + get_residential_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mvgd + ) + ) + + # get CTS buildings with decentral heating systems in both scenarios + buildings_decentral_heating_2035_cts = ( + get_cts_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon2035", mvgd + ) + ) + buildings_decentral_heating_100RE_cts = ( + get_cts_buildings_with_decentral_heat_demand_in_mv_grid( + "eGon100RE", mvgd + ) + ) + + # merge residential and CTS buildings + buildings_decentral_heating_2035 = ( + buildings_decentral_heating_2035_res.append( + buildings_decentral_heating_2035_cts + ).unique() + ) + buildings_decentral_heating_100RE = ( + buildings_decentral_heating_100RE_res.append( + buildings_decentral_heating_100RE_cts + ).unique() + ) + + buildings_decentral_heating = { + "eGon2035": buildings_decentral_heating_2035, + "eGon100RE": buildings_decentral_heating_100RE, + } + + return buildings_decentral_heating + + def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): """ Returns total heat pump capacity per grid that was previously defined @@ -793,24 +960,59 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): Total heat pump capacity in MW in given MV grid. """ - from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply + # TODO temporary commented until table exists + # from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply + # + # with db.session_scope() as session: + # query = ( + # session.query( + # EgonIndividualHeatingSupply.mv_grid_id, + # EgonIndividualHeatingSupply.capacity, + # ) + # .filter(EgonIndividualHeatingSupply.scenario == scenario) + # .filter(EgonIndividualHeatingSupply.carrier == "heat_pump") + # .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) + # ) + # + # hp_cap_mv_grid = pd.read_sql( + # query.statement, query.session.bind, index_col="mv_grid_id" + # ).capacity.values[0] + + # with db.session_scope() as session: + # hp_cap_mv_grid = session.execute( + # EgonIndividualHeatingSupply.capacity + # ).filter( + # EgonIndividualHeatingSupply.scenario == scenario, + # EgonIndividualHeatingSupply.carrier == "heat_pump", + # EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id + # ).scalar() + + # workaround + hp_cap_mv_grid = 50 + return hp_cap_mv_grid + + +def get_heat_peak_demand_per_building(scenario, building_ids): + """""" with db.session_scope() as session: query = ( session.query( - EgonIndividualHeatingSupply.mv_grid_id, - EgonIndividualHeatingSupply.capacity, - ) - .filter(EgonIndividualHeatingSupply.scenario == scenario) - .filter(EgonIndividualHeatingSupply.carrier == "heat_pump") - .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) + BuildingHeatPeakLoads.building_id, + BuildingHeatPeakLoads.peak_load_in_w, + ).filter(BuildingHeatPeakLoads.scenario == scenario) + # .filter(BuildingHeatPeakLoads.sector == "both") + .filter(BuildingHeatPeakLoads.building_id.in_(building_ids)) ) - hp_cap_mv_grid = pd.read_sql( - query.statement, query.session.bind, index_col="mv_grid_id" - ).capacity.values[0] + df_heat_peak_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) - return hp_cap_mv_grid + # TODO remove check + if df_heat_peak_demand.duplicated("building_id").any(): + raise ValueError("Duplicate building_id") + return df_heat_peak_demand def determine_minimum_hp_capacity_per_building( @@ -864,7 +1066,6 @@ def determine_buildings_with_hp_in_mv_grid( # get buildings with PV to give them a higher priority when selecting # buildings a heat pump will be allocated to - engine = db.engine() saio.register_schema("supply", engine) # TODO Adhoc Pv rooftop fix # from saio.supply import egon_power_plants_pv_roof_building @@ -912,23 +1113,32 @@ def determine_buildings_with_hp_in_mv_grid( # choose random heat pumps until remaining heat pumps are larger than remaining # heat pump capacity remaining_hp_cap = ( - hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum()) + hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum() + ) min_cap_buildings_wo_hp = min_hp_cap_per_building.loc[ - building_ids.drop(buildings_with_hp)] + building_ids.drop(buildings_with_hp) + ] possible_buildings = min_cap_buildings_wo_hp[ - min_cap_buildings_wo_hp <= remaining_hp_cap].index + min_cap_buildings_wo_hp <= remaining_hp_cap + ].index while len(possible_buildings) > 0: random.seed(db.credentials()["--random-seed"]) new_hp_building = random.choice(possible_buildings) # add new building to building with HP - buildings_with_hp = buildings_with_hp.append(pd.Index([new_hp_building])) + buildings_with_hp = buildings_with_hp.append( + pd.Index([new_hp_building]) + ) # determine if there are still possible buildings remaining_hp_cap = ( - hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum()) + hp_cap_mv_grid + - min_hp_cap_per_building.loc[buildings_with_hp].sum() + ) min_cap_buildings_wo_hp = min_hp_cap_per_building.loc[ - building_ids.drop(buildings_with_hp)] + building_ids.drop(buildings_with_hp) + ] possible_buildings = min_cap_buildings_wo_hp[ - min_cap_buildings_wo_hp <= remaining_hp_cap].index + min_cap_buildings_wo_hp <= remaining_hp_cap + ].index return buildings_with_hp @@ -999,7 +1209,9 @@ def determine_min_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): return 0.0 -def determine_hp_cap_buildings_eGon2035(mv_grid_id, peak_heat_demand, building_ids): +def determine_hp_cap_buildings_eGon2035( + mv_grid_id, peak_heat_demand, building_ids +): """ Determines which buildings in the MV grid will have a HP (buildings with PV rooftop are more likely to be assigned) in the eGon2035 scenario, as well as @@ -1059,13 +1271,13 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): ) # TODO get peak demand from db - peak_heat_demand = get_peak_demand_per_building( + df_peak_heat_demand = get_heat_peak_demand_per_building( "eGon100RE", building_ids ) # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 + df_peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 ) # distribute total heat pump capacity to all buildings with HP @@ -1077,278 +1289,422 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): ) # ToDo Julian Write desaggregated HP capacity to table (same as for 2035 scenario) + # check columns + write_table_to_postgres( + hp_cap_per_building, + EgonHpCapacityBuildings, + engine=engine, + drop=False, + ) -@timeitlog -def determine_hp_capacity_eGon2035_pypsa_eur_sec(n, max_n=5): - """ - Main function to determine HP capacity per building in eGon2035 scenario and - minimum required HP capacity in MV for pypsa-eur-sec. - Further, creates heat demand time series for all buildings with heat pumps - (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all buildings - with gas boilers (only in eGon2035scenario), used in eTraGo. +def aggregate_residential_and_cts_profiles(mvgd): + """ """ + # ############### get residential heat demand profiles ############### + df_heat_ts = calc_residential_heat_profiles_per_mvgd(mvgd=mvgd) - Parameters - ----------- - n : int - Number between [1;max_n]. - max_n : int - Maximum number of bulks (MV grid sets run in parallel). + # pivot to allow aggregation with CTS profiles + df_heat_ts_2035 = df_heat_ts.loc[ + :, ["building_id", "day_of_year", "hour", "eGon2035"] + ] + df_heat_ts_2035 = df_heat_ts_2035.pivot( + index=["day_of_year", "hour"], + columns="building_id", + values="eGon2035", + ) + df_heat_ts_2035 = df_heat_ts_2035.sort_index().reset_index(drop=True) + + df_heat_ts_100RE = df_heat_ts.loc[ + :, ["building_id", "day_of_year", "hour", "eGon100RE"] + ] + df_heat_ts_100RE = df_heat_ts_100RE.pivot( + index=["day_of_year", "hour"], + columns="building_id", + values="eGon100RE", + ) + df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) - """ + del df_heat_ts - # ========== Register np datatypes with SQLA ========== - register_adapter(np.float64, adapt_numpy_float64) - register_adapter(np.int64, adapt_numpy_int64) - # ===================================================== + # ############### get CTS heat demand profiles ############### + heat_demand_cts_ts_2035 = calc_cts_building_profiles( + bus_ids=[mvgd], + scenario="eGon2035", + sector="heat", + ) + heat_demand_cts_ts_100RE = calc_cts_building_profiles( + bus_ids=[mvgd], + scenario="eGon100RE", + sector="heat", + ) - log_to_file(determine_hp_capacity_eGon2035_pypsa_eur_sec.__qualname__ + f"_{n}") - if n == 0: - raise KeyError("n >= 1") + # ############# aggregate residential and CTS demand profiles ############# + df_heat_ts_2035 = pd.concat( + [df_heat_ts_2035, heat_demand_cts_ts_2035], axis=1 + ) - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + # TODO maybe differentiate between residential, cts and res+cts + # df_heat_ts_2035_agg = df_heat_ts_2035.loc[:, + # df_heat_ts_2035.columns.duplicated(keep=False)] + # df_heat_ts_2035 = df_heat_ts_2035.loc[:, + # ~df_heat_ts_2035.columns.duplicated(keep=False)] - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + df_heat_ts_2035 = df_heat_ts_2035.groupby(axis=1, level=0).sum() - mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) + df_heat_ts_100RE = pd.concat( + [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 + ) + df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() - # TODO mvgd_ids = [kleines mvgd] - for mvgd in [1556]: #mvgd_ids[n - 1]: + # del heat_demand_cts_ts_2035, heat_demand_cts_ts_100RE - logger.trace(f"MVGD={mvgd} | Start") + return df_heat_ts_2035, df_heat_ts_100RE - # ############### get residential heat demand profiles ############### - df_heat_ts = calc_residential_heat_profiles_per_mvgd( - mvgd=mvgd - ) - # pivot to allow aggregation with CTS profiles - df_heat_ts_2035 = df_heat_ts.loc[ - :, ["building_id", "day_of_year", "hour", "eGon2035"]] - df_heat_ts_2035 = df_heat_ts_2035.pivot( - index=["day_of_year", "hour"], - columns="building_id", - values="eGon2035", - ) - df_heat_ts_2035 = df_heat_ts_2035.sort_index().reset_index(drop=True) - - df_heat_ts_100RE = df_heat_ts.loc[ - :, ["building_id", "day_of_year", "hour", "eGon100RE"]] - df_heat_ts_100RE = df_heat_ts_100RE.pivot( - index=["day_of_year", "hour"], - columns="building_id", - values="eGon100RE", - ) - df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) +def determine_peak_loads(df_heat_ts_2035, df_heat_ts_100RE, to_db=False): + """""" + df_peak_loads = pd.concat( + [ + df_heat_ts_2035.max().rename("eGon2035"), + df_heat_ts_100RE.max().rename("eGon100RE"), + ], + axis=1, + ) - del df_heat_ts + if to_db: - # ############### get CTS heat demand profiles ############### - heat_demand_cts_ts_2035 = calc_cts_building_profiles( - bus_ids=[mvgd], - scenario="eGon2035", - sector="heat", - ) - heat_demand_cts_ts_100RE = calc_cts_building_profiles( - bus_ids=[mvgd], - scenario="eGon100RE", - sector="heat", + df_peak_loads_db = df_peak_loads.reset_index().melt( + id_vars="building_id", + var_name="scenario", + value_name="peak_load_in_w", ) - # ############# aggregate residential and CTS demand profiles ############# - df_heat_ts_2035 = pd.concat( - [df_heat_ts_2035, heat_demand_cts_ts_2035], axis=1 + df_peak_loads_db["sector"] = "residential+cts" + # From MW to W + df_peak_loads_db["peak_load_in_w"] = ( + df_peak_loads_db["peak_load_in_w"] * 1e6 ) - df_heat_ts_2035 = df_heat_ts_2035.groupby(axis=1, level=0).sum() - df_heat_ts_100RE = pd.concat( - [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 + write_table_to_postgres( + df_peak_loads_db, BuildingHeatPeakLoads, engine=engine ) - df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() - del heat_demand_cts_ts_2035, heat_demand_cts_ts_100RE + return df_peak_loads - # ##################### export peak loads to DB ################### - df_peak_loads_2035 = df_heat_ts_2035.max() - df_peak_loads_100RE = df_heat_ts_100RE.max() +def determine_hp_capacity( + mvgd, df_peak_loads, buildings_decentral_heating, to_db=False, to_csv=False +): + """""" - df_peak_loads_db_2035 = df_peak_loads_2035.reset_index().melt( - id_vars="building_id", - var_name="scenario", - value_name="peak_load_in_w", + # determine HP capacity per building for NEP2035 scenario + hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( + mvgd, + df_peak_loads["eGon2035"], + buildings_decentral_heating["eGon2035"], + ) + + # TODO buildings_gas_2035 empty? + # determine buildings with gas heating for NEP2035 scenario + buildings_gas_2035 = pd.Index( + buildings_decentral_heating["eGon2035"] + ).drop(hp_cap_per_building_2035.index) + + # determine minimum HP capacity per building for pypsa-eur-sec + hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( + df_peak_loads["eGon100RE"], + buildings_decentral_heating["eGon100RE"] + # TODO 100RE? + ) + # ######################## write HP capacities to DB ###################### + if to_db: + logger.debug(f"MVGD={mvgd} | Write HP capacities to DB.") + + df_hp_cap_per_building_2035 = pd.DataFrame() + df_hp_cap_per_building_2035["hp_capacity"] = hp_cap_per_building_2035 + df_hp_cap_per_building_2035["scenario"] = "eGon2035" + df_hp_cap_per_building_2035 = ( + df_hp_cap_per_building_2035.reset_index().rename( + columns={"index": "building_id"} + ) ) - df_peak_loads_db_2035["scenario"] = "eGon2035" - df_peak_loads_db_100RE = df_peak_loads_100RE.reset_index().melt( - id_vars="building_id", - var_name="scenario", - value_name="peak_load_in_w", + + write_table_to_postgres( + df_hp_cap_per_building_2035, + EgonHpCapacityBuildings, + engine=engine, + drop=False, ) - df_peak_loads_db_100RE["scenario"] = "eGon100RE" - df_peak_loads_db = pd.concat( - [df_peak_loads_db_2035, df_peak_loads_db_100RE]) - del df_peak_loads_db_2035, df_peak_loads_db_100RE + if to_csv: + logger.debug( + f"MVGD={mvgd} | Write pypsa-eur-sec min HP capacities to " f"csv." + ) + folder = Path(".") / "input-pypsa-eur-sec" + file = folder / "minimum_hp_capacity_mv_grid_2035.csv" + # Create the folder, if it does not exists already + if not os.path.exists(folder): + os.mkdir(folder) + # TODO check append + if not file.is_file(): + df_hp_cap_per_building_2035.to_csv(file) + # TODO outsource into separate task incl delete file if clearing + else: + df_hp_cap_per_building_2035.to_csv(file, mode="a", header=False) - df_peak_loads_db["sector"] = "residential+CTS" - # From MW to W - df_peak_loads_db["peak_load_in_w"] = df_peak_loads_db["peak_load_in_w"] * 1e6 + return hp_cap_per_building_2035 # , hp_min_cap_mv_grid_pypsa_eur_sec - logger.trace(f"MVGD={mvgd} | Export to DB") - # TODO export peak loads all buildings both scenarios to db +def determine_mvgd_ts( + mvgd, + df_heat_ts_2035, + df_heat_ts_100RE, + buildings_decentral_heating, + hp_cap_per_building_2035, + to_db=False, +): + """""" + + # heat demand time series for buildings with heat pumps + # ToDo Julian Write aggregated heat demand time series of buildings with HP to + # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # TODO Clara uses this table already + # but will not need it anymore for eTraGo + # EgonEtragoTimeseriesIndividualHeating + df_mvgd_ts_2035_hp = df_heat_ts_2035.loc[ + :, + # buildings_decentral_heating["eGon2035"]].sum( + hp_cap_per_building_2035.index, + ].sum( + axis=1 + ) # TODO davor? buildings_hp_2035 = hp_cap_per_building_2035.index + # TODO nur hp oder auch gas? + df_mvgd_ts_100RE_hp = df_heat_ts_100RE.loc[ + :, buildings_decentral_heating["eGon100RE"] + ].sum(axis=1) + + df_mvgd_ts_2035_gas = df_heat_ts_2035.drop( + columns=hp_cap_per_building_2035.index + ).sum(axis=1) + # heat demand time series for buildings with gas boilers (only 2035 scenario) + # df_heat_ts_100RE_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( + # axis=1 + # ) + + df_mvgd_ts_hp = pd.DataFrame( + data={ + "carrier": ["heat_pump", "heat_pump", "CH4"], + "bus_id": mvgd, + "scenario": ["eGon2035", "eGon100RE", "eGon2035"], + "dist_aggregated_mw": [ + df_mvgd_ts_2035_hp.to_list(), + df_mvgd_ts_100RE_hp.to_list(), + df_mvgd_ts_2035_gas.to_list(), + ], + } + ) + if to_db: # write_table_to_postgres( - # df_peak_loads_db, BuildingHeatPeakLoads, engine=engine + # df_mvgd_ts_hp, + # EgonEtragoTimeseriesIndividualHeating, + # engine=engine, + # drop=False, # ) - # logger.trace(f"MVGD={mvgd} | Done") - - # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ########## - # get residential buildings with decentral heating systems in both scenarios - buildings_decentral_heating_2035_res = ( - get_residential_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon2035", mvgd - ) - ) - buildings_decentral_heating_100RE_res = ( - get_residential_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mvgd - ) + columns = { + column.key: column.type + for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns + } + df_mvgd_ts_hp = df_mvgd_ts_hp.loc[:, columns.keys()] + + df_mvgd_ts_hp.to_sql( + name=EgonEtragoTimeseriesIndividualHeating.__table__.name, + schema=EgonEtragoTimeseriesIndividualHeating.__table__.schema, + con=engine, + if_exists="append", + method="multi", + index=False, + dtype=columns, ) - # get CTS buildings with decentral heating systems in both scenarios - buildings_decentral_heating_2035_cts = ( - get_cts_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon2035", mvgd - ) - ) - buildings_decentral_heating_100RE_cts = ( - get_cts_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mvgd - ) - ) + # # Change format + # # ToDo Julian check columns! especially value column + # df_etrago_ts_individual_heating_hp = pd.DataFrame( + # index=[0, 1], + # columns=["bus_id", "scenario", "dist_aggregated_mw"], + # ) + # df_etrago_ts_individual_heating_hp.loc[ + # 0, "dist_aggregated_mw" + # ] = df_mvgd_ts_2035_hp.values.tolist() + # df_etrago_ts_individual_heating_hp.loc[0, "scenario"] = "eGon2035" + # df_etrago_ts_individual_heating_hp["carrier"] = "heat_pump" + # df_etrago_ts_individual_heating_hp["bus_id"] = mvgd + # # df_etrago_2035_ts_individual_heating_hp.reset_index(inplace=True) + # + # write_table_to_postgres( + # df_etrago_2035_ts_individual_heating_hp, + # EgonEtragoTimeseriesIndividualHeating, + # engine=engine, + # drop=False, + # ) + # + # df_etrago_100RE_ts_individual_heating_hp = pd.DataFrame( + # index=df_heat_ts_100RE_hp.index, + # columns=["scenario", "dist_aggregated_mw"], + # ) + # df_etrago_100RE_ts_individual_heating_hp[ + # "dist_aggregated_mw" + # ] = df_mvgd_ts_100RE_hp.values.tolist() + # df_etrago_100RE_ts_individual_heating_hp["carrier"] = "heat_pump" + # df_etrago_100RE_ts_individual_heating_hp["scenario"] = "eGon100RE" + # df_etrago_100RE_ts_individual_heating_hp.reset_index(inplace=True) + # + # write_table_to_postgres( + # df_etrago_100RE_ts_individual_heating_hp, + # EgonEtragoTimeseriesIndividualHeating, + # engine=engine, + # drop=False, + # ) + # + # # # Drop and recreate Table if exists + # # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), + # # checkfirst=True) + # # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), + # # checkfirst=True) + # # + # # # Write heat ts into db + # # with db.session_scope() as session: + # # session.bulk_insert_mappings( + # # EgonEtragoTimeseriesIndividualHeating, + # # df_etrago_cts_heat_profiles.to_dict(orient="records"), + # # ) + # + # # heat demand time series for buildings with gas boilers (only 2035 scenario) + # df_heat_ts_100RE_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( + # axis=1 + # ) + # # ToDo Julian Write heat demand time series for buildings with gas boiler to + # # database - in gleiche Tabelle wie Zeitreihen für WP Gebäude, falls Clara + # # nichts anderes sagt; wird später weiter aggregiert nach gas voronoi + # # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia + # + # df_etrago_2035_ts_individual_heating_gas = pd.DataFrame( + # index=df_heat_ts_100RE_gas.index, + # columns=["scenario", "dist_aggregated_mw"], + # ) + # df_etrago_2035_ts_individual_heating_gas[ + # "dist_aggregated_mw" + # ] = df_heat_ts_100RE_gas[""].values.tolist() + # df_etrago_2035_ts_individual_heating_gas["carrier"] = "CH4" + # df_etrago_2035_ts_individual_heating_gas["scenario"] = "eGon2035" + # df_etrago_2035_ts_individual_heating_gas.reset_index(inplace=True) + # + # write_table_to_postgres( + # df_etrago_100RE_ts_individual_heating, + # EgonEtragoTimeseriesIndividualHeating, + # engine=engine, + # drop=False, + # ) - # merge residential and CTS buildings - buildings_decentral_heating_2035 = ( - buildings_decentral_heating_2035_res.append( - buildings_decentral_heating_2035_cts - ).unique() - ) - buildings_decentral_heating_100RE = ( - buildings_decentral_heating_100RE_res.append( - buildings_decentral_heating_100RE_cts - ).unique() - ) - # determine HP capacity per building for NEP2035 scenario - hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( - mvgd, df_peak_loads_2035, buildings_decentral_heating_2035) - buildings_hp_2035 = hp_cap_per_building_2035.index - buildings_gas_2035 = pd.Index(buildings_decentral_heating_2035).drop( - buildings_hp_2035) +@timeitlog +def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): + """ + Main function to determine HP capacity per building in eGon2035 scenario + and minimum required HP capacity in MV for pypsa-eur-sec. + Further, creates heat demand time series for all buildings with heat pumps + (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all buildings + with gas boilers (only in eGon2035scenario), used in eTraGo. + + Parameters + ----------- + bulk: list(int) + List of numbers of mvgds - # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( - df_peak_loads_100RE, buildings_decentral_heating_100RE) + """ - # ######################## write HP capacities to DB ###################### + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== - # ToDo Julian Write HP capacity per building in 2035 (hp_cap_per_building_2035) to - # db table - neue Tabelle egon_hp_capacity_buildings + log_to_file( + determine_hp_cap_peak_load_mvgd_ts.__qualname__ + + f"_{min(mvgd_ids)}-{max(mvgd_ids)}" + ) - # ToDo Julian Write minimum required capacity in pypsa-eur-sec - # (hp_min_cap_mv_grid_pypsa_eur_sec) to - # csv for pypsa-eur-sec input - im working directory gibt es directory - # input_pypsa_eur_sec - minimum_hp_capacity_mv_grid.csv + # TODO mvgd_ids = [kleines mvgd] + for mvgd in mvgd_ids: # [1556]: #mvgd_ids[n - 1]: - # ################ write aggregated heat profiles to DB ################### + logger.trace(f"MVGD={mvgd} | Start") - # heat demand time series for buildings with heat pumps - - # ToDo Julian Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating - # TODO Clara uses this table already - # but will not need it anymore for eTraGo - # EgonEtragoTimeseriesIndividualHeating - df_heat_ts_2035.loc[:, buildings_hp_2035].sum(axis=1) # carrier heat_pump - df_heat_ts_100RE.loc[:, buildings_decentral_heating_100RE].sum(axis=1) # carrier heat_pump - - # Change format - # ToDo Julian - # data = CTS_grid.drop(columns="scenario") - # df_etrago_cts_heat_profiles = pd.DataFrame( - # index=data.index, columns=["scn_name", "p_set"] - # ) - # df_etrago_cts_heat_profiles.p_set = data.values.tolist() - # df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"] - # df_etrago_cts_heat_profiles.reset_index(inplace=True) + # ############# aggregate residential and CTS demand profiles ############# - # # Drop and recreate Table if exists - # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), - # checkfirst=True) - # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), - # checkfirst=True) - # - # # Write heat ts into db - # with db.session_scope() as session: - # session.bulk_insert_mappings( - # EgonEtragoTimeseriesIndividualHeating, - # df_etrago_cts_heat_profiles.to_dict(orient="records"), - # ) + ( + df_heat_ts_2035, + df_heat_ts_100RE, + ) = aggregate_residential_and_cts_profiles(mvgd) - # heat demand time series for buildings with gas boilers (only 2035 scenario) - df_heat_ts_2035.loc[:, buildings_gas_2035].sum(axis=1) # carrier gas_boilers - # ToDo Julian Write heat demand time series for buildings with gas boiler to - # database - in gleiche Tabelle wie Zeitreihen für WP Gebäude, falls Clara - # nichts anderes sagt; wird später weiter aggregiert nach gas voronoi - # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia + # ##################### export peak loads to DB ################### + logger.debug(f"MVGD={mvgd} | Determine peak loads.") + df_peak_loads = determine_peak_loads( + df_heat_ts_2035, df_heat_ts_100RE, to_db=True + ) + # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ########## + logger.debug(f"MVGD={mvgd} | Determine HP capacities.") -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(1, max_n=5) + buildings_decentral_heating = ( + get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd) + ) + # ( + # hp_cap_per_building_2035, + # hp_min_cap_mv_grid_pypsa_eur_sec + # ) = \ + hp_cap_per_building_2035 = determine_hp_capacity( + mvgd, + df_peak_loads, + buildings_decentral_heating, + to_db=True, + to_csv=True, + ) -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_2(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(2, max_n=5) + # ################ write aggregated heat profiles to DB ################### + determine_mvgd_ts( + mvgd, + df_heat_ts_2035, + df_heat_ts_100RE, + buildings_decentral_heating, + hp_cap_per_building_2035, + to_db=True, + ) -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_3(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(3, max_n=5) + print("done") -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_4(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(4, max_n=5) +def create_peak_load_table(): + BuildingHeatPeakLoads.__table__.drop(bind=engine, checkfirst=True) + BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) -def determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_5(): - determine_hp_capacity_eGon2035_pypsa_eur_sec(5, max_n=5) +def create_hp_capacity_table(): -def create_peak_load_table(): + EgonHpCapacityBuildings.__table__.drop(bind=engine, checkfirst=True) + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) + +# def create_ def delete_peak_loads_if_existing(): """Remove all entries""" + # TODO check synchronize_session? with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( - BuildingHeatPeakLoads.sector == "residential" + BuildingHeatPeakLoads.sector == "residential+cts" ).delete(synchronize_session=False) - - -if __name__ == "__main__": - determine_hp_capacity_eGon2035_pypsa_eur_sec_bulk_1() From cca22ebf667966bcc0bfcea3d20fddca30b32e17 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:41:11 +0200 Subject: [PATCH 031/119] Comment HP100RE --- src/egon/data/airflow/dags/pipeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 4342cacb2..f2e5e634f 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -584,13 +584,13 @@ ] ) - heat_pumps_2050 = HeatPumps2050( - dependencies=[ - cts_demand_buildings, - DistrictHeatingAreas, - run_pypsaeursec, - ] - ) + # heat_pumps_2050 = HeatPumps2050( + # dependencies=[ + # cts_demand_buildings, + # DistrictHeatingAreas, + # run_pypsaeursec, + # ] + # ) heat_pumps_2035 = HeatPumps2035( dependencies=[ From e0b71ddf41beaae6a8ee818f0e1fd2cd19d9099e Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:44:53 +0200 Subject: [PATCH 032/119] Update dataset import --- src/egon/data/airflow/dags/pipeline.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index f2e5e634f..f362002cf 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -44,9 +44,8 @@ from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable from egon.data.datasets.heat_supply import HeatSupply from egon.data.datasets.heat_supply.individual_heating import ( - HeatPumps2035, + HeatPumpsPypsaEurSecAnd2035, HeatPumps2050, - HeatPumpsEtrago, ) from egon.data.datasets.hydrogen_etrago import ( HydrogenBusEtrago, @@ -592,7 +591,7 @@ # ] # ) - heat_pumps_2035 = HeatPumps2035( + heat_pumps_2035 = HeatPumpsPypsaEurSecAnd2035( dependencies=[ cts_demand_buildings, DistrictHeatingAreas, From b554dce04132aa129d8727dafb25af5abb548f74 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:47:54 +0200 Subject: [PATCH 033/119] Update dataset import --- src/egon/data/airflow/dags/pipeline.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index f362002cf..3309fdf15 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -576,13 +576,6 @@ ] ) - heat_pumps_etrago = HeatPumpsEtrago( - dependencies=[ - cts_demand_buildings, - DistrictHeatingAreas, - ] - ) - # heat_pumps_2050 = HeatPumps2050( # dependencies=[ # cts_demand_buildings, @@ -595,6 +588,8 @@ dependencies=[ cts_demand_buildings, DistrictHeatingAreas, + heat_supply, + heat_time_series, # TODO add PV rooftop ] ) From 14bb8a00bcd5f5346db24222e96ed68d4dbcb246 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:51:22 +0200 Subject: [PATCH 034/119] Fix typo --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 26366fb8a..953daf31a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -81,7 +81,7 @@ def dyn_parallel_tasks(): :func:`egon.data.datasets.heat_supply.individual_heating. determine_hp_capacity_eGon2035_pypsa_eur_sec` """ - parallel_tasks = egon.data.config.datasets()[ + parallel_tasks = config.datasets()[ "demand_timeseries_mvgd" ].get("parallel_tasks", 1) # ========== Register np datatypes with SQLA ========== From 589791a6785d40ca32efbbed036bd4cd028834b4 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 01:57:09 +0200 Subject: [PATCH 035/119] Add parallel_tasks to dataset.yml --- src/egon/data/datasets.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index ada9faef0..b34eedb47 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1087,3 +1087,6 @@ emobility_mit: reduce_memory: True export_results_to_csv: True parallel_tasks: 10 + +demand_timeseries_mvgd: + parallel_tasks: 10 From 9f076927de635857a0c2918c5ade19b5d50f691e Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 02:01:56 +0200 Subject: [PATCH 036/119] Add import python operator --- src/egon/data/datasets/heat_supply/individual_heating.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 953daf31a..68845202f 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -7,6 +7,7 @@ import random import time +from airflow.operators.python_operator import PythonOperator from loguru import logger from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String @@ -81,9 +82,9 @@ def dyn_parallel_tasks(): :func:`egon.data.datasets.heat_supply.individual_heating. determine_hp_capacity_eGon2035_pypsa_eur_sec` """ - parallel_tasks = config.datasets()[ - "demand_timeseries_mvgd" - ].get("parallel_tasks", 1) + parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( + "parallel_tasks", 1 + ) # ========== Register np datatypes with SQLA ========== register_adapter(np.float64, adapt_numpy_float64) register_adapter(np.int64, adapt_numpy_int64) From 72353723a7ce8fd5fd574f29fa0a175d525a9b5b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 10:56:42 +0200 Subject: [PATCH 037/119] Activate task individual_heating for nep hp cap --- src/egon/data/datasets/heat_supply/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/__init__.py b/src/egon/data/datasets/heat_supply/__init__.py index fd356ef86..55511419c 100644 --- a/src/egon/data/datasets/heat_supply/__init__.py +++ b/src/egon/data/datasets/heat_supply/__init__.py @@ -176,8 +176,7 @@ def __init__(self, dependencies): create_tables, { district_heating, - # Temporary drop everything related to rural heat - # individual_heating, + individual_heating, potential_germany, }, ), From 763f559f7335683d8b0c026af5d715e28178a4c8 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 10:58:14 +0200 Subject: [PATCH 038/119] Fix gas_buildings_2035 --- .../data/datasets/heat_supply/individual_heating.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 68845202f..68baca146 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1484,13 +1484,12 @@ def determine_mvgd_ts( :, buildings_decentral_heating["eGon100RE"] ].sum(axis=1) - df_mvgd_ts_2035_gas = df_heat_ts_2035.drop( - columns=hp_cap_per_building_2035.index - ).sum(axis=1) + buildings_gas_2035 = buildings_decentral_heating["eGon2035"].drop( + hp_cap_per_building_2035.index) # heat demand time series for buildings with gas boilers (only 2035 scenario) - # df_heat_ts_100RE_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( - # axis=1 - # ) + df_mvgd_ts_2035_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( + axis=1 + ) df_mvgd_ts_hp = pd.DataFrame( data={ From 28db3b592d5bd0041ecc73b66876dfb5e55a42f6 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 17:51:24 +0200 Subject: [PATCH 039/119] Collect bulk data before export --- .../heat_supply/individual_heating.py | 441 ++++++++---------- 1 file changed, 191 insertions(+), 250 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 68baca146..fdf1eee0d 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -23,7 +23,6 @@ MapZensusDistrictHeatingAreas, ) from egon.data.datasets.electricity_demand_timeseries.cts_buildings import ( - CtsBuildings, calc_cts_building_profiles, ) from egon.data.datasets.electricity_demand_timeseries.mapping import ( @@ -47,6 +46,7 @@ engine = db.engine() Base = declarative_base() + # TODO check column names> class EgonEtragoTimeseriesIndividualHeating(Base): __tablename__ = "egon_etrago_timeseries_individual_heating" @@ -810,8 +810,8 @@ def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heating system in given - MV grid. Type is pandas Index to avoid errors later on when it is + Building IDs (as int) of buildings with decentral heating system in + given MV grid. Type is pandas Index to avoid errors later on when it is used in a query. """ @@ -863,8 +863,8 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heating system in given - MV grid. Type is pandas Index to avoid errors later on when it is + Building IDs (as int) of buildings with decentral heating system in + given MV grid. Type is pandas Index to avoid errors later on when it is used in a query. """ @@ -877,17 +877,12 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( ) # get buildings with decentral heat demand - # ToDo @Julian, sind das alle CTS buildings in der Tabelle? - # ja aber die zensus_population_id stimmt nicht - # boundaries.egon_map_zensus_mvgd_buildings_used benutzen - # with db.session_scope() as session: query = session.query(EgonMapZensusMvgdBuildings.building_id).filter( EgonMapZensusMvgdBuildings.sector == "cts", EgonMapZensusMvgdBuildings.zensus_population_id.in_( zensus_population_ids - ) - # ).unique(EgonMapZensusMvgdBuildings.building_id) + ), ) buildings_with_heat_demand = pd.read_sql( @@ -899,24 +894,28 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd): """""" - # get residential buildings with decentral heating systems in both scenarios + # get residential buildings with decentral heating systems + # scenario eGon2035 buildings_decentral_heating_2035_res = ( get_residential_buildings_with_decentral_heat_demand_in_mv_grid( "eGon2035", mvgd ) ) + # scenario eGon100RE buildings_decentral_heating_100RE_res = ( get_residential_buildings_with_decentral_heat_demand_in_mv_grid( "eGon100RE", mvgd ) ) - # get CTS buildings with decentral heating systems in both scenarios + # get CTS buildings with decentral heating systems + # scenario eGon2035 buildings_decentral_heating_2035_cts = ( get_cts_buildings_with_decentral_heat_demand_in_mv_grid( "eGon2035", mvgd ) ) + # scenario eGon100RE buildings_decentral_heating_100RE_cts = ( get_cts_buildings_with_decentral_heat_demand_in_mv_grid( "eGon100RE", mvgd @@ -961,8 +960,8 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): Total heat pump capacity in MW in given MV grid. """ - # TODO temporary commented until table exists - # from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply + from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply + # # with db.session_scope() as session: # query = ( @@ -979,17 +978,17 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): # query.statement, query.session.bind, index_col="mv_grid_id" # ).capacity.values[0] - # with db.session_scope() as session: - # hp_cap_mv_grid = session.execute( - # EgonIndividualHeatingSupply.capacity - # ).filter( - # EgonIndividualHeatingSupply.scenario == scenario, - # EgonIndividualHeatingSupply.carrier == "heat_pump", - # EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id - # ).scalar() + with db.session_scope() as session: + hp_cap_mv_grid = ( + session.execute(EgonIndividualHeatingSupply.capacity) + .filter( + EgonIndividualHeatingSupply.scenario == scenario, + EgonIndividualHeatingSupply.carrier == "heat_pump", + EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id, + ) + .scalar() + ) - # workaround - hp_cap_mv_grid = 50 return hp_cap_mv_grid @@ -1111,8 +1110,8 @@ def determine_buildings_with_hp_in_mv_grid( hp_cumsum = min_hp_cap_per_building.loc[buildings_with_hp_order].cumsum() buildings_with_hp = hp_cumsum[hp_cumsum <= hp_cap_mv_grid].index - # choose random heat pumps until remaining heat pumps are larger than remaining - # heat pump capacity + # choose random heat pumps until remaining heat pumps are larger than + # remaining heat pump capacity remaining_hp_cap = ( hp_cap_mv_grid - min_hp_cap_per_building.loc[buildings_with_hp].sum() ) @@ -1176,6 +1175,8 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): hp_cap_per_building = ( min_hp_cap_per_building * fac + min_hp_cap_per_building ) + hp_cap_per_building.index.name = "building_id" + return hp_cap_per_building @@ -1190,8 +1191,8 @@ def determine_min_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): Series with peak heat demand per building in MW. Index contains the building ID. building_ids : pd.Index(int) - Building IDs (as int) of buildings with decentral heating system in given - MV grid. + Building IDs (as int) of buildings with decentral heating system in + given MV grid. Returns -------- @@ -1215,8 +1216,8 @@ def determine_hp_cap_buildings_eGon2035( ): """ Determines which buildings in the MV grid will have a HP (buildings with PV - rooftop are more likely to be assigned) in the eGon2035 scenario, as well as - their respective HP capacity in MW. + rooftop are more likely to be assigned) in the eGon2035 scenario, as well + as their respective HP capacity in MW. Parameters ----------- @@ -1252,17 +1253,18 @@ def determine_hp_cap_buildings_eGon2035( min_hp_cap_buildings.loc[buildings_with_hp], hp_cap_grid ) - return hp_cap_per_building + return hp_cap_per_building.rename("hp_capacity") else: - return pd.Series() + return pd.Series().rename("hp_capacity") def determine_hp_cap_buildings_eGon100RE(mv_grid_id): """ Main function to determine HP capacity per building in eGon100RE scenario. - In eGon100RE scenario all buildings without district heating get a heat pump. + In eGon100RE scenario all buildings without district heating get a heat + pump. """ @@ -1289,8 +1291,8 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): min_hp_cap_buildings, hp_cap_grid ) - # ToDo Julian Write desaggregated HP capacity to table (same as for 2035 scenario) - # check columns + # ToDo Julian Write desaggregated HP capacity to table (same as for + # 2035 scenario) check columns write_table_to_postgres( hp_cap_per_building, EgonHpCapacityBuildings, @@ -1344,12 +1346,6 @@ def aggregate_residential_and_cts_profiles(mvgd): [df_heat_ts_2035, heat_demand_cts_ts_2035], axis=1 ) - # TODO maybe differentiate between residential, cts and res+cts - # df_heat_ts_2035_agg = df_heat_ts_2035.loc[:, - # df_heat_ts_2035.columns.duplicated(keep=False)] - # df_heat_ts_2035 = df_heat_ts_2035.loc[:, - # ~df_heat_ts_2035.columns.duplicated(keep=False)] - df_heat_ts_2035 = df_heat_ts_2035.groupby(axis=1, level=0).sum() df_heat_ts_100RE = pd.concat( @@ -1362,40 +1358,7 @@ def aggregate_residential_and_cts_profiles(mvgd): return df_heat_ts_2035, df_heat_ts_100RE -def determine_peak_loads(df_heat_ts_2035, df_heat_ts_100RE, to_db=False): - """""" - df_peak_loads = pd.concat( - [ - df_heat_ts_2035.max().rename("eGon2035"), - df_heat_ts_100RE.max().rename("eGon100RE"), - ], - axis=1, - ) - - if to_db: - - df_peak_loads_db = df_peak_loads.reset_index().melt( - id_vars="building_id", - var_name="scenario", - value_name="peak_load_in_w", - ) - - df_peak_loads_db["sector"] = "residential+cts" - # From MW to W - df_peak_loads_db["peak_load_in_w"] = ( - df_peak_loads_db["peak_load_in_w"] * 1e6 - ) - - write_table_to_postgres( - df_peak_loads_db, BuildingHeatPeakLoads, engine=engine - ) - - return df_peak_loads - - -def determine_hp_capacity( - mvgd, df_peak_loads, buildings_decentral_heating, to_db=False, to_csv=False -): +def determine_hp_capacity(mvgd, df_peak_loads, buildings_decentral_heating): """""" # determine HP capacity per building for NEP2035 scenario @@ -1405,77 +1368,41 @@ def determine_hp_capacity( buildings_decentral_heating["eGon2035"], ) - # TODO buildings_gas_2035 empty? - # determine buildings with gas heating for NEP2035 scenario - buildings_gas_2035 = pd.Index( - buildings_decentral_heating["eGon2035"] - ).drop(hp_cap_per_building_2035.index) - # determine minimum HP capacity per building for pypsa-eur-sec hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( df_peak_loads["eGon100RE"], buildings_decentral_heating["eGon100RE"] # TODO 100RE? ) - # ######################## write HP capacities to DB ###################### - if to_db: - logger.debug(f"MVGD={mvgd} | Write HP capacities to DB.") - - df_hp_cap_per_building_2035 = pd.DataFrame() - df_hp_cap_per_building_2035["hp_capacity"] = hp_cap_per_building_2035 - df_hp_cap_per_building_2035["scenario"] = "eGon2035" - df_hp_cap_per_building_2035 = ( - df_hp_cap_per_building_2035.reset_index().rename( - columns={"index": "building_id"} - ) - ) - - write_table_to_postgres( - df_hp_cap_per_building_2035, - EgonHpCapacityBuildings, - engine=engine, - drop=False, - ) - if to_csv: - logger.debug( - f"MVGD={mvgd} | Write pypsa-eur-sec min HP capacities to " f"csv." - ) - folder = Path(".") / "input-pypsa-eur-sec" - file = folder / "minimum_hp_capacity_mv_grid_2035.csv" - # Create the folder, if it does not exists already - if not os.path.exists(folder): - os.mkdir(folder) - # TODO check append - if not file.is_file(): - df_hp_cap_per_building_2035.to_csv(file) - # TODO outsource into separate task incl delete file if clearing - else: - df_hp_cap_per_building_2035.to_csv(file, mode="a", header=False) - - return hp_cap_per_building_2035 # , hp_min_cap_mv_grid_pypsa_eur_sec + return ( + hp_cap_per_building_2035.rename("hp_capacity"), + hp_min_cap_mv_grid_pypsa_eur_sec, + ) -def determine_mvgd_ts( +def aggregate_heat_profiles( mvgd, df_heat_ts_2035, df_heat_ts_100RE, buildings_decentral_heating, - hp_cap_per_building_2035, - to_db=False, + buildings_gas_2035, ): """""" # heat demand time series for buildings with heat pumps - # ToDo Julian Write aggregated heat demand time series of buildings with HP to - # table to be used in eTraGo - egon_etrago_timeseries_individual_heating + # ToDo Julian Write aggregated heat demand time series of buildings with + # HP to table to be used in eTraGo - + # egon_etrago_timeseries_individual_heating # TODO Clara uses this table already # but will not need it anymore for eTraGo # EgonEtragoTimeseriesIndividualHeating + df_mvgd_ts_2035_hp = df_heat_ts_2035.loc[ :, # buildings_decentral_heating["eGon2035"]].sum( - hp_cap_per_building_2035.index, + # hp_cap_per_building_2035.index, + buildings_decentral_heating["eGon2035"].drop(buildings_gas_2035), ].sum( axis=1 ) # TODO davor? buildings_hp_2035 = hp_cap_per_building_2035.index @@ -1484,14 +1411,13 @@ def determine_mvgd_ts( :, buildings_decentral_heating["eGon100RE"] ].sum(axis=1) - buildings_gas_2035 = buildings_decentral_heating["eGon2035"].drop( - hp_cap_per_building_2035.index) - # heat demand time series for buildings with gas boilers (only 2035 scenario) + # heat demand time series for buildings with gas boiler + # (only 2035 scenario) df_mvgd_ts_2035_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( axis=1 ) - df_mvgd_ts_hp = pd.DataFrame( + df_heat_mvgd_ts = pd.DataFrame( data={ "carrier": ["heat_pump", "heat_pump", "CH4"], "bus_id": mvgd, @@ -1503,108 +1429,70 @@ def determine_mvgd_ts( ], } ) - if to_db: - # write_table_to_postgres( - # df_mvgd_ts_hp, - # EgonEtragoTimeseriesIndividualHeating, - # engine=engine, - # drop=False, - # ) - - columns = { - column.key: column.type - for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns - } - df_mvgd_ts_hp = df_mvgd_ts_hp.loc[:, columns.keys()] - - df_mvgd_ts_hp.to_sql( - name=EgonEtragoTimeseriesIndividualHeating.__table__.name, - schema=EgonEtragoTimeseriesIndividualHeating.__table__.schema, - con=engine, - if_exists="append", - method="multi", - index=False, - dtype=columns, + return df_heat_mvgd_ts + + +def export_to_db( + df_peak_loads_db, df_hp_cap_per_building_2035, df_heat_mvgd_ts_db +): + """""" + + df_peak_loads_db = df_peak_loads_db.reset_index().melt( + id_vars="building_id", + var_name="scenario", + value_name="peak_load_in_w", + ) + df_peak_loads_db["sector"] = "residential+cts" + # From MW to W + df_peak_loads_db["peak_load_in_w"] = ( + df_peak_loads_db["peak_load_in_w"] * 1e6 + ) + write_table_to_postgres( + df_peak_loads_db, BuildingHeatPeakLoads, engine=engine + ) + + df_hp_cap_per_building_2035["scenario"] = "eGon2035" + df_hp_cap_per_building_2035 = ( + df_hp_cap_per_building_2035.reset_index().rename( + columns={"index": "building_id"} ) + ) + write_table_to_postgres( + df_hp_cap_per_building_2035, + EgonHpCapacityBuildings, + engine=engine, + drop=False, + ) + + columns = { + column.key: column.type + for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns + } + df_heat_mvgd_ts_db = df_heat_mvgd_ts_db.loc[:, columns.keys()] + + df_heat_mvgd_ts_db.to_sql( + name=EgonEtragoTimeseriesIndividualHeating.__table__.name, + schema=EgonEtragoTimeseriesIndividualHeating.__table__.schema, + con=engine, + if_exists="append", + method="multi", + index=False, + dtype=columns, + ) - # # Change format - # # ToDo Julian check columns! especially value column - # df_etrago_ts_individual_heating_hp = pd.DataFrame( - # index=[0, 1], - # columns=["bus_id", "scenario", "dist_aggregated_mw"], - # ) - # df_etrago_ts_individual_heating_hp.loc[ - # 0, "dist_aggregated_mw" - # ] = df_mvgd_ts_2035_hp.values.tolist() - # df_etrago_ts_individual_heating_hp.loc[0, "scenario"] = "eGon2035" - # df_etrago_ts_individual_heating_hp["carrier"] = "heat_pump" - # df_etrago_ts_individual_heating_hp["bus_id"] = mvgd - # # df_etrago_2035_ts_individual_heating_hp.reset_index(inplace=True) - # - # write_table_to_postgres( - # df_etrago_2035_ts_individual_heating_hp, - # EgonEtragoTimeseriesIndividualHeating, - # engine=engine, - # drop=False, - # ) - # - # df_etrago_100RE_ts_individual_heating_hp = pd.DataFrame( - # index=df_heat_ts_100RE_hp.index, - # columns=["scenario", "dist_aggregated_mw"], - # ) - # df_etrago_100RE_ts_individual_heating_hp[ - # "dist_aggregated_mw" - # ] = df_mvgd_ts_100RE_hp.values.tolist() - # df_etrago_100RE_ts_individual_heating_hp["carrier"] = "heat_pump" - # df_etrago_100RE_ts_individual_heating_hp["scenario"] = "eGon100RE" - # df_etrago_100RE_ts_individual_heating_hp.reset_index(inplace=True) - # - # write_table_to_postgres( - # df_etrago_100RE_ts_individual_heating_hp, - # EgonEtragoTimeseriesIndividualHeating, - # engine=engine, - # drop=False, - # ) - # - # # # Drop and recreate Table if exists - # # EgonEtragoTimeseriesIndividualHeating.__table__.drop(bind=db.engine(), - # # checkfirst=True) - # # EgonEtragoTimeseriesIndividualHeating.__table__.create(bind=db.engine(), - # # checkfirst=True) - # # - # # # Write heat ts into db - # # with db.session_scope() as session: - # # session.bulk_insert_mappings( - # # EgonEtragoTimeseriesIndividualHeating, - # # df_etrago_cts_heat_profiles.to_dict(orient="records"), - # # ) - # - # # heat demand time series for buildings with gas boilers (only 2035 scenario) - # df_heat_ts_100RE_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( - # axis=1 - # ) - # # ToDo Julian Write heat demand time series for buildings with gas boiler to - # # database - in gleiche Tabelle wie Zeitreihen für WP Gebäude, falls Clara - # # nichts anderes sagt; wird später weiter aggregiert nach gas voronoi - # # (grid.egon_gas_voronoi mit carrier CH4) von Clara oder Amélia - # - # df_etrago_2035_ts_individual_heating_gas = pd.DataFrame( - # index=df_heat_ts_100RE_gas.index, - # columns=["scenario", "dist_aggregated_mw"], - # ) - # df_etrago_2035_ts_individual_heating_gas[ - # "dist_aggregated_mw" - # ] = df_heat_ts_100RE_gas[""].values.tolist() - # df_etrago_2035_ts_individual_heating_gas["carrier"] = "CH4" - # df_etrago_2035_ts_individual_heating_gas["scenario"] = "eGon2035" - # df_etrago_2035_ts_individual_heating_gas.reset_index(inplace=True) - # - # write_table_to_postgres( - # df_etrago_100RE_ts_individual_heating, - # EgonEtragoTimeseriesIndividualHeating, - # engine=engine, - # drop=False, - # ) + +def export_to_csv(df_hp_cap_per_building_2035): + folder = Path(".") / "input-pypsa-eur-sec" + file = folder / "minimum_hp_capacity_mv_grid_2035.csv" + # Create the folder, if it does not exists already + if not os.path.exists(folder): + os.mkdir(folder) + # TODO check append + if not file.is_file(): + df_hp_cap_per_building_2035.to_csv(file) + # TODO outsource into separate task incl delete file if clearing + else: + df_hp_cap_per_building_2035.to_csv(file, mode="a", header=False) @timeitlog @@ -1613,8 +1501,8 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): Main function to determine HP capacity per building in eGon2035 scenario and minimum required HP capacity in MV for pypsa-eur-sec. Further, creates heat demand time series for all buildings with heat pumps - (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all buildings - with gas boilers (only in eGon2035scenario), used in eTraGo. + (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all + buildings with gas boilers (only in eGon2035scenario), used in eTraGo. Parameters ----------- @@ -1634,54 +1522,92 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): ) # TODO mvgd_ids = [kleines mvgd] + df_peak_loads_db = pd.DataFrame() + df_hp_cap_per_building_2035_db = pd.DataFrame() + df_heat_mvgd_ts_db = pd.DataFrame() + for mvgd in mvgd_ids: # [1556]: #mvgd_ids[n - 1]: logger.trace(f"MVGD={mvgd} | Start") - # ############# aggregate residential and CTS demand profiles ############# + # ############# aggregate residential and CTS demand profiles ##### ( df_heat_ts_2035, df_heat_ts_100RE, ) = aggregate_residential_and_cts_profiles(mvgd) - # ##################### export peak loads to DB ################### + # ##################### determine peak loads ################### logger.debug(f"MVGD={mvgd} | Determine peak loads.") - df_peak_loads = determine_peak_loads( - df_heat_ts_2035, df_heat_ts_100RE, to_db=True + df_peak_loads = pd.concat( + [ + df_heat_ts_2035.max().rename("eGon2035"), + df_heat_ts_100RE.max().rename("eGon100RE"), + ], + axis=1, ) - # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ########## + # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ### logger.debug(f"MVGD={mvgd} | Determine HP capacities.") buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd) ) - # ( - # hp_cap_per_building_2035, - # hp_min_cap_mv_grid_pypsa_eur_sec - # ) = \ - hp_cap_per_building_2035 = determine_hp_capacity( + # determine HP capacity per building for NEP2035 scenario + hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( mvgd, - df_peak_loads, - buildings_decentral_heating, - to_db=True, - to_csv=True, + df_peak_loads["eGon2035"], + buildings_decentral_heating["eGon2035"], ) - # ################ write aggregated heat profiles to DB ################### + # determine minimum HP capacity per building for pypsa-eur-sec + hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( + df_peak_loads["eGon100RE"], + buildings_decentral_heating["eGon100RE"] + # TODO 100RE? + ) + + buildings_gas_2035 = pd.Index( + buildings_decentral_heating["eGon2035"] + ).drop(hp_cap_per_building_2035.index) + + # ################ aggregated heat profiles ################### + logger.debug(f"MVGD={mvgd} | Aggregate heat profiles.") - determine_mvgd_ts( + df_heat_mvgd_ts = aggregate_heat_profiles( mvgd, df_heat_ts_2035, df_heat_ts_100RE, buildings_decentral_heating, - hp_cap_per_building_2035, - to_db=True, + buildings_gas_2035, ) - print("done") + # ################ collect results + logger.debug(f"MVGD={mvgd} | Collect results.") + + df_peak_loads_db = pd.concat( + [df_peak_loads_db, df_peak_loads.reset_index()], + axis=0, + ignore_index=True, + ) + df_hp_cap_per_building_2035_db = pd.concat( + [ + df_hp_cap_per_building_2035_db, + hp_cap_per_building_2035.reset_index(), + ], + axis=0, + ) + df_heat_mvgd_ts_db = pd.concat( + [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True + ) + # ################ export to db + logger.debug(" Write data to db.") + export_to_db( + df_peak_loads_db, df_hp_cap_per_building_2035_db, df_heat_mvgd_ts_db + ) + logger.debug(" Write pypsa-eur-sec min HP capacities to csv.") + export_to_csv(hp_min_cap_mv_grid_pypsa_eur_sec) def create_peak_load_table(): @@ -1696,7 +1622,14 @@ def create_hp_capacity_table(): EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) -# def create_ +def create_egon_etrago_timeseries_individual_heating(): + + EgonEtragoTimeseriesIndividualHeating.__table__.drop( + bind=engine, checkfirst=True + ) + EgonEtragoTimeseriesIndividualHeating.__table__.create( + bind=engine, checkfirst=True + ) def delete_peak_loads_if_existing(): @@ -1708,3 +1641,11 @@ def delete_peak_loads_if_existing(): session.query(BuildingHeatPeakLoads).filter( BuildingHeatPeakLoads.sector == "residential+cts" ).delete(synchronize_session=False) + + +if __name__ == "__main__": + create_peak_load_table() + create_hp_capacity_table() + create_egon_etrago_timeseries_individual_heating() + delete_peak_loads_if_existing() + determine_hp_cap_peak_load_mvgd_ts([1556]) From 1e14a7dd1bb6912ed7e0febcb75cd4b8fd5e9bb3 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 17:56:05 +0200 Subject: [PATCH 040/119] Remove if name=main --- src/egon/data/datasets/heat_supply/individual_heating.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index fdf1eee0d..238dcdf7f 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1641,11 +1641,3 @@ def delete_peak_loads_if_existing(): session.query(BuildingHeatPeakLoads).filter( BuildingHeatPeakLoads.sector == "residential+cts" ).delete(synchronize_session=False) - - -if __name__ == "__main__": - create_peak_load_table() - create_hp_capacity_table() - create_egon_etrago_timeseries_individual_heating() - delete_peak_loads_if_existing() - determine_hp_cap_peak_load_mvgd_ts([1556]) From 98b66e3ff9f37d39dfc1bd7efcf79ceb6b702683 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 19:11:37 +0200 Subject: [PATCH 041/119] Rename functions --- .../heat_supply/individual_heating.py | 111 ++++++++++-------- 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 238dcdf7f..fcd9ab81d 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -135,6 +135,7 @@ def dyn_parallel_tasks(): tasks=( create_peak_load_table, create_hp_capacity_table, + create_egon_etrago_timeseries_individual_heating, # delete_peak_loads_if_existing, {*dyn_parallel_tasks()}, ), @@ -147,7 +148,7 @@ def __init__(self, dependencies): name="HeatPumps2050", version="0.0.0", dependencies=dependencies, - tasks=(determine_hp_cap_buildings_eGon100RE,), + tasks=(determine_hp_cap_buildings_eGon100RE), ) @@ -1180,7 +1181,9 @@ def desaggregate_hp_capacity(min_hp_cap_per_building, hp_cap_mv_grid): return hp_cap_per_building -def determine_min_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): +def determine_min_hp_cap_buildings_pypsa_eur_sec( + peak_heat_demand, building_ids +): """ Determines minimum required HP capacity in MV grid in MW as input for pypsa-eur-sec. @@ -1211,7 +1214,7 @@ def determine_min_hp_cap_pypsa_eur_sec(peak_heat_demand, building_ids): return 0.0 -def determine_hp_cap_buildings_eGon2035( +def determine_hp_cap_buildings_eGon2035_per_mvgd( mv_grid_id, peak_heat_demand, building_ids ): """ @@ -1259,7 +1262,7 @@ def determine_hp_cap_buildings_eGon2035( return pd.Series().rename("hp_capacity") -def determine_hp_cap_buildings_eGon100RE(mv_grid_id): +def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): """ Main function to determine HP capacity per building in eGon100RE scenario. @@ -1291,10 +1294,46 @@ def determine_hp_cap_buildings_eGon100RE(mv_grid_id): min_hp_cap_buildings, hp_cap_grid ) - # ToDo Julian Write desaggregated HP capacity to table (same as for - # 2035 scenario) check columns + return hp_cap_per_building.rename("hp_capacity") + + +def determine_hp_cap_buildings_eGon100RE(): + """""" + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + df_hp_cap_per_building_100RE_db = pd.DataFrame() + + for mvgd_id in mvgd_ids: + + hp_cap_per_building_100RE = ( + determine_hp_cap_buildings_eGon100RE_per_mvgd(mvgd_id) + ) + + df_hp_cap_per_building_100RE_db = pd.concat( + [ + df_hp_cap_per_building_100RE_db, + hp_cap_per_building_100RE.reset_index(), + ], + axis=0, + ) + + df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" + write_table_to_postgres( - hp_cap_per_building, + df_hp_cap_per_building_100RE_db, EgonHpCapacityBuildings, engine=engine, drop=False, @@ -1358,29 +1397,6 @@ def aggregate_residential_and_cts_profiles(mvgd): return df_heat_ts_2035, df_heat_ts_100RE -def determine_hp_capacity(mvgd, df_peak_loads, buildings_decentral_heating): - """""" - - # determine HP capacity per building for NEP2035 scenario - hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( - mvgd, - df_peak_loads["eGon2035"], - buildings_decentral_heating["eGon2035"], - ) - - # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( - df_peak_loads["eGon100RE"], - buildings_decentral_heating["eGon100RE"] - # TODO 100RE? - ) - - return ( - hp_cap_per_building_2035.rename("hp_capacity"), - hp_min_cap_mv_grid_pypsa_eur_sec, - ) - - def aggregate_heat_profiles( mvgd, df_heat_ts_2035, @@ -1403,10 +1419,8 @@ def aggregate_heat_profiles( # buildings_decentral_heating["eGon2035"]].sum( # hp_cap_per_building_2035.index, buildings_decentral_heating["eGon2035"].drop(buildings_gas_2035), - ].sum( - axis=1 - ) # TODO davor? buildings_hp_2035 = hp_cap_per_building_2035.index - # TODO nur hp oder auch gas? + ].sum(axis=1) + df_mvgd_ts_100RE_hp = df_heat_ts_100RE.loc[ :, buildings_decentral_heating["eGon100RE"] ].sum(axis=1) @@ -1452,11 +1466,7 @@ def export_to_db( ) df_hp_cap_per_building_2035["scenario"] = "eGon2035" - df_hp_cap_per_building_2035 = ( - df_hp_cap_per_building_2035.reset_index().rename( - columns={"index": "building_id"} - ) - ) + write_table_to_postgres( df_hp_cap_per_building_2035, EgonHpCapacityBuildings, @@ -1521,12 +1531,11 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): + f"_{min(mvgd_ids)}-{max(mvgd_ids)}" ) - # TODO mvgd_ids = [kleines mvgd] df_peak_loads_db = pd.DataFrame() df_hp_cap_per_building_2035_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() - for mvgd in mvgd_ids: # [1556]: #mvgd_ids[n - 1]: + for mvgd in mvgd_ids: # [1556] logger.trace(f"MVGD={mvgd} | Start") @@ -1555,17 +1564,20 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): ) # determine HP capacity per building for NEP2035 scenario - hp_cap_per_building_2035 = determine_hp_cap_buildings_eGon2035( - mvgd, - df_peak_loads["eGon2035"], - buildings_decentral_heating["eGon2035"], + hp_cap_per_building_2035 = ( + determine_hp_cap_buildings_eGon2035_per_mvgd( + mvgd, + df_peak_loads["eGon2035"], + buildings_decentral_heating["eGon2035"], + ) ) # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = determine_min_hp_cap_pypsa_eur_sec( - df_peak_loads["eGon100RE"], - buildings_decentral_heating["eGon100RE"] - # TODO 100RE? + hp_min_cap_mv_grid_pypsa_eur_sec = ( + determine_min_hp_cap_buildings_pypsa_eur_sec( + df_peak_loads["eGon100RE"], + buildings_decentral_heating["eGon100RE"], + ) ) buildings_gas_2035 = pd.Index( @@ -1635,7 +1647,6 @@ def create_egon_etrago_timeseries_individual_heating(): def delete_peak_loads_if_existing(): """Remove all entries""" - # TODO check synchronize_session? with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( From 5bb5e7e3e93316dcd1d97c31ea1eec6965af18b9 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 19:18:18 +0200 Subject: [PATCH 042/119] Add rooftop buildings --- .../heat_supply/individual_heating.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index fcd9ab81d..41f2da447 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1068,19 +1068,18 @@ def determine_buildings_with_hp_in_mv_grid( # get buildings with PV to give them a higher priority when selecting # buildings a heat pump will be allocated to saio.register_schema("supply", engine) - # TODO Adhoc Pv rooftop fix - # from saio.supply import egon_power_plants_pv_roof_building - # - # with db.session_scope() as session: - # query = session.query( - # egon_power_plants_pv_roof_building.building_id - # ).filter( - # egon_power_plants_pv_roof_building.building_id.in_(building_ids) - # ) - # - # buildings_with_pv = pd.read_sql( - # query.statement, query.session.bind, index_col=None - # ).building_id.values + from saio.supply import egon_power_plants_pv_roof_building + + with db.session_scope() as session: + query = session.query( + egon_power_plants_pv_roof_building.building_id + ).filter( + egon_power_plants_pv_roof_building.building_id.in_(building_ids) + ) + + buildings_with_pv = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values buildings_with_pv = [] # set different weights for buildings with PV and without PV weight_with_pv = 1.5 @@ -1416,8 +1415,6 @@ def aggregate_heat_profiles( df_mvgd_ts_2035_hp = df_heat_ts_2035.loc[ :, - # buildings_decentral_heating["eGon2035"]].sum( - # hp_cap_per_building_2035.index, buildings_decentral_heating["eGon2035"].drop(buildings_gas_2035), ].sum(axis=1) From 477455f8f3fe7d62f9d18d4408d373024721af39 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 21:05:09 +0200 Subject: [PATCH 043/119] Seperate pypsa-eur-sec and 2035 --- .../heat_supply/individual_heating.py | 440 +++++++++++------- 1 file changed, 275 insertions(+), 165 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 41f2da447..ee254140d 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -65,7 +65,7 @@ class EgonHpCapacityBuildings(Base): hp_capacity = Column(REAL) -class HeatPumpsPypsaEurSecAnd2035(Dataset): +class HeatPumpsPypsaEurSec(Dataset): def __init__(self, dependencies): def dyn_parallel_tasks(): """Dynamically generate tasks @@ -129,14 +129,87 @@ def dyn_parallel_tasks(): return tasks super().__init__( - name="HeatPumpsPypsaEurSecAnd2035", + name="HeatPumpsPypsaEurSec", version="0.0.0", dependencies=dependencies, tasks=( create_peak_load_table, - create_hp_capacity_table, create_egon_etrago_timeseries_individual_heating, - # delete_peak_loads_if_existing, + {*dyn_parallel_tasks()}, + ), + ) + + +class HeatPumps2035(Dataset): + def __init__(self, dependencies): + def dyn_parallel_tasks(): + """Dynamically generate tasks + + The goal is to speed up tasks by parallelising bulks of mvgds. + + The number of parallel tasks is defined via parameter + `parallel_tasks` in the dataset config `datasets.yml`. + + Returns + ------- + set of airflow.PythonOperators + The tasks. Each element is of + :func:`egon.data.datasets.heat_supply.individual_heating. + determine_hp_capacity_eGon2035_pypsa_eur_sec` + """ + parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( + "parallel_tasks", 1 + ) + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split( + mvgd_ids["bus_id"].values, parallel_tasks + ) + + # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] + tasks = set() + for i, bulk in enumerate(mvgd_ids): + tasks.add( + PythonOperator( + task_id=( + f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" + f"mvgd_{min(bulk)}-{max(bulk)}" + ), + python_callable=determine_hp_cap_peak_load_mvgd_ts, + op_kwargs={ + "mvgd_ids": bulk, + }, + ) + ) + return tasks + + super().__init__( + name="HeatPumps2035", + version="0.0.0", + dependencies=dependencies, + tasks=( + create_hp_capacity_table, + delete_peak_loads_if_existing, {*dyn_parallel_tasks()}, ), ) @@ -437,15 +510,17 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): # @timeitlog -def get_peta_demand(mvgd): +def get_peta_demand(mvgd, scenario): """ - Retrieve annual peta heat demand for residential buildings and both - scenarios. + Retrieve annual peta heat demand for residential buildings and for either + eGon2035 or eGon100RE scenario. Parameters ---------- mvgd : int - ID of MVGD + MV grid ID. + scenario : str + Possible options are eGon2035 or eGon100RE Returns ------- @@ -457,7 +532,7 @@ def get_peta_demand(mvgd): query = ( session.query( MapZensusGridDistricts.zensus_population_id, - EgonPetaHeat.scenario, + # EgonPetaHeat.scenario, EgonPetaHeat.demand, ) .filter(MapZensusGridDistricts.bus_id == mvgd) @@ -465,15 +540,19 @@ def get_peta_demand(mvgd): MapZensusGridDistricts.zensus_population_id == EgonPetaHeat.zensus_population_id ) - .filter(EgonPetaHeat.sector == "residential") + .filter( + EgonPetaHeat.sector == "residential", + EgonPetaHeat.scenario == scenario, + ) ) df_peta_demand = pd.read_sql( query.statement, query.session.bind, index_col=None ) - df_peta_demand = df_peta_demand.pivot( - index="zensus_population_id", columns="scenario", values="demand" - ).reset_index() + + # df_peta_demand = df_peta_demand.pivot( + # index="zensus_population_id", columns="scenario", values="demand" + # ).reset_index() return df_peta_demand @@ -597,15 +676,17 @@ def get_daily_demand_share(mvgd): @timeitlog -def calc_residential_heat_profiles_per_mvgd(mvgd): +def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): """ - Gets residential heat profiles per building in MV grid for both eGon2035 - and eGon100RE scenario. + Gets residential heat profiles per building in MV grid for either eGon2035 + or eGon100RE scenario. Parameters ---------- mvgd : int MV grid ID. + scenario : str + Possible options are eGon2035 or eGon100RE Returns -------- @@ -619,16 +700,21 @@ def calc_residential_heat_profiles_per_mvgd(mvgd): Day of the year (1 - 365). * hour : int Hour of the day (1 - 24). - * eGon2035 : float - Building's residential heat demand in MW, for specified hour - of the year (specified through columns `day_of_year` and - `hour`). - * eGon100RE : float + * demand_ts : float Building's residential heat demand in MW, for specified hour of the year (specified through columns `day_of_year` and `hour`). """ - df_peta_demand = get_peta_demand(mvgd) + # * eGon2035 : float + # Building's residential heat demand in MW, for specified hour + # of the year (specified through columns `day_of_year` and + # `hour`). + # * eGon100RE : float + # Building's residential heat demand in MW, for specified hour + # of the year (specified through columns `day_of_year` and + # `hour`). + + df_peta_demand = get_peta_demand(mvgd, scenario) # TODO maybe return empty dataframe if df_peta_demand.empty: @@ -668,27 +754,28 @@ def calc_residential_heat_profiles_per_mvgd(mvgd): ) # Scale profiles - df_profile_merge["eGon2035"] = ( + df_profile_merge["demand_ts"] = ( df_profile_merge["idp"] .mul(df_profile_merge["daily_demand_share"]) - .mul(df_profile_merge["eGon2035"]) + .mul(df_profile_merge["demand"]) .div(df_profile_merge["buildings"]) ) - df_profile_merge["eGon100RE"] = ( - df_profile_merge["idp"] - .mul(df_profile_merge["daily_demand_share"]) - .mul(df_profile_merge["eGon100RE"]) - .div(df_profile_merge["buildings"]) - ) + # df_profile_merge["eGon100RE"] = ( + # df_profile_merge["idp"] + # .mul(df_profile_merge["daily_demand_share"]) + # .mul(df_profile_merge["eGon100RE"]) + # .div(df_profile_merge["buildings"]) + # ) columns = [ "zensus_population_id", "building_id", "day_of_year", "hour", - "eGon2035", - "eGon100RE", + # "eGon2035", + # "eGon100RE", + "demand_ts", ] return df_profile_merge.loc[:, columns] @@ -893,52 +980,24 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( return pd.Index(buildings_with_heat_demand) -def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd): +def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd, scenario): """""" # get residential buildings with decentral heating systems - # scenario eGon2035 - buildings_decentral_heating_2035_res = ( + buildings_decentral_heating_res = ( get_residential_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon2035", mvgd - ) - ) - # scenario eGon100RE - buildings_decentral_heating_100RE_res = ( - get_residential_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mvgd + scenario, mvgd ) ) # get CTS buildings with decentral heating systems - # scenario eGon2035 - buildings_decentral_heating_2035_cts = ( - get_cts_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon2035", mvgd - ) - ) - # scenario eGon100RE - buildings_decentral_heating_100RE_cts = ( - get_cts_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mvgd - ) + buildings_decentral_heating_cts = ( + get_cts_buildings_with_decentral_heat_demand_in_mv_grid(scenario, mvgd) ) # merge residential and CTS buildings - buildings_decentral_heating_2035 = ( - buildings_decentral_heating_2035_res.append( - buildings_decentral_heating_2035_cts - ).unique() - ) - buildings_decentral_heating_100RE = ( - buildings_decentral_heating_100RE_res.append( - buildings_decentral_heating_100RE_cts - ).unique() - ) - - buildings_decentral_heating = { - "eGon2035": buildings_decentral_heating_2035, - "eGon100RE": buildings_decentral_heating_100RE, - } + buildings_decentral_heating = buildings_decentral_heating_res.append( + buildings_decentral_heating_cts + ).unique() return buildings_decentral_heating @@ -1272,7 +1331,7 @@ def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - "eGon100RE", mv_grid_id + mv_grid_id, "eGon100RE" ) # TODO get peak demand from db @@ -1339,69 +1398,68 @@ def determine_hp_cap_buildings_eGon100RE(): ) -def aggregate_residential_and_cts_profiles(mvgd): +def aggregate_residential_and_cts_profiles(mvgd, scenario): """ """ # ############### get residential heat demand profiles ############### - df_heat_ts = calc_residential_heat_profiles_per_mvgd(mvgd=mvgd) - - # pivot to allow aggregation with CTS profiles - df_heat_ts_2035 = df_heat_ts.loc[ - :, ["building_id", "day_of_year", "hour", "eGon2035"] - ] - df_heat_ts_2035 = df_heat_ts_2035.pivot( - index=["day_of_year", "hour"], - columns="building_id", - values="eGon2035", + df_heat_ts = calc_residential_heat_profiles_per_mvgd( + mvgd=mvgd, scenario=scenario ) - df_heat_ts_2035 = df_heat_ts_2035.sort_index().reset_index(drop=True) - df_heat_ts_100RE = df_heat_ts.loc[ - :, ["building_id", "day_of_year", "hour", "eGon100RE"] - ] - df_heat_ts_100RE = df_heat_ts_100RE.pivot( + # pivot to allow aggregation with CTS profiles + # df_heat_ts = df_heat_ts.loc[ + # :, ["building_id", "day_of_year", "hour", "eGon2035"] + # ] + df_heat_ts = df_heat_ts.pivot( index=["day_of_year", "hour"], columns="building_id", - values="eGon100RE", + values="demand_ts", ) - df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) - - del df_heat_ts + df_heat_ts = df_heat_ts.sort_index().reset_index(drop=True) + + # df_heat_ts_100RE = df_heat_ts.loc[ + # :, ["building_id", "day_of_year", "hour", "eGon100RE"] + # ] + # df_heat_ts_100RE = df_heat_ts_100RE.pivot( + # index=["day_of_year", "hour"], + # columns="building_id", + # values="eGon100RE", + # ) + # df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) + # + # del df_heat_ts # ############### get CTS heat demand profiles ############### - heat_demand_cts_ts_2035 = calc_cts_building_profiles( + heat_demand_cts_ts = calc_cts_building_profiles( bus_ids=[mvgd], - scenario="eGon2035", - sector="heat", - ) - heat_demand_cts_ts_100RE = calc_cts_building_profiles( - bus_ids=[mvgd], - scenario="eGon100RE", + scenario=scenario, sector="heat", ) + # heat_demand_cts_ts_100RE = calc_cts_building_profiles( + # bus_ids=[mvgd], + # scenario="eGon100RE", + # sector="heat", + # ) # ############# aggregate residential and CTS demand profiles ############# - df_heat_ts_2035 = pd.concat( - [df_heat_ts_2035, heat_demand_cts_ts_2035], axis=1 - ) + df_heat_ts = pd.concat([df_heat_ts, heat_demand_cts_ts], axis=1) - df_heat_ts_2035 = df_heat_ts_2035.groupby(axis=1, level=0).sum() + df_heat_ts = df_heat_ts.groupby(axis=1, level=0).sum() - df_heat_ts_100RE = pd.concat( - [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 - ) - df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() + # df_heat_ts_100RE = pd.concat( + # [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 + # ) + # df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() # del heat_demand_cts_ts_2035, heat_demand_cts_ts_100RE - return df_heat_ts_2035, df_heat_ts_100RE + # return df_heat_ts_2035, df_heat_ts_100RE + return df_heat_ts def aggregate_heat_profiles( mvgd, - df_heat_ts_2035, - df_heat_ts_100RE, - buildings_decentral_heating, - buildings_gas_2035, + df_heat_ts, + buildings_hp, ): """""" @@ -1413,13 +1471,9 @@ def aggregate_heat_profiles( # but will not need it anymore for eTraGo # EgonEtragoTimeseriesIndividualHeating - df_mvgd_ts_2035_hp = df_heat_ts_2035.loc[ + df_mvgd_ts_hp = df_heat_ts.loc[ :, - buildings_decentral_heating["eGon2035"].drop(buildings_gas_2035), - ].sum(axis=1) - - df_mvgd_ts_100RE_hp = df_heat_ts_100RE.loc[ - :, buildings_decentral_heating["eGon100RE"] + buildings_decentral_heating.drop(buildings_gas_2035), ].sum(axis=1) # heat demand time series for buildings with gas boiler @@ -1443,12 +1497,10 @@ def aggregate_heat_profiles( return df_heat_mvgd_ts -def export_to_db( - df_peak_loads_db, df_hp_cap_per_building_2035, df_heat_mvgd_ts_db -): +def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): """""" - df_peak_loads_db = df_peak_loads_db.reset_index().melt( + df_peak_loads_db = df_peak_loads_db.melt( id_vars="building_id", var_name="scenario", value_name="peak_load_in_w", @@ -1462,15 +1514,6 @@ def export_to_db( df_peak_loads_db, BuildingHeatPeakLoads, engine=engine ) - df_hp_cap_per_building_2035["scenario"] = "eGon2035" - - write_table_to_postgres( - df_hp_cap_per_building_2035, - EgonHpCapacityBuildings, - engine=engine, - drop=False, - ) - columns = { column.key: column.type for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns @@ -1489,6 +1532,14 @@ def export_to_db( def export_to_csv(df_hp_cap_per_building_2035): + + df_hp_min_cap_mv_grid_pypsa_eur_sec.index.name = "mvgd_id" + df_hp_min_cap_mv_grid_pypsa_eur_sec = ( + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_frame( + name="min_hp_capacity" + ).reset_index() + ) + folder = Path(".") / "input-pypsa-eur-sec" file = folder / "minimum_hp_capacity_mv_grid_2035.csv" # Create the folder, if it does not exists already @@ -1503,7 +1554,7 @@ def export_to_csv(df_hp_cap_per_building_2035): @timeitlog -def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): +def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): """ Main function to determine HP capacity per building in eGon2035 scenario and minimum required HP capacity in MV for pypsa-eur-sec. @@ -1538,65 +1589,66 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): # ############# aggregate residential and CTS demand profiles ##### - ( - df_heat_ts_2035, - df_heat_ts_100RE, - ) = aggregate_residential_and_cts_profiles(mvgd) + df_heat_ts = aggregate_residential_and_cts_profiles(mvgd, scenario) # ##################### determine peak loads ################### logger.debug(f"MVGD={mvgd} | Determine peak loads.") - df_peak_loads = pd.concat( - [ - df_heat_ts_2035.max().rename("eGon2035"), - df_heat_ts_100RE.max().rename("eGon100RE"), - ], - axis=1, - ) + + peak_load = df_heat_ts.max().rename("eGon2035") # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ### logger.debug(f"MVGD={mvgd} | Determine HP capacities.") buildings_decentral_heating = ( - get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd) + get_buildings_with_decentral_heat_demand_in_mv_grid( + mvgd, "eGon2035" + ) ) # determine HP capacity per building for NEP2035 scenario hp_cap_per_building_2035 = ( determine_hp_cap_buildings_eGon2035_per_mvgd( mvgd, - df_peak_loads["eGon2035"], - buildings_decentral_heating["eGon2035"], + peak_load, + buildings_decentral_heating, ) ) - # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = ( - determine_min_hp_cap_buildings_pypsa_eur_sec( - df_peak_loads["eGon100RE"], - buildings_decentral_heating["eGon100RE"], - ) + buildings_gas_2035 = pd.Index(buildings_decentral_heating).drop( + hp_cap_per_building_2035.index ) - buildings_gas_2035 = pd.Index( - buildings_decentral_heating["eGon2035"] - ).drop(hp_cap_per_building_2035.index) - # ################ aggregated heat profiles ################### logger.debug(f"MVGD={mvgd} | Aggregate heat profiles.") - df_heat_mvgd_ts = aggregate_heat_profiles( - mvgd, - df_heat_ts_2035, - df_heat_ts_100RE, - buildings_decentral_heating, - buildings_gas_2035, + df_mvgd_ts_2035_hp = df_heat_ts.loc[ + :, + buildings_decentral_heating.drop(buildings_gas_2035), + ].sum(axis=1) + + # heat demand time series for buildings with gas boiler + # (only 2035 scenario) + df_mvgd_ts_2035_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( + axis=1 + ) + + df_heat_mvgd_ts = pd.DataFrame( + data={ + "carrier": ["heat_pump", "CH4"], + "bus_id": mvgd, + "scenario": ["eGon2035", "eGon2035"], + "dist_aggregated_mw": [ + df_mvgd_ts_2035_hp.to_list(), + df_mvgd_ts_2035_gas.to_list(), + ], + } ) # ################ collect results logger.debug(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( - [df_peak_loads_db, df_peak_loads.reset_index()], + [df_peak_loads_db, peak_load.reset_index()], axis=0, ignore_index=True, ) @@ -1612,11 +1664,70 @@ def determine_hp_cap_peak_load_mvgd_ts(mvgd_ids): ) # ################ export to db logger.debug(" Write data to db.") - export_to_db( - df_peak_loads_db, df_hp_cap_per_building_2035_db, df_heat_mvgd_ts_db + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) + df_hp_cap_per_building_2035["scenario"] = "eGon2035" + + write_table_to_postgres( + df_hp_cap_per_building_2035, + EgonHpCapacityBuildings, + engine=engine, + drop=False, ) + + +def determine_pypsa_eur_sec_min_hp_cap(mvgd_ids): + """""" + for mvgd in mvgd_ids: # [1556] + + df_heat_ts = aggregate_residential_and_cts_profiles(mvgd, scenario) + + peak_load_100RE = df_heat_ts_100RE.max().rename("eGon100RE") + + buildings_decentral_heating = ( + get_buildings_with_decentral_heat_demand_in_mv_grid( + mvgd, "eGon100RE" + ) + ) + + df_mvgd_ts_hp = df_heat_ts.loc[ + :, + buildings_decentral_heating.drop(buildings_gas_2035), + ].sum(axis=1) + + df_heat_mvgd_ts = pd.DataFrame( + data={ + "carrier": "heat_pump", + "bus_id": mvgd, + "scenario": "eGon100RE", + "dist_aggregated_mw": [df_mvgd_ts_hp.to_list()], + } + ) + + # determine minimum HP capacity per building for pypsa-eur-sec + hp_min_cap_mv_grid_pypsa_eur_sec = ( + determine_min_hp_cap_buildings_pypsa_eur_sec( + peak_load_100RE, + buildings_decentral_heating, + ) + ) + df_hp_min_cap_mv_grid_pypsa_eur_sec[ + mvgd + ] = hp_min_cap_mv_grid_pypsa_eur_sec + + df_peak_loads_db = pd.concat( + [df_peak_loads_db, peak_load_100RE.reset_index()], + axis=0, + ignore_index=True, + ) + + df_heat_mvgd_ts_db = pd.concat( + [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True + ) + + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) logger.debug(" Write pypsa-eur-sec min HP capacities to csv.") - export_to_csv(hp_min_cap_mv_grid_pypsa_eur_sec) + + export_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) def create_peak_load_table(): @@ -1643,9 +1754,8 @@ def create_egon_etrago_timeseries_individual_heating(): def delete_peak_loads_if_existing(): """Remove all entries""" - with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( - BuildingHeatPeakLoads.sector == "residential+cts" + BuildingHeatPeakLoads.scenario == "eGon2035" ).delete(synchronize_session=False) From 8db867f34c83cc0c63c20e4cd095f7b004300c42 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 11 Oct 2022 21:05:40 +0200 Subject: [PATCH 044/119] Try pipline fix --- src/egon/data/airflow/dags/pipeline.py | 31 +++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 90df19b9c..b2182d06c 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -51,7 +51,8 @@ from egon.data.datasets.heat_supply import HeatSupply from egon.data.datasets.heat_supply.individual_heating import ( HeatPumps2050, - HeatPumpsPypsaEurSecAnd2035, + HeatPumpsPypsaEurSec, + HeatPumps2035, ) from egon.data.datasets.hydrogen_etrago import ( HydrogenBusEtrago, @@ -343,6 +344,7 @@ data_bundle, electrical_load_etrago, heat_time_series, + heat_pumps_2035, ] ) @@ -591,25 +593,34 @@ tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"], ] ) + heat_pumps_pypsa_eur_sec = HeatPumpsPypsaEurSec( + dependencies=[ + cts_demand_buildings, + DistrictHeatingAreas, + heat_supply, + heat_time_series, + # TODO add PV rooftop + ] + ) - # heat_pumps_2050 = HeatPumps2050( - # dependencies=[ - # cts_demand_buildings, - # DistrictHeatingAreas, - # run_pypsaeursec, - # ] - # ) - - heat_pumps_2035 = HeatPumpsPypsaEurSecAnd2035( + heat_pumps_2035 = HeatPumps2035( dependencies=[ cts_demand_buildings, DistrictHeatingAreas, heat_supply, heat_time_series, + heat_pumps_pypsa_eur_sec # TODO add PV rooftop ] ) + heat_pumps_2050 = HeatPumps2050( + dependencies=[ + run_pypsaeursec, + heat_pumps_2035, + ] + ) + # ########## Keep this dataset at the end # Sanity Checks sanity_checks = SanityChecks( From ce580f5d748b9ca7896e7ae4c478419b70fbb0fb Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 00:58:45 +0200 Subject: [PATCH 045/119] Fix dyn task creation --- .../data/datasets/heat_supply/individual_heating.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index ee254140d..66cbbe2b6 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -117,10 +117,10 @@ def dyn_parallel_tasks(): tasks.add( PythonOperator( task_id=( - f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" + f"determine-hp-capacity-pypsa-eur-sec_" f"mvgd_{min(bulk)}-{max(bulk)}" ), - python_callable=determine_hp_cap_peak_load_mvgd_ts, + python_callable=determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec, op_kwargs={ "mvgd_ids": bulk, }, @@ -192,10 +192,10 @@ def dyn_parallel_tasks(): tasks.add( PythonOperator( task_id=( - f"determine-hp-capacity-eGon2035-pypsa-eur-sec_" + f"determine-hp-capacity-eGon2035_" f"mvgd_{min(bulk)}-{max(bulk)}" ), - python_callable=determine_hp_cap_peak_load_mvgd_ts, + python_callable=determine_hp_cap_peak_load_mvgd_ts_2035, op_kwargs={ "mvgd_ids": bulk, }, @@ -1675,7 +1675,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) -def determine_pypsa_eur_sec_min_hp_cap(mvgd_ids): +def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): """""" for mvgd in mvgd_ids: # [1556] From d77dd43e08c5c5ed90cc8f1c0f8c30ecc943a6db Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:04:29 +0200 Subject: [PATCH 046/119] Remove timeitlog --- .../heat_supply/individual_heating.py | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 66cbbe2b6..bb53aaa17 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -47,7 +47,6 @@ Base = declarative_base() -# TODO check column names> class EgonEtragoTimeseriesIndividualHeating(Base): __tablename__ = "egon_etrago_timeseries_individual_heating" __table_args__ = {"schema": "demand"} @@ -243,22 +242,6 @@ def adapt_numpy_int64(numpy_int64): return AsIs(numpy_int64) -def log_to_file(name): - """Simple only file logger""" - file = os.path.basename(__file__).rstrip(".py") - file_path = Path(f"./{file}_logs") - os.makedirs(file_path, exist_ok=True) - logger.remove() - logger.add( - file_path / Path(f"{name}.log"), - format="{time} {level} {message}", - # filter="my_module", - level="DEBUG", - ) - logger.trace(f"Start logging of: {name}") - return logger - - def timeit(func): """ Decorator for measuring function's running time. @@ -276,31 +259,6 @@ def measure_time(*args, **kw): return measure_time -def timeitlog(func): - """ - Decorator for measuring running time of residential heat peak load and - logging it. - """ - - def measure_time(*args, **kw): - start_time = time.time() - result = func(*args, **kw) - process_time = time.time() - start_time - try: - mvgd = kw["mvgd"] - except KeyError: - mvgd = "bulk" - statement = ( - f"MVGD={mvgd} | Processing time of {func.__qualname__} | " - f"{time.strftime('%H h, %M min, %S s', time.gmtime(process_time))}" - ) - logger.debug(statement) - print(statement) - return result - - return measure_time - - def cascade_per_technology( heat_per_mv, technologies, @@ -509,7 +467,6 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): ) -# @timeitlog def get_peta_demand(mvgd, scenario): """ Retrieve annual peta heat demand for residential buildings and for either @@ -557,7 +514,6 @@ def get_peta_demand(mvgd, scenario): return df_peta_demand -# @timeitlog def get_residential_heat_profile_ids(mvgd): """ Retrieve 365 daily heat profiles ids per residential building and selected @@ -609,7 +565,6 @@ def get_residential_heat_profile_ids(mvgd): return df_profiles_ids -# @timeitlog def get_daily_profiles(profile_ids): """ Parameters @@ -642,7 +597,6 @@ def get_daily_profiles(profile_ids): return df_profiles -# @timeitlog def get_daily_demand_share(mvgd): """per census cell Parameters @@ -675,7 +629,6 @@ def get_daily_demand_share(mvgd): return df_daily_demand_share -@timeitlog def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): """ Gets residential heat profiles per building in MV grid for either eGon2035 @@ -814,7 +767,6 @@ def plot_heat_supply(resulting_capacities): plt.savefig(f"plots/individual_heat_supply_{c}.png", dpi=300) -@timeitlog def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( scenario, mv_grid_id ): @@ -877,7 +829,6 @@ def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( return pd.Index(zensus_population_ids) -@timeitlog def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( scenario, mv_grid_id ): @@ -930,7 +881,6 @@ def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( return pd.Index(buildings_with_heat_demand) -@timeitlog def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( scenario, mv_grid_id ): @@ -1553,7 +1503,6 @@ def export_to_csv(df_hp_cap_per_building_2035): df_hp_cap_per_building_2035.to_csv(file, mode="a", header=False) -@timeitlog def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): """ Main function to determine HP capacity per building in eGon2035 scenario From 2c9af8b939d82e694ed1ea0de4b4fca49d251902 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:08:15 +0200 Subject: [PATCH 047/119] Remove unused function --- .../heat_supply/individual_heating.py | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index bb53aaa17..1e0d4a6ae 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1406,47 +1406,6 @@ def aggregate_residential_and_cts_profiles(mvgd, scenario): return df_heat_ts -def aggregate_heat_profiles( - mvgd, - df_heat_ts, - buildings_hp, -): - """""" - - # heat demand time series for buildings with heat pumps - # ToDo Julian Write aggregated heat demand time series of buildings with - # HP to table to be used in eTraGo - - # egon_etrago_timeseries_individual_heating - # TODO Clara uses this table already - # but will not need it anymore for eTraGo - # EgonEtragoTimeseriesIndividualHeating - - df_mvgd_ts_hp = df_heat_ts.loc[ - :, - buildings_decentral_heating.drop(buildings_gas_2035), - ].sum(axis=1) - - # heat demand time series for buildings with gas boiler - # (only 2035 scenario) - df_mvgd_ts_2035_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( - axis=1 - ) - - df_heat_mvgd_ts = pd.DataFrame( - data={ - "carrier": ["heat_pump", "heat_pump", "CH4"], - "bus_id": mvgd, - "scenario": ["eGon2035", "eGon100RE", "eGon2035"], - "dist_aggregated_mw": [ - df_mvgd_ts_2035_hp.to_list(), - df_mvgd_ts_100RE_hp.to_list(), - df_mvgd_ts_2035_gas.to_list(), - ], - } - ) - return df_heat_mvgd_ts - - def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): """""" From 275ed62b966bb758eaf1b4594f9e95ff5fbff596 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:13:24 +0200 Subject: [PATCH 048/119] Switch to internal logger --- src/egon/data/datasets/heat_supply/individual_heating.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 1e0d4a6ae..c1e97ccee 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -8,7 +8,6 @@ import time from airflow.operators.python_operator import PythonOperator -from loguru import logger from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -17,7 +16,7 @@ import pandas as pd import saio -from egon.data import config, db +from egon.data import config, db, logger from egon.data.datasets import Dataset from egon.data.datasets.district_heating_areas import ( MapZensusDistrictHeatingAreas, @@ -1482,10 +1481,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - log_to_file( - determine_hp_cap_peak_load_mvgd_ts.__qualname__ - + f"_{min(mvgd_ids)}-{max(mvgd_ids)}" - ) df_peak_loads_db = pd.DataFrame() df_hp_cap_per_building_2035_db = pd.DataFrame() @@ -1493,7 +1488,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): for mvgd in mvgd_ids: # [1556] - logger.trace(f"MVGD={mvgd} | Start") + logger.debug(f"MVGD={mvgd} | Start") # ############# aggregate residential and CTS demand profiles ##### From 843e63fc06ec0ef672d74f8c38ffb1d919c0381e Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:15:37 +0200 Subject: [PATCH 049/119] Fix export min cap --- src/egon/data/datasets/heat_supply/individual_heating.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index c1e97ccee..5e620eac3 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1439,7 +1439,7 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): ) -def export_to_csv(df_hp_cap_per_building_2035): +def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): df_hp_min_cap_mv_grid_pypsa_eur_sec.index.name = "mvgd_id" df_hp_min_cap_mv_grid_pypsa_eur_sec = ( @@ -1455,10 +1455,10 @@ def export_to_csv(df_hp_cap_per_building_2035): os.mkdir(folder) # TODO check append if not file.is_file(): - df_hp_cap_per_building_2035.to_csv(file) + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file) # TODO outsource into separate task incl delete file if clearing else: - df_hp_cap_per_building_2035.to_csv(file, mode="a", header=False) + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file, mode="a", header=False) def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): @@ -1481,7 +1481,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - df_peak_loads_db = pd.DataFrame() df_hp_cap_per_building_2035_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() @@ -1630,7 +1629,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) logger.debug(" Write pypsa-eur-sec min HP capacities to csv.") - export_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) + export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) def create_peak_load_table(): From 70e4b6be8933b08a25deceeb49ee69554efed28f Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:20:50 +0200 Subject: [PATCH 050/119] Fix scenario typos --- .../datasets/heat_supply/individual_heating.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 5e620eac3..8a9ff3922 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1280,7 +1280,7 @@ def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): # determine minimum required heat pump capacity per building building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - mv_grid_id, "eGon100RE" + mv_grid_id, scenario="eGon100RE" ) # TODO get peak demand from db @@ -1491,7 +1491,9 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # ############# aggregate residential and CTS demand profiles ##### - df_heat_ts = aggregate_residential_and_cts_profiles(mvgd, scenario) + df_heat_ts = aggregate_residential_and_cts_profiles( + mvgd, scenario="eGon2035" + ) # ##################### determine peak loads ################### logger.debug(f"MVGD={mvgd} | Determine peak loads.") @@ -1503,7 +1505,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( - mvgd, "eGon2035" + mvgd, scenario="eGon2035" ) ) @@ -1581,13 +1583,15 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): """""" for mvgd in mvgd_ids: # [1556] - df_heat_ts = aggregate_residential_and_cts_profiles(mvgd, scenario) + df_heat_ts = aggregate_residential_and_cts_profiles( + mvgd, scenario="eGon100RE" + ) peak_load_100RE = df_heat_ts_100RE.max().rename("eGon100RE") buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( - mvgd, "eGon100RE" + mvgd, scenario="eGon100RE" ) ) From d35935d3c280fbf907e22441af41bd69d0917fde Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:41:07 +0200 Subject: [PATCH 051/119] Fix typos --- .../datasets/heat_supply/individual_heating.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 8a9ff3922..e3078d09f 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1458,7 +1458,9 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file) # TODO outsource into separate task incl delete file if clearing else: - df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file, mode="a", header=False) + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( + file, mode="a", header=False + ) def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): @@ -1532,7 +1534,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # heat demand time series for buildings with gas boiler # (only 2035 scenario) - df_mvgd_ts_2035_gas = df_heat_ts_2035.loc[:, buildings_gas_2035].sum( + df_mvgd_ts_2035_gas = df_heat_ts.loc[:, buildings_gas_2035].sum( axis=1 ) @@ -1569,10 +1571,10 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # ################ export to db logger.debug(" Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) - df_hp_cap_per_building_2035["scenario"] = "eGon2035" + df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" write_table_to_postgres( - df_hp_cap_per_building_2035, + df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, engine=engine, drop=False, @@ -1581,13 +1583,16 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): """""" + + df_peak_loads_db = pd.DataFrame() + df_heat_mvgd_ts_db = pd.DataFrame() for mvgd in mvgd_ids: # [1556] df_heat_ts = aggregate_residential_and_cts_profiles( mvgd, scenario="eGon100RE" ) - peak_load_100RE = df_heat_ts_100RE.max().rename("eGon100RE") + peak_load_100RE = df_heat_ts.max().rename("eGon100RE") buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( @@ -1597,7 +1602,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): df_mvgd_ts_hp = df_heat_ts.loc[ :, - buildings_decentral_heating.drop(buildings_gas_2035), + hp_cap_per_building_2035.index, ].sum(axis=1) df_heat_mvgd_ts = pd.DataFrame( From f59090e0d056e8b5ee514cc1ab940562d3542d2f Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:46:58 +0200 Subject: [PATCH 052/119] Fix typos --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index e3078d09f..3a75f5319 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1534,9 +1534,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # heat demand time series for buildings with gas boiler # (only 2035 scenario) - df_mvgd_ts_2035_gas = df_heat_ts.loc[:, buildings_gas_2035].sum( - axis=1 - ) + df_mvgd_ts_2035_gas = df_heat_ts.loc[:, buildings_gas_2035].sum(axis=1) df_heat_mvgd_ts = pd.DataFrame( data={ @@ -1586,6 +1584,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): df_peak_loads_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() + df_hp_min_cap_mv_grid_pypsa_eur_sec = pd.Series() for mvgd in mvgd_ids: # [1556] df_heat_ts = aggregate_residential_and_cts_profiles( @@ -1621,6 +1620,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): buildings_decentral_heating, ) ) + df_hp_min_cap_mv_grid_pypsa_eur_sec[ mvgd ] = hp_min_cap_mv_grid_pypsa_eur_sec From 0ec02b3358eadea8fd75d9c021083ab851245313 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:47:20 +0200 Subject: [PATCH 053/119] Add todo note --- .../data/datasets/heat_supply/individual_heating.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 3a75f5319..19a5bb46d 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1599,6 +1599,16 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) ) + # TODO birgit: wo bekommen wir die Gebäude mit hp her? + # hp_cap_per_building_2035 nochmal berechnen? reproducable? + # hp_cap_per_building_2035 = ( + # determine_hp_cap_buildings_eGon2035_per_mvgd( + # mvgd, + # peak_load, + # buildings_decentral_heating, + # ) + # ) + df_mvgd_ts_hp = df_heat_ts.loc[ :, hp_cap_per_building_2035.index, From 7fbf360d9b25f4785770a2c2bceef1f7e6abea82 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 01:48:54 +0200 Subject: [PATCH 054/119] Add debugigng --- .../data/datasets/heat_supply/individual_heating.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 19a5bb46d..7195e0e43 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1680,3 +1680,16 @@ def delete_peak_loads_if_existing(): session.query(BuildingHeatPeakLoads).filter( BuildingHeatPeakLoads.scenario == "eGon2035" ).delete(synchronize_session=False) + + +if __name__ == "__main__": + + determine_hp_cap_buildings_eGon100RE() + + create_peak_load_table() + create_egon_etrago_timeseries_individual_heating() + determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec([1556]) + + create_hp_capacity_table() + delete_peak_loads_if_existing() + determine_hp_cap_peak_load_mvgd_ts_2035([1556]) From 7abf75b1ae3561664920867f5c73eb8a87068ea6 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 12 Oct 2022 11:42:40 +0200 Subject: [PATCH 055/119] Fixes in heat pump disaggregation workflow --- .../heat_supply/individual_heating.py | 360 +++++++++--------- 1 file changed, 190 insertions(+), 170 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 7195e0e43..0543f9814 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -468,7 +468,7 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): def get_peta_demand(mvgd, scenario): """ - Retrieve annual peta heat demand for residential buildings and for either + Retrieve annual peta heat demand for residential buildings for either eGon2035 or eGon100RE scenario. Parameters @@ -481,14 +481,15 @@ def get_peta_demand(mvgd, scenario): Returns ------- df_peta_demand : pd.DataFrame - Annual residential heat demand per building and scenario + Annual residential heat demand per building and scenario. Columns of the + dataframe are zensus_population_id and demand. + """ with db.session_scope() as session: query = ( session.query( MapZensusGridDistricts.zensus_population_id, - # EgonPetaHeat.scenario, EgonPetaHeat.demand, ) .filter(MapZensusGridDistricts.bus_id == mvgd) @@ -506,10 +507,6 @@ def get_peta_demand(mvgd, scenario): query.statement, query.session.bind, index_col=None ) - # df_peta_demand = df_peta_demand.pivot( - # index="zensus_population_id", columns="scenario", values="demand" - # ).reset_index() - return df_peta_demand @@ -526,7 +523,10 @@ def get_residential_heat_profile_ids(mvgd): Returns ------- df_profiles_ids : pd.DataFrame - Residential daily heat profile ID's per building + Residential daily heat profile ID's per building. Columns of the dataframe + are zensus_population_id, building_id, selected_idp_profiles, buildings + and day_of_year. + """ with db.session_scope() as session: query = ( @@ -574,7 +574,9 @@ def get_daily_profiles(profile_ids): Returns ------- df_profiles : pd.DataFrame - Residential daily heat profiles + Residential daily heat profiles. Columns of the dataframe are idp, house, + temperature_class and hour. + """ saio.register_schema("demand", db.engine()) from saio.demand import egon_heat_idp_pool @@ -606,7 +608,9 @@ def get_daily_demand_share(mvgd): Returns ------- df_daily_demand_share : pd.DataFrame - Daily annual demand share per cencus cell + Daily annual demand share per cencus cell. Columns of the dataframe + are zensus_population_id, day_of_year and daily_demand_share. + """ with db.session_scope() as session: @@ -638,7 +642,7 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): mvgd : int MV grid ID. scenario : str - Possible options are eGon2035 or eGon100RE + Possible options are eGon2035 or eGon100RE. Returns -------- @@ -657,27 +661,27 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): of the year (specified through columns `day_of_year` and `hour`). """ - # * eGon2035 : float - # Building's residential heat demand in MW, for specified hour - # of the year (specified through columns `day_of_year` and - # `hour`). - # * eGon100RE : float - # Building's residential heat demand in MW, for specified hour - # of the year (specified through columns `day_of_year` and - # `hour`). + + columns = [ + "zensus_population_id", + "building_id", + "day_of_year", + "hour", + "demand_ts", + ] df_peta_demand = get_peta_demand(mvgd, scenario) # TODO maybe return empty dataframe if df_peta_demand.empty: logger.info(f"No demand for MVGD: {mvgd}") - return None + return pd.DataFrame(columns=columns) df_profiles_ids = get_residential_heat_profile_ids(mvgd) if df_profiles_ids.empty: logger.info(f"No profiles for MVGD: {mvgd}") - return None + return pd.DataFrame(columns=columns) df_profiles = get_daily_profiles( df_profiles_ids["selected_idp_profiles"].unique() @@ -713,23 +717,6 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): .div(df_profile_merge["buildings"]) ) - # df_profile_merge["eGon100RE"] = ( - # df_profile_merge["idp"] - # .mul(df_profile_merge["daily_demand_share"]) - # .mul(df_profile_merge["eGon100RE"]) - # .div(df_profile_merge["buildings"]) - # ) - - columns = [ - "zensus_population_id", - "building_id", - "day_of_year", - "hour", - # "eGon2035", - # "eGon100RE", - "demand_ts", - ] - return df_profile_merge.loc[:, columns] @@ -930,7 +917,28 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd, scenario): - """""" + """ + Returns building IDs of buildings with decentral heat demand in + given MV grid. + + As cells with district heating differ between scenarios, this is also + depending on the scenario. + + Parameters + ----------- + mv_grid_id : int + ID of MV grid. + scenario : str + Name of scenario. Can be either "eGon2035" or "eGon100RE". + + Returns + -------- + pd.Index(int) + Building IDs (as int) of buildings with decentral heating system in given + MV grid. Type is pandas Index to avoid errors later on when it is + used in a query. + + """ # get residential buildings with decentral heating systems buildings_decentral_heating_res = ( get_residential_buildings_with_decentral_heat_demand_in_mv_grid( @@ -971,34 +979,24 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): """ from egon.data.datasets.heat_supply import EgonIndividualHeatingSupply - # - # with db.session_scope() as session: - # query = ( - # session.query( - # EgonIndividualHeatingSupply.mv_grid_id, - # EgonIndividualHeatingSupply.capacity, - # ) - # .filter(EgonIndividualHeatingSupply.scenario == scenario) - # .filter(EgonIndividualHeatingSupply.carrier == "heat_pump") - # .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) - # ) - # - # hp_cap_mv_grid = pd.read_sql( - # query.statement, query.session.bind, index_col="mv_grid_id" - # ).capacity.values[0] - with db.session_scope() as session: - hp_cap_mv_grid = ( - session.execute(EgonIndividualHeatingSupply.capacity) - .filter( - EgonIndividualHeatingSupply.scenario == scenario, - EgonIndividualHeatingSupply.carrier == "heat_pump", - EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id, + query = ( + session.query( + EgonIndividualHeatingSupply.mv_grid_id, + EgonIndividualHeatingSupply.capacity, ) - .scalar() + .filter(EgonIndividualHeatingSupply.scenario == scenario) + .filter(EgonIndividualHeatingSupply.carrier == "heat_pump") + .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) ) - return hp_cap_mv_grid + hp_cap_mv_grid = pd.read_sql( + query.statement, query.session.bind, index_col="mv_grid_id" + ) + if hp_cap_mv_grid.empty: + return 0.0 + else: + return hp_cap_mv_grid.capacity.values[0] def get_heat_peak_demand_per_building(scenario, building_ids): @@ -1021,6 +1019,10 @@ def get_heat_peak_demand_per_building(scenario, building_ids): # TODO remove check if df_heat_peak_demand.duplicated("building_id").any(): raise ValueError("Duplicate building_id") + + # convert to series and from W to MW + df_heat_peak_demand = df_heat_peak_demand.set_index("building_id").loc[ + :, "peak_load_in_w"] * 1e6 return df_heat_peak_demand @@ -1088,7 +1090,6 @@ def determine_buildings_with_hp_in_mv_grid( buildings_with_pv = pd.read_sql( query.statement, query.session.bind, index_col=None ).building_id.values - buildings_with_pv = [] # set different weights for buildings with PV and without PV weight_with_pv = 1.5 weight_without_pv = 1.0 @@ -1242,7 +1243,11 @@ def determine_hp_cap_buildings_eGon2035_per_mvgd( """ - if len(building_ids) > 0: + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( + "eGon2035", mv_grid_id + ) + + if len(building_ids) > 0 and hp_cap_grid > 0.0: peak_heat_demand = peak_heat_demand.loc[building_ids] # determine minimum required heat pump capacity per building @@ -1251,9 +1256,6 @@ def determine_hp_cap_buildings_eGon2035_per_mvgd( ) # select buildings that will have a heat pump - hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( - "eGon2035", mv_grid_id - ) buildings_with_hp = determine_buildings_with_hp_in_mv_grid( hp_cap_grid, min_hp_cap_buildings ) @@ -1271,41 +1273,54 @@ def determine_hp_cap_buildings_eGon2035_per_mvgd( def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): """ - Main function to determine HP capacity per building in eGon100RE scenario. + Determines HP capacity per building in eGon100RE scenario. In eGon100RE scenario all buildings without district heating get a heat pump. + Returns + -------- + pd.Series + Pandas series with heat pump capacity per building in MW. + """ - # determine minimum required heat pump capacity per building - building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( - mv_grid_id, scenario="eGon100RE" + hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( + "eGon100RE", mv_grid_id ) - # TODO get peak demand from db - df_peak_heat_demand = get_heat_peak_demand_per_building( - "eGon100RE", building_ids - ) + if hp_cap_grid > 0.0: - # determine minimum required heat pump capacity per building - min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( - df_peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 - ) + # get buildings with decentral heating systems + building_ids = get_buildings_with_decentral_heat_demand_in_mv_grid( + mv_grid_id, scenario="eGon100RE" + ) - # distribute total heat pump capacity to all buildings with HP - hp_cap_grid = get_total_heat_pump_capacity_of_mv_grid( - "eGon100RE", mv_grid_id - ) - hp_cap_per_building = desaggregate_hp_capacity( - min_hp_cap_buildings, hp_cap_grid - ) + # TODO get peak demand from db + df_peak_heat_demand = get_heat_peak_demand_per_building( + "eGon100RE", building_ids + ) - return hp_cap_per_building.rename("hp_capacity") + # determine minimum required heat pump capacity per building + min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( + df_peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 + ) + + # distribute total heat pump capacity to all buildings with HP + hp_cap_per_building = desaggregate_hp_capacity( + min_hp_cap_buildings, hp_cap_grid + ) + + return hp_cap_per_building.rename("hp_capacity") + else: + return pd.Series().rename("hp_capacity") def determine_hp_cap_buildings_eGon100RE(): - """""" + """ + Main function to determine HP capacity per building in eGon100RE scenario. + + """ with db.session_scope() as session: query = ( @@ -1329,16 +1344,16 @@ def determine_hp_cap_buildings_eGon100RE(): determine_hp_cap_buildings_eGon100RE_per_mvgd(mvgd_id) ) - df_hp_cap_per_building_100RE_db = pd.concat( - [ - df_hp_cap_per_building_100RE_db, - hp_cap_per_building_100RE.reset_index(), - ], - axis=0, - ) + if not hp_cap_per_building_100RE.empty: + df_hp_cap_per_building_100RE_db = pd.concat( + [ + df_hp_cap_per_building_100RE_db, + hp_cap_per_building_100RE.reset_index(), + ], + axis=0, + ) df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" - write_table_to_postgres( df_hp_cap_per_building_100RE_db, EgonHpCapacityBuildings, @@ -1348,16 +1363,29 @@ def determine_hp_cap_buildings_eGon100RE(): def aggregate_residential_and_cts_profiles(mvgd, scenario): - """ """ + """ + Gets residential and CTS heat demand profiles per building and aggregates them. + + Parameters + ---------- + mvgd : int + MV grid ID. + scenario : str + Possible options are eGon2035 or eGon100RE. + + Returns + -------- + pd.DataFrame + Table of demand profile per building. Column names are building IDs and index + is hour of the year as int (0-8759). + + """ # ############### get residential heat demand profiles ############### df_heat_ts = calc_residential_heat_profiles_per_mvgd( mvgd=mvgd, scenario=scenario ) # pivot to allow aggregation with CTS profiles - # df_heat_ts = df_heat_ts.loc[ - # :, ["building_id", "day_of_year", "hour", "eGon2035"] - # ] df_heat_ts = df_heat_ts.pivot( index=["day_of_year", "hour"], columns="building_id", @@ -1365,43 +1393,18 @@ def aggregate_residential_and_cts_profiles(mvgd, scenario): ) df_heat_ts = df_heat_ts.sort_index().reset_index(drop=True) - # df_heat_ts_100RE = df_heat_ts.loc[ - # :, ["building_id", "day_of_year", "hour", "eGon100RE"] - # ] - # df_heat_ts_100RE = df_heat_ts_100RE.pivot( - # index=["day_of_year", "hour"], - # columns="building_id", - # values="eGon100RE", - # ) - # df_heat_ts_100RE = df_heat_ts_100RE.sort_index().reset_index(drop=True) - # - # del df_heat_ts - # ############### get CTS heat demand profiles ############### heat_demand_cts_ts = calc_cts_building_profiles( bus_ids=[mvgd], scenario=scenario, sector="heat", ) - # heat_demand_cts_ts_100RE = calc_cts_building_profiles( - # bus_ids=[mvgd], - # scenario="eGon100RE", - # sector="heat", - # ) # ############# aggregate residential and CTS demand profiles ############# df_heat_ts = pd.concat([df_heat_ts, heat_demand_cts_ts], axis=1) df_heat_ts = df_heat_ts.groupby(axis=1, level=0).sum() - # df_heat_ts_100RE = pd.concat( - # [df_heat_ts_100RE, heat_demand_cts_ts_100RE], axis=1 - # ) - # df_heat_ts_100RE = df_heat_ts_100RE.groupby(axis=1, level=0).sum() - - # del heat_demand_cts_ts_2035, heat_demand_cts_ts_100RE - - # return df_heat_ts_2035, df_heat_ts_100RE return df_heat_ts @@ -1450,7 +1453,7 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): folder = Path(".") / "input-pypsa-eur-sec" file = folder / "minimum_hp_capacity_mv_grid_2035.csv" - # Create the folder, if it does not exists already + # Create the folder, if it does not exist already if not os.path.exists(folder): os.mkdir(folder) # TODO check append @@ -1465,16 +1468,14 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): """ - Main function to determine HP capacity per building in eGon2035 scenario - and minimum required HP capacity in MV for pypsa-eur-sec. + Main function to determine HP capacity per building in eGon2035 scenario. Further, creates heat demand time series for all buildings with heat pumps - (in eGon2035 and eGon100RE scenario) in MV grid, as well as for all - buildings with gas boilers (only in eGon2035scenario), used in eTraGo. + in MV grid, as well as for all buildings with gas boilers, used in eTraGo. Parameters ----------- - bulk: list(int) - List of numbers of mvgds + mvgd_ids : list(int) + List of MV grid IDs to determine data for. """ @@ -1487,7 +1488,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_hp_cap_per_building_2035_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() - for mvgd in mvgd_ids: # [1556] + for mvgd in mvgd_ids: logger.debug(f"MVGD={mvgd} | Start") @@ -1502,7 +1503,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): peak_load = df_heat_ts.max().rename("eGon2035") - # ######## determine HP capacity for NEP scenario and pypsa-eur-sec ### + # ######## determine HP capacity per building ######### logger.debug(f"MVGD={mvgd} | Determine HP capacities.") buildings_decentral_heating = ( @@ -1510,8 +1511,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): mvgd, scenario="eGon2035" ) ) - - # determine HP capacity per building for NEP2035 scenario hp_cap_per_building_2035 = ( determine_hp_cap_buildings_eGon2035_per_mvgd( mvgd, @@ -1519,7 +1518,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): buildings_decentral_heating, ) ) - buildings_gas_2035 = pd.Index(buildings_decentral_heating).drop( hp_cap_per_building_2035.index ) @@ -1529,11 +1527,10 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_mvgd_ts_2035_hp = df_heat_ts.loc[ :, - buildings_decentral_heating.drop(buildings_gas_2035), + hp_cap_per_building_2035.index, ].sum(axis=1) # heat demand time series for buildings with gas boiler - # (only 2035 scenario) df_mvgd_ts_2035_gas = df_heat_ts.loc[:, buildings_gas_2035].sum(axis=1) df_heat_mvgd_ts = pd.DataFrame( @@ -1548,7 +1545,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): } ) - # ################ collect results + # ################ collect results ################## logger.debug(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( @@ -1556,6 +1553,11 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): axis=0, ignore_index=True, ) + + df_heat_mvgd_ts_db = pd.concat( + [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True + ) + df_hp_cap_per_building_2035_db = pd.concat( [ df_hp_cap_per_building_2035_db, @@ -1563,14 +1565,12 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ], axis=0, ) - df_heat_mvgd_ts_db = pd.concat( - [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True - ) - # ################ export to db + + # ################ export to db ####################### logger.debug(" Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) - df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" + df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" write_table_to_postgres( df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, @@ -1580,38 +1580,61 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): - """""" + """ + Main function to determine minimum required HP capacity in MV for pypsa-eur-sec. + Further, creates heat demand time series for all buildings with heat pumps in MV + grid in eGon100RE scenario, used in eTraGo. + + Parameters + ----------- + mvgd_ids : list(int) + List of MV grid IDs to determine data for. + + """ + + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== df_peak_loads_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() df_hp_min_cap_mv_grid_pypsa_eur_sec = pd.Series() - for mvgd in mvgd_ids: # [1556] + + for mvgd in mvgd_ids: + + logger.debug(f"MVGD={mvgd} | Start") + + # ############# aggregate residential and CTS demand profiles ##### df_heat_ts = aggregate_residential_and_cts_profiles( mvgd, scenario="eGon100RE" ) + # ##################### determine peak loads ################### + logger.debug(f"MVGD={mvgd} | Determine peak loads.") + peak_load_100RE = df_heat_ts.max().rename("eGon100RE") + # ######## determine minimum HP capacity pypsa-eur-sec ########### buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( mvgd, scenario="eGon100RE" ) ) + hp_min_cap_mv_grid_pypsa_eur_sec = ( + determine_min_hp_cap_buildings_pypsa_eur_sec( + peak_load_100RE, + buildings_decentral_heating, + ) + ) - # TODO birgit: wo bekommen wir die Gebäude mit hp her? - # hp_cap_per_building_2035 nochmal berechnen? reproducable? - # hp_cap_per_building_2035 = ( - # determine_hp_cap_buildings_eGon2035_per_mvgd( - # mvgd, - # peak_load, - # buildings_decentral_heating, - # ) - # ) + # ################ aggregated heat profiles ################### + logger.debug(f"MVGD={mvgd} | Aggregate heat profiles.") df_mvgd_ts_hp = df_heat_ts.loc[ :, - hp_cap_per_building_2035.index, + buildings_decentral_heating, ].sum(axis=1) df_heat_mvgd_ts = pd.DataFrame( @@ -1623,17 +1646,8 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): } ) - # determine minimum HP capacity per building for pypsa-eur-sec - hp_min_cap_mv_grid_pypsa_eur_sec = ( - determine_min_hp_cap_buildings_pypsa_eur_sec( - peak_load_100RE, - buildings_decentral_heating, - ) - ) - - df_hp_min_cap_mv_grid_pypsa_eur_sec[ - mvgd - ] = hp_min_cap_mv_grid_pypsa_eur_sec + # ################ collect results ################## + logger.debug(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( [df_peak_loads_db, peak_load_100RE.reset_index()], @@ -1645,9 +1659,15 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True ) + df_hp_min_cap_mv_grid_pypsa_eur_sec.loc[ + mvgd + ] = hp_min_cap_mv_grid_pypsa_eur_sec + + # ################ export to db and csv ###################### + logger.debug(" Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) - logger.debug(" Write pypsa-eur-sec min HP capacities to csv.") + logger.debug("Write pypsa-eur-sec min HP capacities to csv.") export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) From d6ddddbe8aea3d0f3c50bfeddb7dafbe202ae2d6 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 12 Oct 2022 16:09:13 +0200 Subject: [PATCH 056/119] Fix pipeline dependency for minimum heat pump capacity in pypsa-eur-sec --- src/egon/data/airflow/dags/pipeline.py | 44 ++++++++++++++------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index b2182d06c..e833ebf92 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -334,6 +334,24 @@ ] ) + cts_demand_buildings = CtsDemandBuildings( + dependencies=[ + osm_buildings_streets, + cts_electricity_demand_annual, + hh_demand_buildings_setup, + tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"], + ] + ) + + # Minimum heat pump capacity for pypsa-eur-sec + heat_pumps_pypsa_eur_sec = HeatPumpsPypsaEurSec( + dependencies=[ + cts_demand_buildings, + DistrictHeatingAreas, + heat_time_series, + ] + ) + # run pypsa-eur-sec run_pypsaeursec = PypsaEurSec( dependencies=[ @@ -344,7 +362,7 @@ data_bundle, electrical_load_etrago, heat_time_series, - heat_pumps_2035, + heat_pumps_pypsa_eur_sec, ] ) @@ -585,39 +603,23 @@ dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035] ) - cts_demand_buildings = CtsDemandBuildings( - dependencies=[ - osm_buildings_streets, - cts_electricity_demand_annual, - hh_demand_buildings_setup, - tasks["heat_demand_timeseries.export-etrago-cts-heat-profiles"], - ] - ) - heat_pumps_pypsa_eur_sec = HeatPumpsPypsaEurSec( - dependencies=[ - cts_demand_buildings, - DistrictHeatingAreas, - heat_supply, - heat_time_series, - # TODO add PV rooftop - ] - ) - + # Heat pump disaggregation for eGon2035 heat_pumps_2035 = HeatPumps2035( dependencies=[ cts_demand_buildings, DistrictHeatingAreas, heat_supply, heat_time_series, - heat_pumps_pypsa_eur_sec + heat_pumps_pypsa_eur_sec, # TODO add PV rooftop ] ) + # Heat pump disaggregation for eGon100RE heat_pumps_2050 = HeatPumps2050( dependencies=[ run_pypsaeursec, - heat_pumps_2035, + heat_pumps_pypsa_eur_sec, ] ) From d8f4a9e48850680fc1a1d6a26404ca39d6ca63fc Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 17:27:15 +0200 Subject: [PATCH 057/119] Fix minor bugs --- .../heat_supply/individual_heating.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 0543f9814..a40c088ed 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1268,7 +1268,7 @@ def determine_hp_cap_buildings_eGon2035_per_mvgd( return hp_cap_per_building.rename("hp_capacity") else: - return pd.Series().rename("hp_capacity") + return pd.Series(dtype="float64").rename("hp_capacity") def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): @@ -1313,7 +1313,7 @@ def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): return hp_cap_per_building.rename("hp_capacity") else: - return pd.Series().rename("hp_capacity") + return pd.Series(dtype="float64").rename("hp_capacity") def determine_hp_cap_buildings_eGon100RE(): @@ -1322,6 +1322,12 @@ def determine_hp_cap_buildings_eGon100RE(): """ + # ========== Register np datatypes with SQLA ========== + register_adapter(np.float64, adapt_numpy_float64) + register_adapter(np.int64, adapt_numpy_int64) + # ===================================================== + + with db.session_scope() as session: query = ( session.query( @@ -1334,11 +1340,11 @@ def determine_hp_cap_buildings_eGon100RE(): .distinct(MapZensusGridDistricts.bus_id) ) mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + mvgd_ids = mvgd_ids.sort_values("bus_id") - df_hp_cap_per_building_100RE_db = pd.DataFrame() + df_hp_cap_per_building_100RE_db = pd.DataFrame(columns=["building_id", "hp_capacity"]) - for mvgd_id in mvgd_ids: + for mvgd_id in mvgd_ids["bus_id"].values: hp_cap_per_building_100RE = ( determine_hp_cap_buildings_eGon100RE_per_mvgd(mvgd_id) @@ -1354,6 +1360,7 @@ def determine_hp_cap_buildings_eGon100RE(): ) df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" + write_table_to_postgres( df_hp_cap_per_building_100RE_db, EgonHpCapacityBuildings, @@ -1599,7 +1606,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): df_peak_loads_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() - df_hp_min_cap_mv_grid_pypsa_eur_sec = pd.Series() + df_hp_min_cap_mv_grid_pypsa_eur_sec = pd.Series(dtype="float64") for mvgd in mvgd_ids: From f7de1962fc003e2bd73f6b85511cf783d06eecb1 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 17:27:43 +0200 Subject: [PATCH 058/119] Remove debugging --- .../data/datasets/heat_supply/individual_heating.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index a40c088ed..ff452a935 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1707,16 +1707,3 @@ def delete_peak_loads_if_existing(): session.query(BuildingHeatPeakLoads).filter( BuildingHeatPeakLoads.scenario == "eGon2035" ).delete(synchronize_session=False) - - -if __name__ == "__main__": - - determine_hp_cap_buildings_eGon100RE() - - create_peak_load_table() - create_egon_etrago_timeseries_individual_heating() - determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec([1556]) - - create_hp_capacity_table() - delete_peak_loads_if_existing() - determine_hp_cap_peak_load_mvgd_ts_2035([1556]) From c782488cd4c07c8d8af55d4f914644d0c9d9b2cd Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 17:28:23 +0200 Subject: [PATCH 059/119] Add rooftop dependency to 2035 --- src/egon/data/airflow/dags/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index e833ebf92..03f96b9c0 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -611,7 +611,7 @@ heat_supply, heat_time_series, heat_pumps_pypsa_eur_sec, - # TODO add PV rooftop + tasks["power_plants.pv-rooftop-to-buildings"] ] ) From 6e6557be72194b36e5491c5c3fe2ea4cb626f8a7 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 17:29:23 +0200 Subject: [PATCH 060/119] Black&isort --- .../data/datasets/heat_supply/individual_heating.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index ff452a935..18b1ed6bf 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1021,8 +1021,10 @@ def get_heat_peak_demand_per_building(scenario, building_ids): raise ValueError("Duplicate building_id") # convert to series and from W to MW - df_heat_peak_demand = df_heat_peak_demand.set_index("building_id").loc[ - :, "peak_load_in_w"] * 1e6 + df_heat_peak_demand = ( + df_heat_peak_demand.set_index("building_id").loc[:, "peak_load_in_w"] + * 1e6 + ) return df_heat_peak_demand @@ -1327,7 +1329,6 @@ def determine_hp_cap_buildings_eGon100RE(): register_adapter(np.int64, adapt_numpy_int64) # ===================================================== - with db.session_scope() as session: query = ( session.query( @@ -1342,7 +1343,9 @@ def determine_hp_cap_buildings_eGon100RE(): mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) mvgd_ids = mvgd_ids.sort_values("bus_id") - df_hp_cap_per_building_100RE_db = pd.DataFrame(columns=["building_id", "hp_capacity"]) + df_hp_cap_per_building_100RE_db = pd.DataFrame( + columns=["building_id", "hp_capacity"] + ) for mvgd_id in mvgd_ids["bus_id"].values: From bb28eae3fac0802ff1fa4cdee6b6e414a640232b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 20:43:14 +0200 Subject: [PATCH 061/119] Fix dyamic parallelisation Table excess is moved into function call as during the setup of the pipeline no tables exist yet. --- src/egon/data/airflow/dags/pipeline.py | 4 +- .../heat_supply/individual_heating.py | 102 +++++++----------- 2 files changed, 39 insertions(+), 67 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 03f96b9c0..dc44cfa86 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -50,9 +50,9 @@ from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable from egon.data.datasets.heat_supply import HeatSupply from egon.data.datasets.heat_supply.individual_heating import ( + HeatPumps2035, HeatPumps2050, HeatPumpsPypsaEurSec, - HeatPumps2035, ) from egon.data.datasets.hydrogen_etrago import ( HydrogenBusEtrago, @@ -611,7 +611,7 @@ heat_supply, heat_time_series, heat_pumps_pypsa_eur_sec, - tasks["power_plants.pv-rooftop-to-buildings"] + tasks["power_plants.pv_rooftop.pv-rooftop-to-buildings"], ] ) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 18b1ed6bf..3524dfb00 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -67,7 +67,6 @@ class HeatPumpsPypsaEurSec(Dataset): def __init__(self, dependencies): def dyn_parallel_tasks(): """Dynamically generate tasks - The goal is to speed up tasks by parallelising bulks of mvgds. The number of parallel tasks is defined via parameter @@ -78,49 +77,25 @@ def dyn_parallel_tasks(): set of airflow.PythonOperators The tasks. Each element is of :func:`egon.data.datasets.heat_supply.individual_heating. - determine_hp_capacity_eGon2035_pypsa_eur_sec` + determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec` """ parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( "parallel_tasks", 1 ) - # ========== Register np datatypes with SQLA ========== - register_adapter(np.float64, adapt_numpy_float64) - register_adapter(np.int64, adapt_numpy_int64) - # ===================================================== - - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - - mvgd_ids = np.array_split( - mvgd_ids["bus_id"].values, parallel_tasks - ) - - # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] tasks = set() - for i, bulk in enumerate(mvgd_ids): + for i in range(parallel_tasks): tasks.add( PythonOperator( task_id=( f"determine-hp-capacity-pypsa-eur-sec_" - f"mvgd_{min(bulk)}-{max(bulk)}" + f"mvgd_bulk{i}" ), - python_callable=determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec, + python_callable=split_mvgds_into_bulks, op_kwargs={ - "mvgd_ids": bulk, + "n": i, + "max_n": parallel_tasks, + "func": determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec, }, ) ) @@ -153,49 +128,24 @@ def dyn_parallel_tasks(): set of airflow.PythonOperators The tasks. Each element is of :func:`egon.data.datasets.heat_supply.individual_heating. - determine_hp_capacity_eGon2035_pypsa_eur_sec` + determine_hp_cap_peak_load_mvgd_ts_2035` """ parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( "parallel_tasks", 1 ) - # ========== Register np datatypes with SQLA ========== - register_adapter(np.float64, adapt_numpy_float64) - register_adapter(np.int64, adapt_numpy_int64) - # ===================================================== - - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) - - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - - mvgd_ids = np.array_split( - mvgd_ids["bus_id"].values, parallel_tasks - ) - - # mvgd_bunch_size = divmod(MVGD_MIN_COUNT, parallel_tasks)[0] tasks = set() - for i, bulk in enumerate(mvgd_ids): + for i in range(parallel_tasks): tasks.add( PythonOperator( task_id=( - f"determine-hp-capacity-eGon2035_" - f"mvgd_{min(bulk)}-{max(bulk)}" + f"determine-hp-capacity-pypsa-eur-sec_" + f"mvgd_bulk{i}" ), - python_callable=determine_hp_cap_peak_load_mvgd_ts_2035, + python_callable=split_mvgds_into_bulks, op_kwargs={ - "mvgd_ids": bulk, + "n": i, + "max_n": parallel_tasks, + "func": determine_hp_cap_peak_load_mvgd_ts_2035, }, ) ) @@ -1681,6 +1631,28 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) +def split_mvgds_into_bulks(n, max_n, func): + """""" + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, parallel_tasks) + func(mvgd_ids) + + def create_peak_load_table(): BuildingHeatPeakLoads.__table__.drop(bind=engine, checkfirst=True) From 1097566240bedc1fb9fa694e670519d9c2638315 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 21:02:11 +0200 Subject: [PATCH 062/119] Add logging --- .../heat_supply/individual_heating.py | 81 +++++++++++-------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 3524dfb00..d21dc21f3 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -95,7 +95,7 @@ def dyn_parallel_tasks(): op_kwargs={ "n": i, "max_n": parallel_tasks, - "func": determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec, + "func": determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec, # noqa: E501 }, ) ) @@ -431,8 +431,8 @@ def get_peta_demand(mvgd, scenario): Returns ------- df_peta_demand : pd.DataFrame - Annual residential heat demand per building and scenario. Columns of the - dataframe are zensus_population_id and demand. + Annual residential heat demand per building and scenario. Columns of + the dataframe are zensus_population_id and demand. """ @@ -473,9 +473,9 @@ def get_residential_heat_profile_ids(mvgd): Returns ------- df_profiles_ids : pd.DataFrame - Residential daily heat profile ID's per building. Columns of the dataframe - are zensus_population_id, building_id, selected_idp_profiles, buildings - and day_of_year. + Residential daily heat profile ID's per building. Columns of the + dataframe are zensus_population_id, building_id, + selected_idp_profiles, buildings and day_of_year. """ with db.session_scope() as session: @@ -524,8 +524,8 @@ def get_daily_profiles(profile_ids): Returns ------- df_profiles : pd.DataFrame - Residential daily heat profiles. Columns of the dataframe are idp, house, - temperature_class and hour. + Residential daily heat profiles. Columns of the dataframe are idp, + house, temperature_class and hour. """ saio.register_schema("demand", db.engine()) @@ -884,8 +884,8 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd, scenario): Returns -------- pd.Index(int) - Building IDs (as int) of buildings with decentral heating system in given - MV grid. Type is pandas Index to avoid errors later on when it is + Building IDs (as int) of buildings with decentral heating system in + given MV grid. Type is pandas Index to avoid errors later on when it is used in a query. """ @@ -957,8 +957,8 @@ def get_heat_peak_demand_per_building(scenario, building_ids): session.query( BuildingHeatPeakLoads.building_id, BuildingHeatPeakLoads.peak_load_in_w, - ).filter(BuildingHeatPeakLoads.scenario == scenario) - # .filter(BuildingHeatPeakLoads.sector == "both") + ) + .filter(BuildingHeatPeakLoads.scenario == scenario) .filter(BuildingHeatPeakLoads.building_id.in_(building_ids)) ) @@ -1248,16 +1248,18 @@ def determine_hp_cap_buildings_eGon100RE_per_mvgd(mv_grid_id): mv_grid_id, scenario="eGon100RE" ) - # TODO get peak demand from db + logger.info(f"MVGD={mv_grid_id} | Get peak loads from DB") df_peak_heat_demand = get_heat_peak_demand_per_building( "eGon100RE", building_ids ) + logger.info(f"MVGD={mv_grid_id} | Determine HP capacities.") # determine minimum required heat pump capacity per building min_hp_cap_buildings = determine_minimum_hp_capacity_per_building( df_peak_heat_demand, flexibility_factor=24 / 18, cop=1.7 ) + logger.info(f"MVGD={mv_grid_id} | Desaggregate HP capacities.") # distribute total heat pump capacity to all buildings with HP hp_cap_per_building = desaggregate_hp_capacity( min_hp_cap_buildings, hp_cap_grid @@ -1292,12 +1294,15 @@ def determine_hp_cap_buildings_eGon100RE(): ) mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) mvgd_ids = mvgd_ids.sort_values("bus_id") + mvgd_ids = mvgd_ids["bus_id"].values df_hp_cap_per_building_100RE_db = pd.DataFrame( columns=["building_id", "hp_capacity"] ) - for mvgd_id in mvgd_ids["bus_id"].values: + for mvgd_id in mvgd_ids: + + logger.info(f"MVGD={mvgd_id} | Start") hp_cap_per_building_100RE = ( determine_hp_cap_buildings_eGon100RE_per_mvgd(mvgd_id) @@ -1312,6 +1317,7 @@ def determine_hp_cap_buildings_eGon100RE(): axis=0, ) + logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" write_table_to_postgres( @@ -1324,7 +1330,8 @@ def determine_hp_cap_buildings_eGon100RE(): def aggregate_residential_and_cts_profiles(mvgd, scenario): """ - Gets residential and CTS heat demand profiles per building and aggregates them. + Gets residential and CTS heat demand profiles per building and aggregates + them. Parameters ---------- @@ -1336,8 +1343,8 @@ def aggregate_residential_and_cts_profiles(mvgd, scenario): Returns -------- pd.DataFrame - Table of demand profile per building. Column names are building IDs and index - is hour of the year as int (0-8759). + Table of demand profile per building. Column names are building IDs and + index is hour of the year as int (0-8759). """ # ############### get residential heat demand profiles ############### @@ -1450,7 +1457,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): for mvgd in mvgd_ids: - logger.debug(f"MVGD={mvgd} | Start") + logger.info(f"MVGD={mvgd} | Start") # ############# aggregate residential and CTS demand profiles ##### @@ -1459,12 +1466,12 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) # ##################### determine peak loads ################### - logger.debug(f"MVGD={mvgd} | Determine peak loads.") + logger.info(f"MVGD={mvgd} | Determine peak loads.") peak_load = df_heat_ts.max().rename("eGon2035") # ######## determine HP capacity per building ######### - logger.debug(f"MVGD={mvgd} | Determine HP capacities.") + logger.info(f"MVGD={mvgd} | Determine HP capacities.") buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( @@ -1483,7 +1490,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) # ################ aggregated heat profiles ################### - logger.debug(f"MVGD={mvgd} | Aggregate heat profiles.") + logger.info(f"MVGD={mvgd} | Aggregate heat profiles.") df_mvgd_ts_2035_hp = df_heat_ts.loc[ :, @@ -1506,7 +1513,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) # ################ collect results ################## - logger.debug(f"MVGD={mvgd} | Collect results.") + logger.info(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( [df_peak_loads_db, peak_load.reset_index()], @@ -1527,7 +1534,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) # ################ export to db ####################### - logger.debug(" Write data to db.") + logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" @@ -1541,9 +1548,9 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): """ - Main function to determine minimum required HP capacity in MV for pypsa-eur-sec. - Further, creates heat demand time series for all buildings with heat pumps in MV - grid in eGon100RE scenario, used in eTraGo. + Main function to determine minimum required HP capacity in MV for + pypsa-eur-sec. Further, creates heat demand time series for all buildings + with heat pumps in MV grid in eGon100RE scenario, used in eTraGo. Parameters ----------- @@ -1563,7 +1570,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): for mvgd in mvgd_ids: - logger.debug(f"MVGD={mvgd} | Start") + logger.info(f"MVGD={mvgd} | Start") # ############# aggregate residential and CTS demand profiles ##### @@ -1572,7 +1579,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) # ##################### determine peak loads ################### - logger.debug(f"MVGD={mvgd} | Determine peak loads.") + logger.info(f"MVGD={mvgd} | Determine peak loads.") peak_load_100RE = df_heat_ts.max().rename("eGon100RE") @@ -1590,7 +1597,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) # ################ aggregated heat profiles ################### - logger.debug(f"MVGD={mvgd} | Aggregate heat profiles.") + logger.info(f"MVGD={mvgd} | Aggregate heat profiles.") df_mvgd_ts_hp = df_heat_ts.loc[ :, @@ -1607,7 +1614,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) # ################ collect results ################## - logger.debug(f"MVGD={mvgd} | Collect results.") + logger.info(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( [df_peak_loads_db, peak_load_100RE.reset_index()], @@ -1624,10 +1631,14 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ] = hp_min_cap_mv_grid_pypsa_eur_sec # ################ export to db and csv ###################### - logger.debug(" Write data to db.") + logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) - logger.debug("Write pypsa-eur-sec min HP capacities to csv.") + logger.info( + f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write " + f"pypsa-eur-sec min " + f"HP capacities to csv." + ) export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) @@ -1649,7 +1660,11 @@ def split_mvgds_into_bulks(n, max_n, func): mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, parallel_tasks) + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) + # Only take split n + mvgd_ids = mvgd_ids[n] + + logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} - {max(mvgd_ids)}") func(mvgd_ids) From 8722e9e61318997c5c6219c822dad399858fc69b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 21:12:11 +0200 Subject: [PATCH 063/119] Fix rooftop dependency --- src/egon/data/airflow/dags/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index dc44cfa86..e7592c860 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -611,7 +611,7 @@ heat_supply, heat_time_series, heat_pumps_pypsa_eur_sec, - tasks["power_plants.pv_rooftop.pv-rooftop-to-buildings"], + tasks["power_plants.pv_rooftop_buildings.pv-rooftop-to-buildings"], ] ) From 7227c85848f4e479915b7101d597b8ad869f8ccb Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 21:47:00 +0200 Subject: [PATCH 064/119] Fix cycles dyn parallel --- .../heat_supply/individual_heating.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index d21dc21f3..1382da003 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -88,10 +88,10 @@ def dyn_parallel_tasks(): tasks.add( PythonOperator( task_id=( - f"determine-hp-capacity-pypsa-eur-sec_" - f"mvgd_bulk{i}" + f"determine-hp-capacity-pypsa-eur-sec-" + f"mvgd-bulk{i}" ), - python_callable=split_mvgds_into_bulks, + python_callable=split_mvgds_into_bulks_pypsa_eur_sec, op_kwargs={ "n": i, "max_n": parallel_tasks, @@ -141,7 +141,7 @@ def dyn_parallel_tasks(): f"determine-hp-capacity-pypsa-eur-sec_" f"mvgd_bulk{i}" ), - python_callable=split_mvgds_into_bulks, + python_callable=split_mvgds_into_bulks_2035, op_kwargs={ "n": i, "max_n": parallel_tasks, @@ -1642,7 +1642,32 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) -def split_mvgds_into_bulks(n, max_n, func): +def split_mvgds_into_bulks_2035(n, max_n, func): + """""" + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.bus_id, + ) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .distinct(MapZensusGridDistricts.bus_id) + ) + mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + + mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) + + mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) + # Only take split n + mvgd_ids = mvgd_ids[n] + + logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} - {max(mvgd_ids)}") + func(mvgd_ids) + +def split_mvgds_into_bulks_pypsa_eur_sec(n, max_n, func): """""" with db.session_scope() as session: From 53ef627a4d6d5ec4df8abddd2ec7e4c327536e9a Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 12 Oct 2022 21:49:45 +0200 Subject: [PATCH 065/119] Fix cycles dyn parallel 2 --- src/egon/data/datasets/heat_supply/individual_heating.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 1382da003..618ba545a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -65,7 +65,7 @@ class EgonHpCapacityBuildings(Base): class HeatPumpsPypsaEurSec(Dataset): def __init__(self, dependencies): - def dyn_parallel_tasks(): + def dyn_parallel_tasks_pypsa_eur_sec(): """Dynamically generate tasks The goal is to speed up tasks by parallelising bulks of mvgds. @@ -108,14 +108,14 @@ def dyn_parallel_tasks(): tasks=( create_peak_load_table, create_egon_etrago_timeseries_individual_heating, - {*dyn_parallel_tasks()}, + {*dyn_parallel_tasks_pypsa_eur_sec()}, ), ) class HeatPumps2035(Dataset): def __init__(self, dependencies): - def dyn_parallel_tasks(): + def dyn_parallel_tasks_2035(): """Dynamically generate tasks The goal is to speed up tasks by parallelising bulks of mvgds. @@ -158,7 +158,7 @@ def dyn_parallel_tasks(): tasks=( create_hp_capacity_table, delete_peak_loads_if_existing, - {*dyn_parallel_tasks()}, + {*dyn_parallel_tasks_2035()}, ), ) From b119942a321957d37e61776d1ebd78eb02b370f8 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sat, 15 Oct 2022 22:17:38 +0200 Subject: [PATCH 066/119] Fix missing else --- .../datasets/electricity_demand_timeseries/cts_buildings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index dff0287de..b5bb1a0a6 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -1051,6 +1051,9 @@ def calc_cts_building_profiles( orient="index", ) + else: + raise KeyError("Sector needs to be either 'electricity' or 'heat'") + # TODO remove after #722 df_demand_share.rename(columns={"id": "building_id"}, inplace=True) From 1704b9a43a80c7024727514244781d90181baa45 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 17 Oct 2022 11:36:17 +0200 Subject: [PATCH 067/119] Add logging.info for minimum cp capacity --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 618ba545a..9f9f212b5 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1584,6 +1584,8 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): peak_load_100RE = df_heat_ts.max().rename("eGon100RE") # ######## determine minimum HP capacity pypsa-eur-sec ########### + logger.info(f"MVGD={mvgd} | Determine minimum HP capacity.") + buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( mvgd, scenario="eGon100RE" From a0301647b865098ec54f7f698480040ea613321b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 17 Oct 2022 11:53:47 +0200 Subject: [PATCH 068/119] Add try except if no cts substation profile for bus id --- .../cts_buildings.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index b5bb1a0a6..d09b5a658 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -1005,12 +1005,12 @@ def calc_cts_building_profiles( ) ).filter(EgonEtragoElectricityCts.bus_id.in_(bus_ids)) - df_cts_profiles = pd.read_sql( + df_cts_substation_profiles = pd.read_sql( cells_query.statement, cells_query.session.bind, ) - df_cts_profiles = pd.DataFrame.from_dict( - df_cts_profiles.set_index("bus_id")["p_set"].to_dict(), + df_cts_substation_profiles = pd.DataFrame.from_dict( + df_cts_substation_profiles.set_index("bus_id")["p_set"].to_dict(), orient="index", ) # df_cts_profiles = calc_load_curves_cts(scenario) @@ -1042,12 +1042,12 @@ def calc_cts_building_profiles( ) ).filter(EgonEtragoHeatCts.bus_id.in_(bus_ids)) - df_cts_profiles = pd.read_sql( + df_cts_substation_profiles = pd.read_sql( cells_query.statement, cells_query.session.bind, ) - df_cts_profiles = pd.DataFrame.from_dict( - df_cts_profiles.set_index("bus_id")["p_set"].to_dict(), + df_cts_substation_profiles = pd.DataFrame.from_dict( + df_cts_substation_profiles.set_index("bus_id")["p_set"].to_dict(), orient="index", ) @@ -1061,7 +1061,14 @@ def calc_cts_building_profiles( df_building_profiles = pd.DataFrame() for bus_id, df in df_demand_share.groupby("bus_id"): shares = df.set_index("building_id", drop=True)["profile_share"] - profile_ts = df_cts_profiles.loc[bus_id] + try: + profile_ts = df_cts_substation_profiles.loc[bus_id] + except KeyError: + # This should only happen within the SH cutout + log.info(f"No CTS profile found for substation with bus_id:" + f" {bus_id}") + continue + building_profiles = np.outer(profile_ts, shares) building_profiles = pd.DataFrame( building_profiles, index=profile_ts.index, columns=shares.index From de1ef8b3a9a02a5516d5e1a574e52cbe9c5665f1 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 17 Oct 2022 13:56:45 +0200 Subject: [PATCH 069/119] Catch missing peak load buildings / SH-only Buildings with decentral heating but without peak load are removed. This should only happen in SH cutout --- .../heat_supply/individual_heating.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 9f9f212b5..4a89143d0 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1433,6 +1433,39 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): ) +def catch_missing_buidings(buildings_decentral_heating, peak_load): + """ + Check for missing buildings and reduce the list of buildings with + decentral heating if no peak loads available. This should only happen + in case of cutout SH + + Parameters + ----------- + buildings_decentral_heating : list(int) + Array or list of buildings with decentral heating + + peak_load : pd.Series + Peak loads of all building within the mvgd + + """ + # Catch missing buildings key error + # should only happen within cutout SH + if ( + not all(buildings_decentral_heating.isin(peak_load.index)) + and config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + diff = buildings_decentral_heating.difference(peak_load.index) + logger.warning( + f"Dropped {len(diff)} building ids due to missing peak " + f"loads. {len(buildings_decentral_heating.index)} left." + ) + logger.info(f"Dropped buildings: {diff}") + buildings_decentral_heating.drop(diff, inplace=True) + + return buildings_decentral_heating + + def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): """ Main function to determine HP capacity per building in eGon2035 scenario. @@ -1468,7 +1501,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # ##################### determine peak loads ################### logger.info(f"MVGD={mvgd} | Determine peak loads.") - peak_load = df_heat_ts.max().rename("eGon2035") + peak_load_2035 = df_heat_ts.max().rename("eGon2035") # ######## determine HP capacity per building ######### logger.info(f"MVGD={mvgd} | Determine HP capacities.") @@ -1478,10 +1511,17 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): mvgd, scenario="eGon2035" ) ) + + # Reduce list of decentral heating if no Peak load available + # TODO maybe remove after succesfull DE run + buildings_decentral_heating = catch_missing_buidings( + buildings_decentral_heating, peak_load_2035 + ) + hp_cap_per_building_2035 = ( determine_hp_cap_buildings_eGon2035_per_mvgd( mvgd, - peak_load, + peak_load_2035, buildings_decentral_heating, ) ) @@ -1516,7 +1556,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): logger.info(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( - [df_peak_loads_db, peak_load.reset_index()], + [df_peak_loads_db, peak_load_2035.reset_index()], axis=0, ignore_index=True, ) @@ -1591,6 +1631,13 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): mvgd, scenario="eGon100RE" ) ) + + # Reduce list of decentral heating if no Peak load available + # TODO maybe remove after succesfull DE run + buildings_decentral_heating = catch_missing_buidings( + buildings_decentral_heating, peak_load_100RE + ) + hp_min_cap_mv_grid_pypsa_eur_sec = ( determine_min_hp_cap_buildings_pypsa_eur_sec( peak_load_100RE, @@ -1669,6 +1716,7 @@ def split_mvgds_into_bulks_2035(n, max_n, func): logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} - {max(mvgd_ids)}") func(mvgd_ids) + def split_mvgds_into_bulks_pypsa_eur_sec(n, max_n, func): """""" From 8168a9cc0c546b0f6aeaebfd881b8ed0ba4930d0 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 17 Oct 2022 15:54:34 +0200 Subject: [PATCH 070/119] Fix typo --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 4a89143d0..ffff8189f 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1458,7 +1458,7 @@ def catch_missing_buidings(buildings_decentral_heating, peak_load): diff = buildings_decentral_heating.difference(peak_load.index) logger.warning( f"Dropped {len(diff)} building ids due to missing peak " - f"loads. {len(buildings_decentral_heating.index)} left." + f"loads. {len(buildings_decentral_heating)} left." ) logger.info(f"Dropped buildings: {diff}") buildings_decentral_heating.drop(diff, inplace=True) From 4a5527e53d83107906317df36e29e813ba347177 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 17 Oct 2022 16:21:59 +0200 Subject: [PATCH 071/119] Fix minor bug --- src/egon/data/datasets/heat_supply/individual_heating.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index ffff8189f..244f23ec2 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1460,8 +1460,8 @@ def catch_missing_buidings(buildings_decentral_heating, peak_load): f"Dropped {len(diff)} building ids due to missing peak " f"loads. {len(buildings_decentral_heating)} left." ) - logger.info(f"Dropped buildings: {diff}") - buildings_decentral_heating.drop(diff, inplace=True) + logger.info(f"Dropped buildings: {diff.values}") + buildings_decentral_heating = buildings_decentral_heating.drop(diff) return buildings_decentral_heating From 6badd705917067564ab680daa102e947a012bd57 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 13:43:29 +0200 Subject: [PATCH 072/119] Indent pd.read_sql within session scope --- .../heat_supply/individual_heating.py | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 244f23ec2..924e94579 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -453,9 +453,9 @@ def get_peta_demand(mvgd, scenario): ) ) - df_peta_demand = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) + df_peta_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) return df_peta_demand @@ -492,9 +492,9 @@ def get_residential_heat_profile_ids(mvgd): ) ) - df_profiles_ids = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) + df_profiles_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) # Add building count per cell df_profiles_ids = pd.merge( left=df_profiles_ids, @@ -536,9 +536,9 @@ def get_daily_profiles(profile_ids): egon_heat_idp_pool.index.in_(profile_ids) ) - df_profiles = pd.read_sql( - query.statement, query.session.bind, index_col="index" - ) + df_profiles = pd.read_sql( + query.statement, query.session.bind, index_col="index" + ) # unnest array of profile values per id df_profiles = df_profiles.explode("idp") @@ -576,9 +576,9 @@ def get_daily_demand_share(mvgd): MapZensusGridDistricts.bus_id == mvgd, ) - df_daily_demand_share = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) + df_daily_demand_share = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) return df_daily_demand_share @@ -754,9 +754,9 @@ def get_zensus_cells_with_decentral_heat_demand_in_mv_grid( ), ) - cells_with_dh = pd.read_sql( - query.statement, query.session.bind, index_col=None - ).zensus_population_id.values + cells_with_dh = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).zensus_population_id.values # remove zensus cells with district heating zensus_population_ids = zensus_population_ids.drop( @@ -810,9 +810,9 @@ def get_residential_buildings_with_decentral_heat_demand_in_mv_grid( ) ) - buildings_with_heat_demand = pd.read_sql( - query.statement, query.session.bind, index_col=None - ).building_id.values + buildings_with_heat_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values return pd.Index(buildings_with_heat_demand) @@ -859,9 +859,9 @@ def get_cts_buildings_with_decentral_heat_demand_in_mv_grid( ), ) - buildings_with_heat_demand = pd.read_sql( - query.statement, query.session.bind, index_col=None - ).building_id.values + buildings_with_heat_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values return pd.Index(buildings_with_heat_demand) @@ -940,9 +940,9 @@ def get_total_heat_pump_capacity_of_mv_grid(scenario, mv_grid_id): .filter(EgonIndividualHeatingSupply.mv_grid_id == mv_grid_id) ) - hp_cap_mv_grid = pd.read_sql( - query.statement, query.session.bind, index_col="mv_grid_id" - ) + hp_cap_mv_grid = pd.read_sql( + query.statement, query.session.bind, index_col="mv_grid_id" + ) if hp_cap_mv_grid.empty: return 0.0 else: @@ -962,9 +962,9 @@ def get_heat_peak_demand_per_building(scenario, building_ids): .filter(BuildingHeatPeakLoads.building_id.in_(building_ids)) ) - df_heat_peak_demand = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) + df_heat_peak_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) # TODO remove check if df_heat_peak_demand.duplicated("building_id").any(): @@ -1039,9 +1039,9 @@ def determine_buildings_with_hp_in_mv_grid( egon_power_plants_pv_roof_building.building_id.in_(building_ids) ) - buildings_with_pv = pd.read_sql( - query.statement, query.session.bind, index_col=None - ).building_id.values + buildings_with_pv = pd.read_sql( + query.statement, query.session.bind, index_col=None + ).building_id.values # set different weights for buildings with PV and without PV weight_with_pv = 1.5 weight_without_pv = 1.0 @@ -1292,7 +1292,9 @@ def determine_hp_cap_buildings_eGon100RE(): ) .distinct(MapZensusGridDistricts.bus_id) ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + mvgd_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) mvgd_ids = mvgd_ids.sort_values("bus_id") mvgd_ids = mvgd_ids["bus_id"].values @@ -1705,7 +1707,9 @@ def split_mvgds_into_bulks_2035(n, max_n, func): ) .distinct(MapZensusGridDistricts.bus_id) ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + mvgd_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) @@ -1731,7 +1735,9 @@ def split_mvgds_into_bulks_pypsa_eur_sec(n, max_n, func): ) .distinct(MapZensusGridDistricts.bus_id) ) - mvgd_ids = pd.read_sql(query.statement, query.session.bind, index_col=None) + mvgd_ids = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) From 0b0328a841ee744d926bdaa761302eb6ef81e035 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 14:45:51 +0200 Subject: [PATCH 073/119] Remove heat peak load task from cts-buildings This task is replaced in the HeatPumpPypsaEurSec dataset --- .../cts_buildings.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index d09b5a658..a07b48876 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -252,7 +252,7 @@ def __init__(self, dependencies): tasks=( cts_buildings, {cts_electricity, cts_heat}, - {get_cts_electricity_peak_load, get_cts_heat_peak_load}, + get_cts_electricity_peak_load, ), ) @@ -987,9 +987,7 @@ def calc_cts_building_profiles( EgonCtsElectricityDemandBuildingShare.scenario == scenario ) .filter( - EgonCtsElectricityDemandBuildingShare.bus_id.in_( - bus_ids - ) + EgonCtsElectricityDemandBuildingShare.bus_id.in_(bus_ids) ) ) @@ -1023,11 +1021,7 @@ def calc_cts_building_profiles( EgonCtsHeatDemandBuildingShare, ) .filter(EgonCtsHeatDemandBuildingShare.scenario == scenario) - .filter( - EgonCtsHeatDemandBuildingShare.bus_id.in_( - bus_ids - ) - ) + .filter(EgonCtsHeatDemandBuildingShare.bus_id.in_(bus_ids)) ) df_demand_share = pd.read_sql( @@ -1065,8 +1059,10 @@ def calc_cts_building_profiles( profile_ts = df_cts_substation_profiles.loc[bus_id] except KeyError: # This should only happen within the SH cutout - log.info(f"No CTS profile found for substation with bus_id:" - f" {bus_id}") + log.info( + f"No CTS profile found for substation with bus_id:" + f" {bus_id}" + ) continue building_profiles = np.outer(profile_ts, shares) From fe2b74b35762442ba0ed68a481ccb0a94aca6bb0 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 21:17:35 +0200 Subject: [PATCH 074/119] Fix dtype of column to int --- src/egon/data/datasets/heat_supply/individual_heating.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 924e94579..90b56e714 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1385,6 +1385,9 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): var_name="scenario", value_name="peak_load_in_w", ) + df_peak_loads_db["building_id"] = df_peak_loads_db["building_id"].astype( + int + ) df_peak_loads_db["sector"] = "residential+cts" # From MW to W df_peak_loads_db["peak_load_in_w"] = ( From 2a8e24dd2c8924f39422c78bb8ab2a320f5cf145 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 22:06:25 +0200 Subject: [PATCH 075/119] Inhouse table creation where possible --- .../heat_supply/individual_heating.py | 98 +++++++++++++------ 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 90b56e714..686806250 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,8 +106,6 @@ def dyn_parallel_tasks_pypsa_eur_sec(): version="0.0.0", dependencies=dependencies, tasks=( - create_peak_load_table, - create_egon_etrago_timeseries_individual_heating, {*dyn_parallel_tasks_pypsa_eur_sec()}, ), ) @@ -156,8 +154,7 @@ def dyn_parallel_tasks_2035(): version="0.0.0", dependencies=dependencies, tasks=( - create_hp_capacity_table, - delete_peak_loads_if_existing, + delete_heat_peak_loads_eGon2035, {*dyn_parallel_tasks_2035()}, ), ) @@ -1322,6 +1319,9 @@ def determine_hp_cap_buildings_eGon100RE(): logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) + delete_hp_capacity(scenario="eGon100RE") + write_table_to_postgres( df_hp_cap_per_building_100RE_db, EgonHpCapacityBuildings, @@ -1377,8 +1377,20 @@ def aggregate_residential_and_cts_profiles(mvgd, scenario): return df_heat_ts -def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): - """""" +def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False): + """ + Function to export the collected results of all MVGDs per bulk to DB. + + Parameters + ---------- + df_peak_loads_db : pd.DataFrame + Table of building peak loads of all MVGDs per bulk + df_heat_mvgd_ts_db : pd.DataFrame + Table of all aggregated MVGD profiles per bulk + drop : boolean + Drop and recreate table if True + + """ df_peak_loads_db = df_peak_loads_db.melt( id_vars="building_id", @@ -1394,27 +1406,39 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db): df_peak_loads_db["peak_load_in_w"] * 1e6 ) write_table_to_postgres( - df_peak_loads_db, BuildingHeatPeakLoads, engine=engine + df_peak_loads_db, BuildingHeatPeakLoads, engine=engine, drop=drop ) - columns = { + dtypes = { column.key: column.type for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns } df_heat_mvgd_ts_db = df_heat_mvgd_ts_db.loc[:, columns.keys()] - df_heat_mvgd_ts_db.to_sql( - name=EgonEtragoTimeseriesIndividualHeating.__table__.name, - schema=EgonEtragoTimeseriesIndividualHeating.__table__.schema, - con=engine, - if_exists="append", - method="multi", - index=False, - dtype=columns, - ) + if drop: + logger.info(f"Drop and recreate table " + f"{EgonEtragoTimeseriesIndividualHeating.__table__.name}.") + EgonEtragoTimeseriesIndividualHeating.__table__.drop( + bind=engine, checkfirst=True + ) + EgonEtragoTimeseriesIndividualHeating.__table__.create( + bind=engine, checkfirst=True + ) + + with db.session_scope() as session: + df_heat_mvgd_ts_db.to_sql( + name=EgonEtragoTimeseriesIndividualHeating.__table__.name, + schema=EgonEtragoTimeseriesIndividualHeating.__table__.schema, + con=session.connection(), + if_exists="append", + method="multi", + index=False, + dtype=dtypes, + ) def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): + """Export minimum capacity of heat pumps for pypsa eur sec to csv""" df_hp_min_cap_mv_grid_pypsa_eur_sec.index.name = "mvgd_id" df_hp_min_cap_mv_grid_pypsa_eur_sec = ( @@ -1431,11 +1455,11 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): # TODO check append if not file.is_file(): df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file) - # TODO outsource into separate task incl delete file if clearing else: df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( file, mode="a", header=False ) + # TODO delete file if task is cleared?! def catch_missing_buidings(buildings_decentral_heating, peak_load): @@ -1580,9 +1604,13 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # ################ export to db ####################### logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") - export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" + + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) + delete_hp_capacity(scenario="eGon2035") + write_table_to_postgres( df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, @@ -1686,7 +1714,8 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): # ################ export to db and csv ###################### logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") - export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db) + + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=True) logger.info( f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write " @@ -1752,12 +1781,6 @@ def split_mvgds_into_bulks_pypsa_eur_sec(n, max_n, func): func(mvgd_ids) -def create_peak_load_table(): - - BuildingHeatPeakLoads.__table__.drop(bind=engine, checkfirst=True) - BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) - - def create_hp_capacity_table(): EgonHpCapacityBuildings.__table__.drop(bind=engine, checkfirst=True) @@ -1774,8 +1797,27 @@ def create_egon_etrago_timeseries_individual_heating(): ) -def delete_peak_loads_if_existing(): - """Remove all entries""" +def delete_hp_capacity(scenario): + """Remove all hp capacities for the selected scenario + + Parameters + ----------- + scenario : string + Either eGon2035 or eGon100RE + + """ + + with db.session_scope() as session: + # Buses + session.query(BuildingHeatPeakLoads).filter( + BuildingHeatPeakLoads.scenario == scenario + ).delete(synchronize_session=False) + +def delete_heat_peak_loads_eGon2035(): + """Remove all heat peak loads for eGon2035. + + This is not necessary for eGon100RE as these peak loads are calculated in + HeatPumpsPypsaEurSec and tables are recreated during this dataset.""" with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( From d0bc61907be690b6f2197148b9810dfc6f2fe7d6 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 22:09:39 +0200 Subject: [PATCH 076/119] Use generic mvgd split function --- .../heat_supply/individual_heating.py | 57 ++++++++----------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 686806250..9c4bc5832 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -88,10 +88,11 @@ def dyn_parallel_tasks_pypsa_eur_sec(): tasks.add( PythonOperator( task_id=( + f"individual_heating." f"determine-hp-capacity-pypsa-eur-sec-" f"mvgd-bulk{i}" ), - python_callable=split_mvgds_into_bulks_pypsa_eur_sec, + python_callable=split_mvgds_into_bulks, op_kwargs={ "n": i, "max_n": parallel_tasks, @@ -136,10 +137,11 @@ def dyn_parallel_tasks_2035(): tasks.add( PythonOperator( task_id=( - f"determine-hp-capacity-pypsa-eur-sec_" - f"mvgd_bulk{i}" + "individual_heating." + f"determine-hp-capacity-2035-" + f"mvgd-bulk{i}" ), - python_callable=split_mvgds_into_bulks_2035, + python_callable=split_mvgds_into_bulks, op_kwargs={ "n": i, "max_n": parallel_tasks, @@ -166,7 +168,8 @@ def __init__(self, dependencies): name="HeatPumps2050", version="0.0.0", dependencies=dependencies, - tasks=(determine_hp_cap_buildings_eGon100RE), + tasks=( + determine_hp_cap_buildings_eGon100RE), ) @@ -525,6 +528,7 @@ def get_daily_profiles(profile_ids): house, temperature_class and hour. """ + saio.register_schema("demand", db.engine()) from saio.demand import egon_heat_idp_pool @@ -1725,36 +1729,21 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) -def split_mvgds_into_bulks_2035(n, max_n, func): - """""" - - with db.session_scope() as session: - query = ( - session.query( - MapZensusGridDistricts.bus_id, - ) - .filter( - MapZensusGridDistricts.zensus_population_id - == EgonPetaHeat.zensus_population_id - ) - .distinct(MapZensusGridDistricts.bus_id) - ) - mvgd_ids = pd.read_sql( - query.statement, query.session.bind, index_col=None - ) - - mvgd_ids = mvgd_ids.sort_values("bus_id").reset_index(drop=True) - - mvgd_ids = np.array_split(mvgd_ids["bus_id"].values, max_n) - # Only take split n - mvgd_ids = mvgd_ids[n] - - logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} - {max(mvgd_ids)}") - func(mvgd_ids) - +def split_mvgds_into_bulks(n, max_n, func): + """ + Generic function to split task into multiple parallel tasks, + dividing the number of MVGDs into even bulks. -def split_mvgds_into_bulks_pypsa_eur_sec(n, max_n, func): - """""" + Parameters + ----------- + n : int + Number of bulk + max_n: int + Maximum number of bulks + func : function + The funnction which is then called with the list of MVGD as + parameter. + """ with db.session_scope() as session: query = ( From 8aa3dbc8c42be609fd00a5c9178d3d45e8bb4d1b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 22:10:28 +0200 Subject: [PATCH 077/119] Black&isort --- .../datasets/heat_supply/individual_heating.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 9c4bc5832..2734379b9 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,9 +106,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.0", dependencies=dependencies, - tasks=( - {*dyn_parallel_tasks_pypsa_eur_sec()}, - ), + tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), ) @@ -168,8 +166,7 @@ def __init__(self, dependencies): name="HeatPumps2050", version="0.0.0", dependencies=dependencies, - tasks=( - determine_hp_cap_buildings_eGon100RE), + tasks=(determine_hp_cap_buildings_eGon100RE), ) @@ -1420,8 +1417,10 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False): df_heat_mvgd_ts_db = df_heat_mvgd_ts_db.loc[:, columns.keys()] if drop: - logger.info(f"Drop and recreate table " - f"{EgonEtragoTimeseriesIndividualHeating.__table__.name}.") + logger.info( + f"Drop and recreate table " + f"{EgonEtragoTimeseriesIndividualHeating.__table__.name}." + ) EgonEtragoTimeseriesIndividualHeating.__table__.drop( bind=engine, checkfirst=True ) @@ -1802,6 +1801,7 @@ def delete_hp_capacity(scenario): BuildingHeatPeakLoads.scenario == scenario ).delete(synchronize_session=False) + def delete_heat_peak_loads_eGon2035(): """Remove all heat peak loads for eGon2035. From 65b0071f5036bb41555880d97db6ab22b804c10b Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 22:12:17 +0200 Subject: [PATCH 078/119] Remove inhoused table creation --- .../data/datasets/heat_supply/individual_heating.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 2734379b9..bdaa9213d 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1775,16 +1775,6 @@ def create_hp_capacity_table(): EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) -def create_egon_etrago_timeseries_individual_heating(): - - EgonEtragoTimeseriesIndividualHeating.__table__.drop( - bind=engine, checkfirst=True - ) - EgonEtragoTimeseriesIndividualHeating.__table__.create( - bind=engine, checkfirst=True - ) - - def delete_hp_capacity(scenario): """Remove all hp capacities for the selected scenario From b9b9016ce41b2dab77df6587a8e709e43dd8be50 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 18 Oct 2022 22:19:29 +0200 Subject: [PATCH 079/119] Stop parallelization --- src/egon/data/datasets.yml | 2 +- src/egon/data/datasets/heat_supply/individual_heating.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index 5654f63fe..cebc7ef98 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1090,7 +1090,7 @@ emobility_mit: parallel_tasks: 10 demand_timeseries_mvgd: - parallel_tasks: 10 + parallel_tasks: 1 charging_infrastructure: original_data: diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index bdaa9213d..0e7a673f6 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,7 +106,8 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.0", dependencies=dependencies, - tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), + # tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), + tasks=(*dyn_parallel_tasks_pypsa_eur_sec(),), ) @@ -155,7 +156,8 @@ def dyn_parallel_tasks_2035(): dependencies=dependencies, tasks=( delete_heat_peak_loads_eGon2035, - {*dyn_parallel_tasks_2035()}, + # {*dyn_parallel_tasks_2035()}, + *dyn_parallel_tasks_2035(), ), ) From 07755a8c5bd35a104920bba7ca3aca7b3255965f Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 19 Oct 2022 18:02:43 +0200 Subject: [PATCH 080/119] Fix typo --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 0e7a673f6..3b425a2be 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1416,7 +1416,7 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False): column.key: column.type for column in EgonEtragoTimeseriesIndividualHeating.__table__.columns } - df_heat_mvgd_ts_db = df_heat_mvgd_ts_db.loc[:, columns.keys()] + df_heat_mvgd_ts_db = df_heat_mvgd_ts_db.loc[:, dtypes.keys()] if drop: logger.info( From 564699794dbdd57f262959b4e427e9e86a10a864 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 19 Oct 2022 19:12:34 +0200 Subject: [PATCH 081/119] Remove engine from to_postgres --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 3b425a2be..c50b30a39 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1328,7 +1328,6 @@ def determine_hp_cap_buildings_eGon100RE(): write_table_to_postgres( df_hp_cap_per_building_100RE_db, EgonHpCapacityBuildings, - engine=engine, drop=False, ) @@ -1408,9 +1407,7 @@ def export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False): df_peak_loads_db["peak_load_in_w"] = ( df_peak_loads_db["peak_load_in_w"] * 1e6 ) - write_table_to_postgres( - df_peak_loads_db, BuildingHeatPeakLoads, engine=engine, drop=drop - ) + write_table_to_postgres(df_peak_loads_db, BuildingHeatPeakLoads, drop=drop) dtypes = { column.key: column.type @@ -1619,7 +1616,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): write_table_to_postgres( df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, - engine=engine, drop=False, ) From c42108354e8a57e6150027d39110ef29cc545b86 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 19 Oct 2022 21:39:44 +0200 Subject: [PATCH 082/119] Add missing dependency --- src/egon/data/airflow/dags/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index e7592c860..b0da7b095 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -620,6 +620,7 @@ dependencies=[ run_pypsaeursec, heat_pumps_pypsa_eur_sec, + heat_supply, ] ) From 2af5c7b273e3a7c71b2d56df4393c6b315d7774a Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 19 Oct 2022 21:50:49 +0200 Subject: [PATCH 083/119] Fix wrong table --- src/egon/data/datasets/heat_supply/individual_heating.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index c50b30a39..474670a05 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1785,8 +1785,8 @@ def delete_hp_capacity(scenario): with db.session_scope() as session: # Buses - session.query(BuildingHeatPeakLoads).filter( - BuildingHeatPeakLoads.scenario == scenario + session.query(EgonHpCapacityBuildings).filter( + EgonHpCapacityBuildings.scenario == scenario ).delete(synchronize_session=False) From 9364530e40829b9e9c3ded5b860e342f478c16d8 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 19 Oct 2022 22:36:36 +0200 Subject: [PATCH 084/119] Add additional info --- src/egon/data/datasets/heat_supply/individual_heating.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 474670a05..4abedbd31 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -872,11 +872,13 @@ def get_buildings_with_decentral_heat_demand_in_mv_grid(mvgd, scenario): given MV grid. As cells with district heating differ between scenarios, this is also - depending on the scenario. + depending on the scenario. CTS and residential have to be retrieved + seperatly as some residential buildings only have electricity but no + heat demand. This does not occure in CTS. Parameters ----------- - mv_grid_id : int + mvgd : int ID of MV grid. scenario : str Name of scenario. Can be either "eGon2035" or "eGon100RE". @@ -1545,6 +1547,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # Reduce list of decentral heating if no Peak load available # TODO maybe remove after succesfull DE run + # Might be fixed in #990 buildings_decentral_heating = catch_missing_buidings( buildings_decentral_heating, peak_load_2035 ) From 11e3216ffbafc589e748380a358a889e273978a5 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 16:58:28 +0200 Subject: [PATCH 085/119] Fix split logging message bug #987 #990 --- .../data/datasets/heat_supply/individual_heating.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 4abedbd31..cb40b5537 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1321,7 +1321,7 @@ def determine_hp_cap_buildings_eGon100RE(): axis=0, ) - logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") + logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) @@ -1608,7 +1608,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) # ################ export to db ####################### - logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") + logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" @@ -1717,12 +1717,12 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ] = hp_min_cap_mv_grid_pypsa_eur_sec # ################ export to db and csv ###################### - logger.info(f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write data to db.") + logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=True) logger.info( - f"MVGD={min(mvgd_ids)} - {max(mvgd_ids)} | Write " + f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write " f"pypsa-eur-sec min " f"HP capacities to csv." ) @@ -1766,7 +1766,7 @@ def split_mvgds_into_bulks(n, max_n, func): # Only take split n mvgd_ids = mvgd_ids[n] - logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} - {max(mvgd_ids)}") + logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} : {max(mvgd_ids)}") func(mvgd_ids) From dac6f340be8be6be9abf152d449b7da5286a4645 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 17:43:56 +0200 Subject: [PATCH 086/119] Comment fixed bug workaround --- .../heat_supply/individual_heating.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index cb40b5537..cbec67c3a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1545,12 +1545,12 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) ) - # Reduce list of decentral heating if no Peak load available - # TODO maybe remove after succesfull DE run - # Might be fixed in #990 - buildings_decentral_heating = catch_missing_buidings( - buildings_decentral_heating, peak_load_2035 - ) + # # Reduce list of decentral heating if no Peak load available + # # TODO commmented after successfull fix in #987 #990 + # # Might be fixed in #990 + # buildings_decentral_heating = catch_missing_buidings( + # buildings_decentral_heating, peak_load_2035 + # ) hp_cap_per_building_2035 = ( determine_hp_cap_buildings_eGon2035_per_mvgd( @@ -1669,11 +1669,11 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) ) - # Reduce list of decentral heating if no Peak load available - # TODO maybe remove after succesfull DE run - buildings_decentral_heating = catch_missing_buidings( - buildings_decentral_heating, peak_load_100RE - ) + # # Reduce list of decentral heating if no Peak load available + # # TODO commmented after successfull fix in #987 #990 + # buildings_decentral_heating = catch_missing_buidings( + # buildings_decentral_heating, peak_load_100RE + # ) hp_min_cap_mv_grid_pypsa_eur_sec = ( determine_min_hp_cap_buildings_pypsa_eur_sec( From 2ec492e09df46722f849397e9afd137e30f17185 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 17:48:39 +0200 Subject: [PATCH 087/119] Recreate hp_min_cap.csv if exists --- .../data/datasets/heat_supply/individual_heating.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index cbec67c3a..e61ba540a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1456,14 +1456,18 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): # Create the folder, if it does not exist already if not os.path.exists(folder): os.mkdir(folder) - # TODO check append if not file.is_file(): - df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv(file) + logger.info(f"Create {file}") + df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( + file, mode="w", header=False + ) else: + logger.info(f"Remove {file}") + os.remove(file) + logger.info(f"Create {file}") df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( file, mode="a", header=False ) - # TODO delete file if task is cleared?! def catch_missing_buidings(buildings_decentral_heating, peak_load): From 96026ab4d6f38e9a3735ea8606dd4fa50a59abcd Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 17:49:15 +0200 Subject: [PATCH 088/119] Identify and drop duplicated buildings --- src/egon/data/datasets/heat_supply/individual_heating.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index e61ba540a..a11e688de 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1620,6 +1620,14 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) delete_hp_capacity(scenario="eGon2035") + # TODO debug duplicated building_ids + duplicates = df_hp_cap_per_building_2035_db.loc[ + df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False)] + + logger.info(f"Dropped duplicated buildings: {duplicates.building_id}") + + df_hp_cap_per_building_2035_db.drop_dupliactes("building_id", inplace=True) + write_table_to_postgres( df_hp_cap_per_building_2035_db, EgonHpCapacityBuildings, From ed889b524c94c3f27b0fcd6b5556e39442f156f5 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 17:56:11 +0200 Subject: [PATCH 089/119] Extend workaround --- src/egon/data/datasets/heat_supply/individual_heating.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index a11e688de..a4e1b78c2 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1624,7 +1624,8 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): duplicates = df_hp_cap_per_building_2035_db.loc[ df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False)] - logger.info(f"Dropped duplicated buildings: {duplicates.building_id}") + logger.info(f"Dropped duplicated buildings: " + f"{duplicates.loc['building_id', 'hp_capacity']}") df_hp_cap_per_building_2035_db.drop_dupliactes("building_id", inplace=True) From 32ca398ac9a43f5393e960120528071540979181 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 17:59:37 +0200 Subject: [PATCH 090/119] Fix duplicated ids with scenario filter in pv_rooftop --- .../data/datasets/heat_supply/individual_heating.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index a4e1b78c2..93dc243be 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1038,7 +1038,8 @@ def determine_buildings_with_hp_in_mv_grid( query = session.query( egon_power_plants_pv_roof_building.building_id ).filter( - egon_power_plants_pv_roof_building.building_id.in_(building_ids) + egon_power_plants_pv_roof_building.building_id.in_(building_ids), + egon_power_plants_pv_roof_building.scenario == "eGon2035", ) buildings_with_pv = pd.read_sql( @@ -1622,10 +1623,13 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # TODO debug duplicated building_ids duplicates = df_hp_cap_per_building_2035_db.loc[ - df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False)] + df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False) + ] - logger.info(f"Dropped duplicated buildings: " - f"{duplicates.loc['building_id', 'hp_capacity']}") + logger.info( + f"Dropped duplicated buildings: " + f"{duplicates.loc['building_id', 'hp_capacity']}" + ) df_hp_cap_per_building_2035_db.drop_dupliactes("building_id", inplace=True) From d9e674bfefa7edc7bd928be0a78b9164bb72f10d Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 18:20:48 +0200 Subject: [PATCH 091/119] Bump version number to 0.0.1 --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 93dc243be..8d7ca7cfe 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -104,7 +104,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): super().__init__( name="HeatPumpsPypsaEurSec", - version="0.0.0", + version="0.0.1", dependencies=dependencies, # tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), tasks=(*dyn_parallel_tasks_pypsa_eur_sec(),), @@ -152,7 +152,7 @@ def dyn_parallel_tasks_2035(): super().__init__( name="HeatPumps2035", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=( delete_heat_peak_loads_eGon2035, @@ -166,7 +166,7 @@ class HeatPumps2050(Dataset): def __init__(self, dependencies): super().__init__( name="HeatPumps2050", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=(determine_hp_cap_buildings_eGon100RE), ) From d498ef97e2ba58358fcaf33b63c7a29778b98f30 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Fri, 21 Oct 2022 19:43:58 +0200 Subject: [PATCH 092/119] Revert "Comment fixed bug workaround" This reverts commit dac6f340be8be6be9abf152d449b7da5286a4645. --- .../heat_supply/individual_heating.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 8d7ca7cfe..0791f02bd 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1550,12 +1550,12 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) ) - # # Reduce list of decentral heating if no Peak load available - # # TODO commmented after successfull fix in #987 #990 - # # Might be fixed in #990 - # buildings_decentral_heating = catch_missing_buidings( - # buildings_decentral_heating, peak_load_2035 - # ) + # Reduce list of decentral heating if no Peak load available + # TODO maybe remove after succesfull DE run + # Might be fixed in #990 + buildings_decentral_heating = catch_missing_buidings( + buildings_decentral_heating, peak_load_2035 + ) hp_cap_per_building_2035 = ( determine_hp_cap_buildings_eGon2035_per_mvgd( @@ -1686,11 +1686,11 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): ) ) - # # Reduce list of decentral heating if no Peak load available - # # TODO commmented after successfull fix in #987 #990 - # buildings_decentral_heating = catch_missing_buidings( - # buildings_decentral_heating, peak_load_100RE - # ) + # Reduce list of decentral heating if no Peak load available + # TODO maybe remove after succesfull DE run + buildings_decentral_heating = catch_missing_buidings( + buildings_decentral_heating, peak_load_100RE + ) hp_min_cap_mv_grid_pypsa_eur_sec = ( determine_min_hp_cap_buildings_pypsa_eur_sec( From f17ed9c87a94bf4b51b4991e21486d5e3abd06d5 Mon Sep 17 00:00:00 2001 From: Julian Endres <51374526+nailend@users.noreply.github.com> Date: Mon, 24 Oct 2022 16:08:44 +0200 Subject: [PATCH 093/119] Update individual_heating.py --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 0791f02bd..c6f209407 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1453,7 +1453,7 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): ) folder = Path(".") / "input-pypsa-eur-sec" - file = folder / "minimum_hp_capacity_mv_grid_2035.csv" + file = folder / "minimum_hp_capacity_mv_grid_100RE.csv" # Create the folder, if it does not exist already if not os.path.exists(folder): os.mkdir(folder) From 94b8321a149177204375874a37c5f3f1c96b924a Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 24 Oct 2022 17:15:48 +0200 Subject: [PATCH 094/119] Bump version number to 0.0.2 --- src/egon/data/datasets/heat_supply/individual_heating.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index c6f209407..528bccced 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -104,7 +104,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): super().__init__( name="HeatPumpsPypsaEurSec", - version="0.0.1", + version="0.0.2", dependencies=dependencies, # tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), tasks=(*dyn_parallel_tasks_pypsa_eur_sec(),), @@ -152,7 +152,7 @@ def dyn_parallel_tasks_2035(): super().__init__( name="HeatPumps2035", - version="0.0.1", + version="0.0.2", dependencies=dependencies, tasks=( delete_heat_peak_loads_eGon2035, @@ -166,7 +166,7 @@ class HeatPumps2050(Dataset): def __init__(self, dependencies): super().__init__( name="HeatPumps2050", - version="0.0.1", + version="0.0.2", dependencies=dependencies, tasks=(determine_hp_cap_buildings_eGon100RE), ) From ecce8be6ecc1bb75d0f461140cc4ca6f0843f797 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 25 Oct 2022 15:45:46 +0200 Subject: [PATCH 095/119] Revert "Stop parallelization" This reverts commit b9b9016ce41b2dab77df6587a8e709e43dd8be50. --- src/egon/data/datasets.yml | 2 +- src/egon/data/datasets/heat_supply/individual_heating.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index cebc7ef98..5654f63fe 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1090,7 +1090,7 @@ emobility_mit: parallel_tasks: 10 demand_timeseries_mvgd: - parallel_tasks: 1 + parallel_tasks: 10 charging_infrastructure: original_data: diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 528bccced..b395da129 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,8 +106,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - # tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), - tasks=(*dyn_parallel_tasks_pypsa_eur_sec(),), + tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), ) @@ -156,8 +155,7 @@ def dyn_parallel_tasks_2035(): dependencies=dependencies, tasks=( delete_heat_peak_loads_eGon2035, - # {*dyn_parallel_tasks_2035()}, - *dyn_parallel_tasks_2035(), + {*dyn_parallel_tasks_2035()}, ), ) From 253645df102bd7718ba5ab427c2e13cd3a006411 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 25 Oct 2022 16:53:05 +0200 Subject: [PATCH 096/119] Fix delete hp cap problem --- .../data/datasets/heat_supply/individual_heating.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index b395da129..e954ef2a5 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,7 +106,8 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), + tasks=(delete_hp_capacity_100RE, + {*dyn_parallel_tasks_pypsa_eur_sec()},), ) @@ -155,6 +156,7 @@ def dyn_parallel_tasks_2035(): dependencies=dependencies, tasks=( delete_heat_peak_loads_eGon2035, + delete_hp_capacity_2035, {*dyn_parallel_tasks_2035()}, ), ) @@ -1324,7 +1326,6 @@ def determine_hp_cap_buildings_eGon100RE(): df_hp_cap_per_building_100RE_db["scenario"] = "eGon100RE" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - delete_hp_capacity(scenario="eGon100RE") write_table_to_postgres( df_hp_cap_per_building_100RE_db, @@ -1617,7 +1618,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - delete_hp_capacity(scenario="eGon2035") # TODO debug duplicated building_ids duplicates = df_hp_cap_per_building_2035_db.loc[ @@ -1807,6 +1807,13 @@ def delete_hp_capacity(scenario): EgonHpCapacityBuildings.scenario == scenario ).delete(synchronize_session=False) +def delete_hp_capacity_100RE(): + """Remove all hp capacities for the selected eGon100RE""" + delete_hp_capacity(scenario="eGon100RE") + +def delete_hp_capacity_2035(): + """Remove all hp capacities for the selected eGon2035""" + delete_hp_capacity(scenario="eGon2035") def delete_heat_peak_loads_eGon2035(): """Remove all heat peak loads for eGon2035. From e57f83029a166e29c840fd4148e354f8299c1ebe Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 25 Oct 2022 18:02:49 +0200 Subject: [PATCH 097/119] Add create tabel statement --- .../datasets/heat_supply/individual_heating.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index e954ef2a5..87f06f16a 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,8 +106,10 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - tasks=(delete_hp_capacity_100RE, - {*dyn_parallel_tasks_pypsa_eur_sec()},), + tasks=( + delete_hp_capacity_100RE, + {*dyn_parallel_tasks_pypsa_eur_sec()}, + ), ) @@ -1785,12 +1787,6 @@ def split_mvgds_into_bulks(n, max_n, func): func(mvgd_ids) -def create_hp_capacity_table(): - - EgonHpCapacityBuildings.__table__.drop(bind=engine, checkfirst=True) - EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - - def delete_hp_capacity(scenario): """Remove all hp capacities for the selected scenario @@ -1807,14 +1803,19 @@ def delete_hp_capacity(scenario): EgonHpCapacityBuildings.scenario == scenario ).delete(synchronize_session=False) + def delete_hp_capacity_100RE(): """Remove all hp capacities for the selected eGon100RE""" + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) delete_hp_capacity(scenario="eGon100RE") + def delete_hp_capacity_2035(): """Remove all hp capacities for the selected eGon2035""" + EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) delete_hp_capacity(scenario="eGon2035") + def delete_heat_peak_loads_eGon2035(): """Remove all heat peak loads for eGon2035. From 4f74209322e1a99c4e762b26c9a6f5991ef1f181 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 25 Oct 2022 18:29:33 +0200 Subject: [PATCH 098/119] Fix task assignement --- .../data/datasets/heat_supply/individual_heating.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 87f06f16a..6b60f4cb1 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,10 +106,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - tasks=( - delete_hp_capacity_100RE, - {*dyn_parallel_tasks_pypsa_eur_sec()}, - ), + tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), ) @@ -170,7 +167,10 @@ def __init__(self, dependencies): name="HeatPumps2050", version="0.0.2", dependencies=dependencies, - tasks=(determine_hp_cap_buildings_eGon100RE), + tasks=( + delete_hp_capacity_100RE, + determine_hp_cap_buildings_eGon100RE, + ), ) @@ -1619,8 +1619,6 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" - EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - # TODO debug duplicated building_ids duplicates = df_hp_cap_per_building_2035_db.loc[ df_hp_cap_per_building_2035_db.duplicated("building_id", keep=False) From f0e17de32a4202f349bbdabcd14863f313e34c10 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 26 Oct 2022 00:32:04 +0200 Subject: [PATCH 099/119] Fix typo --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 6b60f4cb1..67dc69fb3 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1626,7 +1626,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): logger.info( f"Dropped duplicated buildings: " - f"{duplicates.loc['building_id', 'hp_capacity']}" + f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" ) df_hp_cap_per_building_2035_db.drop_dupliactes("building_id", inplace=True) From 7580ae2f01fbc638adef1495559541b5867083e6 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 26 Oct 2022 10:31:43 +0200 Subject: [PATCH 100/119] Fix typo --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 67dc69fb3..cd8c834a3 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1629,7 +1629,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" ) - df_hp_cap_per_building_2035_db.drop_dupliactes("building_id", inplace=True) + df_hp_cap_per_building_2035_db.drop_duplicates("building_id", inplace=True) write_table_to_postgres( df_hp_cap_per_building_2035_db, From b1f497a4482a7c75c3b9152b61216cadfa9da8a6 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 26 Oct 2022 14:22:53 +0200 Subject: [PATCH 101/119] Add delete mvgd ts 2035 task --- .../heat_supply/individual_heating.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index cd8c834a3..86424a46e 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -156,6 +156,7 @@ def dyn_parallel_tasks_2035(): tasks=( delete_heat_peak_loads_eGon2035, delete_hp_capacity_2035, + delete_mvgd_ts_2035, {*dyn_parallel_tasks_2035()}, ), ) @@ -1615,6 +1616,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): # ################ export to db ####################### logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) df_hp_cap_per_building_2035_db["scenario"] = "eGon2035" @@ -1802,6 +1804,23 @@ def delete_hp_capacity(scenario): ).delete(synchronize_session=False) +def delete_mvgd_ts(scenario): + """Remove all hp capacities for the selected scenario + + Parameters + ----------- + scenario : string + Either eGon2035 or eGon100RE + + """ + + with db.session_scope() as session: + # Buses + session.query(EgonEtragoTimeseriesIndividualHeating).filter( + EgonEtragoTimeseriesIndividualHeating.scenario == scenario + ).delete(synchronize_session=False) + + def delete_hp_capacity_100RE(): """Remove all hp capacities for the selected eGon100RE""" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) @@ -1814,6 +1833,14 @@ def delete_hp_capacity_2035(): delete_hp_capacity(scenario="eGon2035") +def delete_mvgd_ts_2035(): + """Remove all mvgd ts for the selected eGon2035""" + EgonEtragoTimeseriesIndividualHeating.__table__.create( + bind=engine, checkfirst=True + ) + delete_mvgd_ts(scenario="eGon2035") + + def delete_heat_peak_loads_eGon2035(): """Remove all heat peak loads for eGon2035. From cc7fde106e51f76c1b9a8a1015eceb9194890791 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Thu, 27 Oct 2022 14:02:07 +0200 Subject: [PATCH 102/119] Fix delete table problem --- .../heat_supply/individual_heating.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 86424a46e..f707a8d48 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -106,7 +106,9 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - tasks=({*dyn_parallel_tasks_pypsa_eur_sec()},), + tasks=(delete_mvgd_ts_100RE, + delete_heat_peak_loads_100RE, + {*dyn_parallel_tasks_pypsa_eur_sec()},), ) @@ -154,7 +156,7 @@ def dyn_parallel_tasks_2035(): version="0.0.2", dependencies=dependencies, tasks=( - delete_heat_peak_loads_eGon2035, + delete_heat_peak_loads_2035, delete_hp_capacity_2035, delete_mvgd_ts_2035, {*dyn_parallel_tasks_2035()}, @@ -1736,7 +1738,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): # ################ export to db and csv ###################### logger.info(f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write data to db.") - export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=True) + export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) logger.info( f"MVGD={min(mvgd_ids)} : {max(mvgd_ids)} | Write " @@ -1841,13 +1843,26 @@ def delete_mvgd_ts_2035(): delete_mvgd_ts(scenario="eGon2035") -def delete_heat_peak_loads_eGon2035(): - """Remove all heat peak loads for eGon2035. +def delete_mvgd_ts_100RE(): + """Remove all mvgd ts for the selected eGon100RE""" + EgonEtragoTimeseriesIndividualHeating.__table__.create( + bind=engine, checkfirst=True + ) + delete_mvgd_ts(scenario="eGon100RE") + - This is not necessary for eGon100RE as these peak loads are calculated in - HeatPumpsPypsaEurSec and tables are recreated during this dataset.""" +def delete_heat_peak_loads_2035(): + """Remove all heat peak loads for eGon2035.""" with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( BuildingHeatPeakLoads.scenario == "eGon2035" ).delete(synchronize_session=False) + +def delete_heat_peak_loads_100RE(): + """Remove all heat peak loads for eGon100RE.""" + with db.session_scope() as session: + # Buses + session.query(BuildingHeatPeakLoads).filter( + BuildingHeatPeakLoads.scenario == "eGon100RE" + ).delete(synchronize_session=False) From 13cb752d37890cb9a8d61a23da4d213e703a8220 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 1 Nov 2022 15:01:12 +0100 Subject: [PATCH 103/119] Add create table for heat peak loads delete task --- src/egon/data/datasets/heat_supply/individual_heating.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index f707a8d48..0e1f1b54b 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1853,6 +1853,7 @@ def delete_mvgd_ts_100RE(): def delete_heat_peak_loads_2035(): """Remove all heat peak loads for eGon2035.""" + BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( @@ -1861,6 +1862,7 @@ def delete_heat_peak_loads_2035(): def delete_heat_peak_loads_100RE(): """Remove all heat peak loads for eGon100RE.""" + BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( From 6f66522c291c41aac2019567452b3f679d4132be Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sun, 13 Nov 2022 17:02:04 +0100 Subject: [PATCH 104/119] Update Changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66e22d1d2..7f3945b08 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -225,6 +225,8 @@ Added `#868 `_ * Write simBEV metadata to DB table `PR #978 `_ +* Heat pump desaggregation to buildings + `PR #903 `_ .. _PR #159: https://github.com/openego/eGon-data/pull/159 .. _PR #703: https://github.com/openego/eGon-data/pull/703 From 9ab875de8d6a73702ef812c5cbf0a98f68764977 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sun, 13 Nov 2022 17:17:46 +0100 Subject: [PATCH 105/119] Bump version number for CtsDemandBuildings to 0.0.1 --- .../datasets/electricity_demand_timeseries/cts_buildings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index a07b48876..ca8ed04ae 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -247,7 +247,7 @@ class CtsDemandBuildings(Dataset): def __init__(self, dependencies): super().__init__( name="CtsDemandBuildings", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=( cts_buildings, From ad7febffbc6c6bcf93fc84d11ed9a2a0a091bee5 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sun, 13 Nov 2022 17:33:59 +0100 Subject: [PATCH 106/119] Remove timit decorator --- .../datasets/heat_supply/individual_heating.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 0e1f1b54b..63fb6849c 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -195,23 +195,6 @@ def adapt_numpy_int64(numpy_int64): return AsIs(numpy_int64) -def timeit(func): - """ - Decorator for measuring function's running time. - """ - - def measure_time(*args, **kw): - start_time = time.time() - result = func(*args, **kw) - print( - "Processing time of %s(): %.2f seconds." - % (func.__qualname__, time.time() - start_time) - ) - return result - - return measure_time - - def cascade_per_technology( heat_per_mv, technologies, From e558aac5a949c51bef75c083f8edb6ea0eb1c6dd Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sun, 13 Nov 2022 17:34:25 +0100 Subject: [PATCH 107/119] Adapt docstring --- .../datasets/electricity_demand_timeseries/cts_buildings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index ca8ed04ae..682e69b1f 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -26,9 +26,9 @@ Table including the mv substation heat profile share of all selected cts buildings for scenario eGon2035 and eGon100RE. This table is created within :func:`cts_heat()` -* `demand.egon_building_peak_loads`: - Mapping of demand time series and buildings including cell_id, building - area and peak load. This table is already created within +* `demand.egon_building_electricity_peak_loads`: + Mapping of electricity demand time series and buildings including cell_id, + building area and peak load. This table is already created within :func:`hh_buildings.get_building_peak_loads()` **The following datasets from the database are mainly used for creation:** From 2e40a99be22fa0afe12f8c603c44faa9fcad3940 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Sun, 13 Nov 2022 17:53:20 +0100 Subject: [PATCH 108/119] Setup first draft for module docstring --- .../heat_supply/individual_heating.py | 93 +++++++++++++++++-- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 63fb6849c..3ae6bfbcb 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1,11 +1,89 @@ -"""The central module containing all code dealing with -individual heat supply. +"""The central module containing all code dealing with individual heat supply. +The desaggregation of heat pump capcacities takes place in three separate +datasets: 'HeatPumpsPypsaEurSec', 'HeatPumps2035', 'HeatPumps2050'. +#TODO WHY? + +The resulting data is stored in separate tables + +* `demand.egon_hp_capacity_buildings`: + This table is already created within + :func:`` +* `demand.egon_etrago_timeseries_individual_heating`: + This table is created within + :func:`` +* `demand.egon_building_heat_peak_loads`: + Mapping of heat demand time series and buildings including cell_id, + building, area and peak load. This table is created in + :func:`` + +**The following datasets from the database are mainly used for creation:** + +* `boundaries.egon_map_zensus_grid_districts`: + + +* `boundaries.egon_map_zensus_district_heating_areas`: + + +* `demand.egon_peta_heat`: + Table of annual heat load demand for residential and cts at census cell + level from peta5. +* `demand.egon_heat_timeseries_selected_profiles`: + + +* `demand.egon_heat_idp_pool`: + + +* `demand.egon_daily_heat_demand_per_climate_zone`: + + +* `boundaries.egon_map_zensus_mvgd_buildings`: + A final mapping table including all buildings used for residential and + cts, heat and electricity timeseries. Including census cells, mvgd bus_id, + building type (osm or synthetic) + +* `supply.egon_individual_heating`: + + +* `demand.egon_cts_heat_demand_building_share`: + Table including the mv substation heat profile share of all selected + cts buildings for scenario eGon2035 and eGon100RE. This table is created + within :func:`cts_heat()` + + +**What is the goal?** + + + +**What is the challenge?** + + +**How are these datasets combined?** + + +**What are central assumptions during the data processing?** + + +**Drawbacks and limitations of the data** + + + +Example Query +----- + + +Notes +----- + +This module docstring is rather a dataset documentation. Once, a decision +is made in ... the content of this module docstring needs to be moved to +docs attribute of the respective dataset class. """ + + from pathlib import Path import os import random -import time from airflow.operators.python_operator import PythonOperator from psycopg2.extensions import AsIs, register_adapter @@ -106,9 +184,11 @@ def dyn_parallel_tasks_pypsa_eur_sec(): name="HeatPumpsPypsaEurSec", version="0.0.2", dependencies=dependencies, - tasks=(delete_mvgd_ts_100RE, - delete_heat_peak_loads_100RE, - {*dyn_parallel_tasks_pypsa_eur_sec()},), + tasks=( + delete_mvgd_ts_100RE, + delete_heat_peak_loads_100RE, + {*dyn_parallel_tasks_pypsa_eur_sec()}, + ), ) @@ -1843,6 +1923,7 @@ def delete_heat_peak_loads_2035(): BuildingHeatPeakLoads.scenario == "eGon2035" ).delete(synchronize_session=False) + def delete_heat_peak_loads_100RE(): """Remove all heat peak loads for eGon100RE.""" BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) From 6e8c97e3fb53cccdd8530ac2a7b1ced6e48cc160 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 18 Nov 2022 16:40:43 +0100 Subject: [PATCH 109/119] Add function to obtain total CTS heat demand in MV grid --- .../cts_buildings.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index 682e69b1f..193374db1 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -950,6 +950,50 @@ def calc_building_amenity_share(df_cts_buildings): return df_demand_share +def get_peta_demand(mvgd, scenario): + """ + Retrieve annual peta heat demand for CTS for either + eGon2035 or eGon100RE scenario. + + Parameters + ---------- + mvgd : int + ID of substation for which to get CTS demand. + scenario : str + Possible options are eGon2035 or eGon100RE + + Returns + ------- + df_peta_demand : pd.DataFrame + Annual residential heat demand per building and scenario. Columns of + the dataframe are zensus_population_id and demand. + + """ + + with db.session_scope() as session: + query = ( + session.query( + MapZensusGridDistricts.zensus_population_id, + EgonPetaHeat.demand, + ) + .filter(MapZensusGridDistricts.bus_id == int(mvgd)) + .filter( + MapZensusGridDistricts.zensus_population_id + == EgonPetaHeat.zensus_population_id + ) + .filter( + EgonPetaHeat.sector == "service", + EgonPetaHeat.scenario == scenario, + ) + ) + + df_peta_demand = pd.read_sql( + query.statement, query.session.bind, index_col=None + ) + + return df_peta_demand + + def calc_cts_building_profiles( bus_ids, scenario, From ab717e6f8ca557b068da57b44946d960b3a4bcd7 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 18 Nov 2022 16:41:13 +0100 Subject: [PATCH 110/119] Bug fix scale CTS heat demand profile to total CTS heat demand in grid --- .../electricity_demand_timeseries/cts_buildings.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index 193374db1..c7f23bea1 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -1073,6 +1073,9 @@ def calc_cts_building_profiles( ) # Get substation cts heat load profiles of selected bus_ids + # (this profile only contains zensus cells with individual heating; + # in order to obtain a profile for the whole MV grid it is afterwards + # scaled by the grids total CTS demand from peta) with db.session_scope() as session: cells_query = ( session.query(EgonEtragoHeatCts).filter( @@ -1088,6 +1091,15 @@ def calc_cts_building_profiles( df_cts_substation_profiles.set_index("bus_id")["p_set"].to_dict(), orient="index", ) + for bus_id in bus_ids: + # get peta demand to scale load profile to + peta_cts_demand = get_peta_demand(bus_id, scenario) + scaling_factor = ( + peta_cts_demand.demand.sum() / + df_cts_substation_profiles.loc[bus_id, :].sum() + ) + # scale load profile + df_cts_substation_profiles.loc[bus_id, :] *= scaling_factor else: raise KeyError("Sector needs to be either 'electricity' or 'heat'") From a13930fb20fe8c8b43167c72b931c496e9d8052d Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 21 Nov 2022 14:05:49 +0100 Subject: [PATCH 111/119] Bug fix only scale CTS profile if it exists --- .../cts_buildings.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index c7f23bea1..4e3e389d0 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -1092,14 +1092,15 @@ def calc_cts_building_profiles( orient="index", ) for bus_id in bus_ids: - # get peta demand to scale load profile to - peta_cts_demand = get_peta_demand(bus_id, scenario) - scaling_factor = ( - peta_cts_demand.demand.sum() / - df_cts_substation_profiles.loc[bus_id, :].sum() - ) - # scale load profile - df_cts_substation_profiles.loc[bus_id, :] *= scaling_factor + if bus_id in df_cts_substation_profiles.index: + # get peta demand to scale load profile to + peta_cts_demand = get_peta_demand(bus_id, scenario) + scaling_factor = ( + peta_cts_demand.demand.sum() / + df_cts_substation_profiles.loc[bus_id, :].sum() + ) + # scale load profile + df_cts_substation_profiles.loc[bus_id, :] *= scaling_factor else: raise KeyError("Sector needs to be either 'electricity' or 'heat'") From 2f73d793b76fccd82d82b99b77d4ee4f1fc57df7 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 24 Nov 2022 17:13:13 +0100 Subject: [PATCH 112/119] Add dataset documentation --- .../heat_supply/individual_heating.py | 137 +++++++++++++++--- 1 file changed, 118 insertions(+), 19 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 3ae6bfbcb..4f24d8136 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1,21 +1,96 @@ """The central module containing all code dealing with individual heat supply. -The desaggregation of heat pump capcacities takes place in three separate -datasets: 'HeatPumpsPypsaEurSec', 'HeatPumps2035', 'HeatPumps2050'. -#TODO WHY? - -The resulting data is stored in separate tables +The following main things are done in this module: + +* .. +* Desaggregation of heat pump capacities to individual buildings +* Determination of minimum required heat pump capacity for pypsa-eur-sec + +The determination of the minimum required heat pump capacity for pypsa-eur-sec takes +place in the dataset 'HeatPumpsPypsaEurSec'. The goal is to ensure that the heat pump +capacities determined in pypsa-eur-sec are large enough to serve the heat demand of +individual buildings after the desaggregation from a few nodes in pypsa-eur-sec to the +individual buildings. +To determine minimum required heat pump capacity per building the buildings heat peak +load in the eGon100RE scenario is used (as pypsa-eur-sec serves as the scenario +generator for the eGon100RE scenario; see +:func:`determine_minimum_hp_capacity_per_building` for information on how minimum +required heat pump capacity is determined). As the heat peak load is not previously +determined, it is as well done in the course of this task. +Further, as determining heat peak load requires heat load +profiles of the buildings to be set up, this task is also utilised to set up +heat load profiles of all buildings with heat pumps within a grid in the eGon100RE +scenario used in eTraGo. +The resulting data is stored in separate tables respectively a csv file: + +* `input-pypsa-eur-sec/minimum_hp_capacity_mv_grid_100RE.csv`: + This csv file contains minimum required heat pump capacity per MV grid in MW as + input for pypsa-eur-sec. It is created within :func:`export_min_cap_to_csv`. +* `demand.egon_etrago_timeseries_individual_heating`: + This table contains aggregated heat load profiles of all buildings with heat pumps + within an MV grid in the eGon100RE scenario used in eTraGo. It is created within + :func:`individual_heating_per_mv_grid_tables`. +* `demand.egon_building_heat_peak_loads`: + Mapping of peak heat demand and buildings including cell_id, + building, area and peak load. This table is created in + :func:`delete_heat_peak_loads_100RE`. + +The desaggregation of heat pump capcacities to individual buildings takes place in two +separate datasets: 'HeatPumps2035' for eGon2035 scenario and 'HeatPumps2050' for +eGon100RE. +It is done separately because for one reason in case of the eGon100RE scenario the +minimum required heat pump capacity per building can directly be determined using the +heat peak load per building determined in the dataset 'HeatPumpsPypsaEurSec', whereas +heat peak load data does not yet exist for the eGon2035 scenario. Another reason is, +that in case of the eGon100RE scenario all buildings with individual heating have a +heat pump whereas in the eGon2035 scenario buildings are randomly selected until the +installed heat pump capacity per MV grid is met. All other buildings with individual +heating but no heat pump are assigned a gas boiler. + +In the 'HeatPumps2035' dataset the following things are done. +First, the building's heat peak load in the eGon2035 scenario is determined for sizing +the heat pumps. To this end, heat load profiles per building are set up. +Using the heat peak load per building the minimum required heat pump capacity per +building is determined (see :func:`determine_minimum_hp_capacity_per_building`). +Afterwards, the total heat pump capacity per MV grid is desaggregated to individual +buildings in the MV grid, wherefore buildings are randomly chosen until the MV grid's total +heat pump capacity is reached (see :func:`determine_buildings_with_hp_in_mv_grid`). +Buildings with PV rooftop plants are more likely to be assigned a heat pump. In case +the minimum heat pump capacity of all chosen buildings is smaller than the total +heat pump capacity of the MV grid but adding another building would exceed the total +heat pump capacity of the MV grid, the remaining capacity is distributed to all +buildings with heat pumps proportionally to the size of their respective minimum +heat pump capacity. Therefore, the heat pump capacity of a building can be larger +than the minimum required heat pump capacity. +The generated heat load profiles per building are in a last step utilised to set up +heat load profiles of all buildings with heat pumps within a grid as well as for all +buildings with a gas boiler (i.e. all buildings with decentral heating system minus +buildings with heat pump) needed in eTraGo. +The resulting data is stored in the following tables: * `demand.egon_hp_capacity_buildings`: - This table is already created within - :func:`` + This table contains the heat pump capacity of all buildings with a heat pump. + It is created within :func:`delete_hp_capacity_2035`. * `demand.egon_etrago_timeseries_individual_heating`: - This table is created within - :func:`` + This table contains aggregated heat load profiles of all buildings with heat pumps + within an MV grid as well as of all buildings with gas boilers within an MV grid in + the eGon100RE scenario used in eTraGo. It is created within + :func:`individual_heating_per_mv_grid_tables`. * `demand.egon_building_heat_peak_loads`: Mapping of heat demand time series and buildings including cell_id, building, area and peak load. This table is created in - :func:`` + :func:`delete_heat_peak_loads_2035`. + +In the 'HeatPumps2050' dataset the total heat pump capacity in each MV grid can be +directly desaggregated to individual buildings, as the building's heat peak load was +already determined in the 'HeatPumpsPypsaEurSec' dataset. Also in contrast to the +'HeatPumps2035' dataset, all buildings with decentral heating system are assigned a +heat pump, wherefore no random sampling of buildings needs to be conducted. +The resulting data is stored in the following table: + +* `demand.egon_hp_capacity_buildings`: + This table contains the heat pump capacity of all buildings with a heat pump. + It is created within :func:`delete_hp_capacity_2035`. **The following datasets from the database are mainly used for creation:** @@ -53,24 +128,48 @@ **What is the goal?** - +The goal is threefold. Primarily, heat pump capacity of individual buildings is +determined as it is necessary for distribution grid analysis. Secondly, as heat +demand profiles need to be set up during the process, the heat demand profiles of all +buildings with individual heat pumps respectively gas boilers per MV grid are set up +to be used in eTraGo. Thirdly, minimum heat pump capacity is determined as input for +pypsa-eur-sec to avoid that heat pump capacity per building is too little to meet +the heat demand after desaggregation to individual buildings. **What is the challenge?** - -**How are these datasets combined?** - +The main challenge lies in the set up of heat demand profiles per building in +:func:`aggregate_residential_and_cts_profiles()` as it takes alot of time and +in grids with a high number of buildings requires alot of RAM. Both runtime and +RAM usage needed to be improved several times. To speed up the process, tasks are set +up to run in parallel. This currently leads to alot of connections being opened and +at a certain point to a runtime error due to too many open connections. **What are central assumptions during the data processing?** +Central assumption for determining minimum heat pump capacity and desaggregating +heat pump capacity to individual buildings is that the required heat pump capacity +is determined using an approach from the +`network development plan `_ +(pp.46-47) (see :func:`determine_minimum_hp_capacity_per_building()`). There, the heat +pump capacity is determined by multiplying the heat peak +demand of the building by a minimum assumed COP of 1.7 and a flexibility factor of +24/18, taking into account that power supply of heat pumps can be interrupted for up +to six hours by the local distribution grid operator. +Another central assumption is, that buildings with PV rooftop plants are more likely +to have a heat pump than other buildings (see +:func:`determine_buildings_with_hp_in_mv_grid()` for details) **Drawbacks and limitations of the data** - - -Example Query ------ - +In the eGon2035 scenario buildings with heat pumps are selected randomly with a higher +probability for a heat pump for buildings with PV rooftop (see +:func:`determine_buildings_with_hp_in_mv_grid()` for details). +Another limitation may be the sizing of the heat pumps, as in the eGon2035 scenario +their size rigidly depends on the heat peak load and a fixed flexibility factor. During +the coldest days of the year, heat pump flexibility strongly depends on this +assumption and cannot be dynamically enlarged to provide more flexibility (or only +slightly through larger heat storage units). Notes ----- From f11b5c050adef8dd7ce814e333f03e1a9926ec7a Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 24 Nov 2022 17:51:27 +0100 Subject: [PATCH 113/119] Add function to delete pypsa eur sec csv file with minimum heat pump capacity before rerunning the task --- .../data/datasets/heat_supply/individual_heating.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 4f24d8136..2d55bbffc 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -284,6 +284,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): version="0.0.2", dependencies=dependencies, tasks=( + delete_pypsa_eur_sec_csv_file, delete_mvgd_ts_100RE, delete_heat_peak_loads_100RE, {*dyn_parallel_tasks_pypsa_eur_sec()}, @@ -1637,6 +1638,16 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): ) +def delete_pypsa_eur_sec_csv_file(): + """Delete pypsa eur sec minimum heat pump capacity csv before new run""" + + folder = Path(".") / "input-pypsa-eur-sec" + file = folder / "minimum_hp_capacity_mv_grid_100RE.csv" + if file.is_file(): + logger.info(f"Delete {file}") + os.remove(file) + + def catch_missing_buidings(buildings_decentral_heating, peak_load): """ Check for missing buildings and reduce the list of buildings with From 255f5c4303ee452203f84a646e2c93a756823abc Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 24 Nov 2022 17:52:18 +0100 Subject: [PATCH 114/119] Bug fix remove deletion of pypsa eur sec csv file from function that exports minimum heat pump capacity to csv file --- src/egon/data/datasets/heat_supply/individual_heating.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index 2d55bbffc..d26145816 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -1627,12 +1627,9 @@ def export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec): if not file.is_file(): logger.info(f"Create {file}") df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( - file, mode="w", header=False + file, mode="w", header=True ) else: - logger.info(f"Remove {file}") - os.remove(file) - logger.info(f"Create {file}") df_hp_min_cap_mv_grid_pypsa_eur_sec.to_csv( file, mode="a", header=False ) From eca1a727ce2e7f43616358c8df75bc6b37afd7d0 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 24 Nov 2022 18:08:07 +0100 Subject: [PATCH 115/119] Adapt function creating residential heat load profiles to reduce memory usage --- .../heat_supply/individual_heating.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index d26145816..608a73a7e 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -207,6 +207,9 @@ from egon.data.datasets.electricity_demand_timeseries.tools import ( write_table_to_postgres, ) +from egon.data.datasets.emobility.motorized_individual_travel.helpers import ( + reduce_mem_usage +) from egon.data.datasets.heat_demand import EgonPetaHeat from egon.data.datasets.heat_demand_timeseries.daily import ( EgonDailyHeatDemandPerClimateZone, @@ -789,6 +792,7 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): ] df_peta_demand = get_peta_demand(mvgd, scenario) + df_peta_demand = reduce_mem_usage(df_peta_demand) # TODO maybe return empty dataframe if df_peta_demand.empty: @@ -812,12 +816,19 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): left=df_peta_demand, right=df_profiles_ids, on="zensus_population_id" ) + df_profile_merge.demand = df_profile_merge.demand.div(df_profile_merge.buildings) + df_profile_merge.drop('buildings', axis='columns', inplace=True) + # Merge daily demand to daily profile ids by zensus_population_id and day df_profile_merge = pd.merge( left=df_profile_merge, right=df_daily_demand_share, on=["zensus_population_id", "day_of_year"], ) + df_profile_merge.demand = df_profile_merge.demand.mul( + df_profile_merge.daily_demand_share) + df_profile_merge.drop('daily_demand_share', axis='columns', inplace=True) + df_profile_merge = reduce_mem_usage(df_profile_merge) # Merge daily profiles by profile id df_profile_merge = pd.merge( @@ -826,14 +837,15 @@ def calc_residential_heat_profiles_per_mvgd(mvgd, scenario): left_on="selected_idp_profiles", right_index=True, ) + df_profile_merge = reduce_mem_usage(df_profile_merge) - # Scale profiles - df_profile_merge["demand_ts"] = ( - df_profile_merge["idp"] - .mul(df_profile_merge["daily_demand_share"]) - .mul(df_profile_merge["demand"]) - .div(df_profile_merge["buildings"]) - ) + df_profile_merge.demand = df_profile_merge.demand.mul( + df_profile_merge.idp.astype(float)) + df_profile_merge.drop('idp', axis='columns', inplace=True) + + df_profile_merge.rename({'demand': 'demand_ts'}, axis='columns', inplace=True) + + df_profile_merge = reduce_mem_usage(df_profile_merge) return df_profile_merge.loc[:, columns] From 3111d689f29dea1ac03521f6d8ad517dc25f3297 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 30 Nov 2022 10:53:24 +0100 Subject: [PATCH 116/119] Reduce number of parallel tasks to avoid connection error --- src/egon/data/datasets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index 355453721..d5b63e565 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1093,7 +1093,7 @@ emobility_mit: parallel_tasks: 10 demand_timeseries_mvgd: - parallel_tasks: 10 + parallel_tasks: 5 charging_infrastructure: original_data: From 7aa0aef6d454c0397f1b6904426ed42b4e31833e Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 1 Dec 2022 08:51:27 +0100 Subject: [PATCH 117/119] Add dependency from heat pumps 2035 to heat demand time series for etrago --- src/egon/data/airflow/dags/pipeline.py | 69 +++++++++++++------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 5ae6b5d7a..af48e11ba 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -443,7 +443,7 @@ run_pypsaeursec, foreign_lines, insert_hydrogen_buses, - create_gas_polygons_egon100RE + create_gas_polygons_egon100RE, ] ) @@ -557,6 +557,32 @@ # CHP to eTraGo chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago]) + # Storages to eTraGo + storage_etrago = StorageEtrago( + dependencies=[pumped_hydro, scenario_parameters, setup_etrago] + ) + + mit_charging_infrastructure = MITChargingInfrastructure( + dependencies=[mv_grid_districts, hh_demand_buildings_setup] + ) + + # eMobility: heavy duty transport + heavy_duty_transport = HeavyDutyTransport( + dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035] + ) + + # Heat pump disaggregation for eGon2035 + heat_pumps_2035 = HeatPumps2035( + dependencies=[ + cts_demand_buildings, + DistrictHeatingAreas, + heat_supply, + heat_time_series, + heat_pumps_pypsa_eur_sec, + tasks["power_plants.pv_rooftop_buildings.pv-rooftop-to-buildings"], + ] + ) + # HTS to eTraGo table hts_etrago_table = HtsEtragoTable( dependencies=[ @@ -564,12 +590,17 @@ heat_etrago, heat_time_series, mv_grid_districts, + heat_pumps_2035, ] ) - # Storages to eTraGo - storage_etrago = StorageEtrago( - dependencies=[pumped_hydro, scenario_parameters, setup_etrago] + # Heat pump disaggregation for eGon100RE + heat_pumps_2050 = HeatPumps2050( + dependencies=[ + run_pypsaeursec, + heat_pumps_pypsa_eur_sec, + heat_supply, + ] ) # eMobility: motorized individual travel @@ -597,36 +628,6 @@ ] ) - mit_charging_infrastructure = MITChargingInfrastructure( - dependencies=[mv_grid_districts, hh_demand_buildings_setup] - ) - - # eMobility: heavy duty transport - heavy_duty_transport = HeavyDutyTransport( - dependencies=[vg250, setup_etrago, create_gas_polygons_egon2035] - ) - - # Heat pump disaggregation for eGon2035 - heat_pumps_2035 = HeatPumps2035( - dependencies=[ - cts_demand_buildings, - DistrictHeatingAreas, - heat_supply, - heat_time_series, - heat_pumps_pypsa_eur_sec, - tasks["power_plants.pv_rooftop_buildings.pv-rooftop-to-buildings"], - ] - ) - - # Heat pump disaggregation for eGon100RE - heat_pumps_2050 = HeatPumps2050( - dependencies=[ - run_pypsaeursec, - heat_pumps_pypsa_eur_sec, - heat_supply, - ] - ) - # ########## Keep this dataset at the end # Sanity Checks sanity_checks = SanityChecks( From 7b7aeb136cab9fcd4ae76a7c741951bbd937385a Mon Sep 17 00:00:00 2001 From: nesnoj Date: Mon, 12 Dec 2022 14:19:20 +0100 Subject: [PATCH 118/119] Revert "Reduce number of parallel tasks to avoid connection error" This reverts commit 3111d689f29dea1ac03521f6d8ad517dc25f3297. --- src/egon/data/datasets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index d5b63e565..355453721 100755 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1093,7 +1093,7 @@ emobility_mit: parallel_tasks: 10 demand_timeseries_mvgd: - parallel_tasks: 5 + parallel_tasks: 10 charging_infrastructure: original_data: From 6a588441794a93b2769b223c836d3fbca8683289 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Tue, 10 Jan 2023 14:10:51 +0100 Subject: [PATCH 119/119] Remove old adhoc fixes for #989 --- .../cts_buildings.py | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index f478f1507..3ac66c1ad 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -918,33 +918,6 @@ def calc_building_amenity_share(df_cts_buildings): "building_amenity_share" ].multiply(df_demand_share["cell_share"]) - # # Fix #989 - # # Mismatched bus_id - # # May result in failing sanity checks - # from saio.boundaries import egon_map_zensus_buildings_filtered_all - # - # with db.session_scope() as session: - # query = session.query( - # egon_map_zensus_buildings_filtered_all.id, - # MapZensusGridDistricts.bus_id, - # ).filter( - # egon_map_zensus_buildings_filtered_all.id.in_( - # df_cts_buildings.id.values - # ), - # MapZensusGridDistricts.zensus_population_id - # == egon_map_zensus_buildings_filtered_all.zensus_population_id, - # ) - # - # df_map_bus_id = pd.read_sql( - # query.statement, session.connection(), index_col=None - # ) - # - # df_demand_share = pd.merge( - # left=df_demand_share.drop(columns="bus_id"), - # right=df_map_bus_id, - # on="id", - # ) - # only pass selected columns df_demand_share = df_demand_share[ ["id", "bus_id", "scenario", "profile_share"] @@ -1402,30 +1375,6 @@ def adapt_numpy_int64(numpy_int64): columns={"index": "serial"} ) - # # Fix #989 - # # Mismatched zensus population id - # from saio.boundaries import egon_map_zensus_buildings_filtered_all - # - # with db.session_scope() as session: - # query = session.query( - # egon_map_zensus_buildings_filtered_all.id, - # egon_map_zensus_buildings_filtered_all.zensus_population_id, - # ).filter( - # egon_map_zensus_buildings_filtered_all.id.in_( - # df_cts_buildings.id.values - # ) - # ) - # - # df_map_zensus_population_id = pd.read_sql( - # query.statement, session.connection(), index_col=None - # ) - # - # df_cts_buildings = pd.merge( - # left=df_cts_buildings.drop(columns="zensus_population_id"), - # right=df_map_zensus_population_id, - # on="id", - # ) - # Write table to db for debugging and postprocessing write_table_to_postgis( df_cts_buildings,