Reduce the number of created connections

src/egon/data/datasets/DSM_cts_ind.py

@@ -778,12 +778,11 @@ def delete_dsm_entries(carrier):
         db.execute_sql(sql)
 
     def dsm_cts_ind(
-        con=db.engine(),
+        con=None,
         cts_cool_vent_ac_share=0.22,
         ind_cool_vent_share=0.039,
         ind_vent_share=0.017,
     ):
-
         """
         Execute methodology to create and implement components for DSM considering
         a) CTS per osm-area: combined potentials of cooling, ventilation and air conditioning
@@ -805,6 +804,9 @@ def dsm_cts_ind(
 
         """
 
+        if con is None:
+            con = db.engine()
+
         # CTS per osm-area: cooling, ventilation and air conditioning
 
         print(" ")

src/egon/data/datasets/chp/__init__.py

@@ -3,14 +3,16 @@
 (CHP) plants.
 """
 
+from pathlib import Path
+
 from geoalchemy2 import Geometry
 from shapely.ops import nearest_points
 from sqlalchemy import Boolean, Column, Float, Integer, Sequence, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
 import geopandas as gpd
 import pandas as pd
+import pypsa
 
 from egon.data import config, db
 from egon.data.datasets import Dataset
@@ -19,6 +21,7 @@
     assign_use_case,
     existing_chp_smaller_10mw,
     extension_per_federal_state,
+    extension_to_areas,
     select_target,
 )
 from egon.data.datasets.power_plants import (
@@ -27,9 +30,6 @@
     filter_mastr_geometry,
     scale_prox2now,
 )
-import pypsa
-from egon.data.datasets.chp.small_chp import extension_to_areas
-from pathlib import Path
 
 Base = declarative_base()
 
@@ -130,14 +130,19 @@ def nearest(
     return value
 
 
-def assign_heat_bus(scenario="eGon2035"):
+@db.session_scoped
+def assign_heat_bus(scenario="eGon2035", session=None):
     """Selects heat_bus for chps used in district heating.
 
     Parameters
     ----------
     scenario : str, optional
         Name of the corresponding scenario. The default is 'eGon2035'.
 
+    session : sqlalchemy.orm.Session
+        The session used in this function. Can be ignored because it will be
+        supplied automatically.
+
     Returns
     -------
     None.
@@ -192,7 +197,6 @@ def assign_heat_bus(scenario="eGon2035"):
     )
 
     # Insert district heating CHP with heat_bus_id
-    session = sessionmaker(bind=db.engine())()
     for i, row in chp.iterrows():
         if row.carrier != "biomass":
             entry = EgonChp(
@@ -226,17 +230,21 @@ def assign_heat_bus(scenario="eGon2035"):
                 geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})",
             )
         session.add(entry)
-    session.commit()
 
 
-def insert_biomass_chp(scenario):
+@db.session_scoped
+def insert_biomass_chp(scenario, session=None):
     """Insert biomass chp plants of future scenario
 
     Parameters
     ----------
     scenario : str
         Name of scenario.
 
+    session : sqlalchemy.orm.Session
+        The session used in this function. Can be ignored because it will be
+        supplied automatically.
+
     Returns
     -------
     None.
@@ -283,7 +291,6 @@ def insert_biomass_chp(scenario):
     mastr_loc = assign_use_case(mastr_loc, cfg["sources"])
 
     # Insert entries with location
-    session = sessionmaker(bind=db.engine())()
     for i, row in mastr_loc.iterrows():
         if row.ThermischeNutzleistung > 0:
             entry = EgonChp(
@@ -303,7 +310,6 @@ def insert_biomass_chp(scenario):
                 geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})",
             )
             session.add(entry)
-    session.commit()
 
 
 def insert_chp_egon2035():

src/egon/data/datasets/chp/match_nep.py

@@ -2,7 +2,6 @@
 The module containing all code dealing with large chp from NEP list.
 """
 
-from sqlalchemy.orm import sessionmaker
 import geopandas
 import pandas as pd
 
@@ -312,7 +311,8 @@ def match_nep_chp(
 
 
 ################################################### Final table ###################################################
-def insert_large_chp(sources, target, EgonChp):
+@db.session_scoped
+def insert_large_chp(sources, target, EgonChp, session=None):
     # Select CHP from NEP list
     chp_NEP = select_chp_from_nep(sources)
 
@@ -516,7 +516,6 @@ def insert_large_chp(sources, target, EgonChp):
     )
 
     # Insert into target table
-    session = sessionmaker(bind=db.engine())()
     for i, row in insert_chp.iterrows():
         entry = EgonChp(
             sources={
@@ -536,6 +535,5 @@ def insert_large_chp(sources, target, EgonChp):
             geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})",
         )
         session.add(entry)
-    session.commit()
 
     return MaStR_konv

src/egon/data/datasets/chp/small_chp.py

@@ -1,7 +1,6 @@
 """
 The module containing all code dealing with chp < 10MW.
 """
-from sqlalchemy.orm import sessionmaker
 import geopandas as gpd
 import numpy as np
 
@@ -13,7 +12,8 @@
 )
 
 
-def insert_mastr_chp(mastr_chp, EgonChp):
+@db.session_scoped
+def insert_mastr_chp(mastr_chp, EgonChp, session=None):
     """Insert MaStR data from exising CHPs into database table
 
     Parameters
@@ -22,14 +22,16 @@ def insert_mastr_chp(mastr_chp, EgonChp):
         List of existing CHPs in MaStR.
     EgonChp : class
         Class definition of daabase table for CHPs
+    session : sqlalchemy.orm.Session
+        The session inside which this function operates. Ignore this, because
+        it will be supplied automatically.
 
     Returns
     -------
     None.
 
     """
 
-    session = sessionmaker(bind=db.engine())()
     for i, row in mastr_chp.iterrows():
         entry = EgonChp(
             sources={
@@ -49,7 +51,6 @@ def insert_mastr_chp(mastr_chp, EgonChp):
             geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})",
         )
         session.add(entry)
-    session.commit()
 
 
 def existing_chp_smaller_10mw(sources, MaStR_konv, EgonChp):
@@ -100,6 +101,7 @@ def existing_chp_smaller_10mw(sources, MaStR_konv, EgonChp):
         insert_mastr_chp(mastr_chp, EgonChp)
 
 
+@db.session_scoped
 def extension_to_areas(
     areas,
     additional_capacity,
@@ -108,6 +110,7 @@ def extension_to_areas(
     EgonChp,
     district_heating=True,
     scenario="eGon2035",
+    session=None,
 ):
     """Builds new CHPs on potential industry or district heating areas.
 
@@ -151,14 +154,15 @@ def extension_to_areas(
         ORM-class definition of CHP database-table.
     district_heating : boolean, optional
         State if the areas are district heating areas. The default is True.
+    session : sqlalchemy.orm.Session
+        The session inside which this function operates. Ignore this, because
+        it will be supplied automatically.
 
     Returns
     -------
     None.
 
     """
-    session = sessionmaker(bind=db.engine())()
-
     np.random.seed(seed=config.settings()["egon-data"]["--random-seed"])
 
     # Add new CHP as long as the additional capacity is not reached

src/egon/data/datasets/electrical_neighbours.py

@@ -3,17 +3,17 @@
 
 import zipfile
 
-import geopandas as gpd
-import pandas as pd
 from shapely.geometry import LineString
 from sqlalchemy.orm import sessionmaker
+import geopandas as gpd
+import pandas as pd
 
-import egon.data.datasets.etrago_setup as etrago
-import egon.data.datasets.scenario_parameters.parameters as scenario_parameters
 from egon.data import config, db
 from egon.data.datasets import Dataset
 from egon.data.datasets.fill_etrago_gen import add_marginal_costs
 from egon.data.datasets.scenario_parameters import get_sector_parameters
+import egon.data.datasets.etrago_setup as etrago
+import egon.data.datasets.scenario_parameters.parameters as scenario_parameters
 
 
 class ElectricalNeighbours(Dataset):
@@ -961,6 +961,7 @@ def insert_generators(capacities):
 
         session.add(entry)
         session.commit()
+    session.close()
 
     # assign generators time-series data
     renew_carriers_2035 = ["wind_onshore", "wind_offshore", "solar"]
@@ -1020,9 +1021,11 @@ def insert_generators(capacities):
 
         session.add(entry)
         session.commit()
+    session.close()
 
 
-def insert_storage(capacities):
+@db.session_scoped
+def insert_storage(capacities, session=None):
     """Insert storage units for foreign countries based on TYNDP-data
 
     Parameters
@@ -1052,7 +1055,8 @@ def insert_storage(capacities):
         """
     )
 
-    # Add missing information suitable for eTraGo selected from scenario_parameter table
+    # Add missing information suitable for eTraGo selected from
+    # scenario_parameter table
     parameters_pumped_hydro = scenario_parameters.electricity("eGon2035")[
         "efficiency"
     ]["pumped_hydro"]
@@ -1078,9 +1082,12 @@ def insert_storage(capacities):
     )
 
     # Add columns for additional parameters to df
-    store["dispatch"], store["store"], store["standing_loss"], store[
-        "max_hours"
-    ] = (None, None, None, None)
+    (
+        store["dispatch"],
+        store["store"],
+        store["standing_loss"],
+        store["max_hours"],
+    ) = (None, None, None, None)
 
     # Insert carrier specific parameters
 
@@ -1093,7 +1100,6 @@ def insert_storage(capacities):
         ] = parameters_pumped_hydro[x]
 
     # insert data
-    session = sessionmaker(bind=db.engine())()
     for i, row in store.iterrows():
         entry = etrago.EgonPfHvStorage(
             scn_name="eGon2035",
@@ -1153,7 +1159,8 @@ def tyndp_generation():
     insert_storage(capacities)
 
 
-def tyndp_demand():
+@db.session_scoped
+def tyndp_demand(session=None):
     """Copy load timeseries data from TYNDP 2020.
     According to NEP 2021, the data for 2030 and 2040 is interpolated linearly.
 
@@ -1182,10 +1189,6 @@ def tyndp_demand():
         """
     )
 
-    # Connect to database
-    engine = db.engine()
-    session = sessionmaker(bind=engine)()
-
     nodes = [
         "AT00",
         "BE00",

src/egon/data/datasets/electricity_demand/__init__.py

@@ -21,7 +21,6 @@
 
 # will be later imported from another file ###
 Base = declarative_base()
-engine = db.engine()
 
 
 class HouseholdElectricityDemand(Dataset):
@@ -100,10 +99,11 @@ def get_annual_household_el_demand_cells():
                 == HouseholdElectricityProfilesInCensusCells.cell_id
             )
             .order_by(HouseholdElectricityProfilesOfBuildings.id)
+            .all()
         )
 
-    df_buildings_and_profiles = pd.read_sql(
-        cells_query.statement, cells_query.session.bind, index_col="id"
+    df_buildings_and_profiles = pd.DataFrame.from_records(
+        [db.asdict(row) for row in cells_query], index="id"
     )
 
     # Read demand profiles from egon-data-bundle