Add missing gas stores in eGon100RE

CHANGELOG.rst

@@ -436,16 +436,18 @@ Changed
   `#787 <https://github.com/openego/eGon-data/issues/787>`_
 * Update pypsa-eur-sec fork and store national demand time series
   `#402 <https://github.com/openego/eGon-data/issues/402>`_
+* Add biogas generators ton eGon100RE
+  `#402 <https://github.com/openego/eGon-data/issues/844>`_
 * Move and merge the two assign_gas_bus_id functions to a central place
   `#797 <https://github.com/openego/eGon-data/issues/797>`_
 * Add coordinates to non AC buses abroad in eGon100RE
   `#803 <https://github.com/openego/eGon-data/issues/803>`_
 * Integrate additional industrial electricity demands for eGon100RE
   `#817 <https://github.com/openego/eGon-data/issues/817>`_
-* Set non extendable gas components from p-e-s as so for eGon100RE
-  `#877 <https://github.com/openego/eGon-data/issues/877>`_
 * Integrate new data bundle using zenodo sandbox
   `#866 <https://github.com/openego/eGon-data/issues/866>`_
+* Set non extendable gas links from p-e-s as so for eGon100RE
+  `#877 <https://github.com/openego/eGon-data/issues/877>`_
 * Add noflex scenario for motorized individual travel
   `#821 <https://github.com/openego/eGon-data/issues/821>`_
 * Allocate PV home batteries to mv grid districts
@@ -456,6 +458,10 @@ Changed
   `#882 <https://github.com/openego/eGon-data/issues/882>`_
 * Insert crossboarding gas pipeline with Germany in eGon100RE
   `#881 <https://github.com/openego/eGon-data/issues/881>`_
+* Harmonize H2 carrier names in eGon100RE
+  `#929 <https://github.com/openego/eGon-data/issues/929>`_
+* Add missing gas stores in eGon100RE
+  `#779 <https://github.com/openego/eGon-data/issues/779>`_
 * Rename noflex to lowflex scenario for motorized individual travel
   `#921 <https://github.com/openego/eGon-data/issues/921>`_
 * Update creation of heat demand timeseries

src/egon/data/airflow/dags/pipeline.py

@@ -7,7 +7,6 @@
 from egon.data.datasets import database
 from egon.data.datasets.calculate_dlr import Calculate_dlr
 from egon.data.datasets.ch4_prod import CH4Production
-from egon.data.datasets.ch4_storages import CH4Storages
 from egon.data.datasets.chp import Chp
 from egon.data.datasets.chp_etrago import ChpEtrago
 from egon.data.datasets.data_bundle import DataBundle
@@ -43,6 +42,7 @@
 from egon.data.datasets.gas_areas import GasAreaseGon100RE, GasAreaseGon2035
 from egon.data.datasets.gas_grid import GasNodesAndPipes
 from egon.data.datasets.gas_neighbours import GasNeighbours
+from egon.data.datasets.gas_stores_germany import GasStores
 from egon.data.datasets.heat_demand import HeatDemandImport
 from egon.data.datasets.heat_demand_europe import HeatDemandEurope
 from egon.data.datasets.heat_demand_timeseries import HeatTimeSeries
@@ -454,9 +454,13 @@
         dependencies=[create_gas_polygons_egon2035]
     )
 
-    # Import CH4 storages
-    insert_data_ch4_storages = CH4Storages(
-        dependencies=[create_gas_polygons_egon2035]
+    # Import non extendable gas storages
+    insert_data_gas_stores = GasStores(
+        dependencies=[
+            create_gas_polygons_egon2035,
+            insert_hydrogen_buses,
+            create_gas_polygons_egon100RE,
+        ]
     )
 
     # Assign industrial gas demand eGon2035
@@ -626,7 +630,7 @@
             insert_h2_to_ch4_grid_links,
             create_gas_polygons_egon100RE,
             gas_production_insert_data,
-            insert_data_ch4_storages,
+            insert_data_gas_stores,
         ]
     )
 

src/egon/data/datasets.yml

@@ -955,6 +955,9 @@ gas_prod:
     stores:
       schema: 'grid'
       table: 'egon_etrago_generator'
+    scenario_parameters:
+      schema: 'scenario'
+      table: 'egon_scenario_parameters'
 
 weather_BusID:
   sources:
@@ -1081,6 +1084,9 @@ gas_neighbours:
     links:
       schema: 'grid'
       table: 'egon_etrago_link'
+    stores:
+      schema: 'grid'
+      table: 'egon_etrago_store'
   targets:
     generators:
       schema: 'grid'

src/egon/data/datasets/ch4_prod.py

@@ -14,6 +14,7 @@
 from pathlib import Path
 from urllib.request import urlretrieve
 import ast
+import json
 
 import geopandas as gpd
 import numpy as np
@@ -22,6 +23,7 @@
 from egon.data import config, db
 from egon.data.config import settings
 from egon.data.datasets import Dataset
+from egon.data.datasets.pypsaeursec import read_network
 from egon.data.datasets.scenario_parameters import get_sector_parameters
 
 
@@ -45,18 +47,18 @@ class CH4Production(Dataset):
     name: str = "CH4Production"
     #:
 
-    version: str = "0.0.7"
+    version: str = "0.0.8"
 
     def __init__(self, dependencies):
         super().__init__(
             name=self.name,
             version=self.version,
             dependencies=dependencies,
-            tasks=(import_gas_generators),
+            tasks=(insert_ch4_generators),
         )
 
 
-def load_NG_generators(scn_name):
+def load_NG_generators(scn_name="eGon2035"):
     """
     Define the fossil CH4 production units in Germany
 
@@ -165,6 +167,28 @@ def load_NG_generators(scn_name):
     return NG_generators_list
 
 
+def download_biogas_data():
+    """Download the biogas production units data in Germany
+
+    Parameters
+    ----------
+    None
+
+    Returns
+    -------
+    None
+
+    """
+    basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
+    url = (
+        "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/"
+        + basename
+    )
+    target_file = Path(".") / "datasets" / "gas_data" / basename
+
+    urlretrieve(url, target_file)
+
+
 def load_biogas_generators(scn_name):
     """
     Define the biogas production units in Germany
@@ -190,16 +214,9 @@ def load_biogas_generators(scn_name):
     # read carrier information from scnario parameter data
     scn_params = get_sector_parameters("gas", scn_name)
 
-    # Download file
     basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
-    url = (
-        "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/"
-        + basename
-    )
     target_file = Path(".") / "datasets" / "gas_data" / basename
 
-    urlretrieve(url, target_file)
-
     # Read-in data from csv-file
     biogas_generators_list = pd.read_excel(
         target_file,
@@ -245,7 +262,7 @@ def load_biogas_generators(scn_name):
         )
 
         sql = """SELECT *
-            FROM grid.egon_biogas_generator, boundaries.vg250_sta_union  as vg
+            FROM grid.egon_biogas_generator, boundaries.vg250_sta_union as vg
             WHERE ST_Transform(vg.geometry,4326) && egon_biogas_generator.geom
             AND ST_Contains(ST_Transform(vg.geometry,4326), egon_biogas_generator.geom)"""
 
@@ -280,7 +297,7 @@ def load_biogas_generators(scn_name):
     return biogas_generators_list
 
 
-def import_gas_generators(scn_name="eGon2035"):
+def import_gas_generators(scn_name):
     """
     Insert list of gas production units into the database
 
@@ -317,6 +334,8 @@ def import_gas_generators(scn_name="eGon2035"):
     None
 
     """
+    carrier = "CH4"
+
     # Connect to local database
     engine = db.engine()
 
@@ -328,25 +347,30 @@ def import_gas_generators(scn_name="eGon2035"):
     db.execute_sql(
         f"""
         DELETE FROM {target['stores']['schema']}.{target['stores']['table']}
-        WHERE "carrier" = 'CH4' AND
+        WHERE "carrier" = '{carrier}' AND
         scn_name = '{scn_name}' AND bus not IN (
             SELECT bus_id FROM {source['buses']['schema']}.{source['buses']['table']}
             WHERE scn_name = '{scn_name}' AND country != 'DE'
         );
         """
     )
 
-    CH4_generators_list = pd.concat(
-        [load_NG_generators(scn_name), load_biogas_generators(scn_name)]
-    )
+    if scn_name == "eGon2035":
+        CH4_generators_list = pd.concat(
+            [load_NG_generators(scn_name), load_biogas_generators(scn_name)]
+        )
+
+    elif scn_name == "eGon100RE":
+        CH4_generators_list = load_biogas_generators(scn_name)
+        overwrite_max_gas_generation_overtheyear(scn_name)
 
     # Add missing columns
-    c = {"scn_name": scn_name, "carrier": "CH4"}
+    c = {"scn_name": scn_name, "carrier": carrier}
     CH4_generators_list = CH4_generators_list.assign(**c)
 
     # Match to associated CH4 bus
     CH4_generators_list = db.assign_gas_bus_id(
-        CH4_generators_list, scn_name, "CH4"
+        CH4_generators_list, scn_name, carrier
     )
 
     # Remove useless columns
@@ -374,3 +398,63 @@ def import_gas_generators(scn_name="eGon2035"):
         index=False,
         if_exists="append",
     )
+
+
+def overwrite_max_gas_generation_overtheyear(scn_name):
+    """Overright max_gas_generation_overtheyear in scenario parameter table
+
+    Overright max_gas_generation_overtheyear in scenario parameter
+    table if the value of this parameter has changed in the p-e-s run.
+
+    Parameters
+    ----------
+    scn_name : str
+        Name of the scenario
+
+    """
+    execute_pypsa_eur_sec = True  # False
+
+    # Select source and target from dataset configuration
+    target = config.datasets()["gas_prod"]["target"]
+
+    if execute_pypsa_eur_sec:
+        n = read_network()
+        max_value = n.stores[n.stores["carrier"] == "biogas"].loc[
+            "DE0 0 biogas", "e_initial"
+        ]
+
+        parameters = db.select_dataframe(
+            f"""
+            SELECT *
+            FROM {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
+            WHERE name = '{scn_name}'
+            """
+        )
+
+        gas_param = parameters.loc[0, "gas_parameters"]
+        gas_param["max_gas_generation_overtheyear"] = {"biogas": max_value}
+        gas_param = json.dumps(gas_param)
+
+        # Update data in db
+        db.execute_sql(
+            f"""
+        UPDATE {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
+        SET gas_parameters = '{gas_param}'
+        WHERE name = '{scn_name}';
+        """
+        )
+
+
+def insert_ch4_generators():
+    """Insert gas production units in database for both scenarios
+
+    Parameters
+    ----------
+    None
+
+    Returns
+    -------
+    None
+    """
+    import_gas_generators("eGon2035")
+    import_gas_generators("eGon100RE")