Merge pull request #526 from openego/features/#525-loadareas-to-datasets

Features/#525 loadareas to datasets

CHANGELOG.rst

@@ -248,6 +248,8 @@ Changed
   `#484 <https://github.com/openego/eGon-data/issues/484>`_
 * Migrate dlr script to datasets
   `#508 <https://github.com/openego/eGon-data/issues/508>`_
+* Migrate loadarea scripts to datasets
+  `#525 <https://github.com/openego/eGon-data/issues/525>`_
 * Migrate plot.py to dataset of district heating areas
   `#527 <https://github.com/openego/eGon-data/issues/527>`_
 * Migrate substation scripts to datasets

src/egon/data/airflow/dags/pipeline.py

@@ -40,6 +40,7 @@
 from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
 from egon.data.datasets.industry import IndustrialDemandCurves
+from egon.data.datasets.loadarea import LoadArea
 from egon.data.datasets.mastr import mastr_data_setup
 from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup
 from egon.data.datasets.osm import OpenStreetMap
@@ -62,7 +63,6 @@
 import egon.data.datasets.gas_grid as gas_grid
 import egon.data.importing.zensus as import_zs
 import egon.data.processing.gas_areas as gas_areas
-import egon.data.processing.loadarea as loadarea
 import egon.data.processing.power_to_h2 as power_to_h2
 
 
@@ -263,21 +263,7 @@
     )
 
     # Extract landuse areas from osm data set
-    create_landuse_table = PythonOperator(
-        task_id="create-landuse-table",
-        python_callable=loadarea.create_landuse_table,
-    )
-
-    landuse_extraction = PostgresOperator(
-        task_id="extract-osm_landuse",
-        sql=resources.read_text(loadarea, "osm_landuse_extraction.sql"),
-        postgres_conn_id="egon_data",
-        autocommit=True,
-    )
-    setup >> create_landuse_table
-    create_landuse_table >> landuse_extraction
-    osm_add_metadata >> landuse_extraction
-    vg250_clean_and_prepare >> landuse_extraction
+    load_area = LoadArea(dependencies=[osm, vg250])
 
     # Import weather data
     weather_data = WeatherData(
@@ -382,7 +368,7 @@
             industrial_sites,
             demandregio_demand_cts_ind,
             osm,
-            landuse_extraction,
+            load_area,
         ]
     )
 

src/egon/data/processing/loadarea/__init__.py → src/egon/data/datasets/loadarea/__init__.py

@@ -2,13 +2,16 @@
 extraction.
 """
 
-from egon.data import db
-import egon.data.config
-
-from sqlalchemy import Column, Float, Integer, String
+from airflow.operators.postgres_operator import PostgresOperator
 from geoalchemy2.types import Geometry
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, Float, Integer, String
 from sqlalchemy.dialects.postgresql import HSTORE
+from sqlalchemy.ext.declarative import declarative_base
+import importlib_resources as resources
+
+from egon.data import db
+from egon.data.datasets import Dataset
+import egon.data.config
 
 # will be later imported from another file ###
 Base = declarative_base()
@@ -28,6 +31,26 @@ class OsmPolygonUrban(Base):
     geom = Column(Geometry("MultiPolygon", 3035))
 
 
+class LoadArea(Dataset):
+    def __init__(self, dependencies):
+        super().__init__(
+            name="LoadArea",
+            version="0.0.0",
+            dependencies=dependencies,
+            tasks=(
+                create_landuse_table,
+                PostgresOperator(
+                    task_id="osm_landuse_extraction",
+                    sql=resources.read_text(
+                        __name__, "osm_landuse_extraction.sql"
+                    ),
+                    postgres_conn_id="egon_data",
+                    autocommit=True,
+                ),
+            ),
+        )
+
+
 def create_landuse_table():
     """Create tables for landuse data
     Returns