add lovac queries and dbt configuration

Signed-off-by: Raphaël Courivaud <[email protected]>

.gitignore

@@ -38,3 +38,5 @@ yarn-error.log*
 .pyc
 __pycache__
 analytics/dagster/storage/
+.duckdb
+.jsonl

analytics/dagster/requirements.txt

@@ -1,3 +1,6 @@
+dagster-dbt
+dbt-duckdb
+
 matplotlib
 pandas
-requests
+requests

analytics/dagster/src/zlv_dbt/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.dagster]
+module_name = "zlv_dbt.definitions"
+code_location_name = "zlv_dbt"

analytics/dagster/src/zlv_dbt/setup.py

@@ -0,0 +1,18 @@
+from setuptools import find_packages, setup
+
+setup(
+    name="zlv_dbt",
+    version="0.0.1",
+    packages=find_packages(),
+    install_requires=[
+        "dagster",
+        "dagster-cloud",
+        "dagster-dbt",
+        "dbt-snowflake<1.9",
+    ],
+    extras_require={
+        "dev": [
+            "dagster-webserver",
+        ]
+    },
+)

analytics/dagster/src/zlv_dbt/zlv_dbt/__init__.py

@@ -0,0 +1 @@
+from .definitions import defs as defs

analytics/dagster/src/zlv_dbt/zlv_dbt/assets.py

@@ -0,0 +1,10 @@
+from dagster import AssetExecutionContext
+from dagster_dbt import DbtCliResource, dbt_assets
+
+from .constants import dbt_manifest_path
+
+
+@dbt_assets(manifest=dbt_manifest_path)
+def zlv_dbt_project_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
+    yield from dbt.cli(["build"], context=context).stream()
+

analytics/dagster/src/zlv_dbt/zlv_dbt/constants.py

@@ -0,0 +1,21 @@
+import os
+from pathlib import Path
+
+from dagster_dbt import DbtCliResource
+
+dbt_project_dir = Path(__file__).joinpath("..", "..", "..", "..", "..", "dbt").resolve()
+dbt = DbtCliResource(project_dir=os.fspath(dbt_project_dir))
+
+# If DAGSTER_DBT_PARSE_PROJECT_ON_LOAD is set, a manifest will be created at run time.
+# Otherwise, we expect a manifest to be present in the project's target directory.
+if os.getenv("DAGSTER_DBT_PARSE_PROJECT_ON_LOAD"):
+    dbt_manifest_path = (
+        dbt.cli(
+            ["--quiet", "parse"],
+            target_path=Path("target"),
+        )
+        .wait()
+        .target_path.joinpath("manifest.json")
+    )
+else:
+    dbt_manifest_path = dbt_project_dir.joinpath("target", "manifest.json")

analytics/dagster/src/zlv_dbt/zlv_dbt/definitions.py

@@ -0,0 +1,16 @@
+import os
+
+from dagster import Definitions
+from dagster_dbt import DbtCliResource
+
+from .assets import zlv_dbt_project_dbt_assets
+from .constants import dbt_project_dir
+from .schedules import schedules
+
+defs = Definitions(
+    assets=[zlv_dbt_project_dbt_assets],
+    schedules=schedules,
+    resources={
+        "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)),
+    },
+)

analytics/dagster/src/zlv_dbt/zlv_dbt/schedules.py

@@ -0,0 +1,15 @@
+"""
+To add a daily schedule that materializes your dbt assets, uncomment the following lines.
+"""
+from dagster_dbt import build_schedule_from_dbt_selection
+
+from .assets import zlv_dbt_project_dbt_assets
+
+schedules = [
+#     build_schedule_from_dbt_selection(
+#         [zlv_dbt_project_dbt_assets],
+#         job_name="materialize_dbt_models",
+#         cron_schedule="0 0 * * *",
+#         dbt_select="fqn:*",
+#     ),
+]

analytics/dagster/workspace.yaml

@@ -1,2 +1,3 @@
 load_from:
-  - python_module: src
+  - python_module: src
+  - python_module: src.zlv_dbt.zlv_dbt

analytics/dagster/zlv_dbt/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.dagster]
+module_name = "zlv_dbt.definitions"
+code_location_name = "zlv_dbt"

analytics/dagster/zlv_dbt/setup.py

@@ -0,0 +1,18 @@
+from setuptools import find_packages, setup
+
+setup(
+    name="zlv_dbt",
+    version="0.0.1",
+    packages=find_packages(),
+    install_requires=[
+        "dagster",
+        "dagster-cloud",
+        "dagster-dbt",
+        "dbt-snowflake<1.9",
+    ],
+    extras_require={
+        "dev": [
+            "dagster-webserver",
+        ]
+    },
+)

analytics/dagster/zlv_dbt/zlv_dbt/assets.py

@@ -0,0 +1,10 @@
+from dagster import AssetExecutionContext
+from dagster_dbt import DbtCliResource, dbt_assets
+
+from .constants import dbt_manifest_path
+
+
+@dbt_assets(manifest=dbt_manifest_path)
+def zlv_dbt_project_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
+    yield from dbt.cli(["build"], context=context).stream()
+

analytics/dagster/zlv_dbt/zlv_dbt/constants.py

@@ -0,0 +1,21 @@
+import os
+from pathlib import Path
+
+from dagster_dbt import DbtCliResource
+
+dbt_project_dir = Path(__file__).joinpath("..", "..", "..", "..", "dbt_project").resolve()
+dbt = DbtCliResource(project_dir=os.fspath(dbt_project_dir))
+
+# If DAGSTER_DBT_PARSE_PROJECT_ON_LOAD is set, a manifest will be created at run time.
+# Otherwise, we expect a manifest to be present in the project's target directory.
+if os.getenv("DAGSTER_DBT_PARSE_PROJECT_ON_LOAD"):
+    dbt_manifest_path = (
+        dbt.cli(
+            ["--quiet", "parse"],
+            target_path=Path("target"),
+        )
+        .wait()
+        .target_path.joinpath("manifest.json")
+    )
+else:
+    dbt_manifest_path = dbt_project_dir.joinpath("target", "manifest.json")

analytics/dagster/zlv_dbt/zlv_dbt/definitions.py

@@ -0,0 +1,16 @@
+import os
+
+from dagster import Definitions
+from dagster_dbt import DbtCliResource
+
+from .assets import zlv_dbt_project_dbt_assets
+from .constants import dbt_project_dir
+from .schedules import schedules
+
+defs = Definitions(
+    assets=[zlv_dbt_project_dbt_assets],
+    schedules=schedules,
+    resources={
+        "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)),
+    },
+)

analytics/dagster/zlv_dbt/zlv_dbt/schedules.py

@@ -0,0 +1,15 @@
+"""
+To add a daily schedule that materializes your dbt assets, uncomment the following lines.
+"""
+from dagster_dbt import build_schedule_from_dbt_selection
+
+from .assets import zlv_dbt_project_dbt_assets
+
+schedules = [
+#     build_schedule_from_dbt_selection(
+#         [zlv_dbt_project_dbt_assets],
+#         job_name="materialize_dbt_models",
+#         cron_schedule="0 0 * * *",
+#         dbt_select="fqn:*",
+#     ),
+]

analytics/dbt/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

analytics/dbt/.sqlfluff

@@ -0,0 +1,2 @@
+[sqlfluff]
+dialect = ansi

analytics/dbt/.user.yml

@@ -0,0 +1 @@
+id: 21b741a1-0ed3-4565-8a7c-468d9e923bf0