Merge branch 'main' into dagster-ban-address

.docker/dagster/Dockerfile

@@ -47,4 +47,4 @@ COPY .docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/default
 
 EXPOSE 8080 3000
 
-CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000
+CMD service nginx start && dagster dev -h 0.0.0.0 -p 3000

.github/workflows/github-actions-data-stack.yml

@@ -19,3 +19,48 @@ jobs:
       alias: metabase-production
       branch: main
     secrets: inherit
+
+  build-and-upload-dbt-doc:
+    name: Build and Upload DBT Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dbt
+        run: |
+          python -m pip install --upgrade pip
+          pip install dbt-core dbt-postgres dbt-duckdb
+
+      - name: Install dbt dependencies
+        working-directory: analytics/dbt
+        run: |
+          dbt deps
+
+      - name: Build DBT Docs
+        working-directory: analytics/dbt
+        run: |
+          dbt docs generate
+          mkdir -p ../dbt_docs
+          mv target/* ../dbt_docs/
+
+      - name: Configure AWS CLI
+        env:
+          CELLAR_KEY_ID: ${{ secrets.CELLAR_KEY_ID }}
+          CELLAR_KEY_SECRET: ${{ secrets.CELLAR_KEY_SECRET }}
+        run: |
+          aws configure set aws_access_key_id $CELLAR_KEY_ID
+          aws configure set aws_secret_access_key $CELLAR_KEY_SECRET
+
+      - name: Upload DBT Docs to Cellar
+        env:
+          DBT_DOC_BUCKET_NAME:  ${{ vars.DBT_DOC_BUCKET_NAME }}
+        run: |
+          aws s3 cp analytics/dbt_docs/index.html s3://$DBT_DOC_BUCKET_NAME/index.html --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com
+          aws s3 cp analytics/dbt_docs/manifest.json s3://$DBT_DOC_BUCKET_NAME/manifest.json --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com
+          aws s3 cp analytics/dbt_docs/catalog.json s3://$DBT_DOC_BUCKET_NAME/catalog.json --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com

.github/workflows/review-app.yml

@@ -233,9 +233,6 @@ jobs:
           clever env set CC_PRE_BUILD_HOOK 'corepack enable && corepack yarn workspaces focus $WORKSPACE && corepack yarn workspaces foreach --from=$WORKSPACE -Rt run build' -a $APP_ALIAS
           clever env set CC_WEBROOT "/frontend/build" -a $APP_ALIAS
           clever env set REACT_APP_API_URL https://$PR_NAME-api.cleverapps.io -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_ENABLED "false" -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_SITE_ID "42" -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_URL_BASE "unused" -a $APP_ALIAS
           clever env set WORKSPACE "@zerologementvacant/front" -a $APP_ALIAS
 
       - name: Scale up for deployment

.talismanrc

@@ -9,12 +9,16 @@ fileignoreconfig:
   checksum: b068d130431a26be58c78e4f14346be53d318110c4020c7f7212069cbd190bf2
 - filename: analytics/dagster/src/resources/ban_config.py
   checksum: 034c6924978983da0ca5897bb06b64598a5a813dc93d1d9e8f8a62da952d4d22
+- filename: frontend/.env.example
+  checksum: 7e2a5ff197c49ff9f715b3d189da7282bdb40de53ea49735e9f183ece19168fc
 - filename: frontend/src/components/Draft/DraftSender.tsx
   checksum: cfcc1023edac4d29c305f00a4bd86f27d491e4e9535bec2bd905a917b3f396b7
 - filename: frontend/src/components/Draft/DraftSignature.tsx
   checksum: 8abd2b819601d46cc6a3d0a61a60b76391601f2d89d9126f5167b684878bd2d1
 - filename: frontend/src/index.tsx
   checksum: 25a58ba4ecfb8606daf5efbae6a1f0f2efe4e87a4a8a18e745712b39198d93a5
+- filename: frontend/src/utils/config.ts
+  checksum: 2916d5a89dbd4b6425e9b6980e235007cb0f09a3dd0f2cc8b49f43c0e79d5d5f
 - filename: packages/api-sdk/src/test/campaign-api.test.ts
   checksum: b200d9e1fec310660a91e99cd048b24cbe983e0f9f965861d061a7f0992af43e
 - filename: packages/healthcheck/src/checks/brevo.ts

analytics/dagster/requirements.txt

@@ -13,6 +13,7 @@ psycopg2
 pydantic_settings
 dlt[filesystem]
 dlt[parquet]
-s3fs
+s3fs==2024.12.0
+boto3==1.35.91
 thefuzz==0.22.1
 psutil

analytics/dagster/src/config.py

@@ -30,8 +30,8 @@ class Config:
     USE_MOTHER_DUCK = os.environ.get("USE_MOTHER_DUCK", "True") == "True"
     USE_MOTHER_DUCK_FOR_METABASE = os.environ.get("USE_MOTHER_DUCK_FOR_METABASE", "False") == "True"
 
-
-
+    DAGSTER_RETRY_DELAY = 10 * 60 # 10 minutes
+    DAGSTER_RETRY_MAX_ATTEMPS = 3
 
 
 public_tables = [

analytics/dagster/src/definitions.py

@@ -1,6 +1,7 @@
 from dagster import (
     AssetSelection,
     Definitions,
+    RetryPolicy,
     ScheduleDefinition,
     define_asset_job,
     load_assets_from_modules,
@@ -84,7 +85,7 @@
 )
 
 yearly_ff_refresh_schedule = ScheduleDefinition(
-    job=yearly_update_ff_dwh_job, cron_schedule="0 0 1 1 *"
+    job=yearly_update_ff_dwh_job, cron_schedule="@yearly"
 )
 
 # Load definitions with assets, resources, and schedule

analytics/dbt/macros/lovac/deduplicate_lovac.sql

@@ -0,0 +1,4 @@
+{% macro deduplicate_lovac() %}
+QUALIFY
+ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
+{% endmacro %}

analytics/dbt/macros/lovac/handle_lovac_different_years.sql

@@ -1,4 +1,4 @@
-{% macro handle_lovac_different_years () %}
+{% macro handle_lovac_different_years (new_version=False) %}
 cleaned_data AS (
 SELECT
 annee as data_year,
@@ -14,13 +14,37 @@ TRY_CAST (debutvacance as INTEGER) as debutvacance,
 ccodep,
 lpad (ccodep, 2, '0') || lpad (commune, 3, '0') AS geo_code,
 nature AS housing_kind,
+ff_stoth, 
+CASE 
+      WHEN potentiel_tlv_thlv = ' ' OR potentiel_tlv_thlv IS NULL THEN false
+      WHEN potentiel_tlv_thlv = '*' THEN true
+      ELSE NULL
+END as potentiel_tlv_thlv,
+TRY_CAST(ff_stoth AS NUMERIC) as living_area,
+{% if new_version %}
+    TRY_CAST(ff_dcntpa AS NUMERIC) as plot_area,
+    TRY_CAST(ff_jdatnss_1 AS DATE) AS ff_jdatnss_1,
+    TRY_CAST(ff_jdatnss_2 AS DATE) AS ff_jdatnss_2,
+    TRY_CAST(ff_jdatnss_3 AS DATE) AS ff_jdatnss_3,
+    TRY_CAST(ff_jdatnss_4 AS DATE) AS ff_jdatnss_4,
+    TRY_CAST(ff_jdatnss_5 AS DATE) AS ff_jdatnss_5,
+    TRY_CAST(ff_jdatnss_6 AS DATE) AS ff_jdatnss_6,
+
+{% else %}
+    NULL as plot_area,
+    NULL AS ff_jdatnss_1,
+    NULL AS ff_jdatnss_2,
+    NULL AS ff_jdatnss_3,
+    NULL AS ff_jdatnss_4,
+    NULL AS ff_jdatnss_5,
+    NULL AS ff_jdatnss_6,
+{% endif %}
+
+
 source.*
 
 FROM
 source
 )
-
 SELECT * FROM cleaned_data
-QUALIFY
-ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
 {% endmacro %}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2019.sql

@@ -1,2 +1,4 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}
+

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2019.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/marts/common/marts_common_morphology.sql

@@ -8,7 +8,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe, 
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2024") }}
     UNION ALL
     SELECT
@@ -20,7 +22,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2023") }}
     UNION ALL
     SELECT
@@ -32,7 +36,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2022") }}
     UNION ALL
     SELECT
@@ -44,7 +50,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2021") }}
     UNION ALL
     SELECT
@@ -56,7 +64,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2020") }}
     UNION ALL
     SELECT
@@ -68,7 +78,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2019") }}
 ),
 
@@ -103,6 +115,8 @@ lovac AS (
         local_id
         , year
         , geo_code
+        , plot_area
+        , living_area
         , CASE
             WHEN
                 (housing_kind IN ('APPART', 'MAISON') AND aff = 'H')
@@ -152,6 +166,23 @@ lovac_geo_code_year AS (
                 ELSE 0
             END
         ) AS count_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN living_area
+                ELSE 0
+            END
+            ) as sum_living_area_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN plot_area
+                ELSE 0
+            END) as sum_plot_area_vacant_housing_private_fil_ccthp
     FROM lovac
     GROUP BY year, geo_code
 ),
@@ -204,6 +235,8 @@ SELECT
     , lovac.count_vacant_housing_private
     , lovac.count_vacant_housing_private_fil
     , lovac.count_vacant_housing_private_fil_ccthp
+    , lovac.sum_living_area_vacant_housing_private_fil_ccthp
+    , lovac.sum_plot_area_vacant_housing_private_fil_ccthp
     , ff.count_housing
     , ff.count_housing_private
     , ff.count_housing_private_rented

analytics/dbt/models/marts/public/marts_public_establishments_morphology.sql

@@ -10,8 +10,9 @@ SELECT
     SUM(count_housing) AS count_housing,
     SUM(count_housing_private) AS count_housing_private,
     SUM(count_housing_private_rented) AS count_housing_private_rented,
-    SUM(count_housing_production) AS count_housing_production
-
+    SUM(count_housing_production) AS count_housing_production, 
+    SUM(sum_living_area_vacant_housing_private_fil_ccthp) as sum_living_area_vacant_housing_private_fil_ccthp,
+    SUM(sum_plot_area_vacant_housing_private_fil_ccthp) as sum_plot_area_vacant_housing_private_fil_ccthp
 FROM {{ ref ('int_production_establishments') }} pe
 LEFT JOIN {{ ref ('int_production_establishments_localities') }} pel ON pe.id = pel.establishment_id
 LEFT JOIN {{ ref ('marts_common_morphology') }} mcm ON pel.geo_code = mcm.geo_code

analytics/dbt/models/marts/public/marts_public_establishments_morphology_unpivoted.sql

@@ -37,6 +37,20 @@ WITH base_data AS (
         count_vacant_housing_private_fil_ccthp AS count_value
     FROM base_data
     UNION ALL
+    SELECT
+        establishment_id,
+        year,
+        'sum_living_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_living_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
+    UNION ALL
+    SELECT
+        establishment_id,
+        year,
+        'sum_plot_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_plot_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
+    UNION ALL
     SELECT
         establishment_id,
         year,
@@ -78,6 +92,12 @@ WITH base_data AS (
             WHEN
                 count_type = 'count_vacant_housing_private_fil'
                 THEN 'Logements Vacants du Parc Privé (FIL)'
+            WHEN
+                count_type = 'sum_living_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces habitables Vacants du Parc Privé (FIL+CCTHP)'
+            WHEN
+                count_type = 'sum_plot_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces foncières Vacants du Parc Privé (FIL+CCTHP)'
             WHEN
                 count_type = 'count_vacant_housing_private_fil_ccthp'
                 THEN 'Logements Vacants du Parc Privé (FIL+CCTHP)'

analytics/dbt/models/staging/lovac/stg_lovac_2019.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2019') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years(new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2020.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2020') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}