Merge pull request #1105 from MTES-MCT/main

Mise en production 23/01/2025

.github/workflows/github-actions-data-stack.yml

@@ -19,3 +19,48 @@ jobs:
       alias: metabase-production
       branch: main
     secrets: inherit
+
+  build-and-upload-dbt-doc:
+    name: Build and Upload DBT Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dbt
+        run: |
+          python -m pip install --upgrade pip
+          pip install dbt-core dbt-postgres dbt-duckdb
+
+      - name: Install dbt dependencies
+        working-directory: analytics/dbt
+        run: |
+          dbt deps
+
+      - name: Build DBT Docs
+        working-directory: analytics/dbt
+        run: |
+          dbt docs generate
+          mkdir -p ../dbt_docs
+          mv target/* ../dbt_docs/
+
+      - name: Configure AWS CLI
+        env:
+          CELLAR_KEY_ID: ${{ secrets.CELLAR_KEY_ID }}
+          CELLAR_KEY_SECRET: ${{ secrets.CELLAR_KEY_SECRET }}
+        run: |
+          aws configure set aws_access_key_id $CELLAR_KEY_ID
+          aws configure set aws_secret_access_key $CELLAR_KEY_SECRET
+
+      - name: Upload DBT Docs to Cellar
+        env:
+          DBT_DOC_BUCKET_NAME:  ${{ vars.DBT_DOC_BUCKET_NAME }}
+        run: |
+          aws s3 cp analytics/dbt_docs/index.html s3://$DBT_DOC_BUCKET_NAME/index.html --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com
+          aws s3 cp analytics/dbt_docs/manifest.json s3://$DBT_DOC_BUCKET_NAME/manifest.json --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com
+          aws s3 cp analytics/dbt_docs/catalog.json s3://$DBT_DOC_BUCKET_NAME/catalog.json --acl public-read --endpoint-url https://cellar-c2.services.clever-cloud.com

.github/workflows/review-app.yml

@@ -233,9 +233,6 @@ jobs:
           clever env set CC_PRE_BUILD_HOOK 'corepack enable && corepack yarn workspaces focus $WORKSPACE && corepack yarn workspaces foreach --from=$WORKSPACE -Rt run build' -a $APP_ALIAS
           clever env set CC_WEBROOT "/frontend/build" -a $APP_ALIAS
           clever env set REACT_APP_API_URL https://$PR_NAME-api.cleverapps.io -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_ENABLED "false" -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_SITE_ID "42" -a $APP_ALIAS
-          clever env set REACT_APP_MATOMO_URL_BASE "unused" -a $APP_ALIAS
           clever env set WORKSPACE "@zerologementvacant/front" -a $APP_ALIAS
 
       - name: Scale up for deployment

.talismanrc

@@ -3,12 +3,16 @@ fileignoreconfig:
   checksum: f7719ba0d36160d97e80ee15cb5415b601354576929e36df0596c7d192465cfb
 - filename: README.md
   checksum: df312ccb4c75fc4c2441a1f7f2c7817ee98ffb3065c78d5d7d6addf6ab129176
+- filename: frontend/.env.example
+  checksum: 7e2a5ff197c49ff9f715b3d189da7282bdb40de53ea49735e9f183ece19168fc
 - filename: frontend/src/components/Draft/DraftSender.tsx
   checksum: cfcc1023edac4d29c305f00a4bd86f27d491e4e9535bec2bd905a917b3f396b7
 - filename: frontend/src/components/Draft/DraftSignature.tsx
   checksum: 8abd2b819601d46cc6a3d0a61a60b76391601f2d89d9126f5167b684878bd2d1
 - filename: frontend/src/index.tsx
   checksum: 25a58ba4ecfb8606daf5efbae6a1f0f2efe4e87a4a8a18e745712b39198d93a5
+- filename: frontend/src/utils/config.ts
+  checksum: 2916d5a89dbd4b6425e9b6980e235007cb0f09a3dd0f2cc8b49f43c0e79d5d5f
 - filename: packages/api-sdk/src/test/campaign-api.test.ts
   checksum: b200d9e1fec310660a91e99cd048b24cbe983e0f9f965861d061a7f0992af43e
 - filename: packages/healthcheck/src/checks/brevo.ts

analytics/dbt/macros/lovac/deduplicate_lovac.sql

@@ -0,0 +1,4 @@
+{% macro deduplicate_lovac() %}
+QUALIFY
+ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
+{% endmacro %}

analytics/dbt/macros/lovac/handle_lovac_different_years.sql

@@ -1,4 +1,4 @@
-{% macro handle_lovac_different_years () %}
+{% macro handle_lovac_different_years (new_version=False) %}
 cleaned_data AS (
 SELECT
 annee as data_year,
@@ -14,13 +14,37 @@ TRY_CAST (debutvacance as INTEGER) as debutvacance,
 ccodep,
 lpad (ccodep, 2, '0') || lpad (commune, 3, '0') AS geo_code,
 nature AS housing_kind,
+ff_stoth, 
+CASE 
+      WHEN potentiel_tlv_thlv = ' ' OR potentiel_tlv_thlv IS NULL THEN false
+      WHEN potentiel_tlv_thlv = '*' THEN true
+      ELSE NULL
+END as potentiel_tlv_thlv,
+TRY_CAST(ff_stoth AS NUMERIC) as living_area,
+{% if new_version %}
+    TRY_CAST(ff_dcntpa AS NUMERIC) as plot_area,
+    TRY_CAST(ff_jdatnss_1 AS DATE) AS ff_jdatnss_1,
+    TRY_CAST(ff_jdatnss_2 AS DATE) AS ff_jdatnss_2,
+    TRY_CAST(ff_jdatnss_3 AS DATE) AS ff_jdatnss_3,
+    TRY_CAST(ff_jdatnss_4 AS DATE) AS ff_jdatnss_4,
+    TRY_CAST(ff_jdatnss_5 AS DATE) AS ff_jdatnss_5,
+    TRY_CAST(ff_jdatnss_6 AS DATE) AS ff_jdatnss_6,
+
+{% else %}
+    NULL as plot_area,
+    NULL AS ff_jdatnss_1,
+    NULL AS ff_jdatnss_2,
+    NULL AS ff_jdatnss_3,
+    NULL AS ff_jdatnss_4,
+    NULL AS ff_jdatnss_5,
+    NULL AS ff_jdatnss_6,
+{% endif %}
+
+
 source.*
 
 FROM
 source
 )
-
 SELECT * FROM cleaned_data
-QUALIFY
-ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
 {% endmacro %}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2019.sql

@@ -1,2 +1,4 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}
+

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2019.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/marts/common/marts_common_morphology.sql

@@ -8,7 +8,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe, 
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2024") }}
     UNION ALL
     SELECT
@@ -20,7 +22,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2023") }}
     UNION ALL
     SELECT
@@ -32,7 +36,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2022") }}
     UNION ALL
     SELECT
@@ -44,7 +50,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2021") }}
     UNION ALL
     SELECT
@@ -56,7 +64,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2020") }}
     UNION ALL
     SELECT
@@ -68,7 +78,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2019") }}
 ),
 
@@ -103,6 +115,8 @@ lovac AS (
         local_id
         , year
         , geo_code
+        , plot_area
+        , living_area
         , CASE
             WHEN
                 (housing_kind IN ('APPART', 'MAISON') AND aff = 'H')
@@ -152,6 +166,23 @@ lovac_geo_code_year AS (
                 ELSE 0
             END
         ) AS count_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN living_area
+                ELSE 0
+            END
+            ) as sum_living_area_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN plot_area
+                ELSE 0
+            END) as sum_plot_area_vacant_housing_private_fil_ccthp
     FROM lovac
     GROUP BY year, geo_code
 ),
@@ -204,6 +235,8 @@ SELECT
     , lovac.count_vacant_housing_private
     , lovac.count_vacant_housing_private_fil
     , lovac.count_vacant_housing_private_fil_ccthp
+    , lovac.sum_living_area_vacant_housing_private_fil_ccthp
+    , lovac.sum_plot_area_vacant_housing_private_fil_ccthp
     , ff.count_housing
     , ff.count_housing_private
     , ff.count_housing_private_rented

analytics/dbt/models/marts/public/marts_public_establishments_morphology.sql

@@ -10,8 +10,9 @@ SELECT
     SUM(count_housing) AS count_housing,
     SUM(count_housing_private) AS count_housing_private,
     SUM(count_housing_private_rented) AS count_housing_private_rented,
-    SUM(count_housing_production) AS count_housing_production
-
+    SUM(count_housing_production) AS count_housing_production, 
+    SUM(sum_living_area_vacant_housing_private_fil_ccthp) as sum_living_area_vacant_housing_private_fil_ccthp,
+    SUM(sum_plot_area_vacant_housing_private_fil_ccthp) as sum_plot_area_vacant_housing_private_fil_ccthp
 FROM {{ ref ('int_production_establishments') }} pe
 LEFT JOIN {{ ref ('int_production_establishments_localities') }} pel ON pe.id = pel.establishment_id
 LEFT JOIN {{ ref ('marts_common_morphology') }} mcm ON pel.geo_code = mcm.geo_code

analytics/dbt/models/marts/public/marts_public_establishments_morphology_unpivoted.sql

@@ -36,6 +36,18 @@ WITH base_data AS (
         'count_vacant_housing_private_fil_ccthp' AS count_type,
         count_vacant_housing_private_fil_ccthp AS count_value
     FROM base_data
+    SELECT
+        establishment_id,
+        year,
+        'sum_living_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_living_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
+    SELECT
+        establishment_id,
+        year,
+        'sum_plot_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_plot_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
     UNION ALL
     SELECT
         establishment_id,
@@ -78,6 +90,12 @@ WITH base_data AS (
             WHEN
                 count_type = 'count_vacant_housing_private_fil'
                 THEN 'Logements Vacants du Parc Privé (FIL)'
+            WHEN
+                count_type = 'sum_living_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces habitables Vacants du Parc Privé (FIL+CCTHP)'
+            WHEN
+                count_type = 'sum_plot_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces foncières Vacants du Parc Privé (FIL+CCTHP)'
             WHEN
                 count_type = 'count_vacant_housing_private_fil_ccthp'
                 THEN 'Logements Vacants du Parc Privé (FIL+CCTHP)'

analytics/dbt/models/staging/lovac/stg_lovac_2019.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2019') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years(new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2020.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2020') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2021.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2021') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2022.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2022') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2023.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2023') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years(new_version=True) }}

analytics/dbt/models/staging/lovac/stg_lovac_2024.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2024') }}
 ),
-{{ handle_lovac_different_years() }}
+{{ handle_lovac_different_years(new_version=True) }}

analytics/dbt/profiles.yml

@@ -7,12 +7,7 @@ duckdb_profile:
       extensions:
         - httpfs
         - parquet
-        - s3
         - spatial
-      settings:
-        s3_endpoint: "{{ env_var('S3_ENDPOINT_URL') }}" 
-        s3_access_key_id: "{{ env_var('S3_ACCESS_KEY_ID') }}" 
-        s3_secret_access_key: "{{ env_var('S3_SECRET_ACCESS_KEY') }}" 
       attach:
         - path: production.duckdb
 

frontend/.env.example

@@ -1,6 +1,5 @@
 REACT_APP_API_URL=http://localhost:3001
-REACT_APP_MATOMO_ENABLED=false
-REACT_APP_MATOMO_SITE_ID=1
-REACT_APP_MATOMO_SRC_URL=http://url-matomo
 REACT_APP_METABASE_STATS_DASHBOARD=UUID
 REACT_APP_FEATURE_OCCUPANCY=ct1,ct2,ct3
+REACT_APP_POSTHOG_ENABLED=false
+REACT_APP_POSTHOG_API_KEY=key