refactor lovac cleaning and filtering

Signed-off-by: Raphaël Courivaud <[email protected]>

analytics/dbt/macros/lovac/deduplicate_lovac.sql

@@ -0,0 +1,4 @@
+{% macro deduplicate_lovac() %}
+QUALIFY
+ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
+{% endmacro %}

analytics/dbt/macros/lovac/handle_lovac_different_years.sql

@@ -1,4 +1,4 @@
-{% macro handle_lovac_different_years () %}
+{% macro handle_lovac_different_years (new_version=False) %}
 cleaned_data AS (
 SELECT
 annee as data_year,
@@ -14,13 +14,37 @@ TRY_CAST (debutvacance as INTEGER) as debutvacance,
 ccodep,
 lpad (ccodep, 2, '0') || lpad (commune, 3, '0') AS geo_code,
 nature AS housing_kind,
+ff_stoth, 
+CASE 
+      WHEN potentiel_tlv_thlv = ' ' OR potentiel_tlv_thlv IS NULL THEN false
+      WHEN potentiel_tlv_thlv = '*' THEN true
+      ELSE NULL
+END as potentiel_tlv_thlv,
+TRY_CAST(ff_stoth AS NUMERIC) as living_area,
+{% if new_version %}
+    TRY_CAST(ff_dcntpa AS NUMERIC) as plot_area,
+    TRY_CAST(ff_jdatnss_1 AS DATE) AS ff_jdatnss_1,
+    TRY_CAST(ff_jdatnss_2 AS DATE) AS ff_jdatnss_2,
+    TRY_CAST(ff_jdatnss_3 AS DATE) AS ff_jdatnss_3,
+    TRY_CAST(ff_jdatnss_4 AS DATE) AS ff_jdatnss_4,
+    TRY_CAST(ff_jdatnss_5 AS DATE) AS ff_jdatnss_5,
+    TRY_CAST(ff_jdatnss_6 AS DATE) AS ff_jdatnss_6,
+
+{% else %}
+    0 as plot_area,
+    NULL AS ff_jdatnss_1,
+    NULL AS ff_jdatnss_2,
+    NULL AS ff_jdatnss_3,
+    NULL AS ff_jdatnss_4,
+    NULL AS ff_jdatnss_5,
+    NULL AS ff_jdatnss_6,
+{% endif %}
+
+
 source.*
 
 FROM
 source
 )
-
 SELECT * FROM cleaned_data
-QUALIFY
-ROW_NUMBER () OVER (PARTITION BY local_id ORDER BY debutvacance DESC) = 1
 {% endmacro %}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2019.sql

@@ -1,2 +1,4 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}
+

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_ex_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True, vacancy = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2019.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2019') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2020.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2020') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2021.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2021') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2022.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2022') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2023.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2023') }}
 {{ filter_lovac(ccthp = False) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/intermediate/lovac/int_lovac_fil_2024.sql

@@ -1,2 +1,3 @@
 SELECT * FROM {{ ref ('stg_lovac_2024') }}
 {{ filter_lovac(ccthp = True) }}
+{{ deduplicate_lovac() }}

analytics/dbt/models/marts/common/marts_common_morphology.sql

@@ -8,7 +8,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe, 
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2024") }}
     UNION ALL
     SELECT
@@ -20,7 +22,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2023") }}
     UNION ALL
     SELECT
@@ -32,7 +36,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2022") }}
     UNION ALL
     SELECT
@@ -44,7 +50,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2021") }}
     UNION ALL
     SELECT
@@ -56,7 +64,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2020") }}
     UNION ALL
     SELECT
@@ -68,7 +78,9 @@ WITH all_lovac AS (
         ff_ccthp,
         housing_kind,
         aff,
-        groupe
+        groupe,
+        plot_area, 
+        living_area
     FROM {{ ref ("stg_lovac_2019") }}
 ),
 
@@ -103,6 +115,8 @@ lovac AS (
         local_id
         , year
         , geo_code
+        , plot_area
+        , living_area
         , CASE
             WHEN
                 (housing_kind IN ('APPART', 'MAISON') AND aff = 'H')
@@ -152,6 +166,23 @@ lovac_geo_code_year AS (
                 ELSE 0
             END
         ) AS count_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN living_area
+                ELSE 0
+            END
+            ) as sum_living_area_vacant_housing_private_fil_ccthp
+        , SUM(CASE
+                WHEN
+                    is_housing = 1
+                    AND is_private = 1
+                    AND is_vacant_fil_ccthp = 1
+                    THEN plot_area
+                ELSE 0
+            END) as sum_plot_area_vacant_housing_private_fil_ccthp
     FROM lovac
     GROUP BY year, geo_code
 ),
@@ -204,6 +235,8 @@ SELECT
     , lovac.count_vacant_housing_private
     , lovac.count_vacant_housing_private_fil
     , lovac.count_vacant_housing_private_fil_ccthp
+    , lovac.sum_living_area_vacant_housing_private_fil_ccthp
+    , lovac.sum_plot_area_vacant_housing_private_fil_ccthp
     , ff.count_housing
     , ff.count_housing_private
     , ff.count_housing_private_rented

analytics/dbt/models/marts/public/marts_public_establishments_morphology.sql

@@ -10,8 +10,9 @@ SELECT
     SUM(count_housing) AS count_housing,
     SUM(count_housing_private) AS count_housing_private,
     SUM(count_housing_private_rented) AS count_housing_private_rented,
-    SUM(count_housing_production) AS count_housing_production
-
+    SUM(count_housing_production) AS count_housing_production, 
+    SUM(sum_living_area_vacant_housing_private_fil_ccthp) as sum_living_area_vacant_housing_private_fil_ccthp,
+    SUM(sum_plot_area_vacant_housing_private_fil_ccthp) as sum_plot_area_vacant_housing_private_fil_ccthp
 FROM {{ ref ('int_production_establishments') }} pe
 LEFT JOIN {{ ref ('int_production_establishments_localities') }} pel ON pe.id = pel.establishment_id
 LEFT JOIN {{ ref ('marts_common_morphology') }} mcm ON pel.geo_code = mcm.geo_code

analytics/dbt/models/marts/public/marts_public_establishments_morphology_unpivoted.sql

@@ -36,6 +36,18 @@ WITH base_data AS (
         'count_vacant_housing_private_fil_ccthp' AS count_type,
         count_vacant_housing_private_fil_ccthp AS count_value
     FROM base_data
+    SELECT
+        establishment_id,
+        year,
+        'sum_living_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_living_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
+    SELECT
+        establishment_id,
+        year,
+        'sum_plot_area_vacant_housing_private_fil_ccthp' AS count_type,
+        sum_plot_area_vacant_housing_private_fil_ccthp AS count_value
+    FROM base_data
     UNION ALL
     SELECT
         establishment_id,
@@ -78,6 +90,12 @@ WITH base_data AS (
             WHEN
                 count_type = 'count_vacant_housing_private_fil'
                 THEN 'Logements Vacants du Parc Privé (FIL)'
+            WHEN
+                count_type = 'sum_living_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces habitables Vacants du Parc Privé (FIL+CCTHP)'
+            WHEN
+                count_type = 'sum_plot_area_vacant_housing_private_fil_ccthp'
+                THEN 'Somme des surfaces foncières Vacants du Parc Privé (FIL+CCTHP)'
             WHEN
                 count_type = 'count_vacant_housing_private_fil_ccthp'
                 THEN 'Logements Vacants du Parc Privé (FIL+CCTHP)'

analytics/dbt/models/staging/lovac/stg_lovac_2019.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2019') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years(new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2020.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2020') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2021.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2021') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2022.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2022') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years (new_version=False) }}

analytics/dbt/models/staging/lovac/stg_lovac_2023.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2023') }}
 ),
-{{ handle_lovac_different_years () }}
+{{ handle_lovac_different_years(new_version=True) }}

analytics/dbt/models/staging/lovac/stg_lovac_2024.sql

@@ -1,4 +1,4 @@
 with source as (
 SELECT * FROM {{ source ('duckdb_raw', 'raw_lovac_2024') }}
 ),
-{{ handle_lovac_different_years() }}
+{{ handle_lovac_different_years(new_version=True) }}