Testing ci_incremental_predicate

.github/workflows/dbt_run.yml

@@ -12,6 +12,12 @@ jobs:
   dbt-test:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     timeout-minutes: 90
+    env:
+      CI: "true"
+      CI_LIMITED_RANGE: "true"
+      CI_DATE_RANGE: "30"
+      DBT_ENV_INCREMENTAL_TIME_UNIT: "day"
+      DBT_ENV_INCREMENTAL_TIME: "30"
 
     steps:
       - name: Check out repository code
@@ -24,12 +30,13 @@ jobs:
           echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
           PROJECT_DIR=dbt_subprojects/${{ inputs.project }}
           echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+
       - name: Add git_sha to schema
         run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
 
       - name: Get latest manifest
         run: "aws s3 cp $S3_LOCATION/manifest.json $PROJECT_DIR/manifest.json"
-        continue-on-error: true # Temprorary until all projects have manifest
+        continue-on-error: true # Temporary until all projects have manifest
 
       - name: dbt dependencies
         working-directory: ${{env.PROJECT_DIR}}

dbt_macros/dune/ci_incremental_predicate.sql

@@ -0,0 +1,9 @@
+{% macro ci_incremental_predicate(column) -%}
+{%- if env_var('CI', false) | as_bool and env_var('CI_LIMITED_RANGE', false) | as_bool -%}
+    -- In CI environment, only process data from the last specified days
+    {{column}} >= date_trunc('day', now() - interval '{{ env_var("CI_DATE_RANGE", "30") }}' day)
+{%- else -%}
+    -- In production environment, use the original incremental logic
+    {{column}} >= date_trunc('{{var("DBT_ENV_INCREMENTAL_TIME_UNIT")}}', now() - interval '{{var('DBT_ENV_INCREMENTAL_TIME')}}' {{var('DBT_ENV_INCREMENTAL_TIME_UNIT')}})
+{%- endif -%}
+{%- endmacro -%} 

dbt_macros/dune/incremental_predicate.sql

@@ -1,3 +1,3 @@
 {% macro incremental_predicate(column) -%}
-{{column}} >= date_trunc('{{var("DBT_ENV_INCREMENTAL_TIME_UNIT")}}', now() - interval '{{var('DBT_ENV_INCREMENTAL_TIME')}}' {{var('DBT_ENV_INCREMENTAL_TIME_UNIT')}})
+{{ ci_incremental_predicate(column) }}
 {%- endmacro -%}

dbt_subprojects/daily_spellbook/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"

dbt_subprojects/dex/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"

dbt_subprojects/hourly_spellbook/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"

dbt_subprojects/nft/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"

dbt_subprojects/solana/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"

dbt_subprojects/tokens/profiles.yml

@@ -9,4 +9,9 @@ spellbook-local:
       port: 1234
       database: hive
       schema: wizard
-      threads: 1
+      threads: 1
+      vars:
+        DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}"
+        DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}"
+        CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}"
+        CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}"