From 5f8dd376aa3929af11d201233716f10749bae083 Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Sat, 8 Feb 2025 17:00:27 +0800 Subject: [PATCH] Draft ci_incremental_predicate --- .github/workflows/dbt_run.yml | 9 ++++++++- dbt_macros/dune/ci_incremental_predicate.sql | 9 +++++++++ dbt_macros/dune/incremental_predicate.sql | 2 +- dbt_subprojects/daily_spellbook/profiles.yml | 7 ++++++- dbt_subprojects/dex/profiles.yml | 7 ++++++- dbt_subprojects/hourly_spellbook/profiles.yml | 7 ++++++- dbt_subprojects/nft/profiles.yml | 7 ++++++- dbt_subprojects/solana/profiles.yml | 7 ++++++- dbt_subprojects/tokens/profiles.yml | 7 ++++++- 9 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 dbt_macros/dune/ci_incremental_predicate.sql diff --git a/.github/workflows/dbt_run.yml b/.github/workflows/dbt_run.yml index afc1d89b4c7..132376c5a97 100644 --- a/.github/workflows/dbt_run.yml +++ b/.github/workflows/dbt_run.yml @@ -12,6 +12,12 @@ jobs: dbt-test: runs-on: [ self-hosted, linux, spellbook-trino-ci ] timeout-minutes: 90 + env: + CI: "true" + CI_LIMITED_RANGE: "true" + CI_DATE_RANGE: "30" + DBT_ENV_INCREMENTAL_TIME_UNIT: "day" + DBT_ENV_INCREMENTAL_TIME: "30" steps: - name: Check out repository code @@ -24,12 +30,13 @@ jobs: echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV PROJECT_DIR=dbt_subprojects/${{ inputs.project }} echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV + - name: Add git_sha to schema run: "/runner/change_schema.sh git_dunesql_$GIT_SHA" - name: Get latest manifest run: "aws s3 cp $S3_LOCATION/manifest.json $PROJECT_DIR/manifest.json" - continue-on-error: true # Temprorary until all projects have manifest + continue-on-error: true # Temporary until all projects have manifest - name: dbt dependencies working-directory: ${{env.PROJECT_DIR}} diff --git a/dbt_macros/dune/ci_incremental_predicate.sql b/dbt_macros/dune/ci_incremental_predicate.sql new file mode 100644 index 00000000000..f382335f00f --- /dev/null +++ b/dbt_macros/dune/ci_incremental_predicate.sql @@ -0,0 +1,9 @@ +{% macro ci_incremental_predicate(column) -%} +{%- if env_var('CI', false) | as_bool and env_var('CI_LIMITED_RANGE', false) | as_bool -%} + -- In CI environment, only process data from the last specified days + {{column}} >= date_trunc('day', now() - interval '{{ env_var("CI_DATE_RANGE", "30") }}' day) +{%- else -%} + -- In production environment, use the original incremental logic + {{column}} >= date_trunc('{{var("DBT_ENV_INCREMENTAL_TIME_UNIT")}}', now() - interval '{{var('DBT_ENV_INCREMENTAL_TIME')}}' {{var('DBT_ENV_INCREMENTAL_TIME_UNIT')}}) +{%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/dbt_macros/dune/incremental_predicate.sql b/dbt_macros/dune/incremental_predicate.sql index ad34e730517..8bb02dcc9e2 100644 --- a/dbt_macros/dune/incremental_predicate.sql +++ b/dbt_macros/dune/incremental_predicate.sql @@ -1,3 +1,3 @@ {% macro incremental_predicate(column) -%} -{{column}} >= date_trunc('{{var("DBT_ENV_INCREMENTAL_TIME_UNIT")}}', now() - interval '{{var('DBT_ENV_INCREMENTAL_TIME')}}' {{var('DBT_ENV_INCREMENTAL_TIME_UNIT')}}) +{{ ci_incremental_predicate(column) }} {%- endmacro -%} diff --git a/dbt_subprojects/daily_spellbook/profiles.yml b/dbt_subprojects/daily_spellbook/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/daily_spellbook/profiles.yml +++ b/dbt_subprojects/daily_spellbook/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file diff --git a/dbt_subprojects/dex/profiles.yml b/dbt_subprojects/dex/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/dex/profiles.yml +++ b/dbt_subprojects/dex/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file diff --git a/dbt_subprojects/hourly_spellbook/profiles.yml b/dbt_subprojects/hourly_spellbook/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/hourly_spellbook/profiles.yml +++ b/dbt_subprojects/hourly_spellbook/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file diff --git a/dbt_subprojects/nft/profiles.yml b/dbt_subprojects/nft/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/nft/profiles.yml +++ b/dbt_subprojects/nft/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file diff --git a/dbt_subprojects/solana/profiles.yml b/dbt_subprojects/solana/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/solana/profiles.yml +++ b/dbt_subprojects/solana/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file diff --git a/dbt_subprojects/tokens/profiles.yml b/dbt_subprojects/tokens/profiles.yml index 10d52bd460f..c3a2f358b2b 100644 --- a/dbt_subprojects/tokens/profiles.yml +++ b/dbt_subprojects/tokens/profiles.yml @@ -9,4 +9,9 @@ spellbook-local: port: 1234 database: hive schema: wizard - threads: 1 \ No newline at end of file + threads: 1 + vars: + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'hour') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '7') }}" + CI_LIMITED_RANGE: "{{ env_var('CI_LIMITED_RANGE', 'false') }}" + CI_DATE_RANGE: "{{ env_var('CI_DATE_RANGE', '30') }}" \ No newline at end of file