Merge branch 'main' into upgrade_protobuf

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0b3
+current_version = 1.9.0b4
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b4.md

@@ -0,0 +1,19 @@
+## dbt-core 1.9.0-b4 - November 06, 2024
+
+### Features
+
+- Emit debug logging event whenever artifacts are written ([#10937](https://github.com/dbt-labs/dbt-core/issues/10937))
+- Support --empty for snapshots ([#10372](https://github.com/dbt-labs/dbt-core/issues/10372))
+
+### Fixes
+
+- Ensure KeyboardInterrupt/SystemExit halts microbatch model execution ([#10862](https://github.com/dbt-labs/dbt-core/issues/10862))
+- Handle exceptions in `get_execution_status` more broadly to better ensure `run_results.json` gets written ([#10934](https://github.com/dbt-labs/dbt-core/issues/10934))
+
+### Under the Hood
+
+- Behavior change for mf timespine without yaml configuration ([#10959](https://github.com/dbt-labs/dbt-core/issues/10959))
+- Behavior change for cumulative metric type param ([#10960](https://github.com/dbt-labs/dbt-core/issues/10960))
+
+### Contributors
+- [@DevonFulcher](https://github.com/DevonFulcher) ([#10959](https://github.com/dbt-labs/dbt-core/issues/10959), [#10960](https://github.com/dbt-labs/dbt-core/issues/10960))

.changes/1.9.0/Features-20241031-163149.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support --empty for snapshots
+time: 2024-10-31T16:31:49.926164-04:00
+custom:
+  Author: gshank
+  Issue: "10372"

.changes/1.9.0/Fixes-20241017-153022.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Ensure KeyboardInterrupt/SystemExit halts microbatch model execution
+time: 2024-10-17T15:30:22.781854-07:00
+custom:
+  Author: QMalcolm
+  Issue: "10862"

.changes/1.9.0/Under the Hood-20241031-093251.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Behavior change for mf timespine without yaml configuration
+time: 2024-10-31T09:32:51.166594-05:00
+custom:
+    Author: DevonFulcher
+    Issue: "10959"

.changes/1.9.0/Under the Hood-20241031-094609.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Behavior change for cumulative metric type param
+time: 2024-10-31T09:46:09.757879-05:00
+custom:
+    Author: DevonFulcher
+    Issue: "10960"

.changes/unreleased/Features-20241001-161422.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Change gating of microbatch feature to be behind project flag / behavior flag
+time: 2024-10-01T16:14:22.267253-05:00
+custom:
+  Author: MichelleArk QMalcolm
+  Issue: "10798"

.changes/unreleased/Fixes-20241007-121402.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Support disabling unit tests via config.
+time: 2024-10-07T12:14:02.252744-07:00
+custom:
+    Author: tsturge
+    Issue: 9109 10540

.changes/unreleased/Fixes-20241105-151459.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix 'no attribute .config' error when ref-ing a microbatch model from non-Model
+  context
+time: 2024-11-05T15:14:59.002236-05:00
+custom:
+  Author: michelleark
+  Issue: "10928"

.changes/unreleased/Fixes-20241106-144656.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'Ensure inferred primary_key is a List[str] with no null values '
+time: 2024-11-06T14:46:56.652963-05:00
+custom:
+  Author: michelleark
+  Issue: "10983"

.github/actions/setup-postgres-macos/action.yml

@@ -6,6 +6,7 @@ runs:
     - shell: bash
       run: |
         brew install postgresql@16
+        brew link postgresql@16 --force
         brew services start postgresql@16
         echo "Check PostgreSQL service is running"
         i=10

.github/workflows/main.yml

@@ -139,7 +139,7 @@ jobs:
       - name: generate include
         id: generate-include
         run: |
-          INCLUDE=('"python-version":"3.9","os":"windows-latest"' '"python-version":"3.9","os":"macos-12"' )
+          INCLUDE=('"python-version":"3.9","os":"windows-latest"' '"python-version":"3.9","os":"macos-14"' )
           INCLUDE_GROUPS="["
           for include in ${INCLUDE[@]}; do
               for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do

.github/workflows/test-repeater.yml

@@ -35,7 +35,7 @@ on:
         type: choice
         options:
           - 'ubuntu-latest'
-          - 'macos-12'
+          - 'macos-14'
           - 'windows-latest'
       num_runs_per_batch:
         description: 'Max number of times to run the test per batch.  We always run 10 batches.'
@@ -100,7 +100,7 @@ jobs:
 
         # mac and windows don't use make due to limitations with docker with those runners in GitHub
       - name: "Set up postgres (macos)"
-        if: inputs.os == 'macos-12'
+        if: inputs.os == 'macos-14'
         uses: ./.github/actions/setup-postgres-macos
 
       - name: "Set up postgres (windows)"

.gitignore

@@ -105,3 +105,6 @@ venv/
 
 # poetry
 poetry.lock
+
+# asdf
+.tool-versions

CHANGELOG.md

@@ -5,6 +5,27 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-core 1.9.0-b4 - November 06, 2024
+
+### Features
+
+- Emit debug logging event whenever artifacts are written ([#10937](https://github.com/dbt-labs/dbt-core/issues/10937))
+- Support --empty for snapshots ([#10372](https://github.com/dbt-labs/dbt-core/issues/10372))
+
+### Fixes
+
+- Ensure KeyboardInterrupt/SystemExit halts microbatch model execution ([#10862](https://github.com/dbt-labs/dbt-core/issues/10862))
+- Handle exceptions in `get_execution_status` more broadly to better ensure `run_results.json` gets written ([#10934](https://github.com/dbt-labs/dbt-core/issues/10934))
+
+### Under the Hood
+
+- Behavior change for mf timespine without yaml configuration ([#10959](https://github.com/dbt-labs/dbt-core/issues/10959))
+- Behavior change for cumulative metric type param ([#10960](https://github.com/dbt-labs/dbt-core/issues/10960))
+
+### Contributors
+- [@DevonFulcher](https://github.com/DevonFulcher) ([#10959](https://github.com/dbt-labs/dbt-core/issues/10959), [#10960](https://github.com/dbt-labs/dbt-core/issues/10960))
+
+
 ## dbt-core 1.9.0-b3 - October 30, 2024
 
 ### Features
@@ -44,7 +65,6 @@
 - [@danlsn](https://github.com/danlsn) ([#10915](https://github.com/dbt-labs/dbt-core/issues/10915))
 - [@plypaul](https://github.com/plypaul) ([#10531](https://github.com/dbt-labs/dbt-core/issues/10531))
 
-
 ## dbt-core 1.9.0-b2 - October 07, 2024
 
 ## dbt-core 1.9.0-b1 - October 01, 2024

core/dbt/artifacts/resources/v1/unit_test_definition.py

@@ -20,6 +20,7 @@ class UnitTestConfig(BaseConfig):
         default_factory=dict,
         metadata=MergeBehavior.Update.meta(),
     )
+    enabled: bool = True
 
 
 class UnitTestFormat(StrEnum):

core/dbt/cli/main.py

@@ -716,6 +716,7 @@ def seed(ctx, **kwargs):
 @cli.command("snapshot")
 @click.pass_context
 @global_flags
+@p.empty
 @p.exclude
 @p.profiles_dir
 @p.project_dir

core/dbt/context/providers.py

@@ -238,11 +238,12 @@ def resolve_limit(self) -> Optional[int]:
     def resolve_event_time_filter(self, target: ManifestNode) -> Optional[EventTimeFilter]:
         event_time_filter = None
         if (
-            os.environ.get("DBT_EXPERIMENTAL_MICROBATCH")
-            and (isinstance(target.config, NodeConfig) or isinstance(target.config, SourceConfig))
+            (isinstance(target.config, NodeConfig) or isinstance(target.config, SourceConfig))
             and target.config.event_time
+            and isinstance(self.model, ModelNode)
             and self.model.config.materialized == "incremental"
             and self.model.config.incremental_strategy == "microbatch"
+            and self.manifest.use_microbatch_batches(project_name=self.config.project_name)
         ):
             start = self.model.config.get("__dbt_internal_microbatch_event_time_start")
             end = self.model.config.get("__dbt_internal_microbatch_event_time_end")

core/dbt/contracts/graph/manifest.py

@@ -714,10 +714,10 @@ def __init__(self):
         self._macros_by_name = {}
         self._macros_by_package = {}
 
-    def find_macro_by_name(
+    def find_macro_candidate_by_name(
         self, name: str, root_project_name: str, package: Optional[str]
-    ) -> Optional[Macro]:
-        """Find a macro in the graph by its name and package name, or None for
+    ) -> Optional[MacroCandidate]:
+        """Find a MacroCandidate in the graph by its name and package name, or None for
         any package. The root project name is used to determine priority:
          - locally defined macros come first
          - then imported macros
@@ -735,7 +735,15 @@ def filter(candidate: MacroCandidate) -> bool:
             filter=filter,
         )
 
-        return candidates.last()
+        return candidates.last_candidate()
+
+    def find_macro_by_name(
+        self, name: str, root_project_name: str, package: Optional[str]
+    ) -> Optional[Macro]:
+        macro_candidate = self.find_macro_candidate_by_name(
+            name=name, root_project_name=root_project_name, package=package
+        )
+        return macro_candidate.macro if macro_candidate else None
 
     def find_generate_macro_by_name(
         self, component: str, root_project_name: str, imported_package: Optional[str] = None
@@ -1657,6 +1665,8 @@ def add_disabled(self, source_file: AnySourceFile, node: GraphMemberNode, test_f
                 source_file.semantic_models.append(node.unique_id)
             if isinstance(node, Exposure):
                 source_file.exposures.append(node.unique_id)
+            if isinstance(node, UnitTestDefinition):
+                source_file.unit_tests.append(node.unique_id)
         elif isinstance(source_file, FixtureSourceFile):
             pass
         else:
@@ -1745,6 +1755,24 @@ def __reduce_ex__(self, protocol):
         )
         return self.__class__, args
 
+    def _microbatch_macro_is_core(self, project_name: str) -> bool:
+        microbatch_is_core = False
+        candidate = self.find_macro_candidate_by_name(
+            name="get_incremental_microbatch_sql", root_project_name=project_name, package=None
+        )
+
+        # We want to check for "Core", because "Core" basically means "builtin"
+        if candidate is not None and candidate.locality == Locality.Core:
+            microbatch_is_core = True
+
+        return microbatch_is_core
+
+    def use_microbatch_batches(self, project_name: str) -> bool:
+        return (
+            get_flags().require_batched_execution_for_custom_microbatch_strategy
+            or self._microbatch_macro_is_core(project_name=project_name)
+        )
+
 
 class MacroManifest(MacroMethods):
     def __init__(self, macros) -> None:

core/dbt/contracts/graph/nodes.py

@@ -541,11 +541,20 @@ def infer_primary_key(self, data_tests: List["GenericTestNode"]) -> List[str]:
         columns_with_disabled_unique_tests = set()
         columns_with_not_null_tests = set()
         for test in data_tests:
-            columns = []
-            if "column_name" in test.test_metadata.kwargs:
+            columns: List[str] = []
+            # extract columns from test kwargs, ensuring columns is a List[str] given tests can have custom (user or pacakge-defined) kwarg types
+            if "column_name" in test.test_metadata.kwargs and isinstance(
+                test.test_metadata.kwargs["column_name"], str
+            ):
                 columns = [test.test_metadata.kwargs["column_name"]]
-            elif "combination_of_columns" in test.test_metadata.kwargs:
-                columns = test.test_metadata.kwargs["combination_of_columns"]
+            elif "combination_of_columns" in test.test_metadata.kwargs and isinstance(
+                test.test_metadata.kwargs["combination_of_columns"], list
+            ):
+                columns = [
+                    column
+                    for column in test.test_metadata.kwargs["combination_of_columns"]
+                    if isinstance(column, str)
+                ]
 
             for column in columns:
                 if test.test_metadata.name in ["unique", "unique_combination_of_columns"]:

core/dbt/contracts/graph/semantic_manifest.py

@@ -1,5 +1,6 @@
-from typing import List, Optional
+from typing import List, Optional, Set
 
+from dbt import deprecations
 from dbt.constants import (
     LEGACY_TIME_SPINE_GRANULARITY,
     LEGACY_TIME_SPINE_MODEL_NAME,
@@ -9,6 +10,7 @@
 from dbt.contracts.graph.nodes import ModelNode
 from dbt.events.types import ArtifactWritten, SemanticValidationFailure
 from dbt.exceptions import ParsingError
+from dbt.flags import get_flags
 from dbt_common.clients.system import write_file
 from dbt_common.events.base_types import EventLevel
 from dbt_common.events.functions import fire_event
@@ -34,6 +36,11 @@
 from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
     SemanticManifestValidator,
 )
+from dbt_semantic_interfaces.validations.validator_helpers import (
+    FileContext,
+    ValidationError,
+    ValidationIssueContext,
+)
 
 
 class SemanticManifest:
@@ -58,14 +65,58 @@ def validate(self) -> bool:
         semantic_manifest = self._get_pydantic_semantic_manifest()
         validator = SemanticManifestValidator[PydanticSemanticManifest]()
         validation_results = validator.validate_semantic_manifest(semantic_manifest)
+        validation_result_errors = list(validation_results.errors)
+
+        metrics_using_old_params: Set[str] = set()
+        for metric in semantic_manifest.metrics or []:
+            for field in ("window", "grain_to_date"):
+                type_params_field_value = getattr(metric.type_params, field)
+                # Warn that the old type_params structure has been deprecated.
+                if type_params_field_value:
+                    metrics_using_old_params.add(metric.name)
+        if metrics_using_old_params:
+            if get_flags().require_nested_cumulative_type_params is False:
+                deprecations.warn(
+                    "mf-cumulative-type-params-deprecation",
+                )
+            else:
+                names = ", ".join(metrics_using_old_params)
+                validation_result_errors.append(
+                    ValidationError(
+                        context=ValidationIssueContext(
+                            # We don't have the file context at this point.
+                            file_context=FileContext(),
+                            object_name=names,
+                            object_type="metric",
+                        ),
+                        message=f"Cumulative fields `type_params.window` and `type_params.grain_to_date` should be nested under `type_params.cumulative_type_params.window` and `type_params.cumulative_type_params.grain_to_date`. Invalid metrics: {names}. See documentation on behavior changes: https://docs.getdbt.com/reference/global-configs/behavior-changes.",
+                    )
+                )
+
+        time_spines = semantic_manifest.project_configuration.time_spines
+        legacy_time_spines = (
+            semantic_manifest.project_configuration.time_spine_table_configurations
+        )
+        # If the time spine contains a day grain then it is functionally equivalent to the legacy time spine.
+        time_spines_contain_day = any(
+            c for c in time_spines if c.primary_column.time_granularity == TimeGranularity.DAY
+        )
+        if (
+            get_flags().require_yaml_configuration_for_mf_time_spines is False
+            and legacy_time_spines
+            and not time_spines_contain_day
+        ):
+            deprecations.warn(
+                "mf-timespine-without-yaml-configuration",
+            )
 
         for warning in validation_results.warnings:
             fire_event(SemanticValidationFailure(msg=warning.message))
 
-        for error in validation_results.errors:
+        for error in validation_result_errors:
             fire_event(SemanticValidationFailure(msg=error.message), EventLevel.ERROR)
 
-        return not validation_results.errors
+        return not validation_result_errors
 
     def write_json_to_file(self, file_path: str):
         semantic_manifest = self._get_pydantic_semantic_manifest()