From bb85070a9c582695946b03f30adde6480c58db56 Mon Sep 17 00:00:00 2001 From: Colin Ho Date: Thu, 9 Jan 2025 11:16:46 -0800 Subject: [PATCH] fix: Run filter null join key optimization once (#3657) Only run the filter null join key optimization once before filter pushdown, instead of 3 times together with the other rules, otherwise you end up with something like `[not(is_null(key)) & not(is_null(key))] & not(is_null(key))`. Co-authored-by: Colin Ho --- src/daft-logical-plan/src/optimization/optimizer.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/daft-logical-plan/src/optimization/optimizer.rs b/src/daft-logical-plan/src/optimization/optimizer.rs index cea66fbf13..d1fea88019 100644 --- a/src/daft-logical-plan/src/optimization/optimizer.rs +++ b/src/daft-logical-plan/src/optimization/optimizer.rs @@ -106,11 +106,16 @@ impl Default for OptimizerBuilder { vec![Box::new(SimplifyExpressionsRule::new())], RuleExecutionStrategy::FixedPoint(Some(3)), ), + // --- Filter out null join keys --- + // This rule should be run once, before any filter pushdown rules. + RuleBatch::new( + vec![Box::new(FilterNullJoinKey::new())], + RuleExecutionStrategy::Once, + ), // --- Bulk of our rules --- RuleBatch::new( vec![ Box::new(DropRepartition::new()), - Box::new(FilterNullJoinKey::new()), Box::new(PushDownFilter::new()), Box::new(PushDownProjection::new()), Box::new(EliminateCrossJoin::new()),