fix(schema-compiler): Missing pre-aggregation filter for partitioned …

…pre-aggregation via view, fix #6623 (#7454)

There was an issue when partitioned pre-aggregation was queried via view. Schema compiler could not find a time dimension to get a range of used pre-aggregation from the query.

As a result, it started to union all partitions. See real example from the issue: #6623

packages/cubejs-schema-compiler/package.json

@@ -99,6 +99,7 @@
     ],
     "coveragePathIgnorePatterns": [
       ".*\\.d\\.ts"
-    ]
+    ],
+    "globalSetup": "<rootDir>/dist/test/global-setup.js"
   }
 }

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -63,6 +63,7 @@ export class PreAggregations {
     if (foundPreAggregation.preAggregation.type === 'rollupLambda') {
       preAggregations = foundPreAggregation.referencedPreAggregations;
     }
+
     return preAggregations.map(preAggregation => {
       if (this.canPartitionsBeUsed(preAggregation)) {
         const { dimension, partitionDimension } = this.partitionDimension(preAggregation);
@@ -151,17 +152,35 @@ export class PreAggregations {
     const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation);
     const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries && queryForSqlEvaluation.partitionInvalidateKeyQueries(cube, preAggregation);
 
-    const matchedTimeDimension =
-      preAggregation.partitionGranularity &&
-      !this.hasCumulativeMeasures &&
-      this.query.timeDimensions.find(
-        td => td.dimension === foundPreAggregation.references.timeDimensions[0].dimension && td.dateRange
-      );
-    const filters = preAggregation.partitionGranularity && this.query.filters.filter(
-      td => td.dimension === foundPreAggregation.references.timeDimensions[0].dimension &&
-        td.isDateOperator() &&
-        td.camelizeOperator === 'inDateRange' // TODO support all date operators
-    );
+    const matchedTimeDimension = preAggregation.partitionGranularity && !this.hasCumulativeMeasures &&
+      this.query.timeDimensions.find(td => {
+        if (!td.dateRange) {
+          return false;
+        }
+
+        if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) {
+          return true;
+        }
+
+        // Handling for views
+        const dimension = this.query.cubeEvaluator.byPath('dimensions', td.expressionPath());
+        return dimension?.aliasMember === foundPreAggregation.references.timeDimensions[0].dimension;
+      });
+
+    const filters = preAggregation.partitionGranularity && this.query.filters.filter(td => {
+      // TODO support all date operators
+      if (td.isDateOperator() && td.camelizeOperator === 'inDateRange') {
+        if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) {
+          return true;
+        }
+
+        // Handling for views
+        const dimension = this.query.cubeEvaluator.byPath('dimensions', td.expressionPath());
+        return dimension?.aliasMember === foundPreAggregation.references.timeDimensions[0].dimension && td.dateRange;
+      }
+
+      return false;
+    });
 
     const uniqueKeyColumnsDefault = () => null;
     const uniqueKeyColumns = ({
@@ -557,15 +576,15 @@ export class PreAggregations {
         R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.leafMeasures)
       ));
     };
-    
+
     /**
      * Wrap granularity string into an array.
      * @param {string} granularity
      * @returns {Array<string>}
      */
     const expandGranularity = (granularity) => (
       transformedQuery.granularityHierarchies[granularity] ||
-        [granularity]
+      [granularity]
     );
 
     /**
@@ -666,7 +685,7 @@ export class PreAggregations {
       transformedQuery.leafMeasureAdditive &&
       !transformedQuery.hasMultipliedMeasures
         ? (r) => canUsePreAggregationLeafMeasureAdditive(r) ||
-        canUsePreAggregationNotAdditive(r)
+          canUsePreAggregationNotAdditive(r)
         : canUsePreAggregationNotAdditive;
 
     if (refs) {

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js

@@ -308,13 +308,22 @@ export class CubeEvaluator extends CubeSymbols {
     return cubeAndName[0];
   }
 
+  /**
+   * @param {measures|dimensions|segments} type
+   * @param {string} path
+   * @returns boolean
+   */
   isInstanceOfType(type, path) {
     const cubeAndName = Array.isArray(path) ? path : path.split('.');
     return this.evaluatedCubes[cubeAndName[0]] &&
       this.evaluatedCubes[cubeAndName[0]][type] &&
       this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]];
   }
 
+  /**
+   * @param {string} path
+   * @returns {*}
+   */
   byPathAnyType(path) {
     const type = ['measures', 'dimensions', 'segments'].find(t => this.isInstanceOfType(t, path));
 
@@ -325,6 +334,11 @@ export class CubeEvaluator extends CubeSymbols {
     return this.byPath(type, path);
   }
 
+  /**
+   * @param {measures|dimensions|segments} type
+   * @param {string} path
+   * @returns {*}
+   */
   byPath(type, path) {
     if (!type) {
       throw new Error(`Type can't be undefined for '${path}'`);

packages/cubejs-schema-compiler/test/global-setup.ts

@@ -0,0 +1,3 @@
+export default () => {
+  process.env.TZ = 'UTC';
+};

packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts

@@ -1,4 +1,6 @@
-import { prepareCompiler } from './PrepareCompiler';
+import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler';
+import { createECommerceSchema, createSchemaYaml } from './utils';
+import { PostgresQuery } from '../../src';
 
 describe('pre-aggregations', () => {
   it('rollupJoin scheduledRefresh', async () => {
@@ -89,4 +91,36 @@ describe('pre-aggregations', () => {
     expect(cubeEvaluator.cubeFromPath('Orders').preAggregations.ordersRollup.scheduledRefresh).toEqual(true);
     expect(cubeEvaluator.cubeFromPath('Orders').preAggregations.ordersRollupJoin.scheduledRefresh).toEqual(undefined);
   });
+
+  // @link https://github.com/cube-js/cube/issues/6623
+  it('view and pre-aggregation granularity', async () => {
+    const { compiler, cubeEvaluator, joinGraph } = prepareYamlCompiler(
+      createSchemaYaml(createECommerceSchema())
+    );
+
+    await compiler.compile();
+
+    const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: [
+        'orders_view.count'
+      ],
+      timeDimensions: [{
+        dimension: 'orders_view.created_at',
+        granularity: 'day',
+        dateRange: ['2023-01-01', '2023-01-10']
+      }],
+      timezone: 'America/Los_Angeles'
+    });
+
+    const queryAndParams = query.buildSqlAndParams();
+    console.log(queryAndParams);
+
+    const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
+    console.log(JSON.stringify(preAggregationsDescription, null, 2));
+
+    expect(preAggregationsDescription[0].matchedTimeDimensionDateRange).toEqual([
+      '2023-01-01T00:00:00.000',
+      '2023-01-10T23:59:59.999'
+    ]);
+  });
 });

packages/cubejs-schema-compiler/test/unit/utils.ts

@@ -1,3 +1,5 @@
+import YAML from 'js-yaml';
+
 interface CreateCubeSchemaOptions {
   name: string,
   publicly?: boolean,
@@ -69,6 +71,15 @@ export function createCubeSchema({ name, refreshKey = '', preAggregations = '',
   `;
 }
 
+export type CreateSchemaOptions = {
+  cubes?: unknown[],
+  views?: unknown[]
+};
+
+export function createSchemaYaml(schema: CreateSchemaOptions): string {
+  return YAML.dump(schema);
+}
+
 export function createCubeSchemaYaml({ name, sqlTable }: CreateCubeSchemaOptions): string {
   return ` 
     cubes:
@@ -98,6 +109,47 @@ export function createCubeSchemaYaml({ name, sqlTable }: CreateCubeSchemaOptions
   `;
 }
 
+export function createECommerceSchema() {
+  return {
+    cubes: [{
+      name: 'orders',
+      sql_table: 'orders',
+      measures: [{
+        name: 'count',
+        type: 'count',
+      }],
+      dimensions: [{
+        name: 'created_at',
+        sql: 'created_at',
+        type: 'time',
+      }],
+      preAggregations: [{
+        name: 'orders_by_day_with_day',
+        measures: ['count'],
+        timeDimension: 'created_at',
+        granularity: 'day',
+        partition_granularity: 'day',
+        build_range_start: {
+          sql: 'SELECT NOW() - INTERVAL \'1000 day\'',
+        },
+        build_range_end: {
+          sql: 'SELECT NOW()'
+        },
+      }]
+    }],
+    views: [{
+      name: 'orders_view',
+      cubes: [{
+        join_path: 'orders',
+        includes: [
+          'created_at',
+          'count'
+        ]
+      }]
+    }]
+  };
+}
+
 /**
  * Returns joined test cubes schema. Schema looks like: A -< B -< C >- D >- E.
  * The original data set can be found under the link.