Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(dremio-driver): Correct dremio date interval functions #9150

Closed
wants to merge 13 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 79 additions & 4 deletions packages/cubejs-dremio-driver/driver/DremioQuery.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
const { BaseFilter, BaseQuery } = require('@cubejs-backend/schema-compiler');
const { parseSqlInterval } = require('@cubejs-backend/shared');

const GRANULARITY_TO_INTERVAL = {
week: (date) => `DATE_TRUNC('week', ${date})`,
Expand Down Expand Up @@ -55,15 +56,39 @@ class DremioQuery extends BaseQuery {
}

dateTimeCast(value) {
return `TO_TIMESTAMP(${value})`;
return `TO_TIMESTAMP(${value}, 'YYYY-MM-DD"T"HH24:MI:SS.FFF')`;
}

subtractInterval(date, interval) {
return `DATE_SUB(${date}, INTERVAL ${interval})`;
const formattedTimeIntervals = this.formatInterval(interval);
const intervalFormatted = formattedTimeIntervals[0];
const timeUnit = formattedTimeIntervals[1];
return `DATE_SUB(${date}, CAST(${intervalFormatted} as INTERVAL ${timeUnit}))`;
}

addInterval(date, interval) {
return `DATE_ADD(${date}, INTERVAL ${interval})`;
const formattedTimeIntervals = this.formatInterval(interval);
const intervalFormatted = formattedTimeIntervals[0];
const timeUnit = formattedTimeIntervals[1];
return `DATE_ADD(${date}, CAST(${intervalFormatted} as INTERVAL ${timeUnit}))`;
}

/**
* @param {string} timestamp
* @param {string} interval
* @returns {string}
*/
addTimestampInterval(timestamp, interval) {
return this.addInterval(timestamp, interval);
}

/**
* @param {string} timestamp
* @param {string} interval
* @returns {string}
*/
subtractTimestampInterval(timestamp, interval) {
return this.subtractInterval(timestamp, interval);
}

timeGroupedColumn(granularity, dimension) {
Expand All @@ -78,7 +103,8 @@ class DremioQuery extends BaseQuery {
const values = timeDimension.timeSeries().map(
([from, to]) => `select '${from}' f, '${to}' t`
).join(' UNION ALL ');
return `SELECT TO_TIMESTAMP(dates.f, 'YYYY-MM-DDTHH:MI:SS.FFF') date_from, TO_TIMESTAMP(dates.t, 'YYYY-MM-DDTHH:MI:SS.FFF') date_to FROM (${values}) AS dates`;

return `SELECT TO_TIMESTAMP(dates.f, 'YYYY-MM-DD"T"HH24:MI:SS.FFF') date_from, TO_TIMESTAMP(dates.t, 'YYYY-MM-DD"T"HH24:MI:SS.FFF') date_to FROM (${values}) AS dates`;
}

concatStringsSql(strings) {
Expand All @@ -92,6 +118,55 @@ class DremioQuery extends BaseQuery {
wrapSegmentForDimensionSelect(sql) {
return `IF(${sql}, 1, 0)`;
}

/**
* The input interval with (possible) plural units, like "1 hour 2 minutes", "2 year", "3 months", "4 weeks", "5 days", "3 months 24 days 15 minutes", ...
* will be converted to Dremio dialect.
* @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_ADD/
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at the docs, I see that Dremio supports more intervals for timestamps.

DATE_ADD(timestamp_expression string, time_interval interval) → timestamp
timestamp_expression: A string-formatted timestamp in the format 'YYYY-MM-DD HH24:MI:SS'.
time_interval: A CAST of a number to one of these intervals: SECOND, MINUTE, HOUR, DAY, MONTH, YEAR.

So i think they should be supported here too :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, we also need to support a week method as well in the same way the quarter is calculated.

* @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_SUB/
* It returns a tuple of (formatted interval, timeUnit to use in date functions)
* This function only supports the following scenarios for now:
* ie. n year[s] or n quarter[s] or n month[s] or n week[s] or n day[s]
*/
formatInterval(interval) {
const intervalParsed = parseSqlInterval(interval);
const intKeys = Object.keys(intervalParsed).length;

if (intervalParsed.year && intKeys === 1) {
return [`${intervalParsed.year}`, 'YEAR'];
} else if (intervalParsed.quarter && intKeys === 1) {
// dremio interval does not support quarter. Convert to month
return [`${intervalParsed.quarter * 3}`, 'MONTH'];
} else if (intervalParsed.week && intKeys === 1) {
// dremio interval does not support week. Convert to days
return [`${intervalParsed.week * 7}`, 'DAY'];
} else if (intervalParsed.month && intKeys === 1) {
return [`${intervalParsed.month}`, 'MONTH'];
} else if (intervalParsed.month && intKeys === 1) {
return [`${intervalParsed.day}`, 'DAY'];
} else if (intervalParsed.hour && intKeys === 1) {
return [`${intervalParsed.hour}`, 'HOUR'];
} else if (intervalParsed.minute && intKeys === 1) {
return [`${intervalParsed.minute}`, 'MINUTE'];
} else if (intervalParsed.second && intKeys === 1) {
return [`${intervalParsed.second}`, 'SECOND'];
}

throw new Error(`Cannot transform interval expression "${interval}" to Dremio dialect`);
}

sqlTemplates() {
const templates = super.sqlTemplates();
templates.functions.CURRENTDATE = 'CURRENT_DATE';
templates.functions.DATETRUNC = 'DATE_TRUNC(\'{{ date_part }}\', {{ args_concat }})';
templates.functions.DATEPART = 'DATE_PART(\'{{ date_part }}\', {{ args_concat }})';
// really need the date locale formatting here...
templates.functions.DATE = 'TO_DATE({{ args_concat }},\'YYYY-MM-DD\', 1)';
templates.functions.DATEDIFF = 'DATE_DIFF(DATE, DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}))';
templates.expressions.interval_single_date_part = 'CAST({{ num }} as INTERVAL {{ date_part }})';
templates.quotes.identifiers = '"';
return templates;
}
}

module.exports = DremioQuery;
Loading