-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* feat: init rollup * * * feat: more rollup stuff * test: fix * * * * fix: * * change rollup column name * docs: * * test: add timeout * feat: add coverage for bucket column enforcing * docs: * * feat: add rollup example to sequelize * feat: add checks for nested rollups * docs: *
- Loading branch information
Showing
28 changed files
with
1,527 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| import { TimescaleDB } from '@timescaledb/core'; | ||
| import { AggregateType, RollupFunctionType } from '@timescaledb/schemas'; | ||
|
|
||
| export const DailyPageStats = TimescaleDB.createRollup({ | ||
| continuousAggregateOptions: { | ||
| name: 'daily_page_stats', | ||
| bucket_interval: '1 day', | ||
| refresh_policy: { | ||
| start_offset: '30 days', | ||
| end_offset: '1 day', | ||
| schedule_interval: '1 day', | ||
| }, | ||
| }, | ||
| rollupOptions: { | ||
| sourceView: 'hourly_page_views', | ||
| name: 'daily_page_stats', | ||
| bucketInterval: '1 day', | ||
| materializedOnly: false, | ||
| bucketColumn: { | ||
| source: 'bucket', | ||
| target: 'bucket', | ||
| }, | ||
| rollupRules: [ | ||
| { | ||
| rollupFn: RollupFunctionType.Rollup, | ||
| sourceColumn: 'total_views', | ||
| targetColumn: 'sum_total_views', | ||
| aggregateType: AggregateType.Sum, | ||
| }, | ||
| { | ||
| rollupFn: RollupFunctionType.Rollup, | ||
| sourceColumn: 'unique_users', | ||
| targetColumn: 'avg_unique_users', | ||
| aggregateType: AggregateType.Avg, | ||
| }, | ||
| ], | ||
| }, | ||
| }); |
24 changes: 24 additions & 0 deletions
24
examples/node-sequelize/migrations/20250110064310-add-daily-stats-rollup.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| 'use strict'; | ||
|
|
||
| const path = require('path'); | ||
| const { DailyPageStats } = require(path.join(__dirname, '../dist/config/DailyPageStats')); | ||
|
|
||
| /** @type {import('sequelize-cli').Migration} */ | ||
| module.exports = { | ||
| async up(queryInterface) { | ||
| const sql = DailyPageStats.up().build(); | ||
| await queryInterface.sequelize.query(sql); | ||
|
|
||
| const refreshPolicy = DailyPageStats.up().getRefreshPolicy(); | ||
| if (refreshPolicy) { | ||
| await queryInterface.sequelize.query(refreshPolicy); | ||
| } | ||
| }, | ||
|
|
||
| async down(queryInterface) { | ||
| const statements = DailyPageStats.down().build(); | ||
| for await (const sql of statements) { | ||
| await queryInterface.sequelize.query(sql); | ||
| } | ||
| }, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| import { Model, DataTypes } from 'sequelize'; | ||
| import sequelize from '../database'; | ||
|
|
||
| class DailyPageStats extends Model { | ||
| public bucket!: Date; | ||
| public sumTotalViews!: number; | ||
| public avgUniqueUsers!: number; | ||
| } | ||
|
|
||
| DailyPageStats.init( | ||
| { | ||
| bucket: { | ||
| type: DataTypes.DATE, | ||
| primaryKey: true, | ||
| }, | ||
| sumTotalViews: { | ||
| type: DataTypes.INTEGER, | ||
| field: 'sum_total_views', | ||
| }, | ||
| avgUniqueUsers: { | ||
| type: DataTypes.FLOAT, | ||
| field: 'avg_unique_users', | ||
| }, | ||
| }, | ||
| { | ||
| sequelize, | ||
| tableName: 'daily_page_stats', | ||
| timestamps: false, | ||
| underscored: true, | ||
| }, | ||
| ); | ||
|
|
||
| export default DailyPageStats; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| import { describe, it, expect, beforeEach, afterAll } from '@jest/globals'; | ||
| import { request } from './mock-request'; | ||
| import sequelize from '../src/database'; | ||
| import PageLoad from '../src/models/PageLoad'; | ||
| import { faker } from '@faker-js/faker'; | ||
|
|
||
| describe('GET /api/daily', () => { | ||
| beforeEach(async () => { | ||
| await PageLoad.destroy({ where: {} }); | ||
| }); | ||
|
|
||
| afterAll(async () => { | ||
| await sequelize.close(); | ||
| }); | ||
|
|
||
| it('should return daily stats for a given time range', async () => { | ||
| const baseTime = new Date(); | ||
| baseTime.setHours(0, 0, 0, 0); // Start of day | ||
|
|
||
| // Create test data across 3 days | ||
| for (let day = 0; day < 3; day++) { | ||
| const dayStart = new Date(baseTime.getTime() - day * 24 * 3600000); | ||
|
|
||
| // Create multiple records per day across different hours | ||
| for (let hour = 0; hour < 24; hour += 4) { | ||
| const time = new Date(dayStart.getTime() + hour * 3600000); | ||
|
|
||
| // Create multiple records per hour | ||
| for (let i = 0; i < 5; i++) { | ||
| await PageLoad.create({ | ||
| userAgent: faker.internet.userAgent(), | ||
| time: new Date(time.getTime() + i * 60000), // Spread over minutes | ||
| }); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Manually refresh the continuous aggregate for hourly views | ||
| await sequelize.query(`CALL refresh_continuous_aggregate('hourly_page_views', null, null);`); | ||
|
|
||
| // Wait for hourly refresh to complete | ||
| await new Promise((resolve) => setTimeout(resolve, 2000)); | ||
|
|
||
| // Manually refresh the rollup for daily stats | ||
| await sequelize.query(`CALL refresh_continuous_aggregate('daily_page_stats', null, null);`); | ||
|
|
||
| // Wait for daily refresh to complete | ||
| await new Promise((resolve) => setTimeout(resolve, 2000)); | ||
|
|
||
| const start = new Date(baseTime.getTime() - 4 * 24 * 3600000); // 4 days ago | ||
| const end = baseTime; | ||
|
|
||
| const response = await request().get('/api/daily').query({ | ||
| start: start.toISOString(), | ||
| end: end.toISOString(), | ||
| }); | ||
|
|
||
| expect(response.status).toBe(200); | ||
| expect(response.body).toHaveLength(3); | ||
|
|
||
| const firstDay = response.body[0]; | ||
| expect(firstDay).toHaveProperty('bucket'); | ||
| expect(firstDay).toHaveProperty('sumTotalViews'); | ||
| expect(firstDay).toHaveProperty('avgUniqueUsers'); | ||
|
|
||
| // Each day should have: | ||
| // - 6 time slots (every 4 hours) | ||
| // - 5 views per time slot | ||
| // - 30 total views per day | ||
| response.body.forEach((day: any) => { | ||
| expect(day.sumTotalViews).toBe('30'); | ||
| expect(Number(day.avgUniqueUsers)).toBeGreaterThan(0); | ||
| expect(Number(day.avgUniqueUsers)).toBeLessThanOrEqual(30); | ||
| }); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| import { Rollup, BucketColumn, RollupColumn } from '@timescaledb/typeorm'; | ||
| import { HourlyPageViews } from './HourlyPageViews'; | ||
| import { AggregateType } from '@timescaledb/schemas'; | ||
|
|
||
| @Rollup(HourlyPageViews, { | ||
| name: 'daily_page_stats', | ||
| bucket_interval: '1 day', | ||
| refresh_policy: { | ||
| start_offset: '30 days', | ||
| end_offset: '1 day', | ||
| schedule_interval: '1 day', | ||
| }, | ||
| }) | ||
| export class DailyPageStats { | ||
| @BucketColumn({ | ||
| source_column: 'bucket', | ||
| }) | ||
| bucket!: Date; | ||
|
|
||
| @RollupColumn({ | ||
| type: AggregateType.Sum, | ||
| source_column: 'total_views', | ||
| }) | ||
| sum_total_views!: number; | ||
|
|
||
| @RollupColumn({ | ||
| type: AggregateType.Avg, | ||
| source_column: 'unique_users', | ||
| }) | ||
| avg_unique_users!: number; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.