Skip to content

Commit

Permalink
added more tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Harshit Gangal <[email protected]>
  • Loading branch information
@harshit-gangal @systay
harshit-gangal authored and systay committed Jun 10, 2024
1 parent 134b977 commit 8ec0132
Show file tree
Hide file tree
Showing 3 changed files with 632 additions and 0 deletions.
1 change: 1 addition & 0 deletions t/info_schema.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
describe information_schema.administrable_role_authorizations;
173 changes: 173 additions & 0 deletions t/insights.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
CREATE TABLE `anomaly` (
`id` bigint unsigned NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`period_start` datetime(6) NOT NULL,
`period_end` datetime(6) NOT NULL,
`minutes_in_violation` int NOT NULL,
`active` tinyint(1) NOT NULL,
`correlations_materialized` tinyint(1) NOT NULL DEFAULT '0',
`created_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`updated_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`expires_at` datetime(6) NOT NULL,
`relevant_started_at` datetime(6),
PRIMARY KEY (`id`),
UNIQUE KEY `idx_anomaly_branch_start` (`database_branch_public_id`, `period_start`),
KEY `idx_anomaly_on_expires_at` (`expires_at`),
KEY `idx_anomaly_on_correlations_materialized_period_end` (`correlations_materialized`, `period_end`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `branch_query` (
`id` bigint unsigned NOT NULL,
`fingerprint` varchar(64) NOT NULL,
`username` varchar(255) NOT NULL,
`remote_address` varchar(255),
`statement_type` varchar(255) NOT NULL,
`tables` json,
`keyspace` varchar(255) NOT NULL,
`tablet_type` tinyint unsigned NOT NULL DEFAULT '0',
`boost_query_public_id` varchar(255) NOT NULL DEFAULT '',
`shard_queries` int unsigned NOT NULL DEFAULT '0',
`rows_read` bigint unsigned NOT NULL DEFAULT '0',
`rows_affected` bigint unsigned NOT NULL DEFAULT '0',
`rows_returned` bigint unsigned NOT NULL DEFAULT '0',
`total_duration_millis` bigint unsigned NOT NULL DEFAULT '0',
`error_message` text,
`normalized_sql` text NOT NULL,
`started_at` datetime(6) NOT NULL,
`created_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`updated_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`kafka_partition` int unsigned NOT NULL,
`kafka_offset` bigint unsigned NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`raw_sql` text,
`raw_sql_abbreviation` varchar(255),
`error_fingerprint` varchar(64),
`tables_used` json,
PRIMARY KEY (`id`),
UNIQUE KEY `index_branch_query_on_kafka_partition_and_kafka_offset` (`kafka_partition`, `kafka_offset`),
KEY `index_branch_query_on_branch_fingerprint_started` (`database_branch_public_id`, `fingerprint`, `started_at`),
KEY `index_branch_query_on_database_branch_public_id_and_started_at` (`database_branch_public_id`, `started_at`),
KEY `index_branch_query_on_started_at` (`started_at`),
KEY `index_branch_query_on_branch_error_fingerprint_started` (`database_branch_public_id`, `error_fingerprint`, `started_at`),
KEY `idx_branch_query_branch_username_started_at` (`database_branch_public_id`, `username`, `started_at`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `branch_query_stat` (
`id` bigint unsigned NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`fingerprint` varbinary(32) NOT NULL,
`keyspace` varchar(255) NOT NULL,
`tablet_type` tinyint unsigned NOT NULL DEFAULT '0',
`boost_query_public_id` varchar(255) NOT NULL DEFAULT '',
`bucket` datetime(6) NOT NULL,
`kafka_offsets` json NOT NULL,
`rows_returned` int unsigned NOT NULL,
`rows_read` int unsigned NOT NULL,
`rows_affected` int unsigned NOT NULL,
`query_count` int unsigned NOT NULL DEFAULT '0',
`error_count` int unsigned NOT NULL DEFAULT '0',
`correlation_materialized` tinyint(1) NOT NULL DEFAULT '0',
`total_duration_sketch` blob NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_branch_query_stat_branch_fp_keyspace_bucket_tablet_boost` (`database_branch_public_id`, `fingerprint`, `keyspace`, `bucket`, `tablet_type`, `boost_query_public_id`),
KEY `idx_branch_query_stat_on_bucket_correlation_materialized_db` (`correlation_materialized`, `bucket`, `database_branch_public_id`),
KEY `idx_branch_query_stat_bucket` (`bucket`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `branch_query_tag` (
`id` bigint unsigned NOT NULL,
`branch_query_id` bigint unsigned NOT NULL,
`name` varchar(255) NOT NULL,
`value` varchar(255) NOT NULL,
`created_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`updated_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`database_branch_public_id` varchar(12) NOT NULL,
`new_column` varchar(255),
PRIMARY KEY (`id`),
UNIQUE KEY `index_branch_query_tag_on_branch_query_id_and_name_and_value` (`branch_query_id`, `name`, `value`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `branch_stat` (
`id` bigint unsigned NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`bucket` datetime(6) NOT NULL,
`tablet_type` tinyint unsigned NOT NULL,
`kafka_offsets` json NOT NULL,
`violation_eligible_query_count` bigint unsigned NOT NULL DEFAULT '0',
`violation_count` int unsigned NOT NULL,
`query_count` int unsigned NOT NULL,
`rows_returned` bigint unsigned NOT NULL DEFAULT '0',
`rows_read` bigint unsigned NOT NULL DEFAULT '0',
`rows_affected` bigint unsigned NOT NULL DEFAULT '0',
`error_count` int unsigned NOT NULL DEFAULT '0',
`anomaly_materialized` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `idx_branch_stat_branch_bucket_stablet_type` (`database_branch_public_id`, `bucket`, `tablet_type`),
KEY `index_branch_stat_on_bucket` (`bucket`),
KEY `idx_branch_stat_on_anomaly_bucket_branch` (`anomaly_materialized`, `bucket`, `database_branch_public_id`),
KEY `idx_branch_stat_on_branch_anomaly_bucket` (`database_branch_public_id`, `anomaly_materialized`, `bucket`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `correlation` (
`id` bigint unsigned NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`anomaly_id` bigint unsigned NOT NULL,
`keyspace` varchar(255) NOT NULL,
`fingerprint` varbinary(32) NOT NULL,
`tablet_type` tinyint unsigned NOT NULL,
`r` float NOT NULL,
`normalized_sql` text NOT NULL,
`created_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`updated_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
PRIMARY KEY (`id`),
UNIQUE KEY `idx_correlation_anomaly_keyspace_fp_tablet_type` (`anomaly_id`, `keyspace`, `fingerprint`, `tablet_type`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;
CREATE TABLE `hourly_branch_query` (
`id` bigint unsigned NOT NULL,
`fingerprint` varchar(64) NOT NULL,
`hour` datetime(6) NOT NULL,
`statement_type` varchar(255) NOT NULL,
`tables` json,
`keyspace` varchar(255) NOT NULL,
`tablet_type` tinyint unsigned NOT NULL DEFAULT '0',
`boost_query_public_id` varchar(255) NOT NULL DEFAULT '',
`query_count` bigint unsigned NOT NULL DEFAULT '0',
`error_count` bigint unsigned NOT NULL DEFAULT '0',
`sum_shard_queries` bigint unsigned NOT NULL DEFAULT '0',
`max_shard_queries` int unsigned NOT NULL DEFAULT '0',
`sum_rows_read` bigint unsigned NOT NULL DEFAULT '0',
`sum_rows_affected` bigint unsigned NOT NULL DEFAULT '0',
`sum_rows_returned` bigint unsigned NOT NULL DEFAULT '0',
`sum_total_duration_millis` bigint unsigned NOT NULL DEFAULT '0',
`normalized_sql` text NOT NULL,
`created_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`updated_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`kafka_offsets` json NOT NULL,
`database_branch_public_id` varchar(12) NOT NULL,
`last_run_at` datetime(6) NOT NULL DEFAULT current_timestamp(6),
`tables_used` json,
`correlation_n` bigint unsigned NOT NULL DEFAULT '0',
`correlation_sum_xy` float unsigned NOT NULL DEFAULT '0',
`correlation_sum_x` float unsigned NOT NULL DEFAULT '0',
`correlation_sum_y` float unsigned NOT NULL DEFAULT '0',
`correlation_sum_xsq` float unsigned NOT NULL DEFAULT '0',
`correlation_sum_ysq` float unsigned NOT NULL DEFAULT '0',
`correlation_minutes_set` bigint NOT NULL DEFAULT '0',
`total_duration_sketch` blob,
PRIMARY KEY (`id`),
UNIQUE KEY `idx_hourly_branch_query_branch_hour_fp_keyspace_tablet_boost` (`database_branch_public_id`, `hour`, `fingerprint`, `keyspace`, `tablet_type`, `boost_query_public_id`),
KEY `index_hourly_branch_query_on_hour` (`hour`),
KEY `idx_hourly_branch_query_branch_hour_statement_type` (`database_branch_public_id`, `hour`, `statement_type`),
KEY `idx_hourly_branch_query_branch_fingerprint` (`database_branch_public_id`, `fingerprint`)
) ENGINE InnoDB,
CHARSET utf8mb4,
COLLATE utf8mb4_0900_ai_ci;

select keyspace, fingerprint, tablet_type, normalized_sql, round(correlation_r(correlation_n, correlation_sum_xy, correlation_sum_x, correlation_sum_y, correlation_sum_xsq, correlation_sum_ysq, cast(unix_timestamp(hour) + 0 as unsigned), correlation_minutes_set, (select coalesce(group_concat(bucket_offset, ':', violation_ratio), "") from (select floor((unix_timestamp(bucket) + 0 - 1712746800) / 60) as bucket_offset, sum(violation_count) / sum(violation_eligible_query_count) violation_ratio from branch_stat where database_branch_public_id = '2dn3aald67wy' and bucket >= '2024-04-10 11:00:00' and bucket < '2024-04-10 20:00:00' group by 1 order by bucket_offset) as bucketed_violation_ratios), 1712746800, 1712779200), 4) r from hourly_branch_query where database_branch_public_id = '2dn3aald67wy' and hour between '2024-04-10 11:00:00' and '2024-04-10 19:00:00' and `hourly_branch_query`.`statement_type` IN ('SELECT', 'INSERT', 'UPDATE', 'DELETE') group by keyspace, fingerprint, tablet_type, normalized_sql having r >= 0.3 and sum(sum_total_duration_millis) >= 30000 order by r desc limit 25;
Loading

0 comments on commit 8ec0132

Please sign in to comment.