diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index fe3e2ddea7..e80df0ecf7 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -4,8 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ## [Unreleased] +### Added +- Schema Migration support for Enums, Relations, Subscription (#2251) +- +### Fixed +- Fixed non-atomic schema migration execution (#2244) ## [7.2.1] - 2024-02-07 ### Added @@ -14,9 +18,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Historical queries not using the correct block height (#2243) -### Fixed -- Fixed non-autonomous schema migration execution (#2244) - ## [7.2.0] - 2024-01-30 ### Changed - Update `@subql/apollo-links` and add specific logger for it diff --git a/packages/node-core/src/configure/migration-service/SchemaMigration.service.ts b/packages/node-core/src/configure/migration-service/SchemaMigration.service.ts index 16d19f21a6..b6fafda129 100644 --- a/packages/node-core/src/configure/migration-service/SchemaMigration.service.ts +++ b/packages/node-core/src/configure/migration-service/SchemaMigration.service.ts @@ -11,6 +11,7 @@ import {sortModels} from '../../utils'; import {NodeConfig} from '../NodeConfig'; import {Migration} from './migration'; import { + alignModelOrder, compareEnums, compareModels, compareRelations, @@ -80,39 +81,6 @@ export class SchemaMigrationService { } } - private alignModelOrder( - schemaModels: GraphQLModelsType[], - models: GraphQLModelsType[] | ModifiedModels - ): GraphQLModelsType[] | ModifiedModels { - const orderIndex = schemaModels.reduce((acc: Record, model, index) => { - acc[model.name] = index; - return acc; - }, {}); - - if (Array.isArray(models)) { - return models.sort((a, b) => { - const indexA = orderIndex[a.name] ?? Number.MAX_VALUE; // Place unknown models at the end - const indexB = orderIndex[b.name] ?? Number.MAX_VALUE; - return indexA - indexB; - }); - } else { - const modelNames = Object.keys(models); - const sortedModelNames = modelNames.sort((a, b) => { - const indexA = orderIndex[a] ?? Number.MAX_VALUE; - const indexB = orderIndex[b] ?? Number.MAX_VALUE; - return indexA - indexB; - }); - - const sortedModifiedModels: ModifiedModels = {}; - sortedModelNames.forEach((modelName) => { - sortedModifiedModels[modelName] = models[modelName]; - }); - - return sortedModifiedModels; - } - } - - // eslint-disable-next-line complexity async run( currentSchema: GraphQLSchema | null, nextSchema: GraphQLSchema, @@ -148,61 +116,52 @@ export class SchemaMigrationService { getAllEntitiesRelations(nextSchema).relations ); - const sortedAddedModels = this.alignModelOrder(sortedSchemaModels, addedModels) as GraphQLModelsType[]; - const sortedModifiedModels = this.alignModelOrder(sortedSchemaModels, modifiedModels); + const sortedAddedModels = alignModelOrder(sortedSchemaModels, addedModels); + const sortedModifiedModels = alignModelOrder(sortedSchemaModels, modifiedModels); await this.flushCache(true); const migrationAction = await Migration.create( this.sequelize, this.storeService, this.dbSchema, - nextSchema, currentSchema, this.config, - logger, this.dbType ); + // TODO this should only be printed if schema migration is enabled and not store.service sync schema logger.info(`${schemaChangesLoggerMessage(schemaDifference)}`); try { - if (addedEnums.length) { - for (const enumValue of addedEnums) { - await migrationAction.createEnum(enumValue); - } + for (const enumValue of addedEnums) { + await migrationAction.createEnum(enumValue); } - if (removedModels.length) { - for (const model of removedModels) { - migrationAction.dropTable(model); - } + for (const model of removedModels) { + migrationAction.dropTable(model); } - if (sortedAddedModels.length) { - for (const model of sortedAddedModels) { - await migrationAction.createTable(model, this.withoutForeignKeys); - } + for (const model of sortedAddedModels) { + await migrationAction.createTable(model, this.withoutForeignKeys); } - if (Object.keys(sortedModifiedModels).length) { - const entities = Object.keys(modifiedModels); - for (const model of entities) { - const modelValue = modifiedModels[model]; + const entities = Object.keys(sortedModifiedModels); + for (const model of entities) { + const modelValue = sortedModifiedModels[model]; - for (const index of modelValue.removedIndexes) { - migrationAction.dropIndex(modelValue.model, index); - } + for (const index of modelValue.removedIndexes) { + migrationAction.dropIndex(modelValue.model, index); + } - for (const field of modelValue.removedFields) { - migrationAction.dropColumn(modelValue.model, field); - } + for (const field of modelValue.removedFields) { + migrationAction.dropColumn(modelValue.model, field); + } - for (const field of modelValue.addedFields) { - migrationAction.createColumn(modelValue.model, field); - } + for (const field of modelValue.addedFields) { + migrationAction.createColumn(modelValue.model, field); + } - for (const index of modelValue.addedIndexes) { - migrationAction.createIndex(modelValue.model, index); - } + for (const index of modelValue.addedIndexes) { + migrationAction.createIndex(modelValue.model, index); } } @@ -214,15 +173,11 @@ export class SchemaMigrationService { migrationAction.addRelationComments(); } - if (removedRelations.length) { - for (const relationModel of removedRelations) { - migrationAction.dropRelation(relationModel); - } + for (const relationModel of removedRelations) { + migrationAction.dropRelation(relationModel); } - if (removedEnums.length) { - for (const enumValue of removedEnums) { - migrationAction.dropEnums(enumValue); - } + for (const enumValue of removedEnums) { + migrationAction.dropEnum(enumValue); } return migrationAction.run(transaction); diff --git a/packages/node-core/src/configure/migration-service/migration-helpers.ts b/packages/node-core/src/configure/migration-service/migration-helpers.ts index 27ad04483c..889c8cc6ad 100644 --- a/packages/node-core/src/configure/migration-service/migration-helpers.ts +++ b/packages/node-core/src/configure/migration-service/migration-helpers.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import {addRelationToMap, enumNameToHash, SmartTags} from '@subql/node-core'; import { GraphQLEntityField, GraphQLEntityIndex, @@ -8,6 +9,7 @@ import { GraphQLModelsType, GraphQLRelationsType, } from '@subql/utils'; +import {QueryTypes, Sequelize} from '@subql/x-sequelize'; import {isEqual} from 'lodash'; export type ModifiedModels = Record< @@ -205,3 +207,81 @@ export function schemaChangesLoggerMessage(schemaChanges: SchemaChangesType): st } return logMessage; } +export function alignModelOrder( + schemaModels: GraphQLModelsType[], + models: T +): T { + const orderIndex = schemaModels.reduce((acc: Record, model, index) => { + acc[model.name] = index; + return acc; + }, {}); + + if (Array.isArray(models)) { + return models.sort((a, b) => { + const indexA = orderIndex[a.name] ?? Number.MAX_VALUE; // Place unknown models at the end + const indexB = orderIndex[b.name] ?? Number.MAX_VALUE; + return indexA - indexB; + }) as T; + } else { + const modelNames = Object.keys(models); + const sortedModelNames = modelNames.sort((a, b) => { + const indexA = orderIndex[a] ?? Number.MAX_VALUE; + const indexB = orderIndex[b] ?? Number.MAX_VALUE; + return indexA - indexB; + }); + + const sortedModifiedModels: ModifiedModels = {}; + sortedModelNames.forEach((modelName) => { + sortedModifiedModels[modelName] = models[modelName]; + }); + + return sortedModifiedModels as T; + } +} + +export function loadExistingForeignKeys( + relations: GraphQLRelationsType[], + sequelize: Sequelize +): Map> { + const foreignKeyMap = new Map>(); + for (const relation of relations) { + const model = sequelize.model(relation.from); + const relatedModel = sequelize.model(relation.to); + Object.values(model.associations).forEach(() => { + addRelationToMap(relation, foreignKeyMap, model, relatedModel); + }); + } + return foreignKeyMap; +} + +export async function loadExistingEnums( + enums: GraphQLEnumsType[], + schema: string, + sequelize: Sequelize +): Promise> { + const enumTypeMap = new Map(); + + const results = (await sequelize.query( + ` + SELECT t.typname AS enum_type +FROM pg_type t + JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace +WHERE n.nspname = :schema + AND t.typtype = 'e' +ORDER BY t.typname; + `, + { + replacements: {schema: schema}, + type: QueryTypes.SELECT, + } + )) as {enum_type: string}[]; + + for (const e of enums) { + const enumTypeName = enumNameToHash(e.name); + if (!results.find((en) => en.enum_type === enumTypeName)) { + continue; + } + enumTypeMap.set(e.name, `"${schema}"."${enumTypeName}"`); + } + return enumTypeMap; +} diff --git a/packages/node-core/src/configure/migration-service/migration.ts b/packages/node-core/src/configure/migration-service/migration.ts index 172b80233f..4fa6332bb4 100644 --- a/packages/node-core/src/configure/migration-service/migration.ts +++ b/packages/node-core/src/configure/migration-service/migration.ts @@ -12,29 +12,30 @@ import { hashName, IndexType, } from '@subql/utils'; -import { - IndexesOptions, - ModelAttributes, - ModelStatic, - QueryTypes, - Sequelize, - Transaction, - Utils, -} from '@subql/x-sequelize'; +import {IndexesOptions, ModelAttributes, ModelStatic, Sequelize, Transaction, Utils} from '@subql/x-sequelize'; import {GraphQLSchema} from 'graphql'; import {isEqual} from 'lodash'; -import Pino from 'pino'; import {StoreService} from '../../indexer'; import {getLogger} from '../../logger'; import { addRelationToMap, + commentColumnQuery, commentConstraintQuery, + commentOnEnumStatement, commentTableQuery, + constraintDeferrableQuery, + createColumnStatement, + createEnumStatement, + createIndexStatement, createNotifyTrigger, createSendNotificationTriggerFunction, createUniqueIndexQuery, + dropColumnStatement, + dropEumStatement, + dropIndexStatement, dropNotifyFunction, dropNotifyTrigger, + dropRelationStatement, enumNameToHash, formatAttributes, formatColumnName, @@ -50,16 +51,16 @@ import { SmartTags, smartTags, validateNotifyTriggers, -} from '../../utils'; -import {getColumnOption, modelsTypeToModelAttributes} from '../../utils/graphql'; -import { addBlockRangeColumnToIndexes, addHistoricalIdIndex, addIdAndBlockRangeAttributes, getExistedIndexesQuery, updateIndexesName, -} from '../../utils/sync-helper'; + getColumnOption, + modelsTypeToModelAttributes, +} from '../../utils'; import {NodeConfig} from '../NodeConfig'; +import {loadExistingEnums, loadExistingForeignKeys} from './migration-helpers'; type RemovedIndexes = Record; @@ -67,27 +68,40 @@ const logger = getLogger('db-manager'); export class Migration { private sequelizeModels: ModelStatic[] = []; + /* + mainQueries are used for executions, that are not reliant on any prior db operations + extraQueries are executions, that are reliant on certain db operations, e.g. comments on foreignKeys or comments on tables, should be executed only after the table has been created + */ private mainQueries: string[] = []; - private readonly historical: boolean; private extraQueries: string[] = []; - private foreignKeyMap: Map> = new Map>(); + private readonly historical: boolean; private useSubscription: boolean; - private enumTypeMap: Map = new Map(); + private foreignKeyMap: Map>; + private enumTypeMap: Map; + // TODO i think this can be removed private removedIndexes: RemovedIndexes = {}; - constructor( + private constructor( private sequelize: Sequelize, private storeService: StoreService, private schemaName: string, private config: NodeConfig, - private dbType: SUPPORT_DB + private dbType: SUPPORT_DB, + // TODO refactor load map funcs + private initForeignKeyMap: Map>, + private initEnumTypeMap: Map ) { this.historical = !config.disableHistorical; this.useSubscription = config.subscription; + + // TODO test if this is working, enable, then disable if (this.useSubscription && dbType === SUPPORT_DB.cockRoach) { this.useSubscription = false; logger.warn(`Subscription is not support with ${this.dbType}`); } + + this.foreignKeyMap = this.initForeignKeyMap; + this.enumTypeMap = this.initEnumTypeMap; } static async create( @@ -98,55 +112,17 @@ export class Migration { config: NodeConfig, dbType: SUPPORT_DB ): Promise { - const migration = new Migration(sequelize, storeService, schemaName, config, dbType); - await migration.init(currentSchema); - return migration; - } - async init(currentSchema: GraphQLSchema | null): Promise { - const CurrentModelsRelationsEnums = getAllEntitiesRelations(currentSchema); - - // Should load all keys and enums of currentSchema, as nextSchema will be added from the migration execution - this.loadExistingForeignKeys(CurrentModelsRelationsEnums.relations); - await this.loadExisitingEnums(CurrentModelsRelationsEnums.enums); + const currentModelsRelationsEnums = getAllEntitiesRelations(currentSchema); + const foreignKeyMap = loadExistingForeignKeys(currentModelsRelationsEnums.relations, sequelize); + const enumTypeMap = await loadExistingEnums(currentModelsRelationsEnums.enums, schemaName, sequelize); - if (this.useSubscription) { - this.extraQueries.push(createSendNotificationTriggerFunction(this.schemaName)); - } - } + const migration = new Migration(sequelize, storeService, schemaName, config, dbType, foreignKeyMap, enumTypeMap); - private loadExistingForeignKeys(relations: GraphQLRelationsType[]): void { - for (const relation of relations) { - const model = this.sequelize.model(relation.from); - const relatedModel = this.sequelize.model(relation.to); - Object.values(model.associations).forEach(() => { - addRelationToMap(relation, this.foreignKeyMap, model, relatedModel); - }); + if (migration.useSubscription) { + migration.extraQueries.push(createSendNotificationTriggerFunction(schemaName)); } - } - - private async loadExisitingEnums(enums: GraphQLEnumsType[]): Promise { - const results = (await this.sequelize.query( - ` - SELECT t.typname AS enum_type -FROM pg_type t - JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace -WHERE n.nspname = :schema - AND t.typtype = 'e' -ORDER BY t.typname; - `, - { - replacements: {schema: this.schemaName}, - type: QueryTypes.SELECT, - } - )) as {enum_type: string}[]; - for (const e of enums) { - const enumTypeName = enumNameToHash(e.name); - if (!results.find((en) => en.enum_type === enumTypeName)) { - continue; - } - this.enumTypeMap.set(e.name, `"${this.schemaName}"."${enumTypeName}"`); - } + return migration; } async run(transaction: Transaction | undefined): Promise[]> { @@ -225,6 +201,7 @@ ORDER BY t.typname; this.mainQueries.push(...generateCreateTableStatement(sequelizeModel, this.schemaName, withoutForeignKey)); + // TODO double check if this is getting picked up by the indexes methods if (sequelizeModel.options.indexes) { this.mainQueries.push( ...generateCreateIndexStatement(sequelizeModel.options.indexes, this.schemaName, sequelizeModel.tableName) @@ -278,25 +255,19 @@ ORDER BY t.typname; const dbColumnName = formatColumnName(field.name); const formattedAttributes = formatAttributes(columnOptions, this.schemaName, false); - this.mainQueries.push( - `ALTER TABLE "${this.schemaName}"."${dbTableName}" ADD COLUMN "${dbColumnName}" ${formattedAttributes};` - ); + this.mainQueries.push(createColumnStatement(this.schemaName, dbTableName, dbColumnName, formattedAttributes)); if (columnOptions.comment) { - this.extraQueries.push( - `COMMENT ON COLUMN "${this.schemaName}".${dbTableName}.${dbColumnName} IS '${columnOptions.comment}';` - ); + this.extraQueries.push(commentColumnQuery(this.schemaName, dbTableName, dbColumnName, columnOptions.comment)); } this.addModelToSequelizeCache(sequelizeModel); } dropColumn(model: GraphQLModelsType, field: GraphQLEntityField): void { - this.mainQueries.push( - `ALTER TABLE "${this.schemaName}"."${modelToTableName(model.name)}" DROP COLUMN IF EXISTS ${formatColumnName( - field.name - )};` - ); + const columnName = formatColumnName(field.name); + const tableName = modelToTableName(model.name); + this.mainQueries.push(dropColumnStatement(this.schemaName, columnName, tableName)); this.addModelToSequelizeCache(this.createSequelizeModel(model)); } @@ -312,22 +283,12 @@ ORDER BY t.typname; indexOptions.name = generateHashedIndexName(model.name, indexOptions); - if (!indexOptions.fields || indexOptions.fields.length === 0) { - throw new Error("The 'fields' property is required and cannot be empty."); - } - - const createIndexQuery = - `CREATE ${indexOptions.unique ? 'UNIQUE ' : ''}INDEX IF NOT EXISTS "${indexOptions.name}" ` + - `ON "${this.schemaName}"."${formattedTableName}" ` + - `${indexOptions.using ? `USING ${indexOptions.using} ` : ''}` + - `(${indexOptions.fields.join(', ')})`; - - this.mainQueries.push(createIndexQuery); + this.mainQueries.push(createIndexStatement(indexOptions, formattedTableName, this.schemaName)); } dropIndex(model: GraphQLModelsType, index: GraphQLEntityIndex): void { const hashedIndexName = generateHashedIndexName(model.name, index); - this.mainQueries.push(`DROP INDEX IF EXISTS "${this.schemaName}"."${hashedIndexName}";`); + this.mainQueries.push(dropIndexStatement(this.schemaName, hashedIndexName)); } createRelation(relation: GraphQLRelationsType): void { @@ -344,8 +305,11 @@ ORDER BY t.typname; case 'belongsTo': { model.belongsTo(relatedModel, {foreignKey: relation.foreignKey}); addRelationToMap(relation, this.foreignKeyMap, model, relatedModel); - // TODO cockroach support - // logger.warn(`Relation: ${model.tableName} to ${relatedModel.tableName} is ONLY supported by postgresDB`); + const rel = model.belongsTo(relatedModel, {foreignKey: relation.foreignKey}); + const fkConstraint = getFkConstraint(rel.source.tableName, rel.foreignKey); + if (this.dbType !== SUPPORT_DB.cockRoach) { + this.extraQueries.push(constraintDeferrableQuery(model.getTableName().toString(), fkConstraint)); + } break; } case 'hasOne': { @@ -404,8 +368,7 @@ ORDER BY t.typname; const tableName = modelToTableName(relation.from); const fkConstraint = getFkConstraint(tableName, relation.foreignKey); - const dropFkeyStatement = `ALTER TABLE "${this.schemaName}"."${tableName}" DROP CONSTRAINT ${fkConstraint};`; - this.mainQueries.unshift(dropFkeyStatement); + this.mainQueries.unshift(dropRelationStatement(this.schemaName, tableName, fkConstraint)); // TODO remove from sequelize model // Should update sequelize model cache @@ -414,6 +377,11 @@ ORDER BY t.typname; async createEnum(e: GraphQLEnumsType): Promise { const queries: string[] = []; + // Ref: https://www.graphile.org/postgraphile/enums/ + // Example query for enum name: COMMENT ON TYPE "polkadot-starter_enum_a40fe73329" IS E'@enum\n@enumName TestEnum' + // It is difficult for sequelize use replacement, instead we use escape to avoid injection + // UPDATE: this comment got syntax error with cockroach db, disable it for now. Waiting to be fixed. + // See https://github.com/cockroachdb/cockroach/issues/44135 const enumTypeName = enumNameToHash(e.name); let type = `"${this.schemaName}"."${enumTypeName}"`; let [results] = await this.sequelize.query( @@ -432,8 +400,7 @@ ORDER BY t.typname; if (results.length === 0) { const escapedEnumValues = e.values.map((value) => this.sequelize.escape(value)).join(','); - const createEnumStatement = `CREATE TYPE ${type} as ENUM (${escapedEnumValues});`; - queries.unshift(createEnumStatement); + queries.unshift(createEnumStatement(type, escapedEnumValues)); } else { const currentValues = results.map((v: any) => v.enum_value); @@ -454,17 +421,17 @@ ORDER BY t.typname; const comment = this.sequelize.escape( `@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}` ); - const commentStatement = `COMMENT ON TYPE ${type} IS E${comment}`; - queries.push(commentStatement); + + queries.push(commentOnEnumStatement(type, comment)); } this.mainQueries.unshift(...queries); this.enumTypeMap.set(e.name, type); } - dropEnums(e: GraphQLEnumsType): void { + dropEnum(e: GraphQLEnumsType): void { const enumTypeValue = this.enumTypeMap?.get(e.name); if (enumTypeValue) { - this.mainQueries.push(`DROP TYPE ${enumTypeValue}`); + this.mainQueries.push(dropEumStatement(enumTypeValue)); this.enumTypeMap.delete(e.name); } @@ -486,6 +453,7 @@ ORDER BY t.typname; indexes.forEach((index, i) => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion if (index.using === IndexType.HASH && !existedIndexes.includes(index.name!)) { + // TODO double check with idempotent on cockroach const cockroachDbIndexQuery = `CREATE INDEX "${index.name}" ON "${schema}"."${modelToTableName(modelName)}"(${ index.fields }) USING HASH;`; diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index d953a0e227..a1199b5dc2 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -159,7 +159,12 @@ export class StoreService { this._modelsRelations = modelsRelations; try { + const start = new Date(); await this.syncSchema(schema); + const end = new Date(); + + const diff = end.getTime() - start.getTime(); + console.log(`syncSchema took ${diff} ms`); } catch (e: any) { logger.error(e, `Having a problem when syncing schema`); process.exit(1); @@ -208,6 +213,10 @@ export class StoreService { } } + /* + On SyncSchema, if no schema migration is introduced, it would consider current schema to be null, and go all db operations again + every start up is a migration + */ const schemaMigrationService = new SchemaMigrationService( this.sequelize, this, @@ -216,6 +225,13 @@ export class StoreService { this.config ); await schemaMigrationService.run(null, this.subqueryProject.schema, tx); + // TODO syncSchema should initalize sequelize models regards of if the schema contains changes + /* + + If project restarts wtih DB already, this would fail, as there are many queries that are only for init + + */ + // TODO this should also apply to notifyTriggers on subscription } defineModel( diff --git a/packages/node-core/src/utils/sync-helper.ts b/packages/node-core/src/utils/sync-helper.ts index db1effbddc..55bf13609b 100644 --- a/packages/node-core/src/utils/sync-helper.ts +++ b/packages/node-core/src/utils/sync-helper.ts @@ -1,15 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {SUPPORT_DB} from '@subql/common'; -import { - hashName, - blake2AsHex, - GraphQLEnumsType, - IndexType, - GraphQLModelsType, - GraphQLRelationsType, -} from '@subql/utils'; +import assert from 'assert'; +import {hashName, blake2AsHex, IndexType, GraphQLModelsType, GraphQLRelationsType} from '@subql/utils'; import { DataTypes, IndexesOptions, @@ -24,9 +17,6 @@ import { Utils, } from '@subql/x-sequelize'; import {ModelAttributeColumnReferencesOptions, ModelIndexesOptions} from '@subql/x-sequelize/types/model'; -import {isEqual} from 'lodash'; -import Pino from 'pino'; -import {getEnumDeprecated} from './project'; import {formatAttributes, generateIndexName, modelToTableName} from './sequelizeUtil'; // eslint-disable-next-line @typescript-eslint/no-var-requires const Toposort = require('toposort-class'); @@ -85,6 +75,10 @@ export function commentTableQuery(column: string, comment: string): string { return `COMMENT ON TABLE ${column} IS E'${comment}'`; } +export function commentColumnQuery(schema: string, table: string, column: string, comment: string): string { + return `COMMENT ON COLUMN "${schema}".${table}.${column} IS '${comment}';`; +} + // This is used when historical is disabled so that we can perform bulk updates export function constraintDeferrableQuery(table: string, constraint: string): string { return `ALTER TABLE ${table} ALTER CONSTRAINT ${constraint} DEFERRABLE INITIALLY IMMEDIATE`; @@ -184,14 +178,26 @@ END; $$ LANGUAGE plpgsql;`; } +// TODO use idempotent export function createNotifyTrigger(schema: string, table: string): string { const triggerName = hashName(schema, 'notify_trigger', table); const channelName = hashName(schema, 'notify_channel', table); + // return ` + // CREATE TRIGGER "${triggerName}" + // AFTER INSERT OR UPDATE OR DELETE + // ON "${schema}"."${table}" + // FOR EACH ROW EXECUTE FUNCTION "${schema}".send_notification('${channelName}');`; return ` -CREATE TRIGGER "${triggerName}" - AFTER INSERT OR UPDATE OR DELETE - ON "${schema}"."${table}" - FOR EACH ROW EXECUTE FUNCTION "${schema}".send_notification('${channelName}');`; +DO $$ +BEGIN + CREATE TRIGGER ${triggerName} AFTER INSERT OR UPDATE OR DELETE + ON "${schema}"."${table}" + FOR EACH ROW EXECUTE FUNCTION "${schema}".send_notification('${channelName}'); +EXCEPTION + WHEN duplicate_object THEN + RAISE NOTICE 'Trigger already exists. Ignoring...'; +END$$; + `; } export function dropNotifyTrigger(schema: string, table: string): string { @@ -205,6 +211,7 @@ export function dropNotifyFunction(schema: string): string { } // Hot schema reload, _metadata table +// Should also use Idempotent export function createSchemaTrigger(schema: string, metadataTableName: string): string { const triggerName = hashName(schema, 'schema_trigger', metadataTableName); return ` @@ -217,6 +224,7 @@ export function createSchemaTrigger(schema: string, metadataTableName: string): } export function createSchemaTriggerFunction(schema: string): string { + // TODO i think this should also not use replace return ` CREATE OR REPLACE FUNCTION "${schema}".schema_notification() RETURNS trigger AS $$ @@ -320,65 +328,6 @@ export function addBlockRangeColumnToIndexes(indexes: IndexesOptions[]): void { }); } -// Ref: https://www.graphile.org/postgraphile/enums/ -// Example query for enum name: COMMENT ON TYPE "polkadot-starter_enum_a40fe73329" IS E'@enum\n@enumName TestEnum' -// It is difficult for sequelize use replacement, instead we use escape to avoid injection -// UPDATE: this comment got syntax error with cockroach db, disable it for now. Waiting to be fixed. -// See https://github.com/cockroachdb/cockroach/issues/44135 -export async function syncEnums( - sequelize: Sequelize, - dbType: SUPPORT_DB, - e: GraphQLEnumsType, - schema: string, - enumTypeMap: Map, - logger: Pino.Logger -): Promise { - // We shouldn't set the typename to e.name because it could potentially create SQL injection, - // using a replacement at the type name location doesn't work. - const enumTypeName = enumNameToHash(e.name); - let type = `"${schema}"."${enumTypeName}"`; - let [results] = await sequelize.query( - `SELECT pg_enum.enumlabel as enum_value - FROM pg_type t JOIN pg_enum ON pg_enum.enumtypid = t.oid JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace - WHERE t.typname = ? AND n.nspname = ? order by enumsortorder;`, - {replacements: [enumTypeName, schema]} - ); - - const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(e.name)}`; - const resultsDeprecated = await getEnumDeprecated(sequelize, enumTypeNameDeprecated); - if (resultsDeprecated.length !== 0) { - results = resultsDeprecated; - type = `"${enumTypeNameDeprecated}"`; - } - - if (results.length === 0) { - await sequelize.query(`CREATE TYPE ${type} as ENUM (${e.values.map(() => '?').join(',')});`, { - replacements: e.values, - }); - } else { - const currentValues = results.map((v: any) => v.enum_value); - // Assert the existing enum is same - - // Make it a function to not execute potentially big joins unless needed - if (!isEqual(e.values, currentValues)) { - throw new Error( - `\n * Can't modify enum "${e.name}" between runs: \n * Before: [${currentValues.join( - `,` - )}] \n * After : [${e.values.join(',')}] \n * You must rerun the project to do such a change` - ); - } - } - if (dbType === SUPPORT_DB.cockRoach) { - logger.warn( - `Comment on enum ${e.description} is not supported with ${dbType}, enum name may display incorrectly in query service` - ); - } else { - const comment = sequelize.escape(`@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}`); - await sequelize.query(`COMMENT ON TYPE ${type} IS E${comment}`); - } - enumTypeMap.set(e.name, type); -} - export function addRelationToMap( relation: GraphQLRelationsType, foreignKeys: Map>, @@ -460,7 +409,7 @@ export function generateCreateIndexStatement( const indexUsed = index.using ? `USING ${index.using}` : ''; const indexName = index.name; - const statement = `CREATE ${unique} INDEX "${indexName}" ON "${schema}"."${tableName}" ${indexUsed} (${fieldsList});`; + const statement = `CREATE ${unique} INDEX IF NOT EXISTS "${indexName}" ON "${schema}"."${tableName}" ${indexUsed} (${fieldsList});`; indexStatements.push(statement); }); @@ -497,15 +446,44 @@ export function sortModels(relations: GraphQLRelationsType[], models: GraphQLMod return sortedModels.length > 0 ? sortedModels : null; } +export function validateNotifyTriggers(triggerName: string, triggers: NotifyTriggerPayload[]): void { + if (triggers.length !== NotifyTriggerManipulationType.length) { + throw new Error( + `Found ${triggers.length} ${triggerName} triggers, expected ${NotifyTriggerManipulationType.length} triggers ` + ); + } + triggers.map((t) => { + if (!NotifyTriggerManipulationType.includes(t.eventManipulation)) { + throw new Error(`Found unexpected trigger ${t.triggerName} with manipulation ${t.eventManipulation}`); + } + }); +} + +// enums +export const dropEumStatement = (enumTypeValue: string): string => `DROP TYPE IF EXISTS ${enumTypeValue}`; +export const commentOnEnumStatement = (type: string, comment: string): string => + `COMMENT ON TYPE ${type} IS E${comment}`; +export const createEnumStatement = (type: string, enumValues: string): string => + `CREATE TYPE ${type} as ENUM (${enumValues});`; + +// relations +export const dropRelationStatement = (schemaName: string, tableName: string, fkConstraint: string): string => + `ALTER TABLE "${schemaName}"."${tableName}" DROP CONSTRAINT IF EXISTS ${fkConstraint};`; export function generateForeignKeyStatement(attribute: ModelAttributeColumnOptions, tableName: string): string | void { const references = attribute?.references as ModelAttributeColumnReferencesOptions; if (!references) { return; } const foreignTable = references.model as TableNameWithSchema; + assert(attribute.field, 'Missing field on attribute'); + const fkey = getFkConstraint(tableName, attribute.field); let statement = ` + DO $$ + BEGIN ALTER TABLE "${foreignTable.schema}"."${tableName}" - ADD FOREIGN KEY (${attribute.field}) + ADD + CONSTRAINT ${fkey} + FOREIGN KEY (${attribute.field}) REFERENCES "${foreignTable.schema}"."${foreignTable.tableName}" (${references.key})`; if (attribute.onDelete) { statement += ` ON DELETE ${attribute.onDelete}`; @@ -516,18 +494,37 @@ export function generateForeignKeyStatement(attribute: ModelAttributeColumnOptio if (references.deferrable) { statement += ` DEFERRABLE`; } + statement += `; + EXCEPTION + WHEN duplicate_object THEN + RAISE NOTICE 'Constraint already exists. Ignoring...'; + END$$; +`; return `${statement.trim()};`; } -export function validateNotifyTriggers(triggerName: string, triggers: NotifyTriggerPayload[]): void { - if (triggers.length !== NotifyTriggerManipulationType.length) { - throw new Error( - `Found ${triggers.length} ${triggerName} triggers, expected ${NotifyTriggerManipulationType.length} triggers ` - ); +// indexes +export const dropIndexStatement = (schema: string, hashedIndexName: string): string => + `DROP INDEX IF EXISTS "${schema}"."${hashedIndexName}";`; +export const createIndexStatement = (indexOptions: IndexesOptions, tableName: string, schema: string) => { + if (!indexOptions.fields || indexOptions.fields.length === 0) { + throw new Error("The 'fields' property is required and cannot be empty."); } - triggers.map((t) => { - if (!NotifyTriggerManipulationType.includes(t.eventManipulation)) { - throw new Error(`Found unexpected trigger ${t.triggerName} with manipulation ${t.eventManipulation}`); - } - }); -} + return ( + `CREATE ${indexOptions.unique ? 'UNIQUE ' : ''}INDEX IF NOT EXISTS "${indexOptions.name}" ` + + `ON "${schema}"."${tableName}" ` + + `${indexOptions.using ? `USING ${indexOptions.using} ` : ''}` + + `(${indexOptions.fields.join(', ')})` + ); +}; + +// columns +export const dropColumnStatement = (schema: string, columnName: string, tableName: string): string => + `ALTER TABLE "${schema}"."${modelToTableName(tableName)}" DROP COLUMN IF EXISTS ${columnName};`; + +export const createColumnStatement = ( + schema: string, + tableName: string, + columnName: string, + attributes: string +): string => `ALTER TABLE IF EXISTS "${schema}"."${tableName}" ADD COLUMN IF NOT EXISTS "${columnName}" ${attributes};`;