Update table creation on schema migration

packages/node-core/CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Historical queries not using the correct block height (#2243)
 
+### Fixed
+- Fixed non-autonomous schema migration execution (#2244)
+
 ## [7.2.0] - 2024-01-30
 ### Changed
 - Update `@subql/apollo-links` and add specific logger for it

packages/node-core/package.json

@@ -41,6 +41,7 @@
     "prom-client": "^14.0.1",
     "source-map": "^0.7.4",
     "tar": "^6.1.11",
+    "toposort-class": "^1.0.1",
     "vm2": "^3.9.19",
     "yargs": "^16.2.0"
   },

packages/node-core/src/configure/migration-service/migration.ts

@@ -7,7 +7,15 @@ import {IndexesOptions, ModelAttributes, ModelStatic, Sequelize, Transaction, Ut
 import {GraphQLSchema} from 'graphql';
 import Pino from 'pino';
 import {StoreService} from '../../indexer';
-import {formatAttributes, formatColumnName, generateHashedIndexName, modelToTableName, syncEnums} from '../../utils';
+import {
+  formatAttributes,
+  formatColumnName,
+  generateCreateIndexStatement,
+  generateCreateTableStatement,
+  generateHashedIndexName,
+  modelToTableName,
+  syncEnums,
+} from '../../utils';
 import {getColumnOption, modelsTypeToModelAttributes} from '../../utils/graphql';
 import {
   addBlockRangeColumnToIndexes,
@@ -53,10 +61,6 @@ export class Migration {
   async run(transaction: Transaction | undefined): Promise<ModelStatic<any>[]> {
     const effectiveTransaction = transaction ?? (await this.sequelize.transaction());
 
-    effectiveTransaction.afterCommit(async () => {
-      await Promise.all(this.sequelizeModels.map((m) => m.sync()));
-    });
-
     try {
       for (const query of this.rawQueries) {
         await this.sequelize.query(query, {transaction: effectiveTransaction});
@@ -125,6 +129,14 @@ export class Migration {
 
     const sequelizeModel = this.storeService.defineModel(model, attributes, indexes, this.schemaName);
 
+    this.rawQueries.push(generateCreateTableStatement(sequelizeModel, this.schemaName));
+
+    if (sequelizeModel.options.indexes) {
+      this.rawQueries.push(
+        ...generateCreateIndexStatement(sequelizeModel.options.indexes, this.schemaName, sequelizeModel.tableName)
+      );
+    }
+
     this.addModel(sequelizeModel);
   }
 
@@ -145,7 +157,7 @@ export class Migration {
     const dbTableName = modelToTableName(model.name);
     const dbColumnName = formatColumnName(field.name);
 
-    const formattedAttributes = formatAttributes(columnOptions);
+    const formattedAttributes = formatAttributes(columnOptions, this.schemaName, false);
     this.rawQueries.push(
       `ALTER TABLE "${this.schemaName}"."${dbTableName}" ADD COLUMN "${dbColumnName}" ${formattedAttributes};`
     );

packages/node-core/src/indexer/store.service.ts

@@ -29,6 +29,7 @@ import {NodeConfig} from '../configure';
 import {getLogger} from '../logger';
 import {
   addBlockRangeColumnToIndexes,
+  addForeignKeyStatement,
   addHistoricalIdIndex,
   addIdAndBlockRangeAttributes,
   addRelationToMap,
@@ -44,6 +45,7 @@ import {
   createUniqueIndexQuery,
   dropNotifyFunction,
   dropNotifyTrigger,
+  generateOrderedStatements,
   getExistedIndexesQuery,
   getFkConstraint,
   getTriggers,
@@ -232,6 +234,7 @@ export class StoreService {
       logger.warn(`Subscription is not support with ${this.dbType}`);
     }
 
+    const tx = await this.sequelize.transaction();
     const enumTypeMap = new Map<string, string>();
     if (this.historical) {
       const [results] = await this.sequelize.query(BTREE_GIST_EXTENSION_EXIST_QUERY);
@@ -246,6 +249,7 @@ export class StoreService {
     for (const e of this.modelsRelations.enums) {
       await syncEnums(this.sequelize, this.dbType, e, schema, enumTypeMap, logger);
     }
+    const mainQueries: string[] = [];
     const extraQueries = [];
     // Function need to create ahead of triggers
     if (useSubscription) {
@@ -352,7 +356,29 @@ export class StoreService {
       extraQueries.push(query);
     });
 
-    await this.sequelize.sync();
+    const referenceQueries: string[] = [];
+
+    generateOrderedStatements(
+      this.sequelize.models,
+      this.modelsRelations.relations,
+      schema,
+      mainQueries,
+      referenceQueries
+    );
+
+    try {
+      for (const query of mainQueries) {
+        await this.sequelize.query(query, {transaction: tx});
+      }
+
+      for (const query of referenceQueries) {
+        await this.sequelize.query(query, {transaction: tx});
+      }
+    } catch (e) {
+      await tx.rollback();
+      throw e;
+    }
+    await tx.commit();
 
     for (const query of extraQueries) {
       await this.sequelize.query(query);

packages/node-core/src/utils/sequelizeUtil.ts

@@ -1,9 +1,18 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import assert from 'assert';
 import NodeUtil from 'node:util';
-import {BigInt, Boolean, DateObj, Float, ID, Int, Json, SequelizeTypes, String} from '@subql/utils';
-import {DataType, DataTypes, IndexesOptions, ModelAttributeColumnOptions, TableName, Utils} from '@subql/x-sequelize';
+import {BigInt, Boolean, DateObj, Int, Json, SequelizeTypes, String} from '@subql/utils';
+import {
+  DataType,
+  DataTypes,
+  IndexesOptions,
+  ModelAttributeColumnOptions,
+  TableName,
+  Utils,
+  TableNameWithSchema,
+} from '@subql/x-sequelize';
 import {underscored} from './sync-helper';
 
 // This method is simplified from https://github.com/sequelize/sequelize/blob/066421c00aad61694dcdbb624d4b73dbac7c7b42/packages/core/src/model-definition.ts#L245
@@ -43,35 +52,59 @@ ${NodeUtil.inspect(index)}`);
   return underscored(out);
 }
 
-export function formatAttributes(columnOptions: ModelAttributeColumnOptions): string {
+export function formatReferences(columnOptions: ModelAttributeColumnOptions, schema: string): string {
+  if (!columnOptions.references) {
+    return '';
+  }
+  assert(typeof columnOptions.references !== 'string', 'reference is type string');
+  assert(typeof columnOptions.references.model !== 'string', 'reference.model is type string');
+  let referenceStatement = `REFERENCES "${schema}"."${
+    (columnOptions.references.model as TableNameWithSchema).tableName
+  }" ("${columnOptions.references.key}")`;
+  if (columnOptions.onDelete) {
+    referenceStatement += ` ON DELETE ${columnOptions.onDelete}`;
+  }
+  if (columnOptions.onUpdate) {
+    referenceStatement += ` ON UPDATE ${columnOptions.onUpdate}`;
+  }
+  return referenceStatement;
+}
+
+export function formatAttributes(
+  columnOptions: ModelAttributeColumnOptions,
+  schema: string,
+  withoutForeignKey: boolean
+): string {
   const type = formatDataType(columnOptions.type);
   const allowNull = columnOptions.allowNull === false ? 'NOT NULL' : '';
-  const primaryKey = columnOptions.primaryKey ? 'PRIMARY KEY' : '';
   const unique = columnOptions.unique ? 'UNIQUE' : '';
   const autoIncrement = columnOptions.autoIncrement ? 'AUTO_INCREMENT' : ''; //  PostgreSQL
 
-  // TODO Support relational
-  // const references = options.references ? formatReferences(options.references) :
+  const references = formatReferences(columnOptions, schema);
 
-  return `${type} ${allowNull} ${primaryKey} ${unique} ${autoIncrement}`.trim();
+  return `${type} ${allowNull} ${unique} ${autoIncrement} ${withoutForeignKey ? '' : references}`.trim();
 }
 
 const sequelizeToPostgresTypeMap = {
   [DataTypes.STRING.name]: (dataType: DataType) => String.sequelizeType,
   [DataTypes.INTEGER.name]: () => Int.sequelizeType,
   [DataTypes.BIGINT.name]: () => BigInt.sequelizeType,
-  [DataTypes.UUID.name]: () => ID.sequelizeType,
+  [DataTypes.UUID.name]: () => DataTypes.UUID,
   [DataTypes.BOOLEAN.name]: () => Boolean.sequelizeType,
-  [DataTypes.FLOAT.name]: () => Float.sequelizeType,
+  [DataTypes.FLOAT.name]: () => 'double precision', // to maintain compatibility we will use float8
   [DataTypes.DATE.name]: () => DateObj.sequelizeType,
   [DataTypes.JSONB.name]: () => Json.sequelizeType,
+  [DataTypes.RANGE.name]: () => 'int8range',
 };
 
 export function formatDataType(dataType: DataType): SequelizeTypes {
   if (typeof dataType === 'string') {
     return dataType;
   } else {
     const formatter = sequelizeToPostgresTypeMap[dataType.key];
+    if (!formatter) {
+      throw new Error(`Unsupported data type: ${dataType.key}`);
+    }
     return formatter(dataType);
   }
 }