update topo logic, add tests, add sync logic with cyclic support

packages/node-core/package.json

@@ -41,6 +41,7 @@
     "prom-client": "^14.0.1",
     "source-map": "^0.7.4",
     "tar": "^6.1.11",
+    "toposort-class": "^1.0.1",
     "vm2": "^3.9.19",
     "yargs": "^16.2.0"
   },

packages/node-core/src/configure/migration-service/migration.ts

@@ -172,7 +172,7 @@ export class Migration {
     const dbTableName = modelToTableName(model.name);
     const dbColumnName = formatColumnName(field.name);
 
-    const formattedAttributes = formatAttributes(columnOptions, this.schemaName);
+    const formattedAttributes = formatAttributes(columnOptions, this.schemaName, false);
     this.rawQueries.push(
       `ALTER TABLE "${this.schemaName}"."${dbTableName}" ADD COLUMN "${dbColumnName}" ${formattedAttributes};`
     );

packages/node-core/src/indexer/store.service.ts

@@ -29,6 +29,7 @@ import {NodeConfig} from '../configure';
 import {getLogger} from '../logger';
 import {
   addBlockRangeColumnToIndexes,
+  addForeignKeyStatement,
   addHistoricalIdIndex,
   addIdAndBlockRangeAttributes,
   addRelationToMap,
@@ -52,6 +53,7 @@ import {
   modelsTypeToModelAttributes,
   SmartTags,
   smartTags,
+  sortModels,
   syncEnums,
   updateIndexesName,
 } from '../utils';
@@ -356,20 +358,39 @@ export class StoreService {
       extraQueries.push(query);
     });
 
-    Object.entries(this.sequelize.models).forEach(([_, model]) => {
-      const tableQuery = generateCreateTableStatement(model, schema);
-      mainQueries.push(tableQuery);
+    const referenceQueries: string[] = [];
+    const sortedModels = sortModels(this.modelsRelations.relations, this.sequelize.models);
 
-      if (model.options.indexes) {
-        const indexQuery = generateCreateIndexStatement(model.options.indexes, schema, model.tableName);
-        mainQueries.push(...indexQuery);
-      }
-    });
+    if (sortedModels === null) {
+      Object.values(this.sequelize.models).forEach((model) => {
+        const tableQuery = generateCreateTableStatement(model, schema, true);
+        mainQueries.push(tableQuery);
+        if (model.options.indexes) {
+          const indexQuery = generateCreateIndexStatement(model.options.indexes, schema, model.tableName);
+          mainQueries.push(...indexQuery);
+        }
+        referenceQueries.push(...addForeignKeyStatement(model));
+      });
+    } else {
+      sortedModels.reverse().forEach((model: ModelStatic<any>) => {
+        const tableQuery = generateCreateTableStatement(model, schema);
+        mainQueries.push(tableQuery);
+
+        if (model.options.indexes) {
+          const indexQuery = generateCreateIndexStatement(model.options.indexes, schema, model.tableName);
+          mainQueries.push(...indexQuery);
+        }
+      });
+    }
 
     try {
       for (const query of mainQueries) {
         await this.sequelize.query(query, {transaction: tx});
       }
+
+      for (const query of referenceQueries) {
+        await this.sequelize.query(query, {transaction: tx});
+      }
     } catch (e) {
       await tx.rollback();
       throw e;

packages/node-core/src/utils/sequelizeUtil.ts

@@ -70,15 +70,19 @@ export function formatReferences(columnOptions: ModelAttributeColumnOptions, sch
   return referenceStatement;
 }
 
-export function formatAttributes(columnOptions: ModelAttributeColumnOptions, schema: string): string {
+export function formatAttributes(
+  columnOptions: ModelAttributeColumnOptions,
+  schema: string,
+  withoutForeignKey: boolean
+): string {
   const type = formatDataType(columnOptions.type);
   const allowNull = columnOptions.allowNull === false ? 'NOT NULL' : '';
   const unique = columnOptions.unique ? 'UNIQUE' : '';
   const autoIncrement = columnOptions.autoIncrement ? 'AUTO_INCREMENT' : ''; //  PostgreSQL
 
   const references = formatReferences(columnOptions, schema);
 
-  return `${type} ${allowNull} ${unique} ${autoIncrement} ${references}`.trim();
+  return `${type} ${allowNull} ${unique} ${autoIncrement} ${withoutForeignKey ? '' : references}`.trim();
 }
 
 const sequelizeToPostgresTypeMap = {

packages/node-core/src/utils/sync-helper.spec.ts

@@ -3,7 +3,12 @@
 
 import {Model, ModelAttributeColumnOptions, ModelStatic} from '@subql/x-sequelize';
 import {formatReferences} from './sequelizeUtil';
-import {generateCreateIndexStatement, generateCreateTableStatement} from './sync-helper';
+import {
+  addForeignKeyStatement,
+  generateCreateIndexStatement,
+  generateCreateTableStatement,
+  sortModels,
+} from './sync-helper';
 
 describe('sync-helper', () => {
   const mockModel = {
@@ -146,7 +151,7 @@ COMMENT ON COLUMN "test"."test-table"."last_transfer_block" IS 'The most recent
     ]);
   });
   it('Generate table statement no historical, no multi primary keys', () => {
-    mockModel.getAttributes = jest.fn(() => {
+    jest.spyOn(mockModel, 'getAttributes').mockImplementationOnce(() => {
       return {
         id: {
           type: 'text',
@@ -158,7 +163,7 @@ COMMENT ON COLUMN "test"."test-table"."last_transfer_block" IS 'The most recent
           field: 'id',
         },
       };
-    }) as any;
+    });
     const statement = generateCreateTableStatement(mockModel, 'test');
 
     // Correcting the expected statement to reflect proper SQL syntax
@@ -195,6 +200,84 @@ COMMENT ON COLUMN "test"."test-table"."id" IS 'id field is always required and m
     } as ModelAttributeColumnOptions;
 
     const statement = formatReferences(attribute, 'test');
-    expect(statement).toMatch(`REFERENCES "test"."test-table" ("id") ON DELETE NO ACTION ON UPDATE CASCADE`);
+    expect(statement).toBe(`REFERENCES "test"."accounts" ("id") ON DELETE NO ACTION ON UPDATE CASCADE`);
+  });
+  it('Ensure correct foreignkey statement', () => {
+    jest.spyOn(mockModel, 'getAttributes').mockImplementationOnce(() => {
+      return {
+        transferIdId: {
+          type: 'text',
+          comment: undefined,
+          allowNull: false,
+          primaryKey: false,
+          field: 'transfer_id_id',
+          references: {
+            model: {
+              schema: 'test',
+              tableName: 'transfers',
+            },
+            key: 'id',
+          },
+          onDelete: 'NO ACTION',
+          onUpdate: 'CASCADE',
+        },
+      } as any;
+    });
+
+    const v = addForeignKeyStatement(mockModel);
+    expect(v[0]).toBe(
+      `ALTER TABLE "test"."test-table"
+      ADD FOREIGN KEY (transfer_id_id) 
+      REFERENCES "test"."transfers" (id) ON DELETE NO ACTION ON UPDATE CASCADE;`
+    );
+  });
+  it('sortModel with toposort on cyclic schema', () => {
+    const mockRelations = [
+      {
+        from: 'Transfer',
+        to: 'Account',
+      },
+      {
+        from: 'Account',
+        to: 'Transfer',
+      },
+    ] as any[];
+    const mockModels = new Map([
+      ['Transfer', {} as any],
+      ['Account', {} as any],
+    ]) as any;
+    expect(sortModels(mockRelations, mockModels)).toBe(null);
+  });
+  it('sortModel with toposort on non cyclic schema', () => {
+    const mockRelations = [
+      {
+        from: 'Transfer',
+        to: 'TestEntity',
+      },
+      {
+        from: 'Account',
+        to: 'TestEntity',
+      },
+    ] as any[];
+    const mockModels = {
+      Transfer: {
+        tableName: 'transfers',
+      },
+      Account: {
+        tableName: 'accounts',
+      },
+      TestEntity: {
+        tableName: 'test_entities',
+      },
+      LonelyEntity: {
+        tableName: 'lonely_Entities',
+      },
+    } as Record<string, any>;
+    expect(sortModels(mockRelations, mockModels)?.map((t) => t.tableName)).toStrictEqual([
+      'lonely_Entities',
+      'accounts',
+      'transfers',
+      'test_entities',
+    ]);
   });
 });

packages/node-core/src/utils/sync-helper.ts

@@ -20,13 +20,16 @@ import {
   Op,
   QueryTypes,
   Sequelize,
+  TableNameWithSchema,
   Utils,
 } from '@subql/x-sequelize';
-import {ModelIndexesOptions} from '@subql/x-sequelize/types/model';
+import {ModelAttributeColumnReferencesOptions, ModelIndexesOptions} from '@subql/x-sequelize/types/model';
 import {isEqual} from 'lodash';
 import Pino from 'pino';
 import {getEnumDeprecated} from './project';
 import {formatAttributes, generateIndexName, modelToTableName} from './sequelizeUtil';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const Toposort = require('toposort-class');
 
 export interface SmartTags {
   foreignKey?: string;
@@ -399,7 +402,11 @@ export function addRelationToMap(
   }
 }
 
-export function generateCreateTableStatement(model: ModelStatic<Model<any, any>>, schema: string): string {
+export function generateCreateTableStatement(
+  model: ModelStatic<Model<any, any>>,
+  schema: string,
+  withoutForeignKey = false
+): string {
   const tableName = model.tableName;
 
   const attributes = model.getAttributes();
@@ -414,7 +421,7 @@ export function generateCreateTableStatement(model: ModelStatic<Model<any, any>>
       attr.type = 'timestamp with time zone';
     }
 
-    const columnDefinition = `"${attr.field}" ${formatAttributes(attr, schema)}`;
+    const columnDefinition = `"${attr.field}" ${formatAttributes(attr, schema, withoutForeignKey)}`;
 
     columnDefinitions.push(columnDefinition);
     if (attr.comment) {
@@ -454,3 +461,62 @@ export function generateCreateIndexStatement(
 
   return indexStatements;
 }
+
+export function sortModels(
+  relations: GraphQLRelationsType[],
+  models: {[p: string]: ModelStatic<Model<any, any>>}
+): ModelStatic<any>[] | null {
+  const sorter = new Toposort();
+
+  Object.keys(models).forEach((modelName) => {
+    sorter.add(modelName, []);
+  });
+
+  relations.forEach(({from, to}) => {
+    sorter.add(from, to);
+  });
+
+  let sortedModelNames: string[];
+  try {
+    sortedModelNames = sorter.sort();
+  } catch (error) {
+    // Handle cyclic dependency error by returning null
+    if (error instanceof Error && error.message.startsWith('Cyclic dependency found.')) {
+      return null;
+    } else {
+      throw error;
+    }
+  }
+
+  const sortedModels = sortedModelNames.map((modelName) => models[modelName]).filter(Boolean);
+
+  return sortedModels.length > 0 ? sortedModels : null;
+}
+
+export function addForeignKeyStatement(model: ModelStatic<any>): string[] {
+  const statements: string[] = [];
+  const attributes = model.getAttributes();
+  Object.values(attributes).forEach((columnOptions) => {
+    const references = columnOptions?.references as ModelAttributeColumnReferencesOptions;
+    if (!references) {
+      return;
+    }
+    const foreignTable = references.model as TableNameWithSchema;
+    let statement = `
+    ALTER TABLE "${foreignTable.schema}"."${model.tableName}"
+      ADD FOREIGN KEY (${columnOptions.field}) 
+      REFERENCES "${foreignTable.schema}"."${foreignTable.tableName}" (${references.key})`;
+    if (columnOptions.onDelete) {
+      statement += ` ON DELETE ${columnOptions.onDelete}`;
+    }
+    if (columnOptions.onUpdate) {
+      statement += ` ON UPDATE ${columnOptions.onUpdate}`;
+    }
+    if (references.deferrable) {
+      statement += ` DEFERRABLE`;
+    }
+    statements.push(`${statement.trim()};`);
+  });
+
+  return statements;
+}