Refactor store.service Add support for relations, enums, subscription on Schema Migration

packages/node-core/CHANGELOG.md

@@ -4,8 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-
 ## [Unreleased]
+### Added
+- Schema Migration support for Enums, Relations, Subscription (#2251)
+- 
+### Fixed
+- Fixed non-atomic schema migration execution (#2244)
 
 ## [7.2.1] - 2024-02-07
 ### Added

packages/node-core/package.json

@@ -41,6 +41,7 @@
     "prom-client": "^14.0.1",
     "source-map": "^0.7.4",
     "tar": "^6.1.11",
+    "toposort-class": "^1.0.1",
     "vm2": "^3.9.19",
     "yargs": "^16.2.0"
   },

packages/node-core/src/configure/ProjectUpgrade.service.ts

@@ -230,7 +230,6 @@ export class ProjectUpgradeSevice<P extends ISubqueryProject = ISubqueryProject>
       assert(this.migrationService, 'MigrationService is undefined');
       if (this.config.allowSchemaMigration) {
         const modifiedModels = await this.migrationService.run(project.schema, newProject.schema, transaction);
-
         if (modifiedModels) {
           this.#storeCache?.updateModels(modifiedModels);
         }

packages/node-core/src/configure/SchemaMigration.service.spec.ts

@@ -2,7 +2,8 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import path from 'path';
-import {buildSchemaFromFile} from '@subql/utils';
+import {compareEnums} from '@subql/node-core/configure/migration-service/migration-helpers';
+import {buildSchemaFromFile, GraphQLEnumsType} from '@subql/utils';
 import {SchemaMigrationService} from './migration-service';
 import {ProjectUpgradeSevice} from './ProjectUpgrade.service';
 
@@ -57,4 +58,30 @@ describe('SchemaMigration', () => {
       expect(v).toBe(false);
     });
   });
+  it('compare enums on modified enums', () => {
+    const currentEnums = [
+      {
+        name: 'TestEnum',
+        values: ['GOOD', 'BAD', 'NEUTRAL', 'CHAOS'],
+      } as GraphQLEnumsType,
+    ];
+    const nextEnum = [
+      {
+        name: 'TestEnum',
+        values: ['GOOD', 'BAD', 'NEUTRAL'],
+      } as GraphQLEnumsType,
+    ];
+
+    const changes: any = {
+      addedEnums: [],
+      removedEnums: [],
+      modifiedEnums: [],
+    };
+    compareEnums(currentEnums, nextEnum, changes);
+    expect(changes).toStrictEqual({
+      addedEnums: [],
+      removedEnums: [],
+      modifiedEnums: [{name: 'TestEnum', values: ['GOOD', 'BAD', 'NEUTRAL']}],
+    });
+  });
 });

packages/node-core/src/configure/migration-service/SchemaMigration.service.ts

@@ -1,31 +1,38 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {getAllEntitiesRelations} from '@subql/utils';
+import {SUPPORT_DB} from '@subql/common';
+import {getAllEntitiesRelations, GraphQLModelsType, GraphQLRelationsType} from '@subql/utils';
 import {ModelStatic, Sequelize, Transaction} from '@subql/x-sequelize';
 import {GraphQLSchema} from 'graphql';
 import {StoreService} from '../../indexer';
 import {getLogger} from '../../logger';
+import {sortModels} from '../../utils';
 import {NodeConfig} from '../NodeConfig';
 import {Migration} from './migration';
 import {
+  alignModelOrder,
   compareEnums,
   compareModels,
   compareRelations,
   hasChanged,
+  ModifiedModels,
   schemaChangesLoggerMessage,
   SchemaChangesType,
 } from './migration-helpers';
 
 const logger = getLogger('SchemaMigrationService');
 
 export class SchemaMigrationService {
+  private withoutForeignKeys = false;
+
   constructor(
     private sequelize: Sequelize,
     private storeService: StoreService,
     private flushCache: (flushAll?: boolean) => Promise<void>,
     private dbSchema: string,
-    private config: NodeConfig
+    private config: NodeConfig,
+    private dbType: SUPPORT_DB = SUPPORT_DB.postgres
   ) {}
 
   static validateSchemaChanges(currentSchema: GraphQLSchema, nextSchema: GraphQLSchema): boolean {
@@ -41,7 +48,7 @@ export class SchemaMigrationService {
     return true;
   }
 
-  static schemaComparator(currentSchema: GraphQLSchema, nextSchema: GraphQLSchema): SchemaChangesType {
+  static schemaComparator(currentSchema: GraphQLSchema | null, nextSchema: GraphQLSchema): SchemaChangesType {
     const currentData = getAllEntitiesRelations(currentSchema);
     const nextData = getAllEntitiesRelations(nextSchema);
 
@@ -53,7 +60,7 @@ export class SchemaMigrationService {
       removedRelations: [],
       addedEnums: [],
       removedEnums: [],
-      allEnums: currentData.enums, // TODO support for Enum migration
+      modifiedEnums: [],
     };
 
     compareEnums(currentData.enums, nextData.enums, changes);
@@ -63,84 +70,113 @@ export class SchemaMigrationService {
     return changes;
   }
 
+  private orderModelsByRelations(models: GraphQLModelsType[], relations: GraphQLRelationsType[]): GraphQLModelsType[] {
+    const sortedModels = sortModels(relations, models);
+    if (sortedModels === null) {
+      this.withoutForeignKeys = true;
+      return models;
+    } else {
+      this.withoutForeignKeys = false;
+      return sortedModels.reverse();
+    }
+  }
+
   async run(
-    currentSchema: GraphQLSchema,
+    currentSchema: GraphQLSchema | null,
     nextSchema: GraphQLSchema,
     transaction: Transaction | undefined
   ): Promise<ModelStatic<any>[] | void> {
     const schemaDifference = SchemaMigrationService.schemaComparator(currentSchema, nextSchema);
-
     const {
       addedEnums,
       addedModels,
       addedRelations,
-      allEnums, // TODO enum support
+      modifiedEnums,
       modifiedModels,
       removedEnums,
       removedModels,
       removedRelations,
     } = schemaDifference;
+
     if (!hasChanged(schemaDifference)) {
       logger.info('No Schema changes');
       return;
     }
 
-    if (addedEnums.length > 0 || removedEnums.length > 0) {
-      throw new Error('Schema Migration currently does not support Enum removal and creation');
+    if (modifiedEnums.length > 0) {
+      throw new Error(
+        `Modifying enums is currently not supported. Please revert the changes to the following enums: ${modifiedEnums
+          .map((e) => e.name)
+          .join(', ')}`
+      );
     }
 
-    if (removedRelations.length > 0 || addedRelations.length > 0) {
-      throw new Error('Schema Migration currently does not support Relational removal or creation');
-    }
+    const sortedSchemaModels = this.orderModelsByRelations(
+      getAllEntitiesRelations(nextSchema).models,
+      getAllEntitiesRelations(nextSchema).relations
+    );
+
+    const sortedAddedModels = alignModelOrder<GraphQLModelsType[]>(sortedSchemaModels, addedModels);
+    const sortedModifiedModels = alignModelOrder<ModifiedModels>(sortedSchemaModels, modifiedModels);
 
     await this.flushCache(true);
     const migrationAction = await Migration.create(
       this.sequelize,
       this.storeService,
       this.dbSchema,
-      nextSchema,
+      currentSchema,
       this.config,
-      logger
+      this.dbType
     );
 
+    // TODO this should only be printed if schema migration is enabled and not store.service sync schema
     logger.info(`${schemaChangesLoggerMessage(schemaDifference)}`);
-
     try {
-      if (removedModels.length) {
-        for (const model of removedModels) {
-          migrationAction.dropTable(model);
-        }
+      for (const enumValue of addedEnums) {
+        await migrationAction.createEnum(enumValue);
       }
 
-      if (addedModels.length) {
-        for (const model of addedModels) {
-          await migrationAction.createTable(model);
-        }
+      for (const model of removedModels) {
+        migrationAction.dropTable(model);
+      }
+
+      for (const model of sortedAddedModels) {
+        await migrationAction.createTable(model, this.withoutForeignKeys);
       }
 
-      if (Object.keys(modifiedModels).length) {
-        const entities = Object.keys(modifiedModels);
-        for (const model of entities) {
-          const modelValue = modifiedModels[model];
+      const entities = Object.keys(sortedModifiedModels);
+      for (const model of entities) {
+        const modelValue = sortedModifiedModels[model];
 
-          for (const index of modelValue.removedIndexes) {
-            migrationAction.dropIndex(modelValue.model, index);
-          }
+        for (const index of modelValue.removedIndexes) {
+          migrationAction.dropIndex(modelValue.model, index);
+        }
 
-          for (const field of modelValue.removedFields) {
-            migrationAction.dropColumn(modelValue.model, field);
-          }
+        for (const field of modelValue.removedFields) {
+          migrationAction.dropColumn(modelValue.model, field);
+        }
 
-          for (const field of modelValue.addedFields) {
-            migrationAction.createColumn(modelValue.model, field);
-          }
+        for (const field of modelValue.addedFields) {
+          migrationAction.createColumn(modelValue.model, field);
+        }
 
-          for (const index of modelValue.addedIndexes) {
-            migrationAction.createIndex(modelValue.model, index);
-          }
+        for (const index of modelValue.addedIndexes) {
+          migrationAction.createIndex(modelValue.model, index);
         }
       }
 
+      if (addedRelations.length) {
+        for (const relationModel of addedRelations) {
+          migrationAction.createRelation(relationModel);
+        }
+      }
+
+      for (const relationModel of removedRelations) {
+        migrationAction.dropRelation(relationModel);
+      }
+      for (const enumValue of removedEnums) {
+        migrationAction.dropEnum(enumValue);
+      }
       return migrationAction.run(transaction);
     } catch (e: any) {
       logger.error(e, 'Failed to execute Schema Migration');