Skip to content

Commit

Permalink
performance optimization for migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@Alfagun74
Alfagun74 committed Oct 18, 2024
1 parent 184779f commit b9321fe
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 119 deletions.
83 changes: 23 additions & 60 deletions src/modules/database/migrations/postgres/1728421385000-v13-final.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Logger, NotImplementedException } from "@nestjs/common";
import { randomBytes } from "crypto";
import { existsSync } from "fs";
import { toLower } from "lodash";
import { toLower, uniqBy } from "lodash";
import { setTimeout } from "timers/promises";
import { In, MigrationInterface, QueryRunner } from "typeorm";
import { GamevaultGame } from "../../../games/gamevault-game.entity";
Expand Down Expand Up @@ -1031,7 +1031,10 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateTags(queryRunner: QueryRunner): Promise<void> {
const tags = await queryRunner.manager.find(TagV12, { withDeleted: true });
const tags = uniqBy(
await queryRunner.manager.find(TagV12, { withDeleted: true }),
"rawg_id",
);
this.logger.debug({
message: `Found ${tags.length} tags in the V12 database.`,
});
Expand All @@ -1040,19 +1043,6 @@ export class V13Final1728421385000 implements MigrationInterface {
this.logger.debug({
message: `Migrating tag ID ${tag.id}, Name: ${tag.name}`,
});

if (
await queryRunner.manager.existsBy(TagMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: tag.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Tag ID ${tag.id} already exists. Skipping.`,
});
continue;
}

await queryRunner.query(
`
INSERT INTO "tag_metadata"("id", "created_at", "updated_at", "deleted_at", "entity_version", "provider_slug", "provider_data_id", "name")
Expand All @@ -1073,9 +1063,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateGenres(queryRunner: QueryRunner): Promise<void> {
const genres = await queryRunner.manager.find(GenreV12, {
withDeleted: true,
});
const genres = uniqBy(
await queryRunner.manager.find(GenreV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${genres.length} genres in the V12 database.`,
});
Expand All @@ -1085,18 +1078,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating genre ID ${genre.id}, Name: ${genre.name}`,
});

if (
await queryRunner.manager.existsBy(GenreMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: genre.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Genre ID ${genre.id} already exists. Skipping.`,
});
continue;
}

await queryRunner.query(
`
INSERT INTO "genre_metadata" ("id", "created_at", "updated_at", "deleted_at", "entity_version", "provider_slug", "provider_data_id", "name")
Expand All @@ -1117,9 +1098,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateDevelopers(queryRunner: QueryRunner): Promise<void> {
const developers = await queryRunner.manager.find(DeveloperV12, {
withDeleted: true,
});
const developers = uniqBy(
await queryRunner.manager.find(DeveloperV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${developers.length} developers in the V12 database.`,
});
Expand All @@ -1129,18 +1113,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating developer ID ${developer.id}, Name: ${developer.name}`,
});

if (
await queryRunner.manager.existsBy(DeveloperMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: developer.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Developer ID ${developer.id} already exists. Skipping.`,
});
continue;
}

await queryRunner.query(
`
INSERT INTO "developer_metadata" ("id", "created_at", "updated_at", "deleted_at", "entity_version", "provider_slug", "provider_data_id", "name")
Expand All @@ -1161,9 +1133,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migratePublishers(queryRunner: QueryRunner): Promise<void> {
const publishers = await queryRunner.manager.find(PublisherV12, {
withDeleted: true,
});
const publishers = uniqBy(
await queryRunner.manager.find(PublisherV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${publishers.length} publishers in the V12 database.`,
});
Expand All @@ -1173,18 +1148,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating publisher ID ${publisher.id}, Name: ${publisher.name}`,
});

if (
await queryRunner.manager.existsBy(PublisherMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: publisher.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Publisher ID ${publisher.id} already exists. Skipping.`,
});
continue;
}

await queryRunner.query(
`
INSERT INTO "publisher_metadata" ("id", "created_at", "updated_at", "deleted_at", "entity_version", "provider_slug", "provider_data_id", "name")
Expand Down
82 changes: 23 additions & 59 deletions src/modules/database/migrations/sqlite/1728421385000-v13-final.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Logger, NotImplementedException } from "@nestjs/common";
import { randomBytes } from "crypto";
import { existsSync } from "fs";
import { toLower } from "lodash";
import { toLower, uniqBy } from "lodash";
import { setTimeout } from "timers/promises";
import { In, MigrationInterface, QueryRunner } from "typeorm";
import { GamevaultGame } from "../../../games/gamevault-game.entity";
Expand Down Expand Up @@ -4459,7 +4459,10 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateTags(queryRunner: QueryRunner): Promise<void> {
const tags = await queryRunner.manager.find(TagV12, { withDeleted: true });
const tags = uniqBy(
await queryRunner.manager.find(TagV12, { withDeleted: true }),
"rawg_id",
);
this.logger.debug({
message: `Found ${tags.length} tags in the V12 database.`,
});
Expand All @@ -4469,18 +4472,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating tag ID ${tag.id}, Name: ${tag.name}`,
});

if (
await queryRunner.manager.existsBy(TagMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: tag.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Tag ID ${tag.id} already exists. Skipping.`,
});
continue;
}

const newTag = await queryRunner.manager.save(TagMetadata, {
id: tag.id,
provider_slug: this.legacyProviderSlug,
Expand All @@ -4497,9 +4488,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateGenres(queryRunner: QueryRunner): Promise<void> {
const genres = await queryRunner.manager.find(GenreV12, {
withDeleted: true,
});
const genres = uniqBy(
await queryRunner.manager.find(GenreV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${genres.length} genres in the V12 database.`,
});
Expand All @@ -4509,18 +4503,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating genre ID ${genre.id}, Name: ${genre.name}`,
});

if (
await queryRunner.manager.existsBy(GenreMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: genre.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Genre ID ${genre.id} already exists. Skipping.`,
});
continue;
}

const newGenre = await queryRunner.manager.save(GenreMetadata, {
id: genre.id,
provider_slug: this.legacyProviderSlug,
Expand All @@ -4537,9 +4519,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migrateDevelopers(queryRunner: QueryRunner): Promise<void> {
const developers = await queryRunner.manager.find(DeveloperV12, {
withDeleted: true,
});
const developers = uniqBy(
await queryRunner.manager.find(DeveloperV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${developers.length} developers in the V12 database.`,
});
Expand All @@ -4549,18 +4534,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating developer ID ${developer.id}, Name: ${developer.name}`,
});

if (
await queryRunner.manager.existsBy(DeveloperMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: developer.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Developer ID ${developer.id} already exists. Skipping.`,
});
continue;
}

const newDeveloper = await queryRunner.manager.save(DeveloperMetadata, {
id: developer.id,
provider_slug: this.legacyProviderSlug,
Expand All @@ -4580,9 +4553,12 @@ export class V13Final1728421385000 implements MigrationInterface {
}

private async migratePublishers(queryRunner: QueryRunner): Promise<void> {
const publishers = await queryRunner.manager.find(PublisherV12, {
withDeleted: true,
});
const publishers = uniqBy(
await queryRunner.manager.find(PublisherV12, {
withDeleted: true,
}),
"rawg_id",
);
this.logger.debug({
message: `Found ${publishers.length} publishers in the V12 database.`,
});
Expand All @@ -4592,18 +4568,6 @@ export class V13Final1728421385000 implements MigrationInterface {
message: `Migrating publisher ID ${publisher.id}, Name: ${publisher.name}`,
});

if (
await queryRunner.manager.existsBy(PublisherMetadata, {
provider_slug: this.legacyProviderSlug,
provider_data_id: publisher.rawg_id.toString(),
})
) {
this.logger.debug({
message: `Publisher ID ${publisher.id} already exists. Skipping.`,
});
continue;
}

const newPublisher = await queryRunner.manager.save(PublisherMetadata, {
id: publisher.id,
provider_slug: this.legacyProviderSlug,
Expand Down

0 comments on commit b9321fe

Please sign in to comment.