fix: duplicate memories

server/src/constants.ts

@@ -7,10 +7,6 @@ export const POSTGRES_VERSION_RANGE = '>=14.0.0';
 export const VECTORS_VERSION_RANGE = '>=0.2 <0.4';
 export const VECTOR_VERSION_RANGE = '>=0.5 <1';
 
-export const ASSET_FILE_CONFLICT_KEYS = ['assetId', 'type'] as const;
-export const EXIF_CONFLICT_KEYS = ['assetId'] as const;
-export const JOB_STATUS_CONFLICT_KEYS = ['assetId'] as const;
-
 export const NEXT_RELEASE = 'NEXT_RELEASE';
 export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle';
 export const DEPRECATED_IN_PREFIX = 'This property was deprecated in ';

server/src/db.d.ts

@@ -4,7 +4,7 @@
  */
 
 import type { ColumnType } from 'kysely';
-import { Permission, SyncEntityType } from 'src/enum';
+import { AssetType, Permission, SyncEntityType } from 'src/enum';
 
 export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[] ? U[] : ArrayTypeImpl<T>;
 
@@ -145,7 +145,7 @@ export interface Assets {
   stackId: string | null;
   status: Generated<AssetsStatusEnum>;
   thumbhash: Buffer | null;
-  type: string;
+  type: AssetType;
   updatedAt: Generated<Timestamp>;
   updateId: Generated<string>;
 }
@@ -316,7 +316,6 @@ export interface SessionSyncCheckpoints {
   updateId: Generated<string>;
 }
 
-
 export interface SharedLinkAsset {
   assetsId: string;
   sharedLinksId: string;

server/src/repositories/asset.repository.ts

@@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
 import { Insertable, Kysely, Updateable, sql } from 'kysely';
 import { isEmpty, isUndefined, omitBy } from 'lodash';
 import { InjectKysely } from 'nestjs-kysely';
-import { ASSET_FILE_CONFLICT_KEYS, EXIF_CONFLICT_KEYS, JOB_STATUS_CONFLICT_KEYS } from 'src/constants';
 import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import {
@@ -24,7 +23,7 @@ import {
 } from 'src/entities/asset.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
 import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';
-import { anyUuid, asUuid, mapUpsertColumns } from 'src/utils/database';
+import { anyUuid, asUuid } from 'src/utils/database';
 import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
 
 export type AssetStats = Record<AssetType, number>;
@@ -146,11 +145,6 @@ export interface DuplicateGroup {
   assets: AssetEntity[];
 }
 
-export interface DayOfYearAssets {
-  yearsAgo: number;
-  assets: AssetEntity[];
-}
-
 @Injectable()
 export class AssetRepository {
   constructor(@InjectKysely() private db: Kysely<DB>) {}
@@ -161,7 +155,36 @@ export class AssetRepository {
       .insertInto('exif')
       .values(value)
       .onConflict((oc) =>
-        oc.columns(EXIF_CONFLICT_KEYS).doUpdateSet(() => mapUpsertColumns('exif', value, EXIF_CONFLICT_KEYS)),
+        oc.column('assetId').doUpdateSet((eb) => ({
+          description: eb.ref('excluded.description'),
+          exifImageWidth: eb.ref('excluded.exifImageWidth'),
+          exifImageHeight: eb.ref('excluded.exifImageHeight'),
+          fileSizeInByte: eb.ref('excluded.fileSizeInByte'),
+          orientation: eb.ref('excluded.orientation'),
+          dateTimeOriginal: eb.ref('excluded.dateTimeOriginal'),
+          modifyDate: eb.ref('excluded.modifyDate'),
+          timeZone: eb.ref('excluded.timeZone'),
+          latitude: eb.ref('excluded.latitude'),
+          longitude: eb.ref('excluded.longitude'),
+          projectionType: eb.ref('excluded.projectionType'),
+          city: eb.ref('excluded.city'),
+          livePhotoCID: eb.ref('excluded.livePhotoCID'),
+          autoStackId: eb.ref('excluded.autoStackId'),
+          state: eb.ref('excluded.state'),
+          country: eb.ref('excluded.country'),
+          make: eb.ref('excluded.make'),
+          model: eb.ref('excluded.model'),
+          lensModel: eb.ref('excluded.lensModel'),
+          fNumber: eb.ref('excluded.fNumber'),
+          focalLength: eb.ref('excluded.focalLength'),
+          iso: eb.ref('excluded.iso'),
+          exposureTime: eb.ref('excluded.exposureTime'),
+          profileDescription: eb.ref('excluded.profileDescription'),
+          colorspace: eb.ref('excluded.colorspace'),
+          bitsPerSample: eb.ref('excluded.bitsPerSample'),
+          rating: eb.ref('excluded.rating'),
+          fps: eb.ref('excluded.fps'),
+        })),
       )
       .execute();
   }
@@ -176,9 +199,13 @@ export class AssetRepository {
       .insertInto('asset_job_status')
       .values(values)
       .onConflict((oc) =>
-        oc
-          .columns(JOB_STATUS_CONFLICT_KEYS)
-          .doUpdateSet(() => mapUpsertColumns('asset_job_status', values[0], JOB_STATUS_CONFLICT_KEYS)),
+        oc.column('assetId').doUpdateSet((eb) => ({
+          duplicatesDetectedAt: eb.ref('excluded.duplicatesDetectedAt'),
+          facesRecognizedAt: eb.ref('excluded.facesRecognizedAt'),
+          metadataExtractedAt: eb.ref('excluded.metadataExtractedAt'),
+          previewAt: eb.ref('excluded.previewAt'),
+          thumbnailAt: eb.ref('excluded.thumbnailAt'),
+        })),
       )
       .execute();
   }
@@ -192,7 +219,7 @@ export class AssetRepository {
   }
 
   @GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] })
-  getByDayOfYear(ownerIds: string[], { day, month }: MonthDay): Promise<DayOfYearAssets[]> {
+  getByDayOfYear(ownerIds: string[], { day, month }: MonthDay) {
     return this.db
       .with('res', (qb) =>
         qb
@@ -248,7 +275,7 @@ export class AssetRepository {
       .groupBy(sql`("localDateTime" at time zone 'UTC')::date`)
       .orderBy(sql`("localDateTime" at time zone 'UTC')::date`, 'desc')
       .limit(10)
-      .execute() as any as Promise<DayOfYearAssets[]>;
+      .execute();
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
@@ -914,15 +941,16 @@ export class AssetRepository {
       .execute() as any as Promise<AssetEntity[]>;
   }
 
+  // TODO remove in favor of `upsertFiles`
   async upsertFile(file: Pick<Insertable<AssetFiles>, 'assetId' | 'path' | 'type'>): Promise<void> {
     const value = { ...file, assetId: asUuid(file.assetId) };
     await this.db
       .insertInto('asset_files')
       .values(value)
       .onConflict((oc) =>
-        oc
-          .columns(ASSET_FILE_CONFLICT_KEYS)
-          .doUpdateSet(() => mapUpsertColumns('asset_files', value, ASSET_FILE_CONFLICT_KEYS)),
+        oc.columns(['assetId', 'type']).doUpdateSet((eb) => ({
+          path: eb.ref('excluded.path'),
+        })),
       )
       .execute();
   }
@@ -937,9 +965,9 @@ export class AssetRepository {
       .insertInto('asset_files')
       .values(values)
       .onConflict((oc) =>
-        oc
-          .columns(ASSET_FILE_CONFLICT_KEYS)
-          .doUpdateSet(() => mapUpsertColumns('asset_files', values[0], ASSET_FILE_CONFLICT_KEYS)),
+        oc.columns(['assetId', 'type']).doUpdateSet((eb) => ({
+          path: eb.ref('excluded.path'),
+        })),
       )
       .execute();
   }

server/src/repositories/database.repository.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { InjectDataSource } from '@nestjs/typeorm';
 import AsyncLock from 'async-lock';
-import { Kysely, sql } from 'kysely';
+import { Kysely } from 'kysely';
 import { InjectKysely } from 'nestjs-kysely';
 import semver from 'semver';
 import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
@@ -10,9 +10,8 @@ import { DatabaseExtension, DatabaseLock, VectorIndex } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { ExtensionVersion, VectorExtension, VectorUpdateResult } from 'src/types';
-import { UPSERT_COLUMNS } from 'src/utils/database';
 import { isValidInteger } from 'src/validation';
-import { DataSource, EntityManager, EntityMetadata, QueryRunner } from 'typeorm';
+import { DataSource, EntityManager, QueryRunner } from 'typeorm';
 
 @Injectable()
 export class DatabaseRepository {
@@ -33,13 +32,6 @@ export class DatabaseRepository {
     await this.db.destroy();
   }
 
-  init() {
-    for (const metadata of this.dataSource.entityMetadatas) {
-      const table = metadata.tableName as keyof DB;
-      UPSERT_COLUMNS[table] = this.getUpsertColumns(metadata);
-    }
-  }
-
   async reconnect() {
     try {
       if (this.dataSource.isInitialized) {
@@ -257,10 +249,4 @@ export class DatabaseRepository {
   private async releaseLock(lock: DatabaseLock, queryRunner: QueryRunner): Promise<void> {
     return queryRunner.query('SELECT pg_advisory_unlock($1)', [lock]);
   }
-
-  private getUpsertColumns(metadata: EntityMetadata) {
-    return Object.fromEntries(
-      metadata.ownColumns.map((column) => [column.propertyName, sql<string>`excluded.${sql.ref(column.propertyName)}`]),
-    ) as any;
-  }
 }

server/src/repositories/person.repository.ts

@@ -7,7 +7,6 @@ import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import { SourceType } from 'src/enum';
-import { mapUpsertColumns } from 'src/utils/database';
 import { Paginated, PaginationOptions } from 'src/utils/pagination';
 import { FindOptionsRelations } from 'typeorm';
 
@@ -417,7 +416,17 @@ export class PersonRepository {
     await this.db
       .insertInto('person')
       .values(people)
-      .onConflict((oc) => oc.column('id').doUpdateSet(() => mapUpsertColumns('person', people[0], ['id'])))
+      .onConflict((oc) =>
+        oc.column('id').doUpdateSet((eb) => ({
+          birthDate: eb.ref('excluded.birthDate'),
+          color: eb.ref('excluded.color'),
+          faceAssetId: eb.ref('excluded.faceAssetId'),
+          isFavorite: eb.ref('excluded.isFavorite'),
+          isHidden: eb.ref('excluded.isHidden'),
+          name: eb.ref('excluded.name'),
+          thumbnailPath: eb.ref('excluded.thumbnailPath'),
+        })),
+      )
       .execute();
   }
 

server/src/repositories/search.repository.ts

@@ -404,7 +404,7 @@ export class SearchRepository {
           .where('assets.ownerId', '=', anyUuid(userIds))
           .where('assets.isVisible', '=', true)
           .where('assets.isArchived', '=', false)
-          .where('assets.type', '=', 'IMAGE')
+          .where('assets.type', '=', AssetType.IMAGE)
           .where('assets.deletedAt', 'is', null)
           .orderBy('city')
           .limit(1);
@@ -421,7 +421,7 @@ export class SearchRepository {
                 .where('assets.ownerId', '=', anyUuid(userIds))
                 .where('assets.isVisible', '=', true)
                 .where('assets.isArchived', '=', false)
-                .where('assets.type', '=', 'IMAGE')
+                .where('assets.type', '=', AssetType.IMAGE)
                 .where('assets.deletedAt', 'is', null)
                 .whereRef('exif.city', '>', 'cte.city')
                 .orderBy('city')

server/src/repositories/user.repository.ts

@@ -5,7 +5,7 @@ import { DB, UserMetadata as DbUserMetadata, Users } from 'src/db';
 import { DummyValue, GenerateSql } from 'src/decorators';
 import { UserMetadata, UserMetadataItem } from 'src/entities/user-metadata.entity';
 import { UserEntity, withMetadata } from 'src/entities/user.entity';
-import { UserStatus } from 'src/enum';
+import { AssetType, UserStatus } from 'src/enum';
 import { asUuid } from 'src/utils/database';
 
 const columns = [
@@ -209,11 +209,11 @@ export class UserRepository {
       .select((eb) => [
         eb.fn
           .countAll()
-          .filterWhere((eb) => eb.and([eb('assets.type', '=', 'IMAGE'), eb('assets.isVisible', '=', true)]))
+          .filterWhere((eb) => eb.and([eb('assets.type', '=', AssetType.IMAGE), eb('assets.isVisible', '=', true)]))
           .as('photos'),
         eb.fn
           .countAll()
-          .filterWhere((eb) => eb.and([eb('assets.type', '=', 'VIDEO'), eb('assets.isVisible', '=', true)]))
+          .filterWhere((eb) => eb.and([eb('assets.type', '=', AssetType.VIDEO), eb('assets.isVisible', '=', true)]))
           .as('videos'),
         eb.fn
           .coalesce(eb.fn.sum('exif.fileSizeInByte').filterWhere('assets.libraryId', 'is', null), eb.lit(0))
@@ -222,15 +222,19 @@ export class UserRepository {
           .coalesce(
             eb.fn
               .sum('exif.fileSizeInByte')
-              .filterWhere((eb) => eb.and([eb('assets.libraryId', 'is', null), eb('assets.type', '=', 'IMAGE')])),
+              .filterWhere((eb) =>
+                eb.and([eb('assets.libraryId', 'is', null), eb('assets.type', '=', AssetType.IMAGE)]),
+              ),
             eb.lit(0),
           )
           .as('usagePhotos'),
         eb.fn
           .coalesce(
             eb.fn
               .sum('exif.fileSizeInByte')
-              .filterWhere((eb) => eb.and([eb('assets.libraryId', 'is', null), eb('assets.type', '=', 'VIDEO')])),
+              .filterWhere((eb) =>
+                eb.and([eb('assets.libraryId', 'is', null), eb('assets.type', '=', AssetType.VIDEO)]),
+              ),
             eb.lit(0),
           )
           .as('usageVideos'),

server/src/services/asset.service.spec.ts

@@ -75,7 +75,7 @@ describe(AssetService.name, () => {
           yearsAgo: 15,
           assets: [image4],
         },
-      ]);
+      ] as any);
 
       await expect(sut.getMemoryLane(authStub.admin, { day: 15, month: 1 })).resolves.toEqual([
         { yearsAgo: 1, title: '1 year ago', assets: [mapAsset(image1), mapAsset(image2)] },

server/src/services/asset.service.ts

@@ -42,7 +42,7 @@ export class AssetService extends BaseService {
       yearsAgo,
       // TODO move this to clients
       title: `${yearsAgo} year${yearsAgo > 1 ? 's' : ''} ago`,
-      assets: assets.map((asset) => mapAsset(asset, { auth })),
+      assets: assets.map((asset) => mapAsset(asset as AssetEntity, { auth })),
     }));
   }
 

server/src/services/database.service.ts

@@ -108,7 +108,6 @@ export class DatabaseService extends BaseService {
       if (!database.skipMigrations) {
         await this.databaseRepository.runMigrations();
       }
-      this.databaseRepository.init();
     });
   }
 

server/src/services/memory.service.ts

@@ -30,20 +30,18 @@ export class MemoryService extends BaseService {
     const start = DateTime.utc().startOf('day').minus({ days: DAYS });
 
     const state = await this.systemMetadataRepository.get(SystemMetadataKey.MEMORIES_STATE);
-    let lastOnThisDayDate = state?.lastOnThisDayDate ? DateTime.fromISO(state?.lastOnThisDayDate) : start;
+    const lastOnThisDayDate = state?.lastOnThisDayDate ? DateTime.fromISO(state.lastOnThisDayDate) : start;
 
     // generate a memory +/- X days from today
-    for (let i = 0; i <= DAYS * 2 + 1; i++) {
+    for (let i = 0; i <= DAYS * 2; i++) {
       const target = start.plus({ days: i });
-      if (lastOnThisDayDate > target) {
+      if (lastOnThisDayDate >= target) {
         continue;
       }
 
       const showAt = target.startOf('day').toISO();
       const hideAt = target.endOf('day').toISO();
 
-      this.logger.log(`Creating memories for month=${target.month}, day=${target.day}`);
-
       for (const [userId, userIds] of Object.entries(userMap)) {
         const memories = await this.assetRepository.getByDayOfYear(userIds, target);
 
@@ -67,8 +65,6 @@ export class MemoryService extends BaseService {
         ...state,
         lastOnThisDayDate: target.toISO(),
       });
-
-      lastOnThisDayDate = target;
     }
   }