Import from Donation Backup Service (#1253)

* Added Donation Save Backup Adapter * add backup service import cronjob * add interface of used params from mongo data * Change loading urls for backup service * Refactor donationSaveBackupAdapter and adding mock adapter * Removed unused package script * Refactored createBackupDonation to reuse resolver * Fix createBackupDonation() and write test case for that * Add importError to failed donation mongo backup --------- Co-authored-by: mohammadranjbarz <[email protected]> Co-authored-by: Carlos <[email protected]>
Giveth · Jan 23, 2024 · de14c4b · de14c4b
1 parent a00ed03
commit de14c4b
Show file tree

Hide file tree

Showing 15 changed files with 548 additions and 4 deletions.
diff --git a/config/example.env b/config/example.env
@@ -247,3 +247,16 @@ LOST_DONATIONS_NETWORK_ID=
 
 SOLANA_CHAIN_ID=103
 DISABLE_NOTIFICATION_CENTER=
+ENABLE_IMPORT_DONATION_BACKUP=false
+DONATION_SAVE_BACKUP_API_URL=
+DONATION_SAVE_BACKUP_API_SECRET=
+DONATION_SAVE_BACKUP_CRONJOB_EXPRESSION=
+DONATION_SAVE_BACKUP_DATA_SOURCE=
+
+# default value is failed_donation
+DONATION_SAVE_BACKUP_COLLECTION=
+# default value is failed_donation
+DONATION_SAVE_BACKUP_DATABASE=
+
+# Default value is saveBackup
+DONATION_SAVE_BACKUP_ADAPTER=saveBackup
diff --git a/config/test.env b/config/test.env
@@ -209,3 +209,7 @@ LOST_DONATIONS_QF_ROUND=
 LOST_DONATIONS_NETWORK_ID=10
 
 DISABLE_NOTIFICATION_CENTER=false
+DONATION_SAVE_BACKUP_CRONJOB_EXPRESSION=
+ENABLE_IMPORT_DONATION_BACKUP=false
+DONATION_SAVE_BACKUP_API_URL=
+DONATION_SAVE_BACKUP_API_SECRET=
diff --git a/package.json b/package.json
@@ -114,6 +114,7 @@
     "tslint:fix": "tslint -c tslint.json --fix '{src,test,migration}/**/*.ts'",
     "test": "NODE_ENV=test mocha --config ./.mocharc.all-test.json",
     "test:syncProjectsRequiredForListing": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/cronJobs/syncProjectsRequiredForListing.test.ts",
+    "test:backupDonationImport": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/cronJobs/backupDonationImport.test.ts",
     "test:projectEntity": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/entities/project.test.ts",
     "test:projectValidators": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/utils/validators/projectValidator.test.ts",
     "test:onramperWebhook": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/onramper/webhookHandler.test.ts",

diff --git a/src/adapters/adaptersFactory.ts b/src/adapters/adaptersFactory.ts
@@ -21,6 +21,8 @@ import { GitcoinAdapter } from './gitcoin/gitcoinAdapter';
 import { GitcoinMockAdapter } from './gitcoin/gitcoinMockAdapter';
 import { GivPowerBalanceAggregatorAdapter } from './givPowerBalanceAggregator/givPowerBalanceAggregatorAdapter';
 import { GivPowerBalanceAggregatorAdapterMock } from './givPowerBalanceAggregator/givPowerBalanceAggregatorAdapterMock';
+import { DonationSaveBackupAdapter } from './donationSaveBackup/donationSaveBackupAdapter';
+import { DonationSaveBackupMockAdapter } from './donationSaveBackup/DonationSaveBackupMockAdapter';
 
 const discordAdapter = new DiscordAdapter();
 const googleAdapter = new GoogleAdapter();
@@ -118,3 +120,17 @@ export const getPowerBalanceAggregatorAdapter = () => {
       return mockPowerBalanceAggregator;
   }
 };
+
+const donationSaveBackupAdapter = new DonationSaveBackupAdapter();
+const mockDonationSaveBackupAdapter = new DonationSaveBackupMockAdapter();
+
+export const getDonationSaveBackupAdapter = () => {
+  switch (process.env.DONATION_SAVE_BACKUP_ADAPTER) {
+    case 'saveBackup':
+      return donationSaveBackupAdapter;
+    case 'mock':
+      return mockDonationSaveBackupAdapter;
+    default:
+      return mockDonationSaveBackupAdapter;
+  }
+};
diff --git a/src/adapters/donationSaveBackup/DonationSaveBackupInterface.ts b/src/adapters/donationSaveBackup/DonationSaveBackupInterface.ts
@@ -0,0 +1,43 @@
+export type FetchedSavedFailDonationInterface = {
+  _id: string;
+  txHash: string;
+  imported?: boolean;
+  importError?: string;
+  token: {
+    symbol: string;
+    address: string;
+    networkId: number;
+  };
+  walletAddress: string;
+  amount: number;
+  chainId: number;
+  projectId: number;
+  anonymous: boolean;
+  nonce: number;
+  symbol: string;
+  chainvineReferred?: string;
+  safeTransactionId?: string;
+};
+
+export interface DonationSaveBackupInterface {
+  getNotImportedDonationsFromBackup(params: {
+    limit: number;
+  }): Promise<FetchedSavedFailDonationInterface[]>;
+
+  getSingleDonationFromBackupByTxHash(
+    txHash: string,
+  ): Promise<FetchedSavedFailDonationInterface | null>;
+
+  markDonationAsImported(donationMongoId: string): Promise<void>;
+
+  unmarkDonationAsImported(donationMongoId: string): Promise<void>;
+
+  getSingleDonationFromBackupById(
+    donationMongoId: string,
+  ): Promise<FetchedSavedFailDonationInterface | null>;
+
+  markDonationAsImportError(
+    donationMongoId: string,
+    errorMessage,
+  ): Promise<void>;
+}
diff --git a/src/adapters/donationSaveBackup/DonationSaveBackupMockAdapter.ts b/src/adapters/donationSaveBackup/DonationSaveBackupMockAdapter.ts
@@ -0,0 +1,41 @@
+import {
+  DonationSaveBackupInterface,
+  FetchedSavedFailDonationInterface,
+} from './DonationSaveBackupInterface';
+
+export class DonationSaveBackupMockAdapter
+  implements DonationSaveBackupInterface
+{
+  async getNotImportedDonationsFromBackup(params: {
+    limit: number;
+  }): Promise<FetchedSavedFailDonationInterface[]> {
+    return [];
+  }
+
+  async getSingleDonationFromBackupByTxHash(
+    txHash: string,
+  ): Promise<FetchedSavedFailDonationInterface | null> {
+    return null;
+  }
+
+  async markDonationAsImported(donationMongoId: string): Promise<void> {
+    //
+  }
+
+  async unmarkDonationAsImported(donationMongoId: string): Promise<void> {
+    //
+  }
+
+  async getSingleDonationFromBackupById(
+    donationMongoId: string,
+  ): Promise<FetchedSavedFailDonationInterface | null> {
+    return null;
+  }
+
+  markDonationAsImportError(
+    donationMongoId: string,
+    errorMessage,
+  ): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+}
diff --git a/src/adapters/donationSaveBackup/donationSaveBackupAdapter.ts b/src/adapters/donationSaveBackup/donationSaveBackupAdapter.ts
@@ -0,0 +1,218 @@
+// a method the get objects from mongodb api read from config DONATION_SAVE_BACKUP_API_URL with sercret read from DONATION_SAVE_BACKUP_API_SECRET,
+// it must filter objects by those doesn't have `imported` field with true value
+// also must support pagination
+
+import { logger } from '../../utils/logger';
+import config from '../../config';
+import axios from 'axios';
+import {
+  DonationSaveBackupInterface,
+  FetchedSavedFailDonationInterface,
+} from './DonationSaveBackupInterface';
+
+const DONATION_SAVE_BACKUP_API_URL = config.get(
+  'DONATION_SAVE_BACKUP_API_URL',
+) as string;
+const DONATION_SAVE_BACKUP_API_SECRET = config.get(
+  'DONATION_SAVE_BACKUP_API_SECRET',
+) as string;
+const DONATION_SAVE_BACKUP_DATA_SOURCE = config.get(
+  'DONATION_SAVE_BACKUP_DATA_SOURCE',
+) as string;
+const DONATION_SAVE_BACKUP_COLLECTION =
+  config.get('DONATION_SAVE_BACKUP_COLLECTION') || 'failed_donation';
+const DONATION_SAVE_BACKUP_DATABASE =
+  config.get('DONATION_SAVE_BACKUP_DATABASE') || 'failed_donation';
+
+// add '/' if doesn't exist at the
+const baseUrl = DONATION_SAVE_BACKUP_API_URL.endsWith('/')
+  ? DONATION_SAVE_BACKUP_API_URL
+  : `${DONATION_SAVE_BACKUP_API_URL}/`;
+
+export class DonationSaveBackupAdapter implements DonationSaveBackupInterface {
+  async getNotImportedDonationsFromBackup(params: {
+    limit: number;
+  }): Promise<FetchedSavedFailDonationInterface[]> {
+    const result = await axios.post(
+      `${baseUrl}find`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        limit: params.limit,
+        filter: {
+          imported: { $ne: true },
+          importError: { $ne: true },
+        },
+        sort: { _id: 1 },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('getNotImportedDonationsFromBackup error', result.data);
+      throw new Error('getNotImportedDonationsFromBackup error');
+    }
+    return result.data.documents;
+  }
+
+  async getSingleDonationFromBackupByTxHash(
+    txHash: string,
+  ): Promise<FetchedSavedFailDonationInterface | null> {
+    const result = await axios.post(
+      `${baseUrl}findOne`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        filter: {
+          txHash,
+        },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('getSingleDonationFromBackupByTxHash error', result.data);
+      throw new Error('getSingleDonationFromBackupByTxHash error');
+    }
+    return result.data.document;
+  }
+
+  async markDonationAsImported(donationMongoId: string): Promise<void> {
+    const result = await axios.post(
+      `${baseUrl}updateOne`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        filter: {
+          _id: { $oid: donationMongoId },
+        },
+        update: {
+          $set: {
+            imported: true,
+          },
+        },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('markDonationAsImported error', result.data);
+      throw new Error('markDonationAsImported error');
+    }
+  }
+
+  async markDonationAsImportError(
+    donationMongoId: string,
+    errorMessage,
+  ): Promise<void> {
+    const result = await axios.post(
+      `${baseUrl}updateOne`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        filter: {
+          _id: { $oid: donationMongoId },
+        },
+        update: {
+          $set: {
+            importError: errorMessage,
+          },
+        },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('markDonationAsImportError error', result.data);
+      throw new Error('markDonationAsImportError error');
+    }
+  }
+
+  async getSingleDonationFromBackupById(
+    donationMongoId: string,
+  ): Promise<FetchedSavedFailDonationInterface | null> {
+    const result = await axios.post(
+      `${baseUrl}findOne`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        filter: {
+          _id: { $oid: donationMongoId },
+        },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('getSingleDonationFromBackupById error', result.data);
+      throw new Error('getSingleDonationFromBackupById error');
+    }
+    return result.data.document;
+  }
+
+  async unmarkDonationAsImported(donationMongoId: string): Promise<void> {
+    const result = await axios.post(
+      `${baseUrl}updateOne`,
+      {
+        collection: DONATION_SAVE_BACKUP_COLLECTION,
+        database: DONATION_SAVE_BACKUP_DATABASE,
+        dataSource: DONATION_SAVE_BACKUP_DATA_SOURCE,
+        filter: {
+          _id: { $oid: donationMongoId },
+        },
+        update: {
+          $unset: {
+            imported: '',
+          },
+        },
+      },
+      {
+        headers: {
+          'api-key': DONATION_SAVE_BACKUP_API_SECRET,
+          'Content-Type': 'application/json',
+          'Access-Control-Request-Headers': '*',
+        },
+      },
+    );
+
+    if (result.status !== 200) {
+      logger.error('unmarkDonationAsImported error', result.data);
+      throw new Error('unmarkDonationAsImported error');
+    }
+  }
+}
diff --git a/src/resolvers/donationResolver.ts b/src/resolvers/donationResolver.ts
@@ -192,7 +192,8 @@ class DonationCurrencyStats {
 
 @Resolver(of => User)
 export class DonationResolver {
-  constructor(private readonly donationRepository: Repository<Donation>) {
+  private readonly donationRepository: Repository<Donation>;
+  constructor() {
     this.donationRepository =
       AppDataSource.getDataSource().getRepository(Donation);
   }

diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts
@@ -69,6 +69,7 @@ import { runCheckActiveStatusOfQfRounds } from '../services/cronJobs/checkActive
 import { runUpdateProjectCampaignsCacheJob } from '../services/cronJobs/updateProjectCampaignsCacheJob';
 import { corsOptions } from './cors';
 import { runSyncLostDonations } from '../services/cronJobs/importLostDonationsJob';
+import { runSyncBackupServiceDonations } from '../services/cronJobs/backupDonationImportJob';
 
 Resource.validate = validate;
 
@@ -340,6 +341,10 @@ export async function bootstrap() {
       runSyncLostDonations();
     }
 
+    if ((config.get('ENABLE_IMPORT_DONATION_BACKUP') as string) === 'true') {
+      runSyncBackupServiceDonations();
+    }
+
     if (
       (config.get('FILL_POWER_SNAPSHOT_BALANCE_SERVICE_ACTIVE') as string) ===
       'true'