Merge pull request #1178 from Giveth/automate_exporting_qf_round_sybi…

…ls_analysis_data

Automate exporting qf round sybils analysis data

config/example.env

@@ -77,6 +77,10 @@ GOOGLE_SPREADSHEETS_PRIVATE_KEY=
 GOOGLE_SPREADSHEETS_CLIENT_EMAIL=
 GOOGLE_PROJECT_EXPORTS_SPREADSHEET_ID=
 
+QF_ROUND_GOOGLE_SPREADSHEETS_PRIVATE_KEY=
+QF_ROUND_GOOGLE_SPREADSHEETS_CLIENT_EMAIL=
+QF_ROUND_DONATIONS_GOOGLE_SPREADSHEET_ID=
+
 POIGN_ART_SUBGRAPH_URL=https://api.thegraph.com/subgraphs/name/dan13ram/poignart-rinkeby
 POIGN_ART_SERVICE_ACTIVE=true
 POIGN_ART_RECIPIENT_ADDRESS=0x10E1439455BD2624878b243819E31CfEE9eb721C
@@ -219,3 +223,4 @@ MORDOR_ETC_TESTNET_SCAN_API_URL=https://etc-mordor.blockscout.com/api/v1
 # https://chainlist.org/chain/63
 MORDOR_ETC_TESTNET_NODE_HTTP_URL=https://rpc.mordor.etccooperative.org
 
+QF_ROUND_MAX_REWARD=0.2

migration/1699512751669-add_known_as_sybils_to_user_table.ts

@@ -0,0 +1,27 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class addKnownAsSybilsToUserTable1699512751669
+  implements MigrationInterface
+{
+  async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            DO $$
+              BEGIN
+                BEGIN
+                    ALTER TABLE public.user ADD COLUMN "knownAsSybilAddress" boolean NOT NULL DEFAULT false;
+                EXCEPTION
+                    WHEN duplicate_column THEN 
+                        -- Handle the error, or just do nothing to skip adding the column.
+                        RAISE NOTICE 'Column "isStableCoin" already exists in "public.user".';
+                END;
+              END $$;
+        `);
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            ALTER TABLE public.user
+            DROP COLUMN "knownAsSybilAddress";
+        `);
+  }
+}

migration/1699542566835-project_actual_matching_view.ts

@@ -0,0 +1,93 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class projectActualMatchingView1699542566835
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `
+            DROP MATERIALIZED VIEW IF EXISTS project_actual_matching_view;
+            CREATE MATERIALIZED VIEW project_actual_matching_view AS
+            WITH DonationsBeforeAnalysis AS (
+            SELECT 
+                p.id, 
+                p.slug, 
+                p.title,
+                qr.id as "qfId",
+                STRING_AGG(distinct pa."networkId" || '-' || pa."address", ', ') AS "networkAddresses",
+                COALESCE(SUM(d."valueUsd"), 0) AS "allUsdReceived", 
+                COUNT(DISTINCT d."fromWalletAddress") AS "totalDonors"
+            FROM 
+                public.donation AS d
+                INNER JOIN project p ON p.id = d."projectId"
+                INNER JOIN qf_round qr on qr.id = d."qfRoundId"
+                inner join project_address pa on pa."projectId" = p.id AND pa."networkId" = ANY(qr."eligibleNetworks")
+                inner join "user" u on u.id = d."userId" 
+            GROUP BY 
+                p.id,
+                p.title, 
+                p.slug,
+                qr.id
+            ), DonationsAfterAnalysis AS (
+            SELECT 
+                p2.id, 
+                p2.slug, 
+                p2.title,
+                qr.id as "qfId",
+                COALESCE(SUM(d2."valueUsd"), 0) AS "allUsdReceivedAfterSybilsAnalysis", 
+                COUNT(DISTINCT d2."fromWalletAddress") AS "uniqueDonors",
+                SUM(SQRT(d2."valueUsd")) AS "donationsSqrtRootSum",
+                POWER(SUM(SQRT(d2."valueUsd")), 2) as "donationsSqrtRootSumSquared"
+            FROM 
+                public.donation AS d2
+                INNER JOIN project p2 ON p2.id = d2."projectId"
+                INNER JOIN qf_round qr on qr.id = d2."qfRoundId"
+                inner join project_address pa on pa."projectId" = p2.id AND pa."networkId" = ANY(qr."eligibleNetworks")
+                inner join "user" u on u.id = d2."userId" and u."knownAsSybilAddress" = false
+            WHERE 
+                p2."statusId" = 5 
+                AND LOWER(d2."fromWalletAddress") NOT IN (
+                SELECT DISTINCT LOWER(pa.address) AS "projectAddress"
+                FROM public.project_address pa 
+                JOIN project p3 ON p3.id = pa."projectId" 
+                AND p3."verified" = true 
+                AND p3."statusId" = 5 
+                AND p3."isImported" = false
+                ) 
+                AND d2."qfRoundUserScore" > 5
+            GROUP BY 
+                p2.id,
+                p2.title,
+                p2.slug,
+                qr.id
+            )
+            SELECT 
+                d1.id AS "projectId",
+                d1.title,
+                d1.slug, 
+                d1."networkAddresses",
+                d1."qfId" AS "qfRoundId",
+                d1."allUsdReceived",
+                d1."totalDonors",
+                d2."allUsdReceivedAfterSybilsAnalysis",
+                d2."uniqueDonors",
+                d2."donationsSqrtRootSum",
+                d2."donationsSqrtRootSumSquared"
+            FROM 
+                DonationsBeforeAnalysis d1
+            INNER JOIN DonationsAfterAnalysis d2 ON d1.id = d2.id AND d1.slug = d2.slug and d1."qfId" = d2."qfId";
+            
+            CREATE INDEX idx_project_actual_matching_project_id ON project_actual_matching_view USING hash ("projectId");
+            CREATE INDEX idx_project_actual_matching_qf_round_id ON project_actual_matching_view USING hash ("qfRoundId");
+            `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `
+            DROP MATERIALIZED VIEW project_actual_matching_view;
+            `,
+    );
+  }
+}

src/entities/user.ts

@@ -135,6 +135,12 @@ export class User extends BaseEntity {
   @Column('bool', { default: false })
   isReferrer: boolean;
 
+  @Field(type => Boolean, { nullable: true })
+  @Column('bool', { default: false })
+  // After each QF round Lauren and Griff review the donations and pass me a list of sybil addresses
+  // And then we exclude qfRound donation from those addresses when calculating the real matchingFund
+  knownAsSybilAddress: boolean;
+
   @Field(() => ReferredEvent, { nullable: true })
   @OneToOne(() => ReferredEvent, referredEvent => referredEvent.user, {
     cascade: true,

src/server/adminJs/tabs/qfRoundTab.ts

@@ -5,6 +5,8 @@ import {
   After,
 } from 'adminjs/src/backend/actions/action.interface';
 import {
+  getQfRoundActualDonationDetails,
+  refreshProjectActualMatchingView,
   refreshProjectDonationSummaryView,
   refreshProjectEstimatedMatchingView,
 } from '../../../services/projectViewsService';
@@ -20,6 +22,9 @@ import {
 } from '../../../repositories/qfRoundRepository';
 import { RecordJSON } from 'adminjs/src/frontend/interfaces/record-json.interface';
 import { NETWORK_IDS } from '../../../provider';
+import { logger } from '../../../utils/logger';
+import { messages } from '../../../utils/messages';
+import { addQfRoundDonationsSheetToSpreadsheet } from '../../../services/googleSheets';
 
 export const refreshMaterializedViews = async (
   response,
@@ -49,6 +54,36 @@ export const fillProjects: After<ActionResponse> = async (
   return response;
 };
 
+const returnAllQfRoundDonationAnalysis = async (
+  context: AdminJsContextInterface,
+  request: AdminJsRequestInterface,
+) => {
+  const { record, currentAdmin } = context;
+  try {
+    const qfRoundId = Number(request?.params?.recordId);
+    logger.debug('qfRoundId', qfRoundId);
+
+    const qfRoundDonationsRows = await getQfRoundActualDonationDetails(
+      qfRoundId,
+    );
+    logger.debug('qfRoundDonationsRows', qfRoundDonationsRows);
+    await addQfRoundDonationsSheetToSpreadsheet({
+      rows: qfRoundDonationsRows,
+      qfRoundId,
+    });
+    // TODO Upload to google sheet
+  } catch (error) {
+    throw error;
+  }
+  return {
+    record: record.toJSON(currentAdmin),
+    notice: {
+      message: messages.QF_ROUND_DATA_UPLOAD_IN_GOOGLE_SHEET_SUCCESSFULLY,
+      type: 'success',
+    },
+  };
+};
+
 export const qfRoundTab = {
   resource: QfRound,
   options: {
@@ -166,6 +201,16 @@ export const qfRoundTab = {
         },
         after: refreshMaterializedViews,
       },
+
+      returnAllDonationData: {
+        // https://docs.adminjs.co/basics/action#record-type-actions
+        actionType: 'record',
+        isVisible: true,
+        handler: async (request, response, context) => {
+          return returnAllQfRoundDonationAnalysis(context, request);
+        },
+        component: false,
+      },
     },
   },
 };

src/services/googleSheets.ts

@@ -29,6 +29,19 @@ interface ProjectExport {
   secondWalletAddress: string;
   secondWalletAddressNetwork: string;
 }
+export interface QfRoundDonationRow {
+  projectName: string;
+  // Pattern is networkId-projectAddress,... Example: 1-0x123...456,10,ETH,0x123...456
+  addresses: string;
+  link: string;
+  allUsdReceived?: number;
+  allUsdReceivedAfterSybilsAnalysis?: number;
+  totalDonors: number;
+
+  // We can have 20 donors but after sybils analysis we can have 15 unique donors
+  uniqueDonors: number;
+  realMatchingFund: number;
+}
 
 interface DonationExport {
   id: number;
@@ -75,6 +88,25 @@ export const initExportSpreadsheet = async (): Promise<
   return spreadSheet;
 };
 
+const initQfRoundDonationsSpreadsheet = async (): Promise<
+  typeof GoogleSpreadsheet
+> => {
+  // Initialize the sheet - document ID is the long id in the sheets URL
+  const spreadSheet = new GoogleSpreadsheet(
+    config.get('QF_ROUND_DONATIONS_GOOGLE_SPREADSHEET_ID'),
+  );
+
+  // Initialize Auth - see https://theoephraim.github.io/node-google-spreadsheet/#/getting-started/authentication
+  await spreadSheet.useServiceAccountAuth({
+    // env var values are copied from service account credentials generated by google
+    // see "Authentication" section in docs for more info
+    client_email: config.get('QF_ROUND_GOOGLE_SPREADSHEETS_CLIENT_EMAIL'),
+    private_key: config.get('QF_ROUND_GOOGLE_SPREADSHEETS_PRIVATE_KEY'),
+  });
+
+  return spreadSheet;
+};
+
 export const addDonationsSheetToSpreadsheet = async (
   spreadSheet: GoogleSpreadsheet,
   headers: string[],
@@ -112,3 +144,35 @@ export const addProjectsSheetToSpreadsheet = async (
     throw e;
   }
 };
+
+export const addQfRoundDonationsSheetToSpreadsheet = async (params: {
+  rows: QfRoundDonationRow[];
+  qfRoundId: number;
+}): Promise<void> => {
+  try {
+    const spreadSheet = await initQfRoundDonationsSpreadsheet();
+
+    const currentDate = moment().toDate();
+    const headers = [
+      'projectName',
+      'addresses',
+      'link',
+      'allUsdReceived',
+      'allUsdReceivedAfterSybilsAnalysis',
+      'totalDonors',
+      'uniqueDonors',
+      'realMatchingFund',
+    ];
+    const { rows, qfRoundId } = params;
+
+    const sheet = await spreadSheet.addSheet({
+      headerValues: headers,
+      title: `QfRound -${qfRoundId} - ${currentDate.toDateString()} ${currentDate.getTime()}`,
+    });
+    logger.debug('addQfRoundDonationsSheetToSpreadsheet', params);
+    await sheet.addRows(rows);
+  } catch (e) {
+    logger.error('addQfRoundDonationsSheetToSpreadsheet error', e);
+    throw e;
+  }
+};

src/services/projectViewsService.ts

@@ -1,5 +1,7 @@
+import { QfRound } from '../entities/qfRound';
 import { AppDataSource } from '../orm';
 import { logger } from '../utils/logger';
+import { QfRoundDonationRow } from './googleSheets';
 
 export const refreshProjectEstimatedMatchingView = async (): Promise<void> => {
   logger.debug('Refresh project_estimated_matching_view materialized view');
@@ -10,6 +12,15 @@ export const refreshProjectEstimatedMatchingView = async (): Promise<void> => {
   );
 };
 
+export const refreshProjectActualMatchingView = async (): Promise<void> => {
+  logger.debug('Refresh project_actual_matching_view materialized view');
+  return AppDataSource.getDataSource().query(
+    `
+      REFRESH MATERIALIZED VIEW project_actual_matching_view
+    `,
+  );
+};
+
 export const refreshProjectDonationSummaryView = async (): Promise<void> => {
   logger.debug('Refresh project_donation_summary_view materialized view');
   return AppDataSource.getDataSource().query(
@@ -18,3 +29,54 @@ export const refreshProjectDonationSummaryView = async (): Promise<void> => {
     `,
   );
 };
+
+export const getQfRoundActualDonationDetails = async (
+  qfRoundId: Number,
+): Promise<QfRoundDonationRow[]> => {
+  const qfRound = await QfRound.createQueryBuilder('qfRound')
+    .where('qfRound.id = :id', { id: qfRoundId })
+    .getOne();
+
+  if (!qfRoundId) return [];
+
+  await refreshProjectActualMatchingView();
+
+  const rows = await QfRound.query(`
+    SELECT *
+    FROM project_actual_matching_view
+    WHERE "qfRoundId" = ${qfRoundId}
+  `);
+
+  let totalReward = qfRound!.allocatedFund;
+  const qfRoundMaxReward =
+    totalReward * Number(process.env.QF_ROUND_MAX_REWARD_PERCENTAGE || 0.2);
+  let totalWeight = rows.reduce((accumulator, currentRow) => {
+    return accumulator + currentRow.donationsSqrtRootSumSquared;
+  }, 0);
+
+  for (const row of rows) {
+    const weight = row.donationsSqrtRootSumSquared;
+    const reward = Math.min(
+      (totalReward * weight) / totalWeight,
+      qfRoundMaxReward,
+    );
+    row.actualMatching = reward;
+    totalReward -= reward;
+    totalWeight -= weight;
+  }
+
+  const qfRoundDonationsRows = rows.map(row => {
+    return {
+      projectName: row.title,
+      addresses: row.networkAddresses,
+      link: process.env.GIVETH_IO_DAPP_BASE_URL + '/' + row.slug,
+      allUsdReceived: row.allUsdReceived,
+      allUsdReceivedAfterSybilsAnalysis: row.allUsdReceivedAfterSybilsAnalysis,
+      totalDonors: row.totalDonors,
+      uniqueDonors: row.uniqueDonors,
+      realMatchingFund: row.actualMatching,
+    };
+  });
+
+  return qfRoundDonationsRows;
+};

src/utils/messages.ts

@@ -4,5 +4,7 @@ export const messages = {
   DONATION_EDITED_SUCCESSFULLY: 'Donation successfully Edited',
   PROJECTS_RELATED_TO_ACTIVE_QF_ROUND_SUCCESSFULLY:
     'Projects related to active qfRound successfully',
+  QF_ROUND_DATA_UPLOAD_IN_GOOGLE_SHEET_SUCCESSFULLY:
+    'QF round data uploaded in Google sheet successfully',
   THERE_IS_NOT_ANY_ACTIVE_QF_ROUND: 'There is not any active qfRound',
 };