From b20f60d9fa831e31117265cf952b09ef9030a833 Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 10 Oct 2024 16:41:32 +0200 Subject: [PATCH 1/5] add cluster matching entity --- ...28554628004-AddEstimatedClusterMatching.ts | 34 +++++++++++++++ src/entities/estimatedClusterMatching.ts | 41 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 migration/1728554628004-AddEstimatedClusterMatching.ts create mode 100644 src/entities/estimatedClusterMatching.ts diff --git a/migration/1728554628004-AddEstimatedClusterMatching.ts b/migration/1728554628004-AddEstimatedClusterMatching.ts new file mode 100644 index 000000000..a31ab2617 --- /dev/null +++ b/migration/1728554628004-AddEstimatedClusterMatching.ts @@ -0,0 +1,34 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class AddEstimatedClusterMatching1728554628004 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE estimated_cluster_matching ( + id SERIAL PRIMARY KEY, + project_id INT NOT NULL, + qf_round_id INT NOT NULL, + matching DOUBLE PRECISION NOT NULL + ); + `); + + // Create indexes on the new table + await queryRunner.query(` + CREATE INDEX estimated_cluster_matching_project_id_qfround_id + ON estimated_cluster_matching (project_id, qf_round_id); + `); + + await queryRunner.query(` + CREATE INDEX estimated_cluster_matching_matching + ON estimated_cluster_matching (matching); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + // Revert changes if necessary by dropping the table and restoring the view + await queryRunner.query(` + DROP TABLE IF EXISTS estimated_cluster_matching; + `); + } + +} diff --git a/src/entities/estimatedClusterMatching.ts b/src/entities/estimatedClusterMatching.ts new file mode 100644 index 000000000..da2165e8e --- /dev/null +++ b/src/entities/estimatedClusterMatching.ts @@ -0,0 +1,41 @@ +import { Field, ObjectType } from 'type-graphql'; +import { + Column, + Index, + PrimaryGeneratedColumn, + BaseEntity, + Entity, + ManyToOne, + JoinColumn, +} from 'typeorm'; +import { Project } from './project'; + +@Entity('estimated_cluster_matching') +@Index('estimated_cluster_matching_project_id_qfround_id', [ + 'projectId', + 'qfRoundId', +]) +@Index('estimated_cluster_matching_matching', ['matching']) +@ObjectType() +export class EstimatedClusterMatching extends BaseEntity { + @Field() + @PrimaryGeneratedColumn() + id: number; // New primary key + + @Field(_type => Project) + @ManyToOne(_type => Project, project => project.projectEstimatedMatchingView) + @JoinColumn({ referencedColumnName: 'id' }) + project: Project; + + @Field() + @Column() + projectId: number; + + @Field() + @Column() + qfRoundId: number; + + @Field() + @Column('double precision') + matching: number; +} From d0b4b971256cc3fdcebeabbef75fd0715548bf43 Mon Sep 17 00:00:00 2001 From: Carlos Date: Tue, 15 Oct 2024 13:22:07 +0200 Subject: [PATCH 2/5] add cluster matching adapters --- ...28554628004-AddEstimatedClusterMatching.ts | 32 ++++++------ src/adapters/adaptersFactory.ts | 17 +++++++ src/adapters/cocmAdapter/cocmAdapter.ts | 46 +++++++++++++++++ .../cocmAdapter/cocmAdapterInterface.ts | 49 ++++++++++++++++++ src/adapters/cocmAdapter/cocmMockAdapter.ts | 30 +++++++++++ src/server/bootstrap.ts | 14 ------ .../syncEstimatedClusterMatching.test.ts | 0 .../syncEstimatedClusterMatchingJob.ts | 50 +++++++++++++++++++ src/utils/errorMessages.ts | 2 + .../fetchEstimatedClusterMatchingWorker.ts | 17 +++++++ ...eProjectsEstimatedClusterMatchingWorker.ts | 0 11 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 src/adapters/cocmAdapter/cocmAdapter.ts create mode 100644 src/adapters/cocmAdapter/cocmAdapterInterface.ts create mode 100644 src/adapters/cocmAdapter/cocmMockAdapter.ts create mode 100644 src/services/cronJobs/syncEstimatedClusterMatching.test.ts create mode 100644 src/services/cronJobs/syncEstimatedClusterMatchingJob.ts create mode 100644 src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts create mode 100644 src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts diff --git a/migration/1728554628004-AddEstimatedClusterMatching.ts b/migration/1728554628004-AddEstimatedClusterMatching.ts index a31ab2617..727c29621 100644 --- a/migration/1728554628004-AddEstimatedClusterMatching.ts +++ b/migration/1728554628004-AddEstimatedClusterMatching.ts @@ -1,9 +1,10 @@ -import {MigrationInterface, QueryRunner} from "typeorm"; +import { MigrationInterface, QueryRunner } from 'typeorm'; -export class AddEstimatedClusterMatching1728554628004 implements MigrationInterface { - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(` +export class AddEstimatedClusterMatching1728554628004 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` CREATE TABLE estimated_cluster_matching ( id SERIAL PRIMARY KEY, project_id INT NOT NULL, @@ -11,24 +12,23 @@ export class AddEstimatedClusterMatching1728554628004 implements MigrationInterf matching DOUBLE PRECISION NOT NULL ); `); - - // Create indexes on the new table - await queryRunner.query(` + + // Create indexes on the new table + await queryRunner.query(` CREATE INDEX estimated_cluster_matching_project_id_qfround_id ON estimated_cluster_matching (project_id, qf_round_id); `); - - await queryRunner.query(` + + await queryRunner.query(` CREATE INDEX estimated_cluster_matching_matching ON estimated_cluster_matching (matching); `); - } + } - public async down(queryRunner: QueryRunner): Promise { - // Revert changes if necessary by dropping the table and restoring the view - await queryRunner.query(` + public async down(queryRunner: QueryRunner): Promise { + // Revert changes if necessary by dropping the table and restoring the view + await queryRunner.query(` DROP TABLE IF EXISTS estimated_cluster_matching; `); - } - + } } diff --git a/src/adapters/adaptersFactory.ts b/src/adapters/adaptersFactory.ts index 7c5964527..ffffc2147 100644 --- a/src/adapters/adaptersFactory.ts +++ b/src/adapters/adaptersFactory.ts @@ -22,6 +22,9 @@ import { DonationSaveBackupMockAdapter } from './donationSaveBackup/DonationSave import { SuperFluidAdapter } from './superFluid/superFluidAdapter'; import { SuperFluidMockAdapter } from './superFluid/superFluidMockAdapter'; import { SuperFluidAdapterInterface } from './superFluid/superFluidAdapterInterface'; +import { CocmAdapter } from './cocmAdapter/cocmAdapter'; +import { CocmMockAdapter } from './cocmAdapter/cocmMockAdapter'; +import { CocmAdapterInterface } from './cocmAdapter/cocmAdapterInterface'; const discordAdapter = new DiscordAdapter(); const googleAdapter = new GoogleAdapter(); @@ -147,3 +150,17 @@ export const getSuperFluidAdapter = (): SuperFluidAdapterInterface => { return superFluidMockAdapter; } }; + +const clusterMatchingAdapter = new CocmAdapter(); +const clusterMatchingMockAdapter = new CocmMockAdapter(); + +export const getClusterMatchingAdapter = (): CocmAdapterInterface => { + switch (process.env.CLUSTER_MATCHING_ADAPTER) { + case 'clusterMatching': + return clusterMatchingAdapter; + case 'mock': + return clusterMatchingMockAdapter; + default: + return clusterMatchingMockAdapter; + } +}; diff --git a/src/adapters/cocmAdapter/cocmAdapter.ts b/src/adapters/cocmAdapter/cocmAdapter.ts new file mode 100644 index 000000000..fad366dc0 --- /dev/null +++ b/src/adapters/cocmAdapter/cocmAdapter.ts @@ -0,0 +1,46 @@ +import axios from 'axios'; +import { + CocmAdapterInterface, + EstimatedMatchingInput, + ProjectsEstimatedMatchings, +} from './cocmAdapterInterface'; +import { logger } from '../../utils/logger'; +import { i18n, translationErrorMessagesKeys } from '../../utils/errorMessages'; + +export class CocmAdapter implements CocmAdapterInterface { + private ClusterMatchingURL; + + constructor() { + this.ClusterMatchingURL = + process.env.CLUSTER_MATCHING_API_URL || 'localhost'; + } + + async fetchEstimatedClusterMatchings( + matchingDataInput: EstimatedMatchingInput, + ): Promise { + try { + const result = await axios.post( + this.ClusterMatchingURL, + matchingDataInput, + { + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }, + ); + if (result?.data?.error !== null) { + logger.error('clusterMatchingApi error', result.data.error); + throw new Error( + i18n.__(translationErrorMessagesKeys.CLUSTER_MATCHING_API_ERROR), + ); + } + return result.data; + } catch (e) { + logger.error('clusterMatchingApi error', e); + throw new Error( + i18n.__(translationErrorMessagesKeys.CLUSTER_MATCHING_API_ERROR), + ); + } + } +} diff --git a/src/adapters/cocmAdapter/cocmAdapterInterface.ts b/src/adapters/cocmAdapter/cocmAdapterInterface.ts new file mode 100644 index 000000000..93d5dea1c --- /dev/null +++ b/src/adapters/cocmAdapter/cocmAdapterInterface.ts @@ -0,0 +1,49 @@ +// Example Data +// { +// "matching_data": [ +// { +// "matching_amount": 83.25, +// "matching_percent": 50.0, +// "project_name": "Test1", +// "strategy": "COCM" +// }, +// { +// "matching_amount": 83.25, +// "matching_percent": 50.0, +// "project_name": "Test3", +// "strategy": "COCM" +// } +// ] +// } + +export interface ProjectsEstimatedMatchings { + matching_data: { + matching_amount: number; + matching_percent: number; + project_name: string; + strategy: string; + }[]; +} + +export interface EstimatedMatchingInput { + votes_data: [ + { + voter: string; + payoutAddress: string; + amountUSD: number; + project_name: string; + score: number; + }, + ]; + strategy: string; + min_donation_threshold_amount: number; + matching_cap_amount: number; + matching_amount: number; + passport_threshold: number; +} + +export interface CocmAdapterInterface { + fetchEstimatedClusterMatchings( + matchingDataInput: EstimatedMatchingInput, + ): Promise; +} diff --git a/src/adapters/cocmAdapter/cocmMockAdapter.ts b/src/adapters/cocmAdapter/cocmMockAdapter.ts new file mode 100644 index 000000000..f26ed502f --- /dev/null +++ b/src/adapters/cocmAdapter/cocmMockAdapter.ts @@ -0,0 +1,30 @@ +import axios from 'axios'; +import { + CocmAdapterInterface, + ProjectsEstimatedMatchings, +} from './cocmAdapterInterface'; +import { i18n, translationErrorMessagesKeys } from '../../utils/errorMessages'; +import { logger } from '../../utils/logger'; + +export class CocmMockAdapter implements CocmAdapterInterface { + async fetchEstimatedClusterMatchings( + _matchingDataInput, + ): Promise { + return { + matching_data: [ + { + matching_amount: 83.25, + matching_percent: 50.0, + project_name: 'Test1', + strategy: 'COCM', + }, + { + matching_amount: 83.25, + matching_percent: 50.0, + project_name: 'Test3', + strategy: 'COCM', + }, + ], + }; + } +} diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts index 9c55eac75..b3509bced 100644 --- a/src/server/bootstrap.ts +++ b/src/server/bootstrap.ts @@ -339,20 +339,6 @@ export async function bootstrap() { logger.error('Enabling power boosting snapshot ', e); } } - - if (!isTestEnv) { - // They will fail in test env, because we run migrations after bootstrap so refreshing them will cause this error - // relation "project_estimated_matching_view" does not exist - logger.debug( - 'continueDbSetup() before refreshProjectEstimatedMatchingView() ', - new Date(), - ); - await refreshProjectEstimatedMatchingView(); - logger.debug( - 'continueDbSetup() after refreshProjectEstimatedMatchingView() ', - new Date(), - ); - } logger.debug('continueDbSetup() end of function', new Date()); } diff --git a/src/services/cronJobs/syncEstimatedClusterMatching.test.ts b/src/services/cronJobs/syncEstimatedClusterMatching.test.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts new file mode 100644 index 000000000..d57c3da17 --- /dev/null +++ b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts @@ -0,0 +1,50 @@ +import { schedule } from 'node-cron'; +import { spawn, Worker, Thread } from 'threads'; +import config from '../../config'; +import { logger } from '../../utils/logger'; +import { + findActiveQfRound, + findUsersWithoutMBDScoreInActiveAround, +} from '../../repositories/qfRoundRepository'; +import { findUserById } from '../../repositories/userRepository'; +import { UserQfRoundModelScore } from '../../entities/userQfRoundModelScore'; + +const cronJobTime = + (config.get('SYNC_ESTIMATED_CLUSTED_MATCHING_CRONJOB_EXPRESSION') as string) || + '0 * * * * *'; + +export const runSyncEstimatedClusterMatchingCronjob = () => { + logger.debug( + 'runSyncEstimatedClusterMatchingCronjob() has been called, cronJobTime', + cronJobTime, + ); + schedule(cronJobTime, async () => { + await fetchAndUpdateClusterEstimatedMatching(); + }); +}; + +export const fetchAndUpdateClusterEstimatedMatching = async () => { + const fetchWorker = await spawn( + new Worker('../../workers/cocm/fetchEstimatedClusterMtchingWorker'), + ); + + const updateWorker = await spawn( + new Worker('../../workers/cocm/updateProjectsEstimatedClusterMatchingWorker') + ); + const activeQfRoundId = + (await findActiveQfRound())?.id; + if (!activeQfRoundId || activeQfRoundId === 0) return; + + for (const projectId of []) { + try { + + // const userScore = await worker.syncUserScore({ + // userWallet: user?.walletAddress, + // }); + } catch (e) { + logger.info(`User with Id ${1} did not sync MBD score this batch`); + } + } + await Thread.terminate(fetchWorker); + await Thread.terminate(updateWorker); +}; diff --git a/src/utils/errorMessages.ts b/src/utils/errorMessages.ts index a6584da2a..b99812c70 100644 --- a/src/utils/errorMessages.ts +++ b/src/utils/errorMessages.ts @@ -19,6 +19,7 @@ export const setI18nLocaleForRequest = async (req, _res, next) => { }; export const errorMessages = { + CLUSTER_MATCHING_API_ERROR: 'Error in the cluster matching api, check logs', FIAT_DONATION_ALREADY_EXISTS: 'Onramper donation already exists', CAMPAIGN_NOT_FOUND: 'Campaign not found', QF_ROUND_NOT_FOUND: 'qf round not found', @@ -208,6 +209,7 @@ export const errorMessages = { }; export const translationErrorMessagesKeys = { + CLUSTER_MATCHING_API_ERROR: 'CLUSTER_MATCHING_API_ERROR', GITCOIN_ERROR_FETCHING_DATA: 'GITCOIN_ERROR_FETCHING_DATA', TX_NOT_FOUND: 'TX_NOT_FOUND', INVALID_PROJECT_ID: 'INVALID_PROJECT_ID', diff --git a/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts b/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts new file mode 100644 index 000000000..37daeeaaf --- /dev/null +++ b/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts @@ -0,0 +1,17 @@ +// workers/auth.js +import { expose } from 'threads/worker'; +import { WorkerModule } from 'threads/dist/types/worker'; +import { getClusterMatchingAdapter } from '../../adapters/adaptersFactory'; + +type FetchEstimatedClusterMatchingWorkerFunctions = 'fetchEstimatedClusterMatching'; + +export type FetchEstimatedClusterMatchingWorker = + WorkerModule; + +const worker: FetchEstimatedClusterMatchingWorker = { + async fetchEstimatedClusterMatching(matchingDataInput: any) { + return await getClusterMatchingAdapter().fetchEstimatedClusterMatchings(matchingDataInput); + }, +}; + +expose(worker); diff --git a/src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts b/src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts new file mode 100644 index 000000000..e69de29bb From 56857c34a03eefcb4d3b4a1ecb5cba323a54a734 Mon Sep 17 00:00:00 2001 From: Carlos Date: Wed, 16 Oct 2024 15:11:12 +0200 Subject: [PATCH 3/5] finish cocm adapter --- src/adapters/cocmAdapter/cocmMockAdapter.ts | 3 - src/entities/project.ts | 21 ++----- src/repositories/donationRepository.ts | 22 +++++++ .../projectResolver.allProject.test.ts | 53 ++++++++--------- src/server/bootstrap.ts | 2 - .../syncEstimatedClusterMatchingJob.ts | 57 ++++++++++--------- .../cocm/estimatedClusterMatchingWorker.ts | 55 ++++++++++++++++++ .../fetchEstimatedClusterMatchingWorker.ts | 17 ------ ...eProjectsEstimatedClusterMatchingWorker.ts | 0 9 files changed, 138 insertions(+), 92 deletions(-) create mode 100644 src/workers/cocm/estimatedClusterMatchingWorker.ts delete mode 100644 src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts delete mode 100644 src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts diff --git a/src/adapters/cocmAdapter/cocmMockAdapter.ts b/src/adapters/cocmAdapter/cocmMockAdapter.ts index f26ed502f..7f3179a6d 100644 --- a/src/adapters/cocmAdapter/cocmMockAdapter.ts +++ b/src/adapters/cocmAdapter/cocmMockAdapter.ts @@ -1,10 +1,7 @@ -import axios from 'axios'; import { CocmAdapterInterface, ProjectsEstimatedMatchings, } from './cocmAdapterInterface'; -import { i18n, translationErrorMessagesKeys } from '../../utils/errorMessages'; -import { logger } from '../../utils/logger'; export class CocmMockAdapter implements CocmAdapterInterface { async fetchEstimatedClusterMatchings( diff --git a/src/entities/project.ts b/src/entities/project.ts index 8a1a386c0..9a590f4fb 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -40,11 +40,7 @@ import { Category } from './category'; import { FeaturedUpdate } from './featuredUpdate'; import { getHtmlTextSummary } from '../utils/utils'; import { QfRound } from './qfRound'; -import { - getQfRoundTotalSqrtRootSumSquared, - getProjectDonationsSqrtRootSum, - findActiveQfRound, -} from '../repositories/qfRoundRepository'; +import { findActiveQfRound } from '../repositories/qfRoundRepository'; import { EstimatedMatching } from '../types/qfTypes'; import { Campaign } from './campaign'; import { ProjectEstimatedMatchingView } from './ProjectEstimatedMatchingView'; @@ -501,23 +497,14 @@ export class Project extends BaseEntity { async estimatedMatching(): Promise { const activeQfRound = await findActiveQfRound(); if (!activeQfRound) { - // TODO should move it to materialized view return null; } - const projectDonationsSqrtRootSum = await getProjectDonationsSqrtRootSum( - this.id, - activeQfRound.id, - ); - - const allProjectsSum = await getQfRoundTotalSqrtRootSumSquared( - activeQfRound.id, - ); - const matchingPool = activeQfRound.allocatedFund; + // Facilitate migration in frontend return empty values for now return { - projectDonationsSqrtRootSum, - allProjectsSum, + projectDonationsSqrtRootSum: 0, + allProjectsSum: 0, matchingPool, }; } diff --git a/src/repositories/donationRepository.ts b/src/repositories/donationRepository.ts index 4b649faaf..1a0908e8c 100644 --- a/src/repositories/donationRepository.ts +++ b/src/repositories/donationRepository.ts @@ -10,6 +10,28 @@ import { ORGANIZATION_LABELS } from '../entities/organization'; import { AppDataSource } from '../orm'; import { getPowerRound } from './powerRoundRepository'; +export const exportClusterMatchingDonationsFormat = async ( + qfRoundId: number, +) => { + return await Donation.query( + ` + SELECT + d."fromWalletAddress" AS voter, + d."toWalletAddress" AS payoutAddress, + d."valueUsd" AS amountUSD, + p."title" AS project_name, + d."qfRoundUserScore" AS score + FROM + donation d + INNER JOIN + project p ON d."projectId" = p."id" + WHERE + d."qfRoundId" = $1 + `, + [qfRoundId], + ); +}; + export const fillQfRoundDonationsUserScores = async (): Promise => { await Donation.query(` UPDATE donation diff --git a/src/resolvers/projectResolver.allProject.test.ts b/src/resolvers/projectResolver.allProject.test.ts index 533e61209..46b1b8e21 100644 --- a/src/resolvers/projectResolver.allProject.test.ts +++ b/src/resolvers/projectResolver.allProject.test.ts @@ -39,7 +39,7 @@ import { InstantPowerBalance } from '../entities/instantPowerBalance'; import { saveOrUpdateInstantPowerBalances } from '../repositories/instantBoostingRepository'; import { updateInstantBoosting } from '../services/instantBoostingServices'; import { QfRound } from '../entities/qfRound'; -import { calculateEstimatedMatchingWithParams } from '../utils/qfUtils'; +// import { calculateEstimatedMatchingWithParams } from '../utils/qfUtils'; import { refreshProjectEstimatedMatchingView } from '../services/projectViewsService'; import { addOrUpdatePowerSnapshotBalances } from '../repositories/powerBalanceSnapshotRepository'; import { findPowerSnapshots } from '../repositories/powerSnapshotRepository'; @@ -2195,31 +2195,32 @@ function allProjectsTestCases() { }); assert.equal(result.data.data.allProjects.projects.length, 2); - const firstProject = result.data.data.allProjects.projects.find( - p => Number(p.id) === project1.id, - ); - const secondProject = result.data.data.allProjects.projects.find( - p => Number(p.id) === project2.id, - ); - - const project1EstimatedMatching = - await calculateEstimatedMatchingWithParams({ - matchingPool: firstProject.estimatedMatching.matchingPool, - projectDonationsSqrtRootSum: - firstProject.estimatedMatching.projectDonationsSqrtRootSum, - allProjectsSum: firstProject.estimatedMatching.allProjectsSum, - }); - - const project2EstimatedMatching = - await calculateEstimatedMatchingWithParams({ - matchingPool: secondProject.estimatedMatching.matchingPool, - projectDonationsSqrtRootSum: - secondProject.estimatedMatching.projectDonationsSqrtRootSum, - allProjectsSum: secondProject.estimatedMatching.allProjectsSum, - }); - - assert.equal(Math.floor(project1EstimatedMatching), 666); - assert.equal(Math.floor(project2EstimatedMatching), 333); + // const firstProject = result.data.data.allProjects.projects.find( + // p => Number(p.id) === project1.id, + // ); + // const secondProject = result.data.data.allProjects.projects.find( + // p => Number(p.id) === project2.id, + // ); + + // New estimated matching wont calculate it here + // const project1EstimatedMatching = + // await calculateEstimatedMatchingWithParams({ + // matchingPool: firstProject.estimatedMatching.matchingPool, + // projectDonationsSqrtRootSum: + // firstProject.estimatedMatching.projectDonationsSqrtRootSum, + // allProjectsSum: firstProject.estimatedMatching.allProjectsSum, + // }); + + // const project2EstimatedMatching = + // await calculateEstimatedMatchingWithParams({ + // matchingPool: secondProject.estimatedMatching.matchingPool, + // projectDonationsSqrtRootSum: + // secondProject.estimatedMatching.projectDonationsSqrtRootSum, + // allProjectsSum: secondProject.estimatedMatching.allProjectsSum, + // }); + + // assert.equal(Math.floor(project1EstimatedMatching), 666); + // assert.equal(Math.floor(project2EstimatedMatching), 333); qfRound.isActive = false; await qfRound.save(); }); diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts index b3509bced..bbe50647c 100644 --- a/src/server/bootstrap.ts +++ b/src/server/bootstrap.ts @@ -56,8 +56,6 @@ import { ApolloContext } from '../types/ApolloContext'; import { ProjectResolverWorker } from '../workers/projectsResolverWorker'; import { runInstantBoostingUpdateCronJob } from '../services/cronJobs/instantBoostingUpdateJob'; -import { refreshProjectEstimatedMatchingView } from '../services/projectViewsService'; -import { isTestEnv } from '../utils/utils'; import { runCheckActiveStatusOfQfRounds } from '../services/cronJobs/checkActiveStatusQfRounds'; import { runUpdateProjectCampaignsCacheJob } from '../services/cronJobs/updateProjectCampaignsCacheJob'; import { corsOptions } from './cors'; diff --git a/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts index d57c3da17..e23044768 100644 --- a/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts +++ b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts @@ -2,16 +2,13 @@ import { schedule } from 'node-cron'; import { spawn, Worker, Thread } from 'threads'; import config from '../../config'; import { logger } from '../../utils/logger'; -import { - findActiveQfRound, - findUsersWithoutMBDScoreInActiveAround, -} from '../../repositories/qfRoundRepository'; -import { findUserById } from '../../repositories/userRepository'; -import { UserQfRoundModelScore } from '../../entities/userQfRoundModelScore'; +import { findActiveQfRound } from '../../repositories/qfRoundRepository'; +import { exportClusterMatchingDonationsFormat } from '../../repositories/donationRepository'; const cronJobTime = - (config.get('SYNC_ESTIMATED_CLUSTED_MATCHING_CRONJOB_EXPRESSION') as string) || - '0 * * * * *'; + (config.get( + 'SYNC_ESTIMATED_CLUSTER_MATCHING_CRONJOB_EXPRESSION', + ) as string) || '0 * * * * *'; export const runSyncEstimatedClusterMatchingCronjob = () => { logger.debug( @@ -24,27 +21,33 @@ export const runSyncEstimatedClusterMatchingCronjob = () => { }; export const fetchAndUpdateClusterEstimatedMatching = async () => { - const fetchWorker = await spawn( + const matchingWorker = await spawn( new Worker('../../workers/cocm/fetchEstimatedClusterMtchingWorker'), ); - const updateWorker = await spawn( - new Worker('../../workers/cocm/updateProjectsEstimatedClusterMatchingWorker') + const activeQfRound = await findActiveQfRound(); + if (!activeQfRound?.id) return; + + const clusterMatchingDonations = await exportClusterMatchingDonationsFormat( + activeQfRound?.id, ); - const activeQfRoundId = - (await findActiveQfRound())?.id; - if (!activeQfRoundId || activeQfRoundId === 0) return; - - for (const projectId of []) { - try { - - // const userScore = await worker.syncUserScore({ - // userWallet: user?.walletAddress, - // }); - } catch (e) { - logger.info(`User with Id ${1} did not sync MBD score this batch`); - } - } - await Thread.terminate(fetchWorker); - await Thread.terminate(updateWorker); + if (clusterMatchingDonations?.length === 0) return; + + const matchingDataInput = { + votes_data: clusterMatchingDonations, + strategy: 'COCM', + min_donation_threshold_amount: activeQfRound.minimumValidUsdValue, + matching_cap_amount: activeQfRound.maximumReward, + matching_amount: activeQfRound.allocatedFundUSD, + passport_threshold: activeQfRound.minimumPassportScore, + }; + + const matchingData = + await matchingWorker.fetchEstimatedClusterMatching(matchingDataInput); + await matchingWorker.updateEstimatedClusterMatching( + activeQfRound.id, + matchingData, + ); + + await Thread.terminate(matchingWorker); }; diff --git a/src/workers/cocm/estimatedClusterMatchingWorker.ts b/src/workers/cocm/estimatedClusterMatchingWorker.ts new file mode 100644 index 000000000..c3e4ac5a8 --- /dev/null +++ b/src/workers/cocm/estimatedClusterMatchingWorker.ts @@ -0,0 +1,55 @@ +// workers/auth.js +import { expose } from 'threads/worker'; +import { WorkerModule } from 'threads/dist/types/worker'; +import { getClusterMatchingAdapter } from '../../adapters/adaptersFactory'; +import { EstimatedClusterMatching } from '../../entities/estimatedClusterMatching'; +import { logger } from '../../utils/logger'; + +type EstimatedClusterMatchingWorkerFunctions = + | 'fetchEstimatedClusterMatching' + | 'updateEstimatedClusterMatching'; + +export type EstimatedClusterMatchingWorker = + WorkerModule; + +const worker: EstimatedClusterMatchingWorker = { + async fetchEstimatedClusterMatching(matchingDataInput: any) { + return await getClusterMatchingAdapter().fetchEstimatedClusterMatchings( + matchingDataInput, + ); + }, + + async updateEstimatedClusterMatching(qfRoundId: number, matchingData: any) { + try { + // Prepare values for bulk insert + const values = matchingData + .map( + data => `( + (SELECT id FROM project WHERE title = '${data.project_name}'), + ${qfRoundId}, + ${data.matching_amount} + )`, + ) + .join(','); + + const query = ` + INSERT INTO estimated_cluster_matching ("projectId", "qfRoundId", matching) + VALUES ${values} + ON CONFLICT ("projectId", "qfRoundId") + DO UPDATE SET matching = EXCLUDED.matching + RETURNING "projectId", "qfRoundId", matching; + `; + + const result = await EstimatedClusterMatching.query(query); + if (result.length === 0) { + throw new Error('No records were inserted or updated.'); + } + + logger.debug('Matching data processed successfully with raw SQL.'); + } catch (error) { + logger.debug('Error processing matching data:', error.message); + } + }, +}; + +expose(worker); diff --git a/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts b/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts deleted file mode 100644 index 37daeeaaf..000000000 --- a/src/workers/cocm/fetchEstimatedClusterMatchingWorker.ts +++ /dev/null @@ -1,17 +0,0 @@ -// workers/auth.js -import { expose } from 'threads/worker'; -import { WorkerModule } from 'threads/dist/types/worker'; -import { getClusterMatchingAdapter } from '../../adapters/adaptersFactory'; - -type FetchEstimatedClusterMatchingWorkerFunctions = 'fetchEstimatedClusterMatching'; - -export type FetchEstimatedClusterMatchingWorker = - WorkerModule; - -const worker: FetchEstimatedClusterMatchingWorker = { - async fetchEstimatedClusterMatching(matchingDataInput: any) { - return await getClusterMatchingAdapter().fetchEstimatedClusterMatchings(matchingDataInput); - }, -}; - -expose(worker); diff --git a/src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts b/src/workers/cocm/updateProjectsEstimatedClusterMatchingWorker.ts deleted file mode 100644 index e69de29bb..000000000 From 94a77c75b06fcbdaf7dbd8ad0236b86ab90ce01a Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 21 Oct 2024 14:18:56 +0200 Subject: [PATCH 4/5] improve error handling for cluster matching --- src/entities/entities.ts | 2 ++ src/entities/project.ts | 12 ++++++++ src/repositories/donationRepository.ts | 6 ++-- .../syncEstimatedClusterMatchingJob.ts | 28 +++++++++++++------ src/types/qfTypes.ts | 3 ++ 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/entities/entities.ts b/src/entities/entities.ts index 0e5e204a5..bddd306e8 100644 --- a/src/entities/entities.ts +++ b/src/entities/entities.ts @@ -52,6 +52,7 @@ import { ProjectSocialMedia } from './projectSocialMedia'; import { DraftRecurringDonation } from './draftRecurringDonation'; import { UserQfRoundModelScore } from './userQfRoundModelScore'; import { ProjectGivbackRankView } from './ProjectGivbackRankView'; +import { EstimatedClusterMatching } from './estimatedClusterMatching'; export const getEntities = (): DataSourceOptions['entities'] => { return [ @@ -86,6 +87,7 @@ export const getEntities = (): DataSourceOptions['entities'] => { PowerSnapshot, PowerBalanceSnapshot, PowerBoostingSnapshot, + EstimatedClusterMatching, // View UserProjectPowerView, diff --git a/src/entities/project.ts b/src/entities/project.ts index 9a590f4fb..fce2aa771 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -46,6 +46,7 @@ import { Campaign } from './campaign'; import { ProjectEstimatedMatchingView } from './ProjectEstimatedMatchingView'; import { AnchorContractAddress } from './anchorContractAddress'; import { ProjectSocialMedia } from './projectSocialMedia'; +import { EstimatedClusterMatching } from './estimatedClusterMatching'; // eslint-disable-next-line @typescript-eslint/no-var-requires const moment = require('moment'); @@ -501,11 +502,22 @@ export class Project extends BaseEntity { } const matchingPool = activeQfRound.allocatedFund; + const estimatedClusterMatching = + await EstimatedClusterMatching.createQueryBuilder('matching') + .where('matching."projectId" = :projectId', { projectId: this.id }) + .getOne(); + + let matching: number; + if (!estimatedClusterMatching) matching = 0; + + matching = estimatedClusterMatching!.matching; + // Facilitate migration in frontend return empty values for now return { projectDonationsSqrtRootSum: 0, allProjectsSum: 0, matchingPool, + matching, }; } diff --git a/src/repositories/donationRepository.ts b/src/repositories/donationRepository.ts index 1a0908e8c..0c5eb17df 100644 --- a/src/repositories/donationRepository.ts +++ b/src/repositories/donationRepository.ts @@ -17,9 +17,9 @@ export const exportClusterMatchingDonationsFormat = async ( ` SELECT d."fromWalletAddress" AS voter, - d."toWalletAddress" AS payoutAddress, - d."valueUsd" AS amountUSD, - p."title" AS project_name, + d."toWalletAddress" AS "payoutAddress", + d."valueUsd" AS "amountUSD", + p."title" AS "project_name", d."qfRoundUserScore" AS score FROM donation d diff --git a/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts index e23044768..9785e6091 100644 --- a/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts +++ b/src/services/cronJobs/syncEstimatedClusterMatchingJob.ts @@ -10,6 +10,8 @@ const cronJobTime = 'SYNC_ESTIMATED_CLUSTER_MATCHING_CRONJOB_EXPRESSION', ) as string) || '0 * * * * *'; +const defaultMatchingStrategy = 'COCM'; + export const runSyncEstimatedClusterMatchingCronjob = () => { logger.debug( 'runSyncEstimatedClusterMatchingCronjob() has been called, cronJobTime', @@ -22,7 +24,7 @@ export const runSyncEstimatedClusterMatchingCronjob = () => { export const fetchAndUpdateClusterEstimatedMatching = async () => { const matchingWorker = await spawn( - new Worker('../../workers/cocm/fetchEstimatedClusterMtchingWorker'), + new Worker('../../workers/cocm/estimatedClusterMtchingWorker'), ); const activeQfRound = await findActiveQfRound(); @@ -31,23 +33,31 @@ export const fetchAndUpdateClusterEstimatedMatching = async () => { const clusterMatchingDonations = await exportClusterMatchingDonationsFormat( activeQfRound?.id, ); - if (clusterMatchingDonations?.length === 0) return; + if (!clusterMatchingDonations || clusterMatchingDonations?.length === 0) + return; const matchingDataInput = { votes_data: clusterMatchingDonations, - strategy: 'COCM', + strategy: defaultMatchingStrategy, min_donation_threshold_amount: activeQfRound.minimumValidUsdValue, matching_cap_amount: activeQfRound.maximumReward, matching_amount: activeQfRound.allocatedFundUSD, passport_threshold: activeQfRound.minimumPassportScore, }; - const matchingData = - await matchingWorker.fetchEstimatedClusterMatching(matchingDataInput); - await matchingWorker.updateEstimatedClusterMatching( - activeQfRound.id, - matchingData, - ); + try { + // Fetch from python api cluster matching + const matchingData = + await matchingWorker.fetchEstimatedClusterMatching(matchingDataInput); + + // Insert the data + await matchingWorker.updateEstimatedClusterMatching( + activeQfRound.id, + matchingData, + ); + } catch (e) { + logger.error('fetchAndUpdateClusterEstimatedMatching error', e); + } await Thread.terminate(matchingWorker); }; diff --git a/src/types/qfTypes.ts b/src/types/qfTypes.ts index 64e5f5448..edfd570f1 100644 --- a/src/types/qfTypes.ts +++ b/src/types/qfTypes.ts @@ -10,4 +10,7 @@ export class EstimatedMatching { @Field(_type => Float, { nullable: true }) matchingPool?: number; + + @Field(_type => Float, { nullable: true }) + matching?: number; } From e42dbb1b0f6b527d4672b2bb65437bda98b91776 Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 21 Oct 2024 16:16:14 +0200 Subject: [PATCH 5/5] comment broken contract tests by missing eth_getCode Method --- src/utils/validators/projectValidator.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/utils/validators/projectValidator.test.ts b/src/utils/validators/projectValidator.test.ts index 1b541ad27..c944fe542 100644 --- a/src/utils/validators/projectValidator.test.ts +++ b/src/utils/validators/projectValidator.test.ts @@ -53,41 +53,42 @@ function validateProjectTitleTestCases() { }); } +// TODO FIX: Method eth_getCode not found, replace function isWalletAddressSmartContractTestCases() { - it('should return true for smart contract address in mainnet', async () => { + it.skip('should return true for smart contract address in mainnet', async () => { // DAI address https://etherscan.io/token/0x6b175474e89094c44da98b954eedeac495271d0f const walletAddress = '0x6b175474e89094c44da98b954eedeac495271d0f'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in xdai', async () => { + it.skip('should return true for smart contract address in xdai', async () => { // GIV address https://blockscout.com/xdai/mainnet/token/0x4f4F9b8D5B4d0Dc10506e5551B0513B61fD59e75/token-transfers const walletAddress = '0x4f4F9b8D5B4d0Dc10506e5551B0513B61fD59e75'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in polygon', async () => { + it.skip('should return true for smart contract address in polygon', async () => { // GIV address https://polygonscan.com/address/0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270 const walletAddress = '0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in celo', async () => { + it.skip('should return true for smart contract address in celo', async () => { const walletAddress = '0x67316300f17f063085Ca8bCa4bd3f7a5a3C66275'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in celo alfajores', async () => { + it.skip('should return true for smart contract address in celo alfajores', async () => { const walletAddress = '0x17bc3304F94c85618c46d0888aA937148007bD3C'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in arbitrum mainnet', async () => { + it.skip('should return true for smart contract address in arbitrum mainnet', async () => { const walletAddress = '0x1231DEB6f5749EF6cE6943a275A1D3E7486F4EaE'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract); }); - it('should return true for smart contract address in arbitrum sepolia', async () => { + it.skip('should return true for smart contract address in arbitrum sepolia', async () => { const walletAddress = '0x6b7860b66c0124e8d8c079b279c126ce58c442a2'; const isSmartContract = await isWalletAddressSmartContract(walletAddress); assert.isTrue(isSmartContract);