Skip to content

Commit

Permalink
[Job Launcher] Calculate job bounties (#846)
Browse files Browse the repository at this point in the history 
* Add a method to calculate job bounties
  • Loading branch information
@flopez7
flopez7 authored Aug 30, 2023
1 parent ac97979 commit af1e50d
Show file tree
Hide file tree
Showing 6 changed files with 143 additions and 57 deletions.
38 changes: 38 additions & 0 deletions packages/apps/job-launcher/server/src/common/utils/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,41 @@ export async function getRate(from: string, to: string): Promise<number> {

return reversed ? 1 / rate : rate;
}

export const parseUrl = (
url: string,
): {
endpoint: string;
bucket: string;
port?: number;
} => {
const patterns = [
{
regex: /^https:\/\/storage\.googleapis\.com\/([^/]+)\/?$/,
endpoint: 'storage.googleapis.com',
},
{
regex: /^https:\/\/([^\.]+)\.storage\.googleapis\.com\/?$/,
endpoint: 'storage.googleapis.com',
},
{
regex: /^https?:\/\/([^/:]+)(?::(\d+))?(\/.*)?/,
endpoint: '$1',
port: '$2',
},
];

for (const { regex, endpoint, port } of patterns) {
const match = url.match(regex);
if (match) {
const bucket = match[3] ? match[3].split('/')[1] : '';
return {
endpoint: endpoint.replace('$1', match[1]),
bucket,
port: port ? Number(match[2]) : undefined,
};
}
}

throw new Error('Invalid URL');
};
5 changes: 1 addition & 4 deletions packages/apps/job-launcher/server/src/modules/job/job.dto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,9 @@ export class JobCvatDto extends JobDto {
@IsString()
public gtUrl: string;

@ApiProperty()
@IsEnum(JobRequestType)
type: JobRequestType;

@ApiProperty()
@IsString()
public jobBounty: string;
}

export class JobCancelDto {
Expand Down
127 changes: 76 additions & 51 deletions packages/apps/job-launcher/server/src/modules/job/job.service.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { createMock } from '@golevelup/ts-jest';
import { ChainId, EscrowClient, EscrowStatus, StorageClient } from '@human-protocol/sdk';
import {
ChainId,
EscrowClient,
StorageClient,
EscrowStatus,
} from '@human-protocol/sdk';
import { HttpService } from '@nestjs/axios';
import { BadGatewayException, NotFoundException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
Expand All @@ -23,6 +28,7 @@ import {
} from '../../common/enums/job';
import {
MOCK_ADDRESS,
MOCK_BUCKET_FILES,
MOCK_BUCKET_NAME,
MOCK_CHAIN_ID,
MOCK_EXCHANGE_ORACLE_WEBHOOK_URL,
Expand Down Expand Up @@ -77,10 +83,12 @@ jest.mock('@human-protocol/sdk', () => ({
.mockResolvedValue([
{ key: MOCK_FILE_KEY, url: MOCK_FILE_URL, hash: MOCK_FILE_HASH },
]),
listObjects: jest.fn().mockResolvedValue(MOCK_BUCKET_FILES),
})),
}));

jest.mock('../../common/utils', () => ({
...jest.requireActual('../../common/utils'),
getRate: jest.fn().mockImplementation(() => rate),
}));

Expand Down Expand Up @@ -129,6 +137,8 @@ describe('JobService', () => {
return MOCK_PRIVATE_KEY;
case 'S3_BUCKET':
return MOCK_BUCKET_NAME;
case 'CVAT_JOB_SIZE':
return 1;
}
}),
};
Expand Down Expand Up @@ -301,6 +311,18 @@ describe('JobService', () => {
});
});

describe('calculateJobBounty', () => {
it('should calculate the job bounty correctly', async () => {
const fundAmount = 10;
const result = await jobService['calculateJobBounty'](
MOCK_FILE_URL,
fundAmount,
);

expect(result).toEqual(2);
});
});

describe('createJob with image label binary type', () => {
const userId = 1;
const jobId = 123;
Expand All @@ -313,7 +335,6 @@ describe('JobService', () => {
minQuality: 0.95,
fundAmount: 10,
gtUrl: '',
jobBounty: '1',
type: JobRequestType.IMAGE_LABEL_BINARY,
};

Expand Down Expand Up @@ -462,7 +483,6 @@ describe('JobService', () => {
const userId = 123;

it('should cancel the job', async () => {
const escrowAddress = MOCK_ADDRESS;
const mockJobEntity: Partial<JobEntity> = {
id: jobId,
userId,
Expand All @@ -483,7 +503,9 @@ describe('JobService', () => {
it('should throw not found exception if job not found', async () => {
jobRepository.findOne = jest.fn().mockResolvedValue(undefined);

await expect(jobService.requestToCancelJob(userId, jobId)).rejects.toThrow(NotFoundException);
await expect(
jobService.requestToCancelJob(userId, jobId),
).rejects.toThrow(NotFoundException);
});
});

Expand All @@ -492,7 +514,6 @@ describe('JobService', () => {
const userId = 123;

it('should cancel the job', async () => {
const escrowAddress = MOCK_ADDRESS;
const mockJobEntity: Partial<JobEntity> = {
id: jobId,
userId,
Expand All @@ -513,7 +534,9 @@ describe('JobService', () => {
it('should throw not found exception if job not found', async () => {
jobRepository.findOne = jest.fn().mockResolvedValue(undefined);

await expect(jobService.requestToCancelJob(userId, jobId)).rejects.toThrow(NotFoundException);
await expect(
jobService.requestToCancelJob(userId, jobId),
).rejects.toThrow(NotFoundException);
});
});

Expand Down Expand Up @@ -639,26 +662,28 @@ describe('JobService', () => {
it('should throw not found exception if job not found', async () => {
jobRepository.findOne = jest.fn().mockResolvedValue(undefined);

await expect(jobService.requestToCancelJob(userId, jobId)).rejects.toThrow(NotFoundException);
await expect(
jobService.requestToCancelJob(userId, jobId),
).rejects.toThrow(NotFoundException);
});
});

describe('cancelCronJob', () => {
let escrowClientMock: any,
getManifestMock: any,
jobSaveMock: any,
findOneJobMock: any,
findOnePaymentMock: any,
buildMock: any,
sendWebhookMock: any,
jobEntityMock: Partial<JobEntity>,
paymentEntityMock: Partial<PaymentEntity>
getManifestMock: any,
jobSaveMock: any,
findOneJobMock: any,
findOnePaymentMock: any,
buildMock: any,
sendWebhookMock: any,
jobEntityMock: Partial<JobEntity>,
paymentEntityMock: Partial<PaymentEntity>;

beforeEach(() => {
escrowClientMock = {
cancel: jest.fn().mockResolvedValue(undefined),
getStatus: jest.fn().mockResolvedValue(EscrowStatus.Launched),
getBalance: jest.fn().mockResolvedValue(new Decimal(10))
getBalance: jest.fn().mockResolvedValue(new Decimal(10)),
};

jobEntityMock = {
Expand All @@ -684,7 +709,9 @@ describe('JobService', () => {
buildMock = jest.spyOn(EscrowClient, 'build');
sendWebhookMock = jest.spyOn(jobService, 'sendWebhook');
findOneJobMock.mockResolvedValueOnce(jobEntityMock as JobCvatDto);
findOnePaymentMock.mockResolvedValueOnce(paymentEntityMock as PaymentEntity);
findOnePaymentMock.mockResolvedValueOnce(
paymentEntityMock as PaymentEntity,
);
});

afterEach(() => {
Expand All @@ -694,7 +721,7 @@ describe('JobService', () => {
it('cancels a job successfully', async () => {
jobEntityMock.escrowAddress = undefined;
jobSaveMock.mockResolvedValue(jobEntityMock as JobEntity);

const result = await jobService.cancelCronJob();

expect(result).toBe(true);
Expand All @@ -704,26 +731,26 @@ describe('JobService', () => {
});

it('should throw an error if the escrow has invalid status', async () => {
escrowClientMock.getStatus = jest.fn().mockResolvedValue(EscrowStatus.Complete);
escrowClientMock.getStatus = jest
.fn()
.mockResolvedValue(EscrowStatus.Complete);
jobSaveMock.mockResolvedValue(jobEntityMock as JobEntity);
buildMock.mockResolvedValue(escrowClientMock as any);
buildMock.mockResolvedValue(escrowClientMock as any);

await expect(
jobService.cancelCronJob()
).rejects.toThrowError(
await expect(jobService.cancelCronJob()).rejects.toThrowError(
new BadGatewayException(ErrorEscrow.InvalidStatusCancellation),
);
});

it('should throw an error if the escrow has invalid balance', async () => {
escrowClientMock.getStatus = jest.fn().mockResolvedValue(EscrowStatus.Launched);
escrowClientMock.getStatus = jest
.fn()
.mockResolvedValue(EscrowStatus.Launched);
escrowClientMock.getBalance = jest.fn().mockResolvedValue(new Decimal(0));
jobSaveMock.mockResolvedValue(jobEntityMock as JobEntity);
buildMock.mockResolvedValue(escrowClientMock as any);
buildMock.mockResolvedValue(escrowClientMock as any);

await expect(
jobService.cancelCronJob()
).rejects.toThrowError(
await expect(jobService.cancelCronJob()).rejects.toThrowError(
new BadGatewayException(ErrorEscrow.InvalidBalanceCancellation),
);
});
Expand All @@ -734,27 +761,26 @@ describe('JobService', () => {
requesterTitle: MOCK_REQUESTER_TITLE,
requesterDescription: MOCK_REQUESTER_DESCRIPTION,
fundAmount: 10,
requestType: JobRequestType.FORTUNE
requestType: JobRequestType.FORTUNE,
};

jobSaveMock.mockResolvedValue(jobEntityMock as JobEntity);
getManifestMock.mockResolvedValue(manifest);
buildMock.mockResolvedValue(escrowClientMock as any);
sendWebhookMock.mockResolvedValue(true);

const result = await jobService.cancelCronJob();

expect(result).toBe(true);
expect(escrowClientMock.cancel).toHaveBeenCalledWith(jobEntityMock.escrowAddress);
expect(jobSaveMock).toHaveBeenCalledWith();
expect(sendWebhookMock).toHaveBeenCalledWith(
expect.any(String),
{
escrowAddress: jobEntityMock.escrowAddress,
chainId: jobEntityMock.chainId,
eventType: EventType.ESCROW_CANCELED
}
expect(escrowClientMock.cancel).toHaveBeenCalledWith(
jobEntityMock.escrowAddress,
);
expect(jobSaveMock).toHaveBeenCalledWith();
expect(sendWebhookMock).toHaveBeenCalledWith(expect.any(String), {
escrowAddress: jobEntityMock.escrowAddress,
chainId: jobEntityMock.chainId,
eventType: EventType.ESCROW_CANCELED,
});
});

it('cancels a job successfully with image binary lavel job type and send webhook', async () => {
Expand All @@ -776,25 +802,24 @@ describe('JobService', () => {
},
job_bounty: '1',
};

jobSaveMock.mockResolvedValue(jobEntityMock as JobEntity);
getManifestMock.mockResolvedValue(manifest);
buildMock.mockResolvedValue(escrowClientMock as any);
sendWebhookMock.mockResolvedValue(true);

const result = await jobService.cancelCronJob();

expect(result).toBe(true);
expect(escrowClientMock.cancel).toHaveBeenCalledWith(jobEntityMock.escrowAddress);
expect(jobSaveMock).toHaveBeenCalledWith();
expect(sendWebhookMock).toHaveBeenCalledWith(
expect.any(String),
{
escrowAddress: jobEntityMock.escrowAddress,
chainId: jobEntityMock.chainId,
eventType: EventType.ESCROW_CANCELED
}
expect(escrowClientMock.cancel).toHaveBeenCalledWith(
jobEntityMock.escrowAddress,
);
expect(jobSaveMock).toHaveBeenCalledWith();
expect(sendWebhookMock).toHaveBeenCalledWith(expect.any(String), {
escrowAddress: jobEntityMock.escrowAddress,
chainId: jobEntityMock.chainId,
eventType: EventType.ESCROW_CANCELED,
});
});
});

Expand Down
Empty file removed packages/apps/job-launcher/server/src/modules/job/job.service.test.ts
View file
Empty file.
29 changes: 27 additions & 2 deletions packages/apps/job-launcher/server/src/modules/job/job.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import {
ChainId,
EscrowClient,
Expand Down Expand Up @@ -44,7 +45,7 @@ import {
PaymentType,
TokenId,
} from '../../common/enums/payment';
import { getRate } from '../../common/utils';
import { getRate, parseUrl } from '../../common/utils';
import { add, div, lt, mul } from '../../common/utils/decimal';
import { PaymentRepository } from '../payment/payment.repository';
import { PaymentService } from '../payment/payment.service';
Expand Down Expand Up @@ -166,7 +167,9 @@ export class JobService {
),
gt_url: dto.gtUrl,
},
job_bounty: dto.jobBounty,
job_bounty: (
await this.calculateJobBounty(dto.dataUrl, fundAmount)
).toString(),
}));
}

Expand Down Expand Up @@ -216,6 +219,28 @@ export class JobService {
return jobEntity.id;
}

private async calculateJobBounty(
endpointUrl: string,
fundAmount: number,
): Promise<number> {
const storageData = parseUrl(endpointUrl);
const storageClient = new StorageClient({
endPoint: storageData.endpoint,
port: storageData.port,
useSSL: false,
});

const totalImages = (await storageClient.listObjects(storageData.bucket))
.length;

const totalJobs = Math.ceil(
totalImages /
Number(this.configService.get<number>(ConfigNames.CVAT_JOB_SIZE)!),
);

return fundAmount / totalJobs;
}

public async launchJob(jobEntity: JobEntity): Promise<JobEntity> {
const signer = this.web3Service.getSigner(jobEntity.chainId);

Expand Down
1 change: 1 addition & 0 deletions packages/apps/job-launcher/server/test/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export const MOCK_ADDRESS = '0xCf88b3f1992458C2f5a229573c768D0E9F70C44e';
export const MOCK_FILE_URL = 'http://mockedFileUrl.test';
export const MOCK_FILE_HASH = 'mockedFileHash';
export const MOCK_FILE_KEY = 'manifest.json';
export const MOCK_BUCKET_FILES = ['file0', 'file1', 'file2', 'file3', 'file4'];
export const MOCK_PRIVATE_KEY =
'd334daf65a631f40549cc7de126d5a0016f32a2d00c49f94563f9737f7135e55';
export const MOCK_BUCKET_NAME = 'bucket-name';
Expand Down

1 comment on commit af1e50d

@vercel
Copy link

@vercel vercel bot commented on af1e50d Aug 30, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

job-launcher-server – ./packages/apps/job-launcher/server

job-launcher-server-humanprotocol.vercel.app
job-launcher-server-git-develop-humanprotocol.vercel.app
job-launcher-server-nine.vercel.app

Please sign in to comment.