diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml index 1423a35d9..787d15fe0 100644 --- a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml +++ b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml @@ -26,6 +26,13 @@ paths: type: number minimum: 1 maximum: 200 + - name: order + in: query + required: false + schema: + type: string + enum: [asc, desc] + description: Order of the results. Default is "asc" (earliest verified contract first) responses: "200": description: Chain is available as a full match or partial match in the repository diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml index 3ed0fa99d..df6920bcb 100644 --- a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml +++ b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml @@ -26,6 +26,14 @@ paths: type: number minimum: 1 maximum: 200 + - name: order + in: query + required: false + schema: + type: string + enum: [asc, desc] + description: Order of the results. Default is "asc" (earliest verified contract first) + responses: "200": description: Chain is available as a full match in the repository diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-partial.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-partial.stateless.paths.yaml new file mode 100644 index 000000000..e530f38f9 --- /dev/null +++ b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-partial.stateless.paths.yaml @@ -0,0 +1,69 @@ +openapi: "3.0.0" + +paths: + /files/contracts/partial/{chain}: + get: + summary: Get the contract addresses partially verified on a chain + description: Returns the partially verified contracts from the repository for the desired chain. API is paginated. Limit must be a number between 1 and 200. + tags: + - Repository + parameters: + - name: chain + in: path + required: true + schema: + type: string + format: sourcify-chainId + - name: page + in: query + required: false + schema: + type: number + - name: limit + in: query + required: false + schema: + type: number + minimum: 1 + maximum: 200 + - name: order + in: query + required: false + schema: + type: string + enum: [asc, desc] + description: Order of the results. Default is "asc" (earliest verified contract first) + responses: + "200": + description: Chain is available as a full match in the repository + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: string + example: + [ + "0x1fE5d745beABA808AAdF52057Dd7AAA47b42cFD0", + "0xE9c31091868d68598Ac881738D159A63532d12f9", + ] + pagination: + type: object + properties: + currentPage: + type: number + totalPages: + type: number + resultsPerPage: + type: number + resultsCurrentPage: + type: number + totalResults: + type: number + hasNextPage: + type: boolean + hasPreviousPage: + type: boolean diff --git a/services/server/src/server/controllers/repository/repository.handlers.ts b/services/server/src/server/controllers/repository/repository.handlers.ts index 3829a8eb0..a037ebb11 100644 --- a/services/server/src/server/controllers/repository/repository.handlers.ts +++ b/services/server/src/server/controllers/repository/repository.handlers.ts @@ -21,7 +21,8 @@ type PaginatedConractRetrieveMethod = ( chain: string, match: MatchLevel, page: number, - limit: number + limit: number, + descending: boolean ) => Promise<PaginatedContractData>; export function createEndpoint( @@ -75,7 +76,8 @@ export function createPaginatedContractEndpoint( req.params.chain, match, parseInt((req.query.page as string) || "0"), - parseInt((req.query.limit as string) || "200") + parseInt((req.query.limit as string) || "200"), + req.query.order === "desc" // default is asc ); } catch (err: any) { return next(new NotFoundError(err.message)); diff --git a/services/server/src/server/controllers/repository/repository.routes.ts b/services/server/src/server/controllers/repository/repository.routes.ts index 83b8a0b66..ce0ed6c03 100644 --- a/services/server/src/server/controllers/repository/repository.routes.ts +++ b/services/server/src/server/controllers/repository/repository.routes.ts @@ -52,23 +52,52 @@ const router: Router = Router(); { prefix: "/contracts/full", method: createPaginatedContractEndpoint( - (chain, match, page, limit) => - services.storage.getPaginatedContracts(chain, match, page, limit), + (chain, match, page, limit, descending) => + services.storage.getPaginatedContracts( + chain, + match, + page, + limit, + descending + ), "full_match" ), }, + { + prefix: "/contracts/partial", + method: createPaginatedContractEndpoint( + (chain, match, page, limit, descending) => + services.storage.getPaginatedContracts( + chain, + match, + page, + limit, + descending + ), + "partial_match" + ), + }, { prefix: "/contracts/any", method: createPaginatedContractEndpoint( - (chain, match, page, limit) => - services.storage.getPaginatedContracts(chain, match, page, limit), + (chain, match, page, limit, descending) => + services.storage.getPaginatedContracts( + chain, + match, + page, + limit, + descending + ), "any_match" ), }, { prefix: "", method: createEndpoint( - (chain, address, match) => services.storage.getContent(chain, address, match), "full_match"), + (chain, address, match) => + services.storage.getContent(chain, address, match), + "full_match" + ), }, ].forEach((pair) => { router diff --git a/services/server/src/server/services/StorageService.ts b/services/server/src/server/services/StorageService.ts index 763e7d735..dc8ea8b39 100644 --- a/services/server/src/server/services/StorageService.ts +++ b/services/server/src/server/services/StorageService.ts @@ -277,14 +277,16 @@ export class StorageService { chainId: string, match: MatchLevel, page: number, - limit: number + limit: number, + descending: boolean = false ): Promise<PaginatedContractData> => { try { return this.sourcifyDatabase!.getPaginatedContracts( chainId, match, page, - limit + limit, + descending ); } catch (error) { logger.error("Error while getting paginated contracts from database", { @@ -292,6 +294,7 @@ export class StorageService { match, page, limit, + descending, error, }); throw new Error("Error while getting paginated contracts from database"); diff --git a/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts b/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts index b4961f3f0..0ac67ba4d 100644 --- a/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts +++ b/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts @@ -153,7 +153,7 @@ export class SourcifyDatabaseService } ); throw new BadRequestError( - `Cannot fetch more than ${MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS} contracts (${fullTotal} full matches, ${partialTotal} partial matches), please use /contracts/{full|any}/${chainId} with pagination` + `Cannot fetch more than ${MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS} contracts (${fullTotal} full matches, ${partialTotal} partial matches), please use /contracts/{full|any|partial}/${chainId} with pagination` ); } @@ -198,7 +198,8 @@ export class SourcifyDatabaseService chainId: string, match: MatchLevel, page: number, - limit: number + limit: number, + descending: boolean = false ): Promise<PaginatedContractData> => { await this.init(); @@ -233,29 +234,31 @@ export class SourcifyDatabaseService matchAddressesCountResult.rows[0].partial_total ); const anyTotal = fullTotal + partialTotal; - if (match === "full_match") { - if (fullTotal === 0) { - return res; - } - res.pagination.totalResults = fullTotal; - } else if (match === "any_match") { - if (anyTotal === 0) { - return res; - } - res.pagination.totalResults = anyTotal; + const matchTotals: Record<MatchLevel, number> = { + full_match: fullTotal, + partial_match: partialTotal, + any_match: anyTotal, + }; + + // return empty res if requested `match` total is zero + if (matchTotals[match] === 0) { + return res; } + res.pagination.totalResults = matchTotals[match]; res.pagination.totalPages = Math.ceil( res.pagination.totalResults / res.pagination.resultsPerPage ); + // Now make the real query for addresses const matchAddressesResult = await Database.getSourcifyMatchAddressesByChainAndMatch( this.databasePool, parseInt(chainId), match, page, - limit + limit, + descending ); if (matchAddressesResult.rowCount > 0) { diff --git a/services/server/src/server/services/utils/database-util.ts b/services/server/src/server/services/utils/database-util.ts index e458e7abb..62269a70a 100644 --- a/services/server/src/server/services/utils/database-util.ts +++ b/services/server/src/server/services/utils/database-util.ts @@ -430,52 +430,6 @@ export async function insertSourcifyMatch( ); } -export async function insertSourcifySync( - pool: Pool, - { chain_id, address, match_type }: Tables.SourcifySync -) { - // I'm doing this because before passing the match_type I'm calling getMatchStatus(match) - // but then I need to convert the status to full_match | partial_match - let matchType; - switch (match_type) { - case "perfect": - matchType = "full_match"; - break; - case "partial": - matchType = "partial_match"; - break; - } - await pool.query( - `INSERT INTO sourcify_sync - (chain_id, address, match_type, synced) - VALUES ($1, $2, $3, true)`, - [chain_id, address, match_type] - ); -} - -export async function updateSourcifySync( - pool: Pool, - { chain_id, address, match_type }: Tables.SourcifySync -) { - // I'm doing this because before passing the match_type I'm calling getMatchStatus(match) - // but then I need to convert the status to full_match | partial_match - let matchType; - switch (match_type) { - case "perfect": - matchType = "full_match"; - break; - case "partial": - matchType = "partial_match"; - break; - } - await pool.query( - `UPDATE sourcify_sync SET - match_type=$3 - WHERE chain_id=$1 AND address=$2;`, - [chain_id, address, match_type] - ); -} - // Update sourcify_matches to the latest (and better) match in verified_contracts, // you need to pass the old verified_contract_id to be updated. // The old verified_contracts are not deleted from the verified_contracts table. @@ -579,17 +533,17 @@ export function normalizeRecompiledBytecodes( export async function countSourcifyMatchAddresses(pool: Pool, chain: number) { return await pool.query( ` - SELECT - contract_deployments.chain_id, - SUM(CASE WHEN sourcify_matches.creation_match = 'perfect' OR sourcify_matches.runtime_match = 'perfect' THEN 1 ELSE 0 END) AS full_total, - SUM(CASE WHEN sourcify_matches.creation_match != 'perfect' AND sourcify_matches.runtime_match != 'perfect' THEN 1 ELSE 0 END) AS partial_total - FROM sourcify_matches - JOIN verified_contracts ON verified_contracts.id = sourcify_matches.verified_contract_id - JOIN contract_deployments ON - contract_deployments.id = verified_contracts.deployment_id - AND contract_deployments.chain_id = $1 - GROUP BY contract_deployments.chain_id; - `, + SELECT + contract_deployments.chain_id, + SUM(CASE + WHEN COALESCE(sourcify_matches.creation_match, '') = 'perfect' OR sourcify_matches.runtime_match = 'perfect' THEN 1 ELSE 0 END) AS full_total, + SUM(CASE + WHEN COALESCE(sourcify_matches.creation_match, '') != 'perfect' AND sourcify_matches.runtime_match != 'perfect' THEN 1 ELSE 0 END) AS partial_total + FROM sourcify_matches + JOIN verified_contracts ON verified_contracts.id = sourcify_matches.verified_contract_id + JOIN contract_deployments ON contract_deployments.id = verified_contracts.deployment_id + WHERE contract_deployments.chain_id = $1 + GROUP BY contract_deployments.chain_id;`, [chain] ); } @@ -599,18 +553,19 @@ export async function getSourcifyMatchAddressesByChainAndMatch( chain: number, match: "full_match" | "partial_match" | "any_match", page: number, - paginationSize: number + paginationSize: number, + descending: boolean = false ) { let queryWhere = ""; switch (match) { case "full_match": { queryWhere = - "WHERE sourcify_matches.creation_match = 'perfect' OR sourcify_matches.runtime_match = 'perfect'"; + "WHERE COALESCE(sourcify_matches.creation_match, '') = 'perfect' OR sourcify_matches.runtime_match = 'perfect'"; break; } case "partial_match": { queryWhere = - "WHERE sourcify_matches.creation_match != 'perfect' AND sourcify_matches.runtime_match != 'perfect'"; + "WHERE COALESCE(sourcify_matches.creation_match, '') != 'perfect' AND sourcify_matches.runtime_match != 'perfect'"; break; } case "any_match": { @@ -621,6 +576,11 @@ export async function getSourcifyMatchAddressesByChainAndMatch( throw new Error("Match type not supported"); } } + + const orderBy = descending + ? "ORDER BY verified_contracts.id DESC" + : "ORDER BY verified_contracts.id ASC"; + return await pool.query( ` SELECT @@ -631,6 +591,7 @@ export async function getSourcifyMatchAddressesByChainAndMatch( contract_deployments.id = verified_contracts.deployment_id AND contract_deployments.chain_id = $1 ${queryWhere} + ${orderBy} OFFSET $2 LIMIT $3 `, [chain, page * paginationSize, paginationSize] diff --git a/services/server/src/server/types.ts b/services/server/src/server/types.ts index 7df515faf..c7db9151e 100644 --- a/services/server/src/server/types.ts +++ b/services/server/src/server/types.ts @@ -1,9 +1,9 @@ // Types used internally by the server. /** - * A type for specfifying the strictness level of querying (only full or any kind of matches) + * A type for specfifying the strictness level of querying (only full, partial or any kind of matches) */ -export type MatchLevel = "full_match" | "any_match"; +export type MatchLevel = "full_match" | "partial_match" | "any_match"; /** * An array wrapper with info properties. diff --git a/services/server/test/helpers/LocalChainFixture.ts b/services/server/test/helpers/LocalChainFixture.ts index 355d44c0b..e40a23299 100644 --- a/services/server/test/helpers/LocalChainFixture.ts +++ b/services/server/test/helpers/LocalChainFixture.ts @@ -11,6 +11,7 @@ import { LOCAL_CHAINS } from "../../src/sourcify-chains"; import nock from "nock"; import storageContractArtifact from "../testcontracts/Storage/Storage.json"; import storageContractMetadata from "../testcontracts/Storage/metadata.json"; +import storageContractMetadataModified from "../testcontracts/Storage/metadataModified.json"; const storageContractSourcePath = path.join( __dirname, @@ -33,7 +34,10 @@ export class LocalChainFixture { defaultContractMetadata = Buffer.from( JSON.stringify(storageContractMetadata) ); - defaultContractModifiedIpfsMetadata = getModifiedIpfsMetadata(); + defaultContractModifiedMetadata = Buffer.from( + JSON.stringify(storageContractMetadataModified) + ); + defaultContractModifiedSourceIpfs = getModifiedSourceIpfs(); defaultContractArtifact = storageContractArtifact; private _chainId?: string; @@ -124,7 +128,8 @@ export class LocalChainFixture { } } -function getModifiedIpfsMetadata(): Buffer { +// Changes the IPFS hash inside the metadata file to make the source unfetchable +function getModifiedSourceIpfs(): Buffer { const ipfsAddress = storageContractMetadata.sources["project:/contracts/Storage.sol"].urls[1]; // change the last char in ipfs hash of the source file diff --git a/services/server/test/helpers/ServerFixture.ts b/services/server/test/helpers/ServerFixture.ts index a84718984..31ff2d6f8 100644 --- a/services/server/test/helpers/ServerFixture.ts +++ b/services/server/test/helpers/ServerFixture.ts @@ -83,6 +83,7 @@ export class ServerFixture { beforeEach(async () => { rimraf.sync(this.server.repository); await resetDatabase(this.storageService); + console.log("Resetting the StorageService"); }); after(() => { diff --git a/services/server/test/helpers/helpers.ts b/services/server/test/helpers/helpers.ts index 6851ac96e..16eacbc60 100644 --- a/services/server/test/helpers/helpers.ts +++ b/services/server/test/helpers/helpers.ts @@ -17,6 +17,7 @@ import { promises as fs } from "fs"; import { StorageService } from "../../src/server/services/StorageService"; import { ServerFixture } from "./ServerFixture"; import type { Done } from "mocha"; +import { LocalChainFixture } from "./LocalChainFixture"; chai.use(chaiHttp); @@ -66,6 +67,35 @@ export async function deployFromAbiAndBytecodeForCreatorTxHash( return { contractAddress, txHash: creationTx.hash }; } + +export async function deployAndVerifyContract( + chai: Chai.ChaiStatic, + chainFixture: LocalChainFixture, + serverFixture: ServerFixture, + partial: boolean = false +) { + const contractAddress = await deployFromAbiAndBytecode( + chainFixture.localSigner, + chainFixture.defaultContractArtifact.abi, + chainFixture.defaultContractArtifact.bytecode, + [] + ); + await chai + .request(serverFixture.server.app) + .post("/") + .field("address", contractAddress) + .field("chain", chainFixture.chainId) + .attach( + "files", + partial + ? chainFixture.defaultContractModifiedMetadata + : chainFixture.defaultContractMetadata, + "metadata.json" + ) + .attach("files", chainFixture.defaultContractSource); + return contractAddress; +} + /** * Function to deploy contracts from an external account with private key */ diff --git a/services/server/test/integration/repository-handlers/files.spec.ts b/services/server/test/integration/repository-handlers/files.spec.ts index 09955a7b4..688a34a58 100644 --- a/services/server/test/integration/repository-handlers/files.spec.ts +++ b/services/server/test/integration/repository-handlers/files.spec.ts @@ -1,9 +1,6 @@ import chai from "chai"; import chaiHttp from "chai-http"; -import { - deployFromAbiAndBytecodeForCreatorTxHash, - waitSecs, -} from "../../helpers/helpers"; +import { deployAndVerifyContract, waitSecs } from "../../helpers/helpers"; import { LocalChainFixture } from "../../helpers/LocalChainFixture"; import { ServerFixture } from "../../helpers/ServerFixture"; @@ -43,46 +40,84 @@ describe("Verify repository endpoints", function () { chai.expect(res4.body.full).has.a.lengthOf(1); }); - it("should handle pagination in /files/contracts/{chain}", async function () { - for (let i = 0; i < 5; i++) { - const { contractAddress } = - await deployFromAbiAndBytecodeForCreatorTxHash( - chainFixture.localSigner, - chainFixture.defaultContractArtifact.abi, - chainFixture.defaultContractArtifact.bytecode, - [] - ); - await chai - .request(serverFixture.server.app) - .post("/") - .field("address", contractAddress) - .field("chain", chainFixture.chainId) - .attach("files", chainFixture.defaultContractMetadata, "metadata.json") - .attach("files", chainFixture.defaultContractSource); + describe(`Pagination in /files/contracts/{full|any|partial}/${chainFixture.chainId}`, async function () { + const endpointMatchTypes = ["full", "any", "partial"]; + for (const endpointMatchType of endpointMatchTypes) { + it(`should handle pagination in /files/contracts/${endpointMatchType}/${chainFixture.chainId}`, async function () { + const contractAddresses: string[] = []; + + // Deploy 5 contracts + for (let i = 0; i < 5; i++) { + // Deploy partial matching contract if endpoint is partial or choose randomly if endpointMachtype is any. 'any' endpoint results should be consistent regardless. + const shouldDeployPartial = + endpointMatchType === "partial" || + (endpointMatchType === "any" && Math.random() > 0.5); + + const address = await deployAndVerifyContract( + chai, + chainFixture, + serverFixture, + shouldDeployPartial + ); + contractAddresses.push(address); + } + + // Test pagination + const res0 = await chai + .request(serverFixture.server.app) + .get( + `/files/contracts/${endpointMatchType}/${chainFixture.chainId}?page=1&limit=2` + ); + chai.expect(res0.body.pagination).to.deep.equal({ + currentPage: 1, + hasNextPage: true, + hasPreviousPage: true, + resultsCurrentPage: 2, + resultsPerPage: 2, + totalPages: 3, + totalResults: 5, + }); + const res1 = await chai + .request(serverFixture.server.app) + .get( + `/files/contracts/${endpointMatchType}/${chainFixture.chainId}?limit=5` + ); + chai.expect(res1.body.pagination).to.deep.equal({ + currentPage: 0, + hasNextPage: false, + hasPreviousPage: false, + resultsCurrentPage: 5, + resultsPerPage: 5, + totalPages: 1, + totalResults: 5, + }); + + // Test ascending order + const resAsc = await chai + .request(serverFixture.server.app) + .get( + `/files/contracts/${endpointMatchType}/${chainFixture.chainId}?order=asc` + ); + chai + .expect(resAsc.body.results) + .to.deep.equal( + contractAddresses, + "Contract addresses are not in ascending order" + ); + + // Test descending order + const resDesc = await chai + .request(serverFixture.server.app) + .get( + `/files/contracts/${endpointMatchType}/${chainFixture.chainId}?order=desc` + ); + chai + .expect(resDesc.body.results) + .to.deep.equal( + contractAddresses.reverse(), + "Contract addresses are not in reverse order" + ); + }); } - const res0 = await chai - .request(serverFixture.server.app) - .get(`/files/contracts/any/${chainFixture.chainId}?page=1&limit=2`); - chai.expect(res0.body.pagination).to.deep.equal({ - currentPage: 1, - hasNextPage: true, - hasPreviousPage: true, - resultsCurrentPage: 2, - resultsPerPage: 2, - totalPages: 3, - totalResults: 5, - }); - const res1 = await chai - .request(serverFixture.server.app) - .get(`/files/contracts/any/${chainFixture.chainId}?limit=5`); - chai.expect(res1.body.pagination).to.deep.equal({ - currentPage: 0, - hasNextPage: false, - hasPreviousPage: false, - resultsCurrentPage: 5, - resultsPerPage: 5, - totalPages: 1, - totalResults: 5, - }); }); }); diff --git a/services/server/test/integration/verification-handlers/verify.session.spec.ts b/services/server/test/integration/verification-handlers/verify.session.spec.ts index 13d331582..1ec9193f3 100644 --- a/services/server/test/integration/verification-handlers/verify.session.spec.ts +++ b/services/server/test/integration/verification-handlers/verify.session.spec.ts @@ -297,7 +297,7 @@ describe("/session", function () { const agent = chai.request.agent(serverFixture.server.app); agent .post("/session/input-files") - .attach("files", chainFixture.defaultContractModifiedIpfsMetadata) + .attach("files", chainFixture.defaultContractModifiedSourceIpfs) .then((res) => { assertAddressAndChainMissing(res, [], { "project:/contracts/Storage.sol": { diff --git a/services/server/test/integration/verification-handlers/verify.stateless.spec.ts b/services/server/test/integration/verification-handlers/verify.stateless.spec.ts index 990d44cae..db87eaaac 100644 --- a/services/server/test/integration/verification-handlers/verify.stateless.spec.ts +++ b/services/server/test/integration/verification-handlers/verify.stateless.spec.ts @@ -137,7 +137,7 @@ describe("/", function () { .field("chain", chainFixture.chainId) .attach( "files", - chainFixture.defaultContractModifiedIpfsMetadata, + chainFixture.defaultContractModifiedSourceIpfs, "metadata.json" ) .end((err, res) => {