diff --git a/services/server/src/config/local-test.js b/services/server/src/config/local-test.js index 682c9836f..5a14bfe85 100644 --- a/services/server/src/config/local-test.js +++ b/services/server/src/config/local-test.js @@ -1,6 +1,9 @@ module.exports = { repositoryV1: { - path: "/tmp/repository-test/", + path: "/tmp/repositoryV1-test/", + }, + repositoryV2: { + path: "/tmp/repositoryV2-test/", }, session: { storeType: "database", diff --git a/services/server/src/openapi.yaml b/services/server/src/openapi.yaml index 7cfe8bf05..004eeee7b 100644 --- a/services/server/src/openapi.yaml +++ b/services/server/src/openapi.yaml @@ -34,6 +34,10 @@ paths: $ref: "server/controllers/repository/get-file-static.stateless.paths.yaml#/paths/~1repository~1contracts~1{full_match | partial_match}~1{chain}~1{address}~1{filePath}" /files/contracts/{chain}: $ref: "server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml#/paths/~1files~1contracts~1{chain}" + /files/contracts/any/{chain}: + $ref: "server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml#/paths/~1files~1contracts~1any~1{chain}" + /files/contracts/full/{chain}: + $ref: "server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml#/paths/~1files~1contracts~1full~1{chain}" /files/tree/any/{chain}/{address}: $ref: "server/controllers/repository/get-file-tree-all.stateless.paths.yaml#/paths/~1files~1tree~1any~1{chain}~1{address}" /files/tree/{chain}/{address}: @@ -75,10 +79,10 @@ paths: enum: [error, warn, info, debug, silly] examples: change to debug: - value: + value: level: "debug" change to info: - value: + value: level: "info" responses: "200": diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml index 7324e157e..57cd0bf11 100644 --- a/services/server/src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml +++ b/services/server/src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml @@ -3,8 +3,8 @@ openapi: "3.0.0" paths: /files/contracts/{chain}: get: - summary: Get all contract addresses verified on a chain (full or partial match) - description: Returns all verified contracts from the repository for the desired chain. Searches for full and partial matches. + summary: (Deprecated) Get the first 200 contract addresses verified on a chain (full or partial match) + description: Returns the first 200 verified contracts from the repository for the desired chain. Searches for full and partial matches. This endpoint is deprecated and only retuns 200 addresses. Use `/files/contracts/any/` and `/files/contracts/full/` instead tags: - Repository parameters: @@ -50,3 +50,13 @@ paths: error: type: string example: "Contracts have not been found!" + "400": + description: Cannot use this API if there are more than 200 contracts + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: "Cannot fetch more than 200 contracts (1521 full matches, 3453 partial matches), please use /contracts/{full|any}/4 with pagination" diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml new file mode 100644 index 000000000..1423a35d9 --- /dev/null +++ b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml @@ -0,0 +1,62 @@ +openapi: "3.0.0" + +paths: + /files/contracts/any/{chain}: + get: + summary: Get the contract addresses verified on a chain (full or partial match) + description: Returns the verified contracts from the repository for the desired chain. Searches for full and partial matches. API is paginated. Limit must be a number between 1 and 200. + tags: + - Repository + parameters: + - name: chain + in: path + required: true + schema: + type: string + format: sourcify-chainId + - name: page + in: query + required: false + schema: + type: number + - name: limit + in: query + required: false + schema: + type: number + minimum: 1 + maximum: 200 + responses: + "200": + description: Chain is available as a full match or partial match in the repository + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: string + example: + [ + "0x1fE5d745beABA808AAdF52057Dd7AAA47b42cFD0", + "0xE9c31091868d68598Ac881738D159A63532d12f9", + ] + pagination: + type: object + properties: + currentPage: + type: number + totalPages: + type: number + resultsPerPage: + type: number + resultsCurrentPage: + type: number + totalResults: + type: number + hasNextPage: + type: boolean + hasPreviousPage: + type: boolean diff --git a/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml new file mode 100644 index 000000000..3ed0fa99d --- /dev/null +++ b/services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml @@ -0,0 +1,62 @@ +openapi: "3.0.0" + +paths: + /files/contracts/full/{chain}: + get: + summary: Get the contract addresses perfectly verified on a chain + description: Returns the perfectly verified contracts from the repository for the desired chain. API is paginated. Limit must be a number between 1 and 200. + tags: + - Repository + parameters: + - name: chain + in: path + required: true + schema: + type: string + format: sourcify-chainId + - name: page + in: query + required: false + schema: + type: number + - name: limit + in: query + required: false + schema: + type: number + minimum: 1 + maximum: 200 + responses: + "200": + description: Chain is available as a full match in the repository + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: string + example: + [ + "0x1fE5d745beABA808AAdF52057Dd7AAA47b42cFD0", + "0xE9c31091868d68598Ac881738D159A63532d12f9", + ] + pagination: + type: object + properties: + currentPage: + type: number + totalPages: + type: number + resultsPerPage: + type: number + resultsCurrentPage: + type: number + totalResults: + type: number + hasNextPage: + type: boolean + hasPreviousPage: + type: boolean diff --git a/services/server/src/server/controllers/repository/repository.handlers.ts b/services/server/src/server/controllers/repository/repository.handlers.ts index aab2ec829..3829a8eb0 100644 --- a/services/server/src/server/controllers/repository/repository.handlers.ts +++ b/services/server/src/server/controllers/repository/repository.handlers.ts @@ -1,6 +1,11 @@ import { Response, Request, NextFunction } from "express"; import { StatusCodes } from "http-status-codes"; -import { ContractData, FilesInfo, MatchLevel } from "../../types"; +import { + ContractData, + FilesInfo, + MatchLevel, + PaginatedContractData, +} from "../../types"; import { NotFoundError } from "../../../common/errors"; import { Match } from "@ethereum-sourcify/lib-sourcify"; import { services } from "../../services/services"; @@ -12,6 +17,12 @@ type RetrieveMethod = ( match: MatchLevel ) => Promise>; type ConractRetrieveMethod = (chain: string) => Promise; +type PaginatedConractRetrieveMethod = ( + chain: string, + match: MatchLevel, + page: number, + limit: number +) => Promise; export function createEndpoint( retrieveMethod: RetrieveMethod, @@ -53,6 +64,26 @@ export function createContractEndpoint( }; } +export function createPaginatedContractEndpoint( + paginatedContractRetrieveMethod: PaginatedConractRetrieveMethod, + match: MatchLevel +) { + return async (req: Request, res: Response, next: NextFunction) => { + let retrieved: PaginatedContractData; + try { + retrieved = await paginatedContractRetrieveMethod( + req.params.chain, + match, + parseInt((req.query.page as string) || "0"), + parseInt((req.query.limit as string) || "200") + ); + } catch (err: any) { + return next(new NotFoundError(err.message)); + } + return res.status(StatusCodes.OK).json(retrieved); + }; +} + export async function checkAllByChainAndAddressEndpoint( req: any, res: Response @@ -96,6 +127,29 @@ export async function checkAllByChainAndAddressEndpoint( res.send(resultArray); } +export async function getMetadataEndpoint(req: any, res: Response) { + const { match, chain, address } = req.params; + const file = await services.storage.getMetadata(chain, address, match); + if (file === false) { + res.status(404).send(); + } + res.json(JSON.parse(file as string)); +} + +export async function getFileEndpoint(req: any, res: Response) { + const { match, chain, address } = req.params; + const file = await services.storage.getFile( + chain, + address, + match, + req.params[0] + ); + if (!file) { + res.status(404).send(); + } + res.send(file); +} + export async function checkByChainAndAddressesEnpoint(req: any, res: Response) { const map: Map = new Map(); const addresses = req.query.addresses.split(","); diff --git a/services/server/src/server/controllers/repository/repository.routes.ts b/services/server/src/server/controllers/repository/repository.routes.ts index de3548b62..3e375b736 100644 --- a/services/server/src/server/controllers/repository/repository.routes.ts +++ b/services/server/src/server/controllers/repository/repository.routes.ts @@ -6,6 +6,9 @@ import { createContractEndpoint, checkAllByChainAndAddressEndpoint, checkByChainAndAddressesEnpoint, + getFileEndpoint, + getMetadataEndpoint, + createPaginatedContractEndpoint, } from "./repository.handlers"; import { safeHandler } from "../controllers.common"; @@ -16,39 +19,42 @@ const router: Router = Router(); [ { prefix: "/tree/any", - method: createEndpoint( - services.storage.repositoryV1.getTree, - "any_match", - true - ), + method: createEndpoint(services.storage.getTree, "any_match", true), }, { prefix: "/any", - method: createEndpoint( - services.storage.repositoryV1.getContent, - "any_match", - true - ), + method: createEndpoint(services.storage.getContent, "any_match", true), }, { prefix: "/tree", - method: createEndpoint(services.storage.repositoryV1.getTree, "full_match"), + method: createEndpoint(services.storage.getTree, "full_match"), }, { prefix: "/contracts", - method: createContractEndpoint(services.storage.repositoryV1.getContracts), + method: createContractEndpoint(services.storage.getContracts), }, { - prefix: "", - method: createEndpoint( - services.storage.repositoryV1.getContent, + prefix: "/contracts/full", + method: createPaginatedContractEndpoint( + services.storage.getPaginatedContracts, "full_match" ), }, + { + prefix: "/contracts/any", + method: createPaginatedContractEndpoint( + services.storage.getPaginatedContracts, + "any_match" + ), + }, + { + prefix: "", + method: createEndpoint(services.storage.getContent, "full_match"), + }, ].forEach((pair) => { router .route( - pair.prefix != "/contracts" + !pair.prefix.startsWith("/contracts") ? REPOSITORY_CONTROLLER_PREFIX + pair.prefix + "/:chain/:address" : REPOSITORY_CONTROLLER_PREFIX + pair.prefix + "/:chain" ) @@ -60,6 +66,22 @@ router .route("/check-all-by-addresses") .get(safeHandler(checkAllByChainAndAddressEndpoint)); +/** + * The following two routes are the replacement for the removed static file route that exposed RepositoryV1 + * The function getFileEndpoint get the sources from compiled_contracts.sources + * We need both of these routes because compiled_contracts.sources doesn't contain the metadata file + */ + +// This route covers the metadata.json files, fetching them from RepositoryV2 +router + .route("/repository/contracts/:match/:chain/:address/metadata.json") + .get(safeHandler(getMetadataEndpoint)); + +// This route covers the the sources files, fetching them from SourcifyDatabase.compiled_contracts.sources +router + .route("/repository/contracts/:match/:chain/:address/sources/*") + .get(safeHandler(getFileEndpoint)); + router .route("/check-by-addresses") .get(safeHandler(checkByChainAndAddressesEnpoint)); diff --git a/services/server/src/server/server.ts b/services/server/src/server/server.ts index cf701357b..c1eb40197 100644 --- a/services/server/src/server/server.ts +++ b/services/server/src/server/server.ts @@ -266,12 +266,6 @@ export class Server { // Enable session only for session endpoints this.app.use("/*session*", getSessionMiddleware()); - this.app.use( - "/repository", - express.static(this.repository), - serveIndex(this.repository, { icons: true }) - ); - this.app.use("/", routes); this.app.use(genericErrorHandler); } diff --git a/services/server/src/server/services/StorageService.ts b/services/server/src/server/services/StorageService.ts index 53f2879c1..763e7d735 100644 --- a/services/server/src/server/services/StorageService.ts +++ b/services/server/src/server/services/StorageService.ts @@ -17,6 +17,16 @@ import { } from "./storageServices/AllianceDatabaseService"; import logger from "../../common/logger"; import { getMatchStatus } from "../common"; +import { + ContractData, + FileObject, + FilesInfo, + MatchLevel, + PaginatedContractData, +} from "../types"; +import { getFileRelativePath } from "./utils/util"; +import config from "config"; +import { BadRequestError } from "../../common/errors"; export interface IStorageService { init(): Promise; @@ -95,6 +105,199 @@ export class StorageService { } } + getMetadata = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise => { + return this.repositoryV2!.getMetadata(chainId, address, match); + }; + + /** + * This function inject the metadata file in FilesInfo + * SourcifyDatabase.getTree and SourcifyDatabase.getContent read files from + * `compiled_contracts.sources` where the metadata file is not available + */ + pushMetadataInFilesInfo = async ( + responseWithoutMetadata: FilesInfo, + chainId: string, + address: string, + match: MatchLevel + ) => { + const metadata = await this.getMetadata( + chainId, + address, + responseWithoutMetadata.status === "full" ? "full_match" : "any_match" + ); + + if (!metadata) { + logger.error("Contract exists in the database but not in RepositoryV2", { + chainId, + address, + match, + }); + throw new Error( + "Contract exists in the database but not in RepositoryV2" + ); + } + + const relativePath = getFileRelativePath( + chainId, + address, + match === "full_match" ? "full" : "partial", + "metadata.json" + ); + + if (typeof responseWithoutMetadata.files[0] === "string") { + // If this function is called with T == string + responseWithoutMetadata.files.push( + (config.get("repositoryV1.serverUrl") + relativePath) as T + ); + } else { + // If this function is called with T === FileObject + // It's safe to handle this case in the else because of + responseWithoutMetadata.files.push({ + name: "metadata.json", + path: relativePath, + content: metadata, + } as T); + } + }; + + getFile = async ( + chainId: string, + address: string, + match: MatchLevel, + path: string + ): Promise => { + try { + return this.sourcifyDatabase!.getFile(chainId, address, match, path); + } catch (error) { + logger.error("Error while getting file from database", { + chainId, + address, + match, + path, + error, + }); + throw new Error("Error while getting file from database"); + } + }; + + getTree = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise> => { + let responseWithoutMetadata; + try { + responseWithoutMetadata = await this.sourcifyDatabase!.getTree( + chainId, + address, + match + ); + } catch (error) { + logger.error("Error while getting tree from database", { + chainId, + address, + match, + error, + }); + throw new Error("Error while getting tree from database"); + } + + // if files is empty it means that the contract doesn't exist + if (responseWithoutMetadata.files.length === 0) { + return responseWithoutMetadata; + } + + await this.pushMetadataInFilesInfo( + responseWithoutMetadata, + chainId, + address, + match + ); + + return responseWithoutMetadata; + }; + + getContent = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise>> => { + let responseWithoutMetadata; + try { + responseWithoutMetadata = await this.sourcifyDatabase!.getContent( + chainId, + address, + match + ); + } catch (error) { + logger.error("Error while getting content from database", { + chainId, + address, + match, + error, + }); + throw new Error("Error while getting content from database"); + } + + // if files is empty it means that the contract doesn't exist + if (responseWithoutMetadata.files.length === 0) { + return responseWithoutMetadata; + } + + await this.pushMetadataInFilesInfo( + responseWithoutMetadata, + chainId, + address, + match + ); + + return responseWithoutMetadata; + }; + + getContracts = async (chainId: string): Promise => { + try { + return this.sourcifyDatabase!.getContracts(chainId); + } catch (error) { + if (error instanceof BadRequestError) { + throw error; + } + logger.error("Error while getting contracts from database", { + chainId, + error, + }); + throw new Error("Error while getting contracts from database"); + } + }; + + getPaginatedContracts = ( + chainId: string, + match: MatchLevel, + page: number, + limit: number + ): Promise => { + try { + return this.sourcifyDatabase!.getPaginatedContracts( + chainId, + match, + page, + limit + ); + } catch (error) { + logger.error("Error while getting paginated contracts from database", { + chainId, + match, + page, + limit, + error, + }); + throw new Error("Error while getting paginated contracts from database"); + } + }; + /* async init() { try { await this.repositoryV1?.init(); @@ -124,8 +327,11 @@ export class StorageService { chainId: string ): Promise { return ( - (await this.repositoryV1?.checkByChainAndAddress?.(address, chainId)) || - [] + (await this.sourcifyDatabase?.checkByChainAndAddress?.( + address, + chainId, + true + )) || [] ); } @@ -134,9 +340,10 @@ export class StorageService { chainId: string ): Promise { return ( - (await this.repositoryV1?.checkAllByChainAndAddress?.( + (await this.sourcifyDatabase?.checkByChainAndAddress?.( address, - chainId + chainId, + false )) || [] ); } @@ -150,6 +357,14 @@ export class StorageService { creationMatch: match.creationMatch, }); + // Sourcify Database and RepositoryV2 must be enabled + if (!this.sourcifyDatabase) { + throw new Error("SourcifyDatabase must be enabled"); + } + if (!this.repositoryV2) { + throw new Error("RepositoryV2 must be enabled"); + } + const existingMatch = await this.checkAllByChainAndAddress( match.address, match.chainId @@ -192,31 +407,32 @@ export class StorageService { } } - if (this.sourcifyDatabase) { + // @deprecated + if (this.repositoryV1) { promises.push( - this.sourcifyDatabase.storeMatch(contract, match).catch((e) => - logger.error("Error storing to SourcifyDatabase: ", { - error: e, - }) - ) + this.repositoryV1 + .storeMatch(contract, match) + .catch((e) => + logger.error("Error storing to RepositoryV1: ", { error: e }) + ) ); } - if (this.repositoryV2) { - promises.push( - this.repositoryV2.storeMatch(contract, match).catch((e) => - logger.error("Error storing to RepositoryV2: ", { - error: e, - }) - ) - ); - } + // Add by default both sourcifyDatabase and repositoryV2 + promises.push( + this.sourcifyDatabase.storeMatch(contract, match).catch((e) => { + logger.error("Error storing to SourcifyDatabase: ", { + error: e, + }); + throw e; + }) + ); - // Always include repositoryV1 promises.push( - this.repositoryV1.storeMatch(contract, match).catch((e) => { - logger.error("Error storing to RepositoryV1: ", { error: e }); - // For repositoryV1 we throw + this.repositoryV2.storeMatch(contract, match).catch((e) => { + logger.error("Error storing to RepositoryV2: ", { + error: e, + }); throw e; }) ); diff --git a/services/server/src/server/services/storageServices/RepositoryV1Service.ts b/services/server/src/server/services/storageServices/RepositoryV1Service.ts index b909b4e88..82dd10a06 100644 --- a/services/server/src/server/services/storageServices/RepositoryV1Service.ts +++ b/services/server/src/server/services/storageServices/RepositoryV1Service.ts @@ -7,7 +7,15 @@ import { StringMap, CheckedContract, } from "@ethereum-sourcify/lib-sourcify"; -import { MatchLevel, MatchQuality, RepositoryTag } from "../../types"; +import { + ContractData, + FileObject, + FilesInfo, + MatchLevel, + MatchQuality, + PathConfig, + RepositoryTag, +} from "../../types"; import { create as createIpfsClient, IPFSHTTPClient, @@ -19,20 +27,6 @@ import { getAddress } from "ethers"; import { getMatchStatus } from "../../common"; import { IStorageService } from "../StorageService"; import config from "config"; -import { PathConfig } from "../utils/repository-util"; - -type FilesInfo = { status: MatchQuality; files: Array }; - -interface FileObject { - name: string; - path: string; - content?: string; -} - -declare interface ContractData { - full: string[]; - partial: string[]; -} export interface RepositoryV1ServiceOptions { ipfsApi: string; @@ -40,17 +34,6 @@ export interface RepositoryV1ServiceOptions { repositoryServerUrl: string; } -interface FileObject { - name: string; - path: string; - content?: string; -} - -declare interface ContractData { - full: string[]; - partial: string[]; -} - export class RepositoryV1Service implements IStorageService { repositoryPath: string; private ipfsClient?: IPFSHTTPClient; @@ -100,7 +83,7 @@ export class RepositoryV1Service implements IStorageService { * * @example [ * { name: '0x1234.sol', - * path: '/home/.../repository/contracts/full_match/1/0x1234/0x1234.sol, + * path: '/contracts/full_match/1/0x1234/0x1234.sol, * content: "pragma solidity ^0.5.0; contract A { ... }" * }, * ... ] @@ -137,9 +120,14 @@ export class RepositoryV1Service implements IStorageService { const fullPath = this.repositoryPath + `/contracts/full_match/${chain}/`; const partialPath = this.repositoryPath + `/contracts/partial_match/${chain}/`; + + const full = fs.existsSync(fullPath) ? fs.readdirSync(fullPath) : []; + const partial = fs.existsSync(partialPath) + ? fs.readdirSync(partialPath) + : []; return { - full: fs.existsSync(fullPath) ? fs.readdirSync(fullPath) : [], - partial: fs.existsSync(partialPath) ? fs.readdirSync(partialPath) : [], + full, + partial, }; }; @@ -147,7 +135,7 @@ export class RepositoryV1Service implements IStorageService { chainId: string, address: string, match: MatchLevel - ): Promise> => { + ): Promise> => { const fullMatchesTree = this.fetchAllFileUrls( chainId, address, @@ -165,7 +153,7 @@ export class RepositoryV1Service implements IStorageService { chainId: string, address: string, match: MatchLevel - ): Promise> => { + ): Promise>> => { const fullMatchesFiles = this.fetchAllFileContents( chainId, address, diff --git a/services/server/src/server/services/storageServices/RepositoryV2Service.ts b/services/server/src/server/services/storageServices/RepositoryV2Service.ts index 3667b0263..de6795014 100644 --- a/services/server/src/server/services/storageServices/RepositoryV2Service.ts +++ b/services/server/src/server/services/storageServices/RepositoryV2Service.ts @@ -13,7 +13,14 @@ import { StringMap, CheckedContract, } from "@ethereum-sourcify/lib-sourcify"; -import { MatchQuality, RepositoryTag } from "../../types"; +import { + FileObject, + FilesInfo, + MatchLevel, + MatchQuality, + PathConfig, + RepositoryTag, +} from "../../types"; import { create as createIpfsClient, IPFSHTTPClient, @@ -23,7 +30,6 @@ import logger from "../../../common/logger"; import { getAddress, id as keccak256 } from "ethers"; import { getMatchStatus } from "../../common"; import { IStorageService } from "../StorageService"; -import { PathConfig } from "../utils/repository-util"; export interface RepositoryV2ServiceOptions { ipfsApi: string; @@ -49,6 +55,26 @@ export class RepositoryV2Service implements IStorageService { return true; } + getMetadata = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise => { + try { + return fs.readFileSync( + this.generateAbsoluteFilePath({ + matchQuality: match === "full_match" ? "full" : "partial", + chainId: chainId, + address: address, + fileName: "metadata.json", + }), + { encoding: "utf-8" } + ); + } catch (e) { + return false; + } + }; + // /home/user/sourcify/data/repository/contracts/full_match/5/0x00878Ac0D6B8d981ae72BA7cDC967eA0Fae69df4/sources/filename public generateAbsoluteFilePath(pathConfig: PathConfig) { return Path.join( diff --git a/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts b/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts index 5b9fc262d..dfa85da2b 100644 --- a/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts +++ b/services/server/src/server/services/storageServices/SourcifyDatabaseService.ts @@ -10,6 +10,20 @@ import AbstractDatabaseService from "./AbstractDatabaseService"; import { IStorageService } from "../StorageService"; import { bytesFromString } from "../utils/database-util"; import { getMatchStatus } from "../../common"; +import { + ContractData, + FileObject, + FilesInfo, + FilesRaw, + MatchLevel, + MatchQuality, + PaginatedContractData, +} from "../../types"; +import config from "config"; +import Path from "path"; +import { getFileRelativePath } from "../utils/util"; +import { getAddress } from "ethers"; +import { BadRequestError } from "../../../common/errors"; export interface SourcifyDatabaseServiceOptions { postgres: { @@ -21,6 +35,8 @@ export interface SourcifyDatabaseServiceOptions { }; } +const MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS = 200; + export class SourcifyDatabaseService extends AbstractDatabaseService implements IStorageService @@ -88,15 +104,295 @@ export class SourcifyDatabaseService address, chainId, runtimeMatch: existingVerifiedContractResult.rows[0] - .runtime_match as Status, + .runtime_match_status as Status, creationMatch: existingVerifiedContractResult.rows[0] - .creation_match as Status, + .creation_match_status as Status, storageTimestamp: existingVerifiedContractResult.rows[0] .created_at as Date, }, ]; } + getContracts = async (chainId: string): Promise => { + await this.init(); + + const res: ContractData = { + full: [], + partial: [], + }; + const matchAddressesCountResult = + await Database.countSourcifyMatchAddresses( + this.databasePool, + parseInt(chainId) + ); + + if (matchAddressesCountResult.rowCount === 0) { + return res; + } + + const fullTotal = parseInt(matchAddressesCountResult.rows[0].full_total); + const partialTotal = parseInt( + matchAddressesCountResult.rows[0].partial_total + ); + if ( + fullTotal > MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS || + partialTotal > MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS + ) { + logger.info( + "Requested more than MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS contracts", + { + maxReturnedContracts: MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS, + chainId, + } + ); + throw new BadRequestError( + `Cannot fetch more than ${MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS} contracts (${fullTotal} full matches, ${partialTotal} partial matches), please use /contracts/{full|any}/${chainId} with pagination` + ); + } + + if (fullTotal > 0) { + const perfectMatchAddressesResult = + await Database.getSourcifyMatchAddressesByChainAndMatch( + this.databasePool, + parseInt(chainId), + "full_match", + 0, + MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS + ); + + if (perfectMatchAddressesResult.rowCount > 0) { + res.full = perfectMatchAddressesResult.rows.map((row) => + getAddress(row.address) + ); + } + } + + if (partialTotal > 0) { + const partialMatchAddressesResult = + await Database.getSourcifyMatchAddressesByChainAndMatch( + this.databasePool, + parseInt(chainId), + "partial_match", + 0, + MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS + ); + + if (partialMatchAddressesResult.rowCount > 0) { + res.partial = partialMatchAddressesResult.rows.map((row) => + getAddress(row.address) + ); + } + } + + return res; + }; + + getPaginatedContracts = async ( + chainId: string, + match: MatchLevel, + page: number, + limit: number + ): Promise => { + await this.init(); + + // Initialize empty result + const res: PaginatedContractData = { + results: [], + pagination: { + currentPage: page, + resultsPerPage: limit, + resultsCurrentPage: 0, + totalResults: 0, + totalPages: 0, + hasNextPage: false, + hasPreviousPage: false, + }, + }; + + // Count perfect and partial matches + const matchAddressesCountResult = + await Database.countSourcifyMatchAddresses( + this.databasePool, + parseInt(chainId) + ); + + if (matchAddressesCountResult.rowCount === 0) { + return res; + } + + // Calculate totalResults, return empty res if there are no contracts + const fullTotal = parseInt(matchAddressesCountResult.rows[0].full_total); + const partialTotal = parseInt( + matchAddressesCountResult.rows[0].partial_total + ); + const anyTotal = fullTotal + partialTotal; + if (match === "full_match") { + if (fullTotal === 0) { + return res; + } + res.pagination.totalResults = fullTotal; + } else if (match === "any_match") { + if (anyTotal === 0) { + return res; + } + res.pagination.totalResults = anyTotal; + } + + res.pagination.totalPages = Math.ceil( + res.pagination.totalResults / res.pagination.resultsPerPage + ); + + const matchAddressesResult = + await Database.getSourcifyMatchAddressesByChainAndMatch( + this.databasePool, + parseInt(chainId), + match, + page, + limit + ); + + if (matchAddressesResult.rowCount > 0) { + res.pagination.resultsCurrentPage = matchAddressesResult.rowCount; + res.pagination.hasNextPage = + res.pagination.currentPage * res.pagination.resultsPerPage + + matchAddressesResult.rowCount < + res.pagination.totalResults; + res.pagination.hasPreviousPage = + res.pagination.currentPage === 0 ? false : true; + res.results = matchAddressesResult.rows.map((row) => + getAddress(row.address) + ); + } + + return res; + }; + + /** + * getFiles extracts the files from the database `compiled_contracts.sources` + * and store them into FilesInfo.files, this object is then going to be formatted + * by getTree, getContent and getFile. + */ + getFiles = async ( + chainId: string, + address: string + ): Promise> => { + await this.init(); + + const sourcifyMatchResult = await Database.getSourcifyMatchByChainAddress( + this.databasePool, + parseInt(chainId), + bytesFromString(address)! + ); + + if (sourcifyMatchResult.rowCount === 0) { + // This is how you handle a non existing contract + return { status: "partial", files: {} }; + } + + const sourcifyMatch = sourcifyMatchResult.rows[0]; + + // If either one of sourcify_matches.creation_match or sourcify_matches.runtime_match is perfect then "full" status + const contractStatus = + sourcifyMatch.creation_match_status === "perfect" || + sourcifyMatch.runtime_match_status === "perfect" + ? "full" + : "partial"; + + return { status: contractStatus, files: sourcifyMatch.sources }; + }; + + getFile = async ( + chainId: string, + address: string, + match: MatchLevel, + path: string + ): Promise => { + const { status, files } = await this.getFiles(chainId, address); + if (Object.keys(files).length === 0) { + return false; + } + if (match === "full_match" && status === "partial") { + return false; + } + return files[path]; + }; + + /** + * getTree returns FilesInfo in which files contains for each source its repository url + */ + getTree = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise> => { + const { status: contractStatus, files: filesRaw } = await this.getFiles( + chainId, + address + ); + + // If "full_match" files are requested but the contractStatus if partial return empty + if (match === "full_match" && contractStatus === "partial") { + return { + status: "full", + files: [], + }; + } + + // Calculate the the repository's url for each file + const files = Object.keys(filesRaw).map((file) => { + const relativePath = getFileRelativePath( + chainId, + address, + contractStatus, + file + ); + return `${config.get("repositoryV1.serverUrl")}/${relativePath}`; + }); + + return { status: contractStatus, files }; + }; + + /** + * getContent returns FilesInfo in which files contains for each source its FileObject, + * an object that includes the content of the file. + */ + getContent = async ( + chainId: string, + address: string, + match: MatchLevel + ): Promise>> => { + const { status: contractStatus, files: filesRaw } = await this.getFiles( + chainId, + address + ); + + // If "full_match" files are requestd but the contractStatus if partial return empty + if (match === "full_match" && contractStatus === "partial") { + return { + status: "full", + files: [], + }; + } + + // Calculate the the repository's url for each file + const files = Object.keys(filesRaw).map((file) => { + const relativePath = getFileRelativePath( + chainId, + address, + contractStatus, + file + ); + + return { + name: Path.basename(file), + path: relativePath, + content: filesRaw[file], + } as FileObject; + }); + + return { status: contractStatus, files }; + }; + validateBeforeStoring( recompiledContract: CheckedContract, match: Match diff --git a/services/server/src/server/services/utils/database-util.ts b/services/server/src/server/services/utils/database-util.ts index 6d6343039..b356580cc 100644 --- a/services/server/src/server/services/utils/database-util.ts +++ b/services/server/src/server/services/utils/database-util.ts @@ -142,9 +142,13 @@ export async function getSourcifyMatchByChainAddress( return await pool.query( ` SELECT - sourcify_matches.* + sourcify_matches.created_at, + sourcify_matches.creation_match as creation_match_status, + sourcify_matches.runtime_match as runtime_match_status, + compiled_contracts.sources FROM sourcify_matches JOIN verified_contracts ON verified_contracts.id = sourcify_matches.verified_contract_id + JOIN compiled_contracts ON compiled_contracts.id = verified_contracts.compilation_id JOIN contract_deployments ON contract_deployments.id = verified_contracts.deployment_id AND contract_deployments.chain_id = $1 @@ -567,3 +571,64 @@ export function normalizeRecompiledBytecodes( }); } } + +export async function countSourcifyMatchAddresses(pool: Pool, chain: number) { + return await pool.query( + ` + SELECT + contract_deployments.chain_id, + SUM(CASE WHEN sourcify_matches.creation_match = 'perfect' OR sourcify_matches.runtime_match = 'perfect' THEN 1 ELSE 0 END) AS full_total, + SUM(CASE WHEN sourcify_matches.creation_match != 'perfect' AND sourcify_matches.runtime_match != 'perfect' THEN 1 ELSE 0 END) AS partial_total + FROM sourcify_matches + JOIN verified_contracts ON verified_contracts.id = sourcify_matches.verified_contract_id + JOIN contract_deployments ON + contract_deployments.id = verified_contracts.deployment_id + AND contract_deployments.chain_id = $1 + GROUP BY contract_deployments.chain_id; + `, + [chain] + ); +} + +export async function getSourcifyMatchAddressesByChainAndMatch( + pool: Pool, + chain: number, + match: "full_match" | "partial_match" | "any_match", + page: number, + paginationSize: number +) { + let queryWhere = ""; + switch (match) { + case "full_match": { + queryWhere = + "WHERE sourcify_matches.creation_match = 'perfect' OR sourcify_matches.runtime_match = 'perfect'"; + break; + } + case "partial_match": { + queryWhere = + "WHERE sourcify_matches.creation_match != 'perfect' AND sourcify_matches.runtime_match != 'perfect'"; + break; + } + case "any_match": { + queryWhere = ""; + break; + } + default: { + throw new Error("Match type not supported"); + } + } + return await pool.query( + ` + SELECT + concat('0x',encode(contract_deployments.address, 'hex')) as address + FROM sourcify_matches + JOIN verified_contracts ON verified_contracts.id = sourcify_matches.verified_contract_id + JOIN contract_deployments ON + contract_deployments.id = verified_contracts.deployment_id + AND contract_deployments.chain_id = $1 + ${queryWhere} + OFFSET $2 LIMIT $3 + `, + [chain, page * paginationSize, paginationSize] + ); +} diff --git a/services/server/src/server/services/utils/repository-util.ts b/services/server/src/server/services/utils/repository-util.ts deleted file mode 100644 index 3138de818..000000000 --- a/services/server/src/server/services/utils/repository-util.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { MatchQuality } from "../../types"; - -export type PathConfig = { - matchQuality: MatchQuality; - chainId: string; - address: string; - fileName?: string; - source?: boolean; -}; diff --git a/services/server/src/server/services/utils/util.ts b/services/server/src/server/services/utils/util.ts new file mode 100644 index 000000000..277fec8f0 --- /dev/null +++ b/services/server/src/server/services/utils/util.ts @@ -0,0 +1,18 @@ +import Path from "path"; +import { MatchQuality } from "../../types"; + +export const getFileRelativePath = ( + chainId: string, + address: string, + contractStatus: MatchQuality, + file: string +): string => { + return Path.join( + "contracts", + contractStatus === "full" ? "full_match" : "partial_match", + chainId, + address, + "sources", + file + ); +}; diff --git a/services/server/src/server/types.ts b/services/server/src/server/types.ts index 0188e99d3..7df515faf 100644 --- a/services/server/src/server/types.ts +++ b/services/server/src/server/types.ts @@ -8,7 +8,7 @@ export type MatchLevel = "full_match" | "any_match"; /** * An array wrapper with info properties. */ -export type FilesInfo = { status: MatchQuality; files: Array }; +export type FilesInfo = { status: MatchQuality; files: T }; /** * A type for specifying the match quality of files. @@ -20,6 +20,37 @@ export declare interface ContractData { partial: string[]; } +export declare interface PaginatedContractData { + results: string[]; + pagination: { + currentPage: number; + totalPages: number; + resultsCurrentPage: number; + resultsPerPage: number; + totalResults: number; + hasNextPage: boolean; + hasPreviousPage: boolean; + }; +} + export type RepositoryTag = { timestamp: any; }; + +export type PathConfig = { + matchQuality: MatchQuality; + chainId: string; + address: string; + fileName?: string; + source?: boolean; +}; + +export type FilesRaw = { + [path: string]: string; +}; + +export type FileObject = { + name: string; + path: string; + content?: string; +}; diff --git a/services/server/test/server.spec.ts b/services/server/test/server.spec.ts index a6bc44907..164a439b0 100644 --- a/services/server/test/server.spec.ts +++ b/services/server/test/server.spec.ts @@ -82,6 +82,7 @@ const storageService = new StorageService({ }, repositoryV2ServiceOptions: { ipfsApi: process.env.IPFS_API || "", + repositoryPath: config.get("repositoryV2.path"), }, sourcifyDatabaseServiceOptions: { postgres: { @@ -2054,15 +2055,18 @@ describe("Server", async function () { `Original path ${originalPath} not found in metadata` ) .to.include.keys(originalPath); + + const relativeFilePath = path.join( + "contracts", + "full_match", + defaultContractChain, + toBeSanitizedContractAddress, + "sources", + pathTranslationJSON[originalPath] + ); // The path from the server response must be translated const translatedContractObject = fetchedContractFiles.find( - (obj: any) => - obj.path === - path.join( - contractSavedPath, - "sources", - pathTranslationJSON[originalPath] - ) + (obj: any) => obj.path === relativeFilePath ); chai.expect(translatedContractObject).to.exist; // And the saved file must be the same as in the metadata @@ -2129,7 +2133,6 @@ describe("Server", async function () { .field("chain", defaultContractChain) .attach("files", metadataBuffer, "metadata.json") .attach("files", sourceBuffer, "Storage.sol"); - console.log(res.body); const res0 = await agent.get( `/files/${defaultContractChain}/${defaultContractAddress}` ); @@ -2149,6 +2152,48 @@ describe("Server", async function () { const res4 = await agent.get(`/files/contracts/${defaultContractChain}`); chai.expect(res4.body.full).has.a.lengthOf(1); }); + it("should handle pagination in /files/contracts/{chain}", async function () { + for (let i = 0; i < 5; i++) { + const { contractAddress } = + await deployFromAbiAndBytecodeForCreatorTxHash( + localSigner, + artifact.abi, + artifact.bytecode, + [] + ); + await chai + .request(server.app) + .post("/") + .field("address", contractAddress) + .field("chain", defaultContractChain) + .attach("files", metadataBuffer, "metadata.json") + .attach("files", sourceBuffer); + } + const res0 = await chai + .request(server.app) + .get(`/files/contracts/any/${defaultContractChain}?page=1&limit=2`); + chai.expect(res0.body.pagination).to.deep.equal({ + currentPage: 1, + hasNextPage: true, + hasPreviousPage: true, + resultsCurrentPage: 2, + resultsPerPage: 2, + totalPages: 3, + totalResults: 5, + }); + const res1 = await chai + .request(server.app) + .get(`/files/contracts/any/${defaultContractChain}?limit=5`); + chai.expect(res1.body.pagination).to.deep.equal({ + currentPage: 0, + hasNextPage: false, + hasPreviousPage: false, + resultsCurrentPage: 5, + resultsPerPage: 5, + totalPages: 1, + totalResults: 5, + }); + }); }); describe("Verify server status endpoint", function () { it("should check server's health", async function () {