Use Sourcify database as default StorageService

services/server/src/config/local-test.js

@@ -1,6 +1,9 @@
 module.exports = {
   repositoryV1: {
-    path: "/tmp/repository-test/",
+    path: "/tmp/repositoryV1-test/",
+  },
+  repositoryV2: {
+    path: "/tmp/repositoryV2-test/",
   },
   session: {
     storeType: "database",

services/server/src/openapi.yaml

@@ -34,6 +34,10 @@ paths:
     $ref: "server/controllers/repository/get-file-static.stateless.paths.yaml#/paths/~1repository~1contracts~1{full_match | partial_match}~1{chain}~1{address}~1{filePath}"
   /files/contracts/{chain}:
     $ref: "server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml#/paths/~1files~1contracts~1{chain}"
+  /files/contracts/any/{chain}:
+    $ref: "server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml#/paths/~1files~1contracts~1any~1{chain}"
+  /files/contracts/full/{chain}:
+    $ref: "server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml#/paths/~1files~1contracts~1full~1{chain}"
   /files/tree/any/{chain}/{address}:
     $ref: "server/controllers/repository/get-file-tree-all.stateless.paths.yaml#/paths/~1files~1tree~1any~1{chain}~1{address}"
   /files/tree/{chain}/{address}:
@@ -75,10 +79,10 @@ paths:
                   enum: [error, warn, info, debug, silly]
             examples:
               change to debug:
-                value: 
+                value:
                   level: "debug"
               change to info:
-                value: 
+                value:
                   level: "info"
       responses:
         "200":

services/server/src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml

@@ -3,8 +3,8 @@ openapi: "3.0.0"
 paths:
   /files/contracts/{chain}:
     get:
-      summary: Get all contract addresses verified on a chain (full or partial match)
-      description: Returns all verified contracts from the repository for the desired chain. Searches for full and partial matches.
+      summary: (Deprecated) Get the first 200 contract addresses verified on a chain (full or partial match)
+      description: Returns the first 200 verified contracts from the repository for the desired chain. Searches for full and partial matches. This endpoint is deprecated and only retuns 200 addresses. Use `/files/contracts/any/` and `/files/contracts/full/` instead
       tags:
         - Repository
       parameters:
@@ -50,3 +50,13 @@ paths:
                   error:
                     type: string
                     example: "Contracts have not been found!"
+        "400":
+          description: Cannot use this API if there are more than 200 contracts
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  error:
+                    type: string
+                    example: "Cannot fetch more than 200 contracts (1521 full matches, 3453 partial matches), please use /contracts/{full|any}/4 with pagination"

services/server/src/server/controllers/repository/get-contract-addresses-paginated-all.stateless.paths.yaml

@@ -0,0 +1,62 @@
+openapi: "3.0.0"
+
+paths:
+  /files/contracts/any/{chain}:
+    get:
+      summary: Get the contract addresses verified on a chain (full or partial match)
+      description: Returns the verified contracts from the repository for the desired chain. Searches for full and partial matches. API is paginated. Limit must be a number between 1 and 200.
+      tags:
+        - Repository
+      parameters:
+        - name: chain
+          in: path
+          required: true
+          schema:
+            type: string
+            format: sourcify-chainId
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: number
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: number
+            minimum: 1
+            maximum: 200
+      responses:
+        "200":
+          description: Chain is available as a full match or partial match in the repository
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  results:
+                    type: array
+                    items:
+                      type: string
+                    example:
+                      [
+                        "0x1fE5d745beABA808AAdF52057Dd7AAA47b42cFD0",
+                        "0xE9c31091868d68598Ac881738D159A63532d12f9",
+                      ]
+                  pagination:
+                    type: object
+                    properties:
+                      currentPage:
+                        type: number
+                      totalPages:
+                        type: number
+                      resultsPerPage:
+                        type: number
+                      resultsCurrentPage:
+                        type: number
+                      totalResults:
+                        type: number
+                      hasNextPage:
+                        type: boolean
+                      hasPreviousPage:
+                        type: boolean

services/server/src/server/controllers/repository/get-contract-addresses-paginated-full.stateless.paths.yaml

@@ -0,0 +1,62 @@
+openapi: "3.0.0"
+
+paths:
+  /files/contracts/full/{chain}:
+    get:
+      summary: Get the contract addresses perfectly verified on a chain
+      description: Returns the perfectly verified contracts from the repository for the desired chain. API is paginated. Limit must be a number between 1 and 200.
+      tags:
+        - Repository
+      parameters:
+        - name: chain
+          in: path
+          required: true
+          schema:
+            type: string
+            format: sourcify-chainId
+        - name: page
+          in: query
+          required: false
+          schema:
+            type: number
+        - name: limit
+          in: query
+          required: false
+          schema:
+            type: number
+            minimum: 1
+            maximum: 200
+      responses:
+        "200":
+          description: Chain is available as a full match in the repository
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  results:
+                    type: array
+                    items:
+                      type: string
+                    example:
+                      [
+                        "0x1fE5d745beABA808AAdF52057Dd7AAA47b42cFD0",
+                        "0xE9c31091868d68598Ac881738D159A63532d12f9",
+                      ]
+                  pagination:
+                    type: object
+                    properties:
+                      currentPage:
+                        type: number
+                      totalPages:
+                        type: number
+                      resultsPerPage:
+                        type: number
+                      resultsCurrentPage:
+                        type: number
+                      totalResults:
+                        type: number
+                      hasNextPage:
+                        type: boolean
+                      hasPreviousPage:
+                        type: boolean

services/server/src/server/controllers/repository/repository.handlers.ts

@@ -1,6 +1,11 @@
 import { Response, Request, NextFunction } from "express";
 import { StatusCodes } from "http-status-codes";
-import { ContractData, FilesInfo, MatchLevel } from "../../types";
+import {
+  ContractData,
+  FilesInfo,
+  MatchLevel,
+  PaginatedContractData,
+} from "../../types";
 import { NotFoundError } from "../../../common/errors";
 import { Match } from "@ethereum-sourcify/lib-sourcify";
 import { services } from "../../services/services";
@@ -12,6 +17,12 @@ type RetrieveMethod = (
   match: MatchLevel
 ) => Promise<FilesInfo<any>>;
 type ConractRetrieveMethod = (chain: string) => Promise<ContractData>;
+type PaginatedConractRetrieveMethod = (
+  chain: string,
+  match: MatchLevel,
+  page: number,
+  limit: number
+) => Promise<PaginatedContractData>;
 
 export function createEndpoint(
   retrieveMethod: RetrieveMethod,
@@ -53,6 +64,26 @@ export function createContractEndpoint(
   };
 }
 
+export function createPaginatedContractEndpoint(
+  paginatedContractRetrieveMethod: PaginatedConractRetrieveMethod,
+  match: MatchLevel
+) {
+  return async (req: Request, res: Response, next: NextFunction) => {
+    let retrieved: PaginatedContractData;
+    try {
+      retrieved = await paginatedContractRetrieveMethod(
+        req.params.chain,
+        match,
+        parseInt((req.query.page as string) || "0"),
+        parseInt((req.query.limit as string) || "200")
+      );
+    } catch (err: any) {
+      return next(new NotFoundError(err.message));
+    }
+    return res.status(StatusCodes.OK).json(retrieved);
+  };
+}
+
 export async function checkAllByChainAndAddressEndpoint(
   req: any,
   res: Response
@@ -96,6 +127,29 @@ export async function checkAllByChainAndAddressEndpoint(
   res.send(resultArray);
 }
 
+export async function getMetadataEndpoint(req: any, res: Response) {
+  const { match, chain, address } = req.params;
+  const file = await services.storage.getMetadata(chain, address, match);
+  if (file === false) {
+    res.status(404).send();
+  }
+  res.json(JSON.parse(file as string));
+}
+
+export async function getFileEndpoint(req: any, res: Response) {
+  const { match, chain, address } = req.params;
+  const file = await services.storage.getFile(
+    chain,
+    address,
+    match,
+    req.params[0]
+  );
+  if (!file) {
+    res.status(404).send();
+  }
+  res.send(file);
+}
+
 export async function checkByChainAndAddressesEnpoint(req: any, res: Response) {
   const map: Map<string, any> = new Map();
   const addresses = req.query.addresses.split(",");

services/server/src/server/controllers/repository/repository.routes.ts

@@ -6,6 +6,9 @@ import {
   createContractEndpoint,
   checkAllByChainAndAddressEndpoint,
   checkByChainAndAddressesEnpoint,
+  getFileEndpoint,
+  getMetadataEndpoint,
+  createPaginatedContractEndpoint,
 } from "./repository.handlers";
 import { safeHandler } from "../controllers.common";
 
@@ -16,39 +19,42 @@ const router: Router = Router();
 [
   {
     prefix: "/tree/any",
-    method: createEndpoint(
-      services.storage.repositoryV1.getTree,
-      "any_match",
-      true
-    ),
+    method: createEndpoint(services.storage.getTree, "any_match", true),
   },
   {
     prefix: "/any",
-    method: createEndpoint(
-      services.storage.repositoryV1.getContent,
-      "any_match",
-      true
-    ),
+    method: createEndpoint(services.storage.getContent, "any_match", true),
   },
   {
     prefix: "/tree",
-    method: createEndpoint(services.storage.repositoryV1.getTree, "full_match"),
+    method: createEndpoint(services.storage.getTree, "full_match"),
   },
   {
     prefix: "/contracts",
-    method: createContractEndpoint(services.storage.repositoryV1.getContracts),
+    method: createContractEndpoint(services.storage.getContracts),
   },
   {
-    prefix: "",
-    method: createEndpoint(
-      services.storage.repositoryV1.getContent,
+    prefix: "/contracts/full",
+    method: createPaginatedContractEndpoint(
+      services.storage.getPaginatedContracts,
       "full_match"
     ),
   },
+  {
+    prefix: "/contracts/any",
+    method: createPaginatedContractEndpoint(
+      services.storage.getPaginatedContracts,
+      "any_match"
+    ),
+  },
+  {
+    prefix: "",
+    method: createEndpoint(services.storage.getContent, "full_match"),
+  },
 ].forEach((pair) => {
   router
     .route(
-      pair.prefix != "/contracts"
+      !pair.prefix.startsWith("/contracts")
         ? REPOSITORY_CONTROLLER_PREFIX + pair.prefix + "/:chain/:address"
         : REPOSITORY_CONTROLLER_PREFIX + pair.prefix + "/:chain"
     )
@@ -60,6 +66,22 @@ router
   .route("/check-all-by-addresses")
   .get(safeHandler(checkAllByChainAndAddressEndpoint));
 
+/**
+ * The following two routes are the replacement for the removed static file route that exposed RepositoryV1
+ * The function getFileEndpoint get the sources from compiled_contracts.sources
+ * We need both of these routes because compiled_contracts.sources doesn't contain the metadata file
+ */
+
+// This route covers the metadata.json files, fetching them from RepositoryV2
+router
+  .route("/repository/contracts/:match/:chain/:address/metadata.json")
+  .get(safeHandler(getMetadataEndpoint));
+
+// This route covers the the sources files, fetching them from SourcifyDatabase.compiled_contracts.sources
+router
+  .route("/repository/contracts/:match/:chain/:address/sources/*")
+  .get(safeHandler(getFileEndpoint));
+
 router
   .route("/check-by-addresses")
   .get(safeHandler(checkByChainAndAddressesEnpoint));

services/server/src/server/server.ts

@@ -266,12 +266,6 @@ export class Server {
     // Enable session only for session endpoints
     this.app.use("/*session*", getSessionMiddleware());
 
-    this.app.use(
-      "/repository",
-      express.static(this.repository),
-      serveIndex(this.repository, { icons: true })
-    );
-
     this.app.use("/", routes);
     this.app.use(genericErrorHandler);
   }