Fix bytecode hash calculation

packages/lib-sourcify/src/lib/SourcifyChain.ts

@@ -369,15 +369,18 @@
     );
   };
 
-  getContractCreationBytecode = async (
+  getContractCreationBytecodeAndReceipt = async (
     address: string,
     transactionHash: string,
     creatorTx?: TransactionResponse
-  ): Promise<string> => {
+  ): Promise<{
+    creationBytecode: string;
+    txReceipt: TransactionReceipt;
+  }> => {
     const txReceipt = await this.getTxReceipt(transactionHash);
     if (!creatorTx) creatorTx = await this.getTx(transactionHash);
-    let creationBytecode = '';
 
+    let creationBytecode;
     // Non null txreceipt.contractAddress means that the contract was created with an EOA
     if (txReceipt.contractAddress !== null) {
       if (txReceipt.contractAddress !== address) {
@@ -416,6 +419,13 @@
       }
     }
 
-    return creationBytecode;
+    if (!creationBytecode) {
+      throw new Error('Cannot get creation bytecode');
+    }
+
+    return {
+      creationBytecode,
+      txReceipt,
+    };
   };
 }

packages/lib-sourcify/src/lib/verification.ts

@@ -533,36 +533,35 @@ export async function matchWithCreationTx(
     return;
   }
 
-  const creatorTx = await sourcifyChain.getTx(creatorTxHash);
-  let onchainCreationBytecode = '';
+  // Call rpc to find creatorTx, txReceipt and onchainContractCreationBytecode
+  // return null creationMatch if fail
+  let creatorTx;
   try {
-    onchainCreationBytecode =
-      (await sourcifyChain.getContractCreationBytecode(
+    creatorTx = await sourcifyChain.getTx(creatorTxHash);
+    match.creatorTxHash = creatorTxHash;
+    match.blockNumber = creatorTx.blockNumber;
+    match.deployer = creatorTx.from;
+
+    const { creationBytecode, txReceipt } =
+      await sourcifyChain.getContractCreationBytecodeAndReceipt(
         address,
         creatorTxHash,
         creatorTx
-      )) || '';
+      );
+    match.onchainCreationBytecode = creationBytecode;
+    match.txIndex = txReceipt.index;
   } catch (e: any) {
     logWarn('Failed to fetch creation bytecode', {
       address,
       txHash: creatorTxHash,
       chainId: sourcifyChain.chainId.toString(),
+      error: e,
     });
     match.creationMatch = null;
     match.message = `Failed to match with creation bytecode: couldn't get the creation bytecode.`;
     return;
   }
 
-  // txIndex is available only in the receipt
-  const txReceipt = await sourcifyChain.getTxReceipt(creatorTxHash);
-  match.txIndex = txReceipt.index;
-
-  match.creatorTxHash = creatorTxHash;
-  match.blockNumber = creatorTx.blockNumber;
-  match.deployer = creatorTx.from;
-
-  match.onchainCreationBytecode = onchainCreationBytecode;
-
   // Initialize the transformations array if undefined
   if (match.creationTransformations === undefined) {
     match.creationTransformations = [];
@@ -575,7 +574,7 @@ export async function matchWithCreationTx(
   // Replace the library placeholders in the recompiled bytecode with values from the deployed bytecode
   const { replaced, libraryMap } = addLibraryAddresses(
     recompiledCreationBytecode,
-    onchainCreationBytecode,
+    match.onchainCreationBytecode,
     match.creationTransformations
   );
   recompiledCreationBytecode = replaced;
@@ -589,7 +588,7 @@ export async function matchWithCreationTx(
     }, {});
   }
 
-  if (onchainCreationBytecode.startsWith(recompiledCreationBytecode)) {
+  if (match.onchainCreationBytecode.startsWith(recompiledCreationBytecode)) {
     // if the bytecode doesn't end with metadata then "partial" match
     if (endsWithMetadataHash(recompiledCreationBytecode)) {
       match.creationMatch = 'perfect';
@@ -626,7 +625,7 @@ export async function matchWithCreationTx(
         creationTransformationsValuesCborAuxdata,
     } = normalizeBytecodesAuxdata(
       recompiledCreationBytecode,
-      onchainCreationBytecode,
+      match.onchainCreationBytecode,
       cborAuxdataPositions
     )!;
 
@@ -648,7 +647,7 @@ export async function matchWithCreationTx(
   if (match.creationMatch) {
     const abiEncodedConstructorArguments =
       extractAbiEncodedConstructorArguments(
-        onchainCreationBytecode,
+        match.onchainCreationBytecode,
         recompiledCreationBytecode
       );
     const constructorAbiParamInputs = (

services/database/scripts.mjs

@@ -105,7 +105,8 @@ program
       });
 
       for (const contract of contractsSorted) {
-        await databasePool.query(
+        await executeQueryWithRetry(
+          databasePool,
           `INSERT INTO sourcify_sync (chain_id, address, match_type, created_at) VALUES ($1, $2, $3, $4) ON CONFLICT (chain_id, address) DO NOTHING`,
           [
             contract.chainId,
@@ -151,7 +152,11 @@ program
     "Path to repository v1 contracts folder (e.g. /Users/user/sourcify/repository/contracts)"
   )
   .option(
-    "-c, --chainsExceptions [items]",
+    "-c, --chains [items]",
+    "List of chains to sync separated by comma (e.g. 1,5,...)"
+  )
+  .option(
+    "-ce, --chainsExceptions [items]",
     "List of chains exceptions separated by comma (e.g. 1,5,...)"
   )
   .option("-sf, --start-from [number]", "Start from a specific timestamp (ms)")
@@ -176,15 +181,24 @@ program
     // Get chains from database
     let chains = [];
     const query = "SELECT DISTINCT chain_id FROM sourcify_sync";
-    const chainsResult = await databasePool.query(query);
+    const chainsResult = await executeQueryWithRetry(databasePool, query);
     if (chainsResult.rowCount > 0) {
       chains = chainsResult.rows.map(({ chain_id }) => chain_id);
     }
 
+    // Specify which chain to sync
+    if (options.chains) {
+      chains = chains.filter((chain) =>
+        options.chains.split(",").includes(`${chain}`)
+      );
+    }
+
     // Remove exceptions using --chainsException and 0
-    chains = chains.filter(
-      (chain) => !options.chainsExceptions?.split(",").includes(`${chain}`)
-    );
+    if (options.chainsExceptions) {
+      chains = chains.filter(
+        (chain) => !options.chainsExceptions.split(",").includes(`${chain}`)
+      );
+    }
 
     let monitoring = {
       totalSynced: 0,
@@ -315,7 +329,8 @@ const processContract = async (
       if (request.status === 200) {
         const response = await request.json();
         if (response.result[0].status !== null) {
-          await databasePool.query(
+          await executeQueryWithRetry(
+            databasePool,
             `
           UPDATE sourcify_sync
           SET 
@@ -368,7 +383,7 @@ const fetchNextContract = async (databasePool, options, chainId) => {
       ORDER BY id ASC
       LIMIT 1
     `;
-    const contractResult = await databasePool.query(query, [
+    const contractResult = await executeQueryWithRetry(databasePool, query, [
       options.startFrom ? options.startFrom : 0,
       chainId,
     ]);
@@ -388,4 +403,29 @@ function isNumber(value) {
   return !isNaN(parseFloat(value)) && isFinite(value);
 }
 
+const executeQueryWithRetry = async (
+  databasePool,
+  query,
+  params,
+  maxRetries = 5,
+  delay = 5000
+) => {
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const result = await databasePool.query(query, params);
+      return result;
+    } catch (error) {
+      if (error.code === "ECONNREFUSED" || error.code === "ETIMEDOUT") {
+        console.error(
+          `Database connection error. Retrying attempt ${attempt + 1}...`
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      } else {
+        throw error;
+      }
+    }
+  }
+  throw new Error("Max retries reached. Could not connect to the database.");
+};
+
 program.parse(process.argv);

services/server/src/server/services/StorageService.ts

@@ -42,13 +42,15 @@
   repositoryV1ServiceOptions: RepositoryV1ServiceOptions;
   repositoryV2ServiceOptions: RepositoryV2ServiceOptions;
   sourcifyDatabaseServiceOptions?: SourcifyDatabaseServiceOptions;
+  sourcifyFixedDatabaseServiceOptions?: SourcifyDatabaseServiceOptions;
   allianceDatabaseServiceOptions?: AllianceDatabaseServiceOptions;
 }
 
 export class StorageService {
   repositoryV1: RepositoryV1Service;
   repositoryV2?: RepositoryV2Service;
   sourcifyDatabase?: SourcifyDatabaseService;
+  sourcifyFixedDatabase?: SourcifyDatabaseService;
   allianceDatabase?: AllianceDatabaseService;
 
   constructor(options: StorageServiceOptions) {
@@ -86,6 +88,23 @@
       );
     }
 
+    // SourcifyFixedDatabase
+    if (
+      options.sourcifyFixedDatabaseServiceOptions?.postgres?.host &&
+      options.sourcifyFixedDatabaseServiceOptions?.postgres?.database &&
+      options.sourcifyFixedDatabaseServiceOptions?.postgres?.user &&
+      options.sourcifyFixedDatabaseServiceOptions?.postgres?.password
+    ) {
+      this.sourcifyFixedDatabase = new SourcifyDatabaseService(
+        options.sourcifyFixedDatabaseServiceOptions
+      );
+    } else {
+      logger.warn(
+        "Won't use SourcifyFixedDatabase, options not complete",
+        options.sourcifyFixedDatabaseServiceOptions
+      );
+    }
+
     // AllianceDatabase
     if (
       options.allianceDatabaseServiceOptions?.googleCloudSql ||
@@ -431,6 +450,17 @@
       })
     );
 
+    if (this.sourcifyFixedDatabase) {
+      promises.push(
+        this.sourcifyFixedDatabase.storeMatch(contract, match).catch((e) => {
+          logger.error("Error storing to SourcifyFixedDatabase: ", {
+            error: e,
+          });
+          throw e;
+        })
+      );
+    }
+
     promises.push(
       this.repositoryV2.storeMatch(contract, match).catch((e) => {
         logger.error("Error storing to RepositoryV2: ", {

services/server/src/server/services/services.ts

@@ -45,6 +45,15 @@ export class Services {
           port: parseInt(process.env.SOURCIFY_POSTGRES_PORT || "5432"),
         },
       },
+      sourcifyFixedDatabaseServiceOptions: {
+        postgres: {
+          host: process.env.SOURCIFY_FIXED_POSTGRES_HOST as string,
+          database: process.env.SOURCIFY_FIXED_POSTGRES_DB as string,
+          user: process.env.SOURCIFY_FIXED_POSTGRES_USER as string,
+          password: process.env.SOURCIFY_FIXED_POSTGRES_PASSWORD as string,
+          port: parseInt(process.env.SOURCIFY_FIXED_POSTGRES_PORT || "5432"),
+        },
+      },
       allianceDatabaseServiceOptions: {
         postgres: {
           host: process.env.ALLIANCE_POSTGRES_HOST as string,

services/server/src/server/services/storageServices/AbstractDatabaseService.ts

@@ -121,10 +121,12 @@ export default abstract class AbstractDatabaseService {
 
     return {
       bytecodeHashes: {
-        recompiledCreation: bytesFromString(keccak256OnchainCreationBytecode),
-        recompiledRuntime: bytesFromString(keccak256OnchainRuntimeBytecode)!,
-        onchainCreation: bytesFromString(keccak256RecompiledCreationBytecode),
-        onchainRuntime: bytesFromString(keccak256RecompiledRuntimeBytecode)!,
+        recompiledCreation: bytesFromString(
+          keccak256RecompiledCreationBytecode
+        ),
+        recompiledRuntime: bytesFromString(keccak256RecompiledRuntimeBytecode)!,
+        onchainCreation: bytesFromString(keccak256OnchainCreationBytecode),
+        onchainRuntime: bytesFromString(keccak256OnchainRuntimeBytecode)!,
       },
       compiledContract: {
         language,
@@ -212,7 +214,8 @@ export default abstract class AbstractDatabaseService {
           compilation_artifacts:
             databaseColumns.compiledContract.compilation_artifacts!,
           sources: recompiledContract.solidity,
-          compiler_settings: recompiledContract.metadata.settings,
+          compiler_settings:
+            Database.prepareCompilerSettings(recompiledContract),
           creation_code_hash: databaseColumns.bytecodeHashes.recompiledCreation,
           runtime_code_hash: databaseColumns.bytecodeHashes.recompiledRuntime,
           creation_code_artifacts:

services/server/src/server/services/utils/database-util.ts

@@ -2,6 +2,7 @@ import {
   CheckedContract,
   CompiledContractCborAuxdata,
   ImmutableReferences,
+  Libraries,
   Match,
   Transformation,
   TransformationValues,
@@ -534,6 +535,40 @@ export function normalizeRecompiledBytecodes(
   }
 }
 
+export function prepareCompilerSettings(recompiledContract: CheckedContract) {
+  // The metadata.settings contains recompiledContract that is not a field of compiler input
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { compilationTarget, ...restSettings } =
+    recompiledContract.metadata.settings;
+
+  const metadataLibraries =
+    recompiledContract.metadata.settings?.libraries || {};
+  restSettings.libraries = Object.keys(metadataLibraries || {}).reduce(
+    (libraries, libraryKey) => {
+      // Before Solidity v0.7.5: { "ERC20": "0x..."}
+      if (!libraryKey.includes(":")) {
+        if (!libraries[""]) {
+          libraries[""] = {};
+        }
+        // try using the global method, available for pre 0.7.5 versions
+        libraries[""][libraryKey] = metadataLibraries[libraryKey];
+        return libraries;
+      }
+
+      // After Solidity v0.7.5: { "ERC20.sol:ERC20": "0x..."}
+      const [contractPath, contractName] = libraryKey.split(":");
+      if (!libraries[contractPath]) {
+        libraries[contractPath] = {};
+      }
+      libraries[contractPath][contractName] = metadataLibraries[libraryKey];
+      return libraries;
+    },
+    {} as Libraries
+  ) as any;
+
+  return restSettings;
+}
+
 export async function countSourcifyMatchAddresses(pool: Pool, chain: number) {
   return await pool.query(
     `

services/server/test/helpers/helpers.ts

@@ -65,7 +65,16 @@ export async function deployFromAbiAndBytecodeForCreatorTxHash(
     `Deployed contract at ${contractAddress} with tx ${creationTx.hash}`
   );
 
-  return { contractAddress, txHash: creationTx.hash };
+  const txReceipt = await signer.provider.getTransactionReceipt(
+    creationTx.hash
+  );
+
+  return {
+    contractAddress,
+    txHash: creationTx.hash,
+    blockNumber: creationTx.blockNumber,
+    txIndex: txReceipt?.index,
+  };
 }
 
 export async function deployAndVerifyContract(