Using fully qualified names in library Transformations as identifiers (…

…#1487)

* Add library transformation logic to accomodate the VerA format. Modify the test.

The VerA format uses FQNs as identifiers in the transformationValues object instead of the keccak placeholder.

* Fix build

* Fix not iterating over libraries array. Fix library format conversion when using DB as the backend.

* Fix replacing all library occurances

* Fix build error in lib-sourcify tests

* Change callProtection to have offset 1 and only the changed address as value

Prev. we were starting from offset zero and having values like: 0x73abcd... However the bytecode already has 0x73 when comparing for a call protection. So it's actually an offset 1 byte and just 0xabcd

* Add tests to check a library with <0.5.0 placeholder format. Add fixes

* Handle case when libName is longer than 38 chars in SourcifyDBService conversion from library-map.json

packages/lib-sourcify/src/lib/CheckedContract.ts

@@ -478,11 +478,10 @@ export class CheckedContract {
       metadata: this.metadataRaw,
       // Sometimes the compiler returns empty object (not falsey). Convert it to undefined (falsey).
       immutableReferences:
-        contract.evm?.deployedBytecode?.immutableReferences &&
-        Object.keys(contract.evm.deployedBytecode.immutableReferences).length >
-          0
-          ? contract.evm.deployedBytecode.immutableReferences
-          : {},
+        contract.evm?.deployedBytecode?.immutableReferences || {},
+      creationLinkReferences: contract?.evm?.bytecode?.linkReferences || {},
+      runtimeLinkReferences:
+        contract?.evm?.deployedBytecode?.linkReferences || {},
     };
   }
 

packages/lib-sourcify/src/lib/types.ts

@@ -153,11 +153,25 @@ export interface ImmutableReferences {
     start: number;
   }>;
 }
+
+export interface LinkReferences {
+  [filePath: string]: {
+    [libraryName: string]: [
+      {
+        length: number;
+        start: number;
+      },
+    ];
+  };
+}
+
 export interface RecompilationResult {
   creationBytecode: string;
   runtimeBytecode: string;
   metadata: string;
   immutableReferences: ImmutableReferences;
+  creationLinkReferences: LinkReferences;
+  runtimeLinkReferences: LinkReferences;
 }
 
 export type Transformation = {
@@ -176,7 +190,7 @@ export type Transformation = {
 export const CallProtectionTransformation = (): Transformation => ({
   type: 'replace',
   reason: 'callProtection',
-  offset: 0,
+  offset: 1, // 1 byte is always the PUSH20 opcode 0x73
 });
 
 // TransformationValues only has one ConstructorTransformatino so no id field is needed

packages/lib-sourcify/src/lib/verification.ts

@@ -14,12 +14,18 @@ import {
   CallProtectionTransformation,
   TransformationValues,
   CompiledContractCborAuxdata,
+  LinkReferences,
 } from './types';
 import {
   decode as bytecodeDecode,
   splitAuxdata,
 } from '@ethereum-sourcify/bytecode-utils';
-import { getAddress, getCreateAddress, keccak256 } from 'ethers';
+import {
+  getAddress,
+  getCreateAddress,
+  keccak256,
+  id as keccak256Str,
+} from 'ethers';
 import { hexZeroPad, isHexString } from '@ethersproject/bytes';
 import { BigNumber } from '@ethersproject/bignumber';
 import { defaultAbiCoder as abiCoder, ParamType } from '@ethersproject/abi';
@@ -114,6 +120,7 @@ export async function verifyDeployed(
       runtimeBytecode,
       generateRuntimeCborAuxdataPositions,
       recompiled.immutableReferences,
+      recompiled.runtimeLinkReferences,
     );
     if (match.runtimeMatch === 'partial') {
       logDebug('Matched with deployed bytecode', {
@@ -132,6 +139,7 @@ export async function verifyDeployed(
             runtimeBytecode,
             generateRuntimeCborAuxdataPositions,
             recompiled.immutableReferences,
+            recompiled.runtimeLinkReferences,
           );
         },
         'runtimeMatch',
@@ -169,6 +177,7 @@ export async function verifyDeployed(
         creatorTxHash,
         recompiledMetadata,
         generateCreationCborAuxdataPositions,
+        recompiled.creationLinkReferences,
       );
       if (match.runtimeMatch === 'partial') {
         logDebug('Matched partial with creation tx', {
@@ -190,6 +199,7 @@ export async function verifyDeployed(
               creatorTxHash,
               recompiledMetadata,
               generateCreationCborAuxdataPositions,
+              recompiled.creationLinkReferences,
             );
           },
           'creationMatch',
@@ -413,7 +423,8 @@ export async function matchWithRuntimeBytecode(
   recompiledRuntimeBytecode: string,
   onchainRuntimeBytecode: string,
   generateCborAuxdataPositions: () => Promise<CompiledContractCborAuxdata>,
-  immutableReferences?: ImmutableReferences,
+  immutableReferences: ImmutableReferences,
+  linkReferences: LinkReferences,
 ) {
   // Updating the `match.onchainRuntimeBytecode` here so we are sure to always update it
   match.onchainRuntimeBytecode = onchainRuntimeBytecode;
@@ -436,30 +447,21 @@ export async function matchWithRuntimeBytecode(
   );
 
   // Replace the library placeholders in the recompiled bytecode with values from the deployed bytecode
-  const { replaced, libraryMap } = addLibraryAddresses(
+  const { replaced, libraryMap } = handleLibraries(
     recompiledRuntimeBytecode,
     onchainRuntimeBytecode,
+    linkReferences,
     match.runtimeTransformations,
+    match.runtimeTransformationValues,
   );
   recompiledRuntimeBytecode = replaced;
-  if (Object.keys(libraryMap).length > 0) {
-    // Prepend the library addresses with "0x", this is the format for the DB. FS library-map is without "0x"
-    match.runtimeTransformationValues.libraries = Object.keys(
-      libraryMap,
-    ).reduce((libMap: any, lib) => {
-      libMap[lib] = `0x${libraryMap[lib]}`;
-      return libMap;
-    }, {});
-  }
 
-  if (immutableReferences) {
-    onchainRuntimeBytecode = replaceImmutableReferences(
-      immutableReferences,
-      onchainRuntimeBytecode,
-      match.runtimeTransformations,
-      match.runtimeTransformationValues,
-    );
-  }
+  onchainRuntimeBytecode = replaceImmutableReferences(
+    immutableReferences,
+    onchainRuntimeBytecode,
+    match.runtimeTransformations,
+    match.runtimeTransformationValues,
+  );
 
   // We call generateCborAuxdataPositions before returning because we always need
   // to fill cborAuxdata in creation_code_artifacts and runtime_code_artifacts
@@ -525,6 +527,7 @@ export async function matchWithCreationTx(
   creatorTxHash: string,
   recompiledMetadata: Metadata,
   generateCborAuxdataPositions: () => Promise<CompiledContractCborAuxdata>,
+  linkReferences: LinkReferences,
 ) {
   if (recompiledCreationBytecode === '0x') {
     match.creationMatch = null;
@@ -569,24 +572,17 @@ export async function matchWithCreationTx(
     match.creationTransformationValues = {};
   }
 
-  // The reason why this uses `startsWith` instead of `===` is that creationTxData may contain constructor arguments at the end part.
   // Replace the library placeholders in the recompiled bytecode with values from the deployed bytecode
-  const { replaced, libraryMap } = addLibraryAddresses(
+  const { replaced, libraryMap } = handleLibraries(
     recompiledCreationBytecode,
     match.onchainCreationBytecode,
+    linkReferences,
     match.creationTransformations,
+    match.creationTransformationValues,
   );
   recompiledCreationBytecode = replaced;
-  if (Object.keys(libraryMap).length > 0) {
-    // Prepend the library addresses with "0x", this is the format for the DB. FS library-map is without "0x"
-    match.creationTransformationValues.libraries = Object.keys(
-      libraryMap,
-    ).reduce((libMap: any, lib) => {
-      libMap[lib] = `0x${libraryMap[lib]}`;
-      return libMap;
-    }, {});
-  }
 
+  // The reason why this uses `startsWith` instead of `===` is that creationTxData may contain constructor arguments at the end part.
   if (match.onchainCreationBytecode.startsWith(recompiledCreationBytecode)) {
     // if the bytecode doesn't end with metadata then "partial" match
     if (endsWithMetadataHash(recompiledCreationBytecode)) {
@@ -702,34 +698,62 @@ export async function matchWithCreationTx(
   }
 }
 
-export function addLibraryAddresses(
+export function handleLibraries(
   template: string,
   real: string,
+  linkReferences: LinkReferences,
   transformationsArray: Transformation[],
+  transformationValues: TransformationValues,
 ): {
   replaced: string;
   libraryMap: StringMap;
 } {
-  const PLACEHOLDER_START = '__';
-  const PLACEHOLDER_LENGTH = 40;
-
   const libraryMap: StringMap = {};
+  for (const file in linkReferences) {
+    for (const lib in linkReferences[file]) {
+      for (const linkRefObj of linkReferences[file][lib]) {
+        const fqn = `${file}:${lib}`; // Fully Qualified (FQ) name
+
+        const { start, length } = linkRefObj;
+        const strStart = start * 2 + 2; // Each byte 2 chars and +2 for 0x
+        const strLength = length * 2;
+        const placeholder = template.slice(strStart, strStart + strLength);
+
+        // slice(2) removes 0x
+        const calculatedPlaceholder =
+          '__$' + keccak256Str(fqn).slice(2).slice(0, 34) + '$__';
+        // Placeholder format was different pre v0.5.0 https://docs.soliditylang.org/en/v0.4.26/contracts.html#libraries
+        const trimmedFQN = fqn.slice(0, 36); // in case the fqn is too long
+        const calculatedPreV050Placeholder = '__' + trimmedFQN.padEnd(38, '_');
+
+        if (
+          !(
+            placeholder === calculatedPlaceholder ||
+            placeholder === calculatedPreV050Placeholder
+          )
+        )
+          throw new Error(
+            `Library placeholder mismatch: ${placeholder} vs ${calculatedPlaceholder} or ${calculatedPreV050Placeholder}`,
+          );
 
-  let index = template.indexOf(PLACEHOLDER_START);
-  while (index !== -1) {
-    const placeholder = template.slice(index, index + PLACEHOLDER_LENGTH);
-    const address = real.slice(index, index + PLACEHOLDER_LENGTH);
-    libraryMap[placeholder] = address;
+        const address = real.slice(strStart, strStart + strLength);
+        libraryMap[placeholder] = address;
 
-    // Replace regex with simple string replacement
-    template = template.split(placeholder).join(address);
+        // Replace the specific occurrence of the placeholder
+        template =
+          template.slice(0, strStart) +
+          address +
+          template.slice(strStart + strLength);
 
-    transformationsArray.push(
-      // we divide by 2 because we store the length in bytes (without 0x)
-      LibraryTransformation((index - 2) / 2, placeholder),
-    );
+        transformationsArray.push(LibraryTransformation(start, fqn));
 
-    index = template.indexOf(PLACEHOLDER_START);
+        if (!transformationValues.libraries) {
+          transformationValues.libraries = {};
+        }
+        // Prepend the library addresses with "0x", this is the format for the DB. FS library-map is without "0x"
+        transformationValues.libraries[fqn] = '0x' + address;
+      }
+    }
   }
 
   return {
@@ -750,8 +774,9 @@ export function checkCallProtectionAndReplaceAddress(
 
   if (template.startsWith(callProtection)) {
     const replacedCallProtection = real.slice(0, 0 + callProtection.length);
+    const callProtectionAddress = replacedCallProtection.slice(4); // remove 0x73
     transformationsArray.push(CallProtectionTransformation());
-    transformationValues.callProtection = replacedCallProtection;
+    transformationValues.callProtection = '0x' + callProtectionAddress;
 
     return replacedCallProtection + template.substring(callProtection.length);
   }

packages/lib-sourcify/test/verification.spec.ts

@@ -711,6 +711,7 @@ describe('lib-sourcify tests', () => {
           wrongCreatorTxHash,
           recompiledMetadata,
           generateCreationCborAuxdataPositions,
+          recompiled.creationLinkReferences,
         );
       } catch (err) {
         if (err instanceof Error) {
@@ -772,6 +773,7 @@ describe('lib-sourcify tests', () => {
         txHash,
         recompiledMetadata,
         generateCreationCborAuxdataPositions,
+        recompiled.creationLinkReferences,
       );
       expectMatch(match, null, contractAddress, undefined); // status is null
     });
@@ -817,6 +819,7 @@ describe('lib-sourcify tests', () => {
         creatorTxHash,
         recompiledMetadata,
         generateCreationCborAuxdataPositions,
+        recompiled.creationLinkReferences,
       );
       expectMatch(match, null, contractAddress, undefined); // status is null
     });
@@ -854,6 +857,7 @@ describe('lib-sourcify tests', () => {
         creatorTxHash,
         recompiledMetadata,
         generateCreationCborAuxdataPositions,
+        recompiled.creationLinkReferences,
       );
       try {
         expect(match.creationMatch).to.equal('perfect');
@@ -897,6 +901,7 @@ describe('lib-sourcify tests', () => {
         creatorTxHash,
         recompiledMetadata,
         generateCreationCborAuxdataPositions,
+        recompiled.creationLinkReferences,
       );
       try {
         expect(match.creationMatch).to.equal('partial');

services/server/package.json

@@ -70,6 +70,7 @@
     "json-refs": "3.0.15",
     "memorystore": "1.6.7",
     "pg": "8.12.0",
+    "semver": "7.6.2",
     "serve-index": "1.9.1",
     "solc": "0.8.26",
     "swagger-ui-express": "5.0.1",

services/server/src/server/services/storageServices/SourcifyDatabaseService.ts

@@ -2,6 +2,7 @@ import {
   Match,
   CheckedContract,
   Status,
+  StringMap,
 } from "@ethereum-sourcify/lib-sourcify";
 import logger from "../../../common/logger";
 import * as Database from "../utils/database-util";
@@ -22,11 +23,12 @@ import {
   PaginatedContractData,
 } from "../../types";
 import config from "config";
-import Path from "path";
+import Path, { format } from "path";
 import { getFileRelativePath } from "../utils/util";
-import { getAddress } from "ethers";
+import { getAddress, id as keccak256Str } from "ethers";
 import { BadRequestError } from "../../../common/errors";
 import { RWStorageIdentifiers } from "./identifiers";
+import semver from "semver";
 
 const MAX_RETURNED_CONTRACTS_BY_GETCONTRACTS = 200;
 
@@ -307,9 +309,25 @@ export class SourcifyDatabaseService
       sourcifyMatch?.runtime_values?.libraries &&
       Object.keys(sourcifyMatch.runtime_values.libraries).length > 0
     ) {
-      files["library-map.json"] = JSON.stringify(
+      // Must convert "contracts/file.sol:MyLib" FQN format to the placeholder format __$keccak256(file.sol:MyLib)$___ or  __MyLib__________
+      const formattedLibraries: StringMap = {};
+      for (const [key, value] of Object.entries(
         sourcifyMatch.runtime_values.libraries,
-      );
+      )) {
+        let formattedKey;
+        // Solidity >= 0.5.0 is __$keccak256(file.sol:MyLib)$__ (total 40 characters)
+        if (semver.gte(sourcifyMatch.metadata.compiler.version, "0.5.0")) {
+          formattedKey =
+            "__$" + keccak256Str(key).slice(2).slice(0, 34) + "$__";
+        } else {
+          // Solidity < 0.5.0 is __MyLib__________ (total 40 characters)
+          const libName = key.split(":")[1];
+          const trimmedLibName = libName.slice(0, 36); // in case it's longer
+          formattedKey = "__" + trimmedLibName.padEnd(38, "_");
+        }
+        formattedLibraries[formattedKey] = value;
+      }
+      files["library-map.json"] = JSON.stringify(formattedLibraries);
     }
 
     if (