Release (#1136)

- [prevent running create2 tests for external
PRs](ae2903c)
- For solcjs compiler, use Nodejs Worker for earlier versions
[#1130](1882079)
- For platform binaries, Async compiler process
[#1135](239ce7a)
- New chains:
  - Edgeware EdgeEVM (2021)
  - Filecoin Testnet (314159)
  - Fantom Opera Mainnet (250)
  - Arbitrum Nova (42170)
  - Taiko L3 (167006)
  - Beam Testnet (13337)
- Updated chain: Stratos Testnet (2047)

.vscode/launch.json

@@ -39,6 +39,15 @@
       "console": "integratedTerminal",
       "outputCapture": "std",
     },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Docker: Attach to Server",
+      "remoteRoot": "/home/app",
+      "localRoot": "${workspaceFolder}",
+      "port": 9229,
+      "address": "localhost",
+    },
     {
       // Make sure the UI webserver is already running with `npm start`
       "name": "UI",

environments/server.debug.linux-amd64.yaml

@@ -0,0 +1,44 @@
+# Runs the server in a linux-amd64 container with debug port exposed
+# Used in debugging solidity linux-amd64 binary executables  
+version: "3.7"
+x-project-repository-mount: &project-repository-mount
+  type: bind
+  source: $REPOSITORY_PATH
+
+x-project-base: &project-base
+  env_file:
+    - .env
+  restart: always
+  networks:
+    - source-verify
+
+networks:
+  source-verify:
+
+services:
+  server:
+    <<: *project-base
+    # image: ethereum/source-verify:server-${TAG}
+    build:
+      context: ../
+      dockerfile: src/Dockerfile.server.debug
+    container_name: server-${TAG}
+    platform: linux/amd64
+    ports:
+      - "${SERVER_EXTERNAL_PORT}:${SERVER_PORT}"
+      - "9229:9229"      # Debug port
+    volumes:
+      - ../:/home/app
+      - <<: *project-repository-mount
+        target: /home/data/repository
+      - type: bind
+        source: $SOLC_REPO_HOST
+        target: $SOLC_REPO
+      - type: bind
+        source: $SOLJSON_REPO_HOST
+        target: $SOLJSON_REPO
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${SERVER_PORT}/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 10

openapi.yaml

@@ -47,7 +47,7 @@ paths:
   /files/contracts/{chain}:
     $ref: "src/server/controllers/repository/get-contract-addresses-all.stateless.paths.yaml#/paths/~1files~1contracts~1{chain}"
   /contracts/{full_match | partial_match}/{chain}/{address}/{filePath}:
-    $ref: "src/server/controllers/repository/get-file-static.stateless.paths.yaml#/paths/~1contracts~1{full_match | partial_match}~1{chain}~1{address}~1{filePath}"
+    $ref: "src/server/controllers/repository/get-file-static.stateless.paths.yaml#/paths/~1repository~1contracts~1{full_match | partial_match}~1{chain}~1{address}~1{filePath}"
   /files/tree/any/{chain}/{address}:
     $ref: "src/server/controllers/repository/get-file-tree-all.stateless.paths.yaml#/paths/~1files~1tree~1any~1{chain}~1{address}"
   /files/tree/{chain}/{address}:

packages/lib-sourcify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ethereum-sourcify/lib-sourcify",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Library for Sourcify's contract verification methods, contract validation, types, and interfaces.",
   "main": "build/main/index.js",
   "typings": "build/main/index.d.ts",

packages/lib-sourcify/src/lib/compilerWorker.ts

@@ -0,0 +1,13 @@
+import { workerData, parentPort } from 'worker_threads';
+import { getSolcJs } from './solidityCompiler';
+
+async function runUseCompiler(version: string, inputStringified: string) {
+  const solJson = await getSolcJs(version);
+  const result = solJson.compile(inputStringified);
+  if (parentPort === null) {
+    throw new Error('Parent port is null; cannot send compilation result');
+  }
+  parentPort.postMessage(result);
+}
+
+runUseCompiler(workerData.version, workerData.inputStringified);

packages/lib-sourcify/src/lib/solidityCompiler.ts

@@ -1,17 +1,17 @@
 // TODO: Handle nodejs only dependencies
 import path from 'path';
 import fs from 'fs';
-import { spawnSync } from 'child_process';
+import { exec, spawnSync } from 'child_process';
 import { fetchWithTimeout } from './utils';
 import { StatusCodes } from 'http-status-codes';
 import { JsonInput, PathBuffer } from './types';
-import { logError, logInfo, logWarn } from './logger';
+import { logDebug, logError, logInfo, logWarn } from './logger';
+import semver from 'semver';
+import { Worker, WorkerOptions } from 'worker_threads';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const solc = require('solc');
 
 const GITHUB_SOLC_REPO = 'https://github.com/ethereum/solc-bin/raw/gh-pages/';
-const RECOMPILATION_ERR_MSG =
-  'Recompilation error (probably caused by invalid metadata)';
 
 export function findSolcPlatform(): string | false {
   if (process.platform === 'darwin' && process.arch === 'x64') {
@@ -48,35 +48,45 @@ export async function useCompiler(version: string, solcJsonInput: JsonInput) {
   if (solcPlatform) {
     solcPath = await getSolcExecutable(solcPlatform, version);
   }
-  const startCompilation = Date.now();
+  let startCompilation: number;
   if (solcPath) {
-    const shellOutputBuffer = spawnSync(solcPath, ['--standard-json'], {
-      input: inputStringified,
-      maxBuffer: 1000 * 1000 * 10,
-    });
-
-    // Handle errors.
-    let error: false | Error = false;
-    if (shellOutputBuffer.error) {
-      const typedError: NodeJS.ErrnoException = shellOutputBuffer.error;
-      // Handle compilation output size > stdout buffer
-      if (typedError.code === 'ENOBUFS') {
-        error = new Error('Compilation output size too large');
+    logDebug(`Compiling with solc binary ${version} at ${solcPath}`);
+    startCompilation = Date.now();
+    try {
+      compiled = await asyncExecSolc(inputStringified, solcPath);
+    } catch (error: any) {
+      if (error?.code === 'ENOBUFS') {
+        throw new Error('Compilation output size too large');
       }
-      error = new Error('Compilation Error');
-    }
-    if (!shellOutputBuffer.stdout) {
-      error = new Error(RECOMPILATION_ERR_MSG);
-    }
-    if (error) {
       logWarn(error.message);
       throw error;
     }
-    compiled = shellOutputBuffer.stdout.toString();
   } else {
-    const soljson = await getSolcJs(version);
-    if (soljson) {
-      compiled = soljson.compile(inputStringified);
+    const solJson = await getSolcJs(version);
+    startCompilation = Date.now();
+    logDebug(`Compiling with solc-js ${version}`);
+    if (solJson) {
+      const coercedVersion =
+        semver.coerce(new semver.SemVer(version))?.version ?? '';
+      // Run Worker for solc versions < 0.4.0 for clean compiler context. See https://github.com/ethereum/sourcify/issues/1099
+      if (semver.lt(coercedVersion, '0.4.0')) {
+        compiled = await new Promise((resolve, reject) => {
+          const worker = importWorker(
+            path.resolve(__dirname, './compilerWorker.ts'),
+            {
+              workerData: { version, inputStringified },
+            }
+          );
+          worker.once('message', (result) => {
+            resolve(result);
+          });
+          worker.once('error', (error) => {
+            reject(error);
+          });
+        });
+      } else {
+        compiled = solJson.compile(inputStringified);
+      }
     }
   }
 
@@ -108,6 +118,7 @@ export async function getAllMetadataAndSourcesFromSolcJson(
     throw new Error(
       'Only Solidity is supported, the json has language: ' + solcJson.language
     );
+
   const outputSelection = {
     '*': {
       '*': ['metadata'],
@@ -148,9 +159,17 @@ export async function getSolcExecutable(
   const repoPath = process.env.SOLC_REPO || path.join('/tmp', 'solc-repo');
   const solcPath = path.join(repoPath, fileName);
   if (fs.existsSync(solcPath) && validateSolcPath(solcPath)) {
+    logDebug(`Found solc ${version} with platform ${platform} at ${solcPath}`);
     return solcPath;
   }
+
+  logDebug(
+    `Downloading solc ${version} with platform ${platform} at ${solcPath}`
+  );
   const success = await fetchAndSaveSolc(platform, solcPath, version, fileName);
+  logDebug(
+    `Downloaded solc ${version} with platform ${platform} at ${solcPath}`
+  );
   if (success && !validateSolcPath(solcPath)) {
     logError(`Cannot validate solc ${version}.`);
     return null;
@@ -187,6 +206,9 @@ async function fetchAndSaveSolc(
 ): Promise<boolean> {
   const encodedURIFilename = encodeURIComponent(fileName);
   const githubSolcURI = `${GITHUB_SOLC_REPO}${platform}/${encodedURIFilename}`;
+  logDebug(
+    `Fetching solc ${version} on platform ${platform} from GitHub: ${githubSolcURI}`
+  );
   let res = await fetchWithTimeout(githubSolcURI);
   let status = res.status;
   let buffer;
@@ -206,6 +228,9 @@ async function fetchAndSaveSolc(
   }
 
   if (status === StatusCodes.OK && buffer) {
+    logDebug(
+      `Fetched solc ${version} on platform ${platform} from GitHub: ${githubSolcURI}`
+    );
     fs.mkdirSync(path.dirname(solcPath), { recursive: true });
 
     try {
@@ -260,3 +285,48 @@ export async function getSolcJs(version = 'latest'): Promise<any> {
   const solcjsImports = await import(soljsonPath);
   return solc.setupMethods(solcjsImports);
 }
+
+function asyncExecSolc(
+  inputStringified: string,
+  solcPath: string
+): Promise<string> {
+  // check if input is valid JSON. The input is untrusted and potentially cause arbitrary execution.
+  JSON.parse(inputStringified);
+
+  return new Promise((resolve, reject) => {
+    const child = exec(
+      `${solcPath} --standard-json`,
+      {
+        maxBuffer: 1000 * 1000 * 10,
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          reject(error);
+        } else if (stderr) {
+          reject(
+            new Error(`Compiler process returned with errors:\n ${stderr}`)
+          );
+        } else {
+          resolve(stdout);
+        }
+      }
+    );
+    if (!child.stdin) {
+      throw new Error('No stdin on child process');
+    }
+    // Write input to child process's stdin
+    child.stdin.write(inputStringified);
+    child.stdin.end();
+  });
+}
+
+// https://stackoverflow.com/questions/71795469/ts-node-using-worker-thread-cause-cannot-use-import-statement-outside-a-module
+function importWorker(path: string, options: WorkerOptions) {
+  const resolvedPath = require.resolve(path);
+  return new Worker(resolvedPath, {
+    ...options,
+    execArgv: /\.ts$/.test(resolvedPath)
+      ? ['--require', 'ts-node/register']
+      : undefined,
+  });
+}

packages/lib-sourcify/test/functions.spec.ts

@@ -16,7 +16,8 @@ import storageMetadata from './sources/Storage/metadata.json';
 import { Metadata, MissingSources } from '../src/lib/types';
 import WrongMetadata from './sources/WrongMetadata/metadata.json';
 import SimplyLog from './sources/WrongMetadata/SimplyLog.json';
-
+import earlyCompilerInput from './sources/json-input/pre-v0.4.0/input.json';
+import { keccak256 } from 'ethers';
 describe('Verify Solidity Compiler', () => {
   it('Should fetch latest SolcJS compiler', async () => {
     expect(await getSolcJs()).not.equals(null);
@@ -111,6 +112,24 @@ describe('Verify Solidity Compiler', () => {
       });
     }
   });
+
+  // See https://github.com/ethereum/sourcify/issues/1099
+  it(`Should should use a clean compiler context with pre 0.4.0 versions`, async () => {
+    // Run compiler once to change compiler "context"
+    await useCompiler('0.1.5+commit.23865e3', earlyCompilerInput);
+
+    // A second run needs to produce the same result
+    const compilerResult = await useCompiler(
+      '0.1.5+commit.23865e3',
+      earlyCompilerInput
+    );
+    const compiledBytecode = compilerResult.contracts[''].GroveLib.evm
+      .deployedBytecode.object as string;
+    const compiledHash = keccak256('0x' + compiledBytecode);
+    expect(compiledHash).equals(
+      '0xc778f3d42ce4a7ee21a2e93d45265cf771e5970e0e36f882310f4491d0ca889d'
+    );
+  });
 });
 
 describe('Checked contract', () => {

services/ipfs/server/Dockerfile.ipfs

@@ -1,6 +1,6 @@
 # Explicit setting for Mac ARM
 FROM --platform=linux/amd64 ubuntu:20.04 
-RUN apt-get update && apt-get install -y cron curl
+RUN apt-get update && apt-get install -y cron curl jq
 
 RUN mkdir /sourcify
 WORKDIR /sourcify

services/ipfs/server/delete-old-pins.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+BASE_URL="https://api.estuary.tech" # Replace with your API base URL
+DAYS_BACK=2
+LIMIT=1000
+# ESTUARY_PINNING_SECRET should be already set in your environment
+
+# Header for authorization
+AUTH_HEADER="Authorization: Bearer $ESTUARY_PINNING_SECRET"
+
+# Calculate the date in the required format
+calculate_date() {
+    # date -u -d "${DAYS_BACK} days ago" +"%Y-%m-%dT00:00:%SZ"
+    # MacOS has a different date command
+    date -v-${DAYS_BACK}d +"%Y-%m-%dT00:00:%SZ"
+}
+
+# Fetch pins before a certain date
+fetch_pins_before_date() {
+    local before_date="$1"
+    curl -s -H "$AUTH_HEADER" "${BASE_URL}/pinning/pins?before=${before_date}&limit=${LIMIT}"
+}
+
+# Delete a pin by pinid
+delete_pin() {
+    local pinid="$1"
+    curl -s -H "$AUTH_HEADER" -X DELETE "${BASE_URL}/pinning/pins/${pinid}"
+    echo "Deleted pin with ID: ${pinid}"
+}
+
+# Main function to cleanup old pins
+cleanup_old_pins() {
+    local before_date
+    before_date=$(calculate_date)
+
+    local pins
+    pins=$(fetch_pins_before_date "$before_date")
+
+    # Assuming the response is a JSON array and you only need pinid to delete them.
+    local pinids
+    pinids=$(echo "$pins" | jq -r '.results[] | .requestid')
+
+    # Loop through the pins and delete them
+    while IFS= read -r pinid; do
+        if [ -n "$pinid" ]; then
+            delete_pin $pinid
+        fi
+    done <<< "$pinids"
+}
+
+# Execute the main function
+cleanup_old_pins

src/Dockerfile.server.debug

@@ -0,0 +1,9 @@
+# Runs the server in a linux-amd64 container with debug port exposed
+# Used in debugging solidity linux-amd64 binary executables  
+FROM node:16
+WORKDIR /home/app
+
+# Install puppeteer dependencies.
+RUN apt-get update && apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget 
+
+CMD ["node", "--inspect=0.0.0.0:9229", "./dist/server/server.js"]