Async compiler process (#1135)

Runs the process async without blocking the event loop. Expected to improve the current high server response times. Also adds Docker container to debug the linux-amd64 binaries (our server default) on host machines with different platforms.
ethereum · Aug 9, 2023 · 239ce7a · 239ce7a
2 parents 48114e1 + 43afd94
commit 239ce7a
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 22 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -39,6 +39,15 @@
       "console": "integratedTerminal",
       "outputCapture": "std",
     },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Docker: Attach to Server",
+      "remoteRoot": "/home/app",
+      "localRoot": "${workspaceFolder}",
+      "port": 9229,
+      "address": "localhost",
+    },
     {
       // Make sure the UI webserver is already running with `npm start`
       "name": "UI",

diff --git a/environments/server.debug.linux-amd64.yaml b/environments/server.debug.linux-amd64.yaml
@@ -0,0 +1,44 @@
+# Runs the server in a linux-amd64 container with debug port exposed
+# Used in debugging solidity linux-amd64 binary executables  
+version: "3.7"
+x-project-repository-mount: &project-repository-mount
+  type: bind
+  source: $REPOSITORY_PATH
+
+x-project-base: &project-base
+  env_file:
+    - .env
+  restart: always
+  networks:
+    - source-verify
+
+networks:
+  source-verify:
+
+services:
+  server:
+    <<: *project-base
+    # image: ethereum/source-verify:server-${TAG}
+    build:
+      context: ../
+      dockerfile: src/Dockerfile.server.debug
+    container_name: server-${TAG}
+    platform: linux/amd64
+    ports:
+      - "${SERVER_EXTERNAL_PORT}:${SERVER_PORT}"
+      - "9229:9229"      # Debug port
+    volumes:
+      - ../:/home/app
+      - <<: *project-repository-mount
+        target: /home/data/repository
+      - type: bind
+        source: $SOLC_REPO_HOST
+        target: $SOLC_REPO
+      - type: bind
+        source: $SOLJSON_REPO_HOST
+        target: $SOLJSON_REPO
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${SERVER_PORT}/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
diff --git a/packages/lib-sourcify/src/lib/solidityCompiler.ts b/packages/lib-sourcify/src/lib/solidityCompiler.ts
@@ -1,7 +1,7 @@
 // TODO: Handle nodejs only dependencies
 import path from 'path';
 import fs from 'fs';
-import { spawnSync } from 'child_process';
+import { exec, spawnSync } from 'child_process';
 import { fetchWithTimeout } from './utils';
 import { StatusCodes } from 'http-status-codes';
 import { JsonInput, PathBuffer } from './types';
@@ -10,8 +10,6 @@ import { logDebug, logError, logInfo, logWarn } from './logger';
 const solc = require('solc');
 
 const GITHUB_SOLC_REPO = 'https://github.com/ethereum/solc-bin/raw/gh-pages/';
-const RECOMPILATION_ERR_MSG =
-  'Recompilation error (probably caused by invalid metadata)';
 
 export function findSolcPlatform(): string | false {
   if (process.platform === 'darwin' && process.arch === 'x64') {
@@ -52,29 +50,15 @@ export async function useCompiler(version: string, solcJsonInput: JsonInput) {
   if (solcPath) {
     logDebug(`Compiling with solc binary ${version} at ${solcPath}`);
     startCompilation = Date.now();
-    const shellOutputBuffer = spawnSync(solcPath, ['--standard-json'], {
-      input: inputStringified,
-      maxBuffer: 1000 * 1000 * 10,
-    });
-
-    // Handle errors.
-    let error: false | Error = false;
-    if (shellOutputBuffer.error) {
-      const typedError: NodeJS.ErrnoException = shellOutputBuffer.error;
-      // Handle compilation output size > stdout buffer
-      if (typedError.code === 'ENOBUFS') {
-        error = new Error('Compilation output size too large');
+    try {
+      compiled = await asyncExecSolc(inputStringified, solcPath);
+    } catch (error: any) {
+      if (error?.code === 'ENOBUFS') {
+        throw new Error('Compilation output size too large');
       }
-      error = new Error('Compilation Error');
-    }
-    if (!shellOutputBuffer.stdout) {
-      error = new Error(RECOMPILATION_ERR_MSG);
-    }
-    if (error) {
       logWarn(error.message);
       throw error;
     }
-    compiled = shellOutputBuffer.stdout.toString();
   } else {
     const soljson = await getSolcJs(version);
     startCompilation = Date.now();
@@ -278,3 +262,37 @@ export async function getSolcJs(version = 'latest'): Promise<any> {
   const solcjsImports = await import(soljsonPath);
   return solc.setupMethods(solcjsImports);
 }
+
+function asyncExecSolc(
+  inputStringified: string,
+  solcPath: string
+): Promise<string> {
+  // check if input is valid JSON. The input is untrusted and potentially cause arbitrary execution.
+  JSON.parse(inputStringified);
+
+  return new Promise((resolve, reject) => {
+    const child = exec(
+      `${solcPath} --standard-json`,
+      {
+        maxBuffer: 1000 * 1000 * 10,
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          reject(error);
+        } else if (stderr) {
+          reject(
+            new Error(`Compiler process returned with errors:\n ${stderr}`)
+          );
+        } else {
+          resolve(stdout);
+        }
+      }
+    );
+    if (!child.stdin) {
+      throw new Error('No stdin on child process');
+    }
+    // Write input to child process's stdin
+    child.stdin.write(inputStringified);
+    child.stdin.end();
+  });
+}
diff --git a/src/Dockerfile.server.debug b/src/Dockerfile.server.debug
@@ -0,0 +1,9 @@
+# Runs the server in a linux-amd64 container with debug port exposed
+# Used in debugging solidity linux-amd64 binary executables  
+FROM node:16
+WORKDIR /home/app
+
+# Install puppeteer dependencies.
+RUN apt-get update && apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget 
+
+CMD ["node", "--inspect=0.0.0.0:9229", "./dist/server/server.js"]