Stream lambda function compiler responses

serverless/compiler-lambda/index.js

@@ -5,6 +5,8 @@ const { exec, spawnSync } = require("child_process");
 const solc = require("solc");
 const fetch = (...args) =>
   import("node-fetch").then(({ default: fetch }) => fetch(...args));
+const pipeline = require("util").promisify(require("stream").pipeline);
+const { Blob } = require("buffer");
 
 async function fetchWithTimeout(resource, options = {}) {
   const { timeout = 30000 } = options;
@@ -298,23 +300,36 @@ function asyncExecSolc(inputStringified, solcPath) {
   });
 }
 
-exports.handler = async (event) => {
-  try {
-    return {
-      success: true,
-      body: await useCompiler(
+function getOutputBlob(responseObject) {
+  return new Blob([JSON.stringify(responseObject)]);
+}
+
+exports.handler = awslambda.streamifyResponse(
+  async (event, responseStream, _context) => {
+    let output;
+    try {
+      output = await useCompiler(
         event.version,
         event.solcJsonInput,
         event.forceEmscripten
-      ),
-    };
-  } catch (e) {
-    return {
-      success: true,
-      body: e.message,
-    };
+      );
+    } catch (e) {
+      output = { error: e.message };
+    }
+    console.debug("Compilation output: ", output);
+
+    let outputBlob = getOutputBlob(output);
+
+    // Handle AWS lambda's max stream response size of 20 MiB
+    if (outputBlob.size > 20 * 2 ** 20) {
+      console.error("Compilation output exceeded 20 MiB");
+      output = { error: "Stream response limit exceeded" };
+      outputBlob = getOutputBlob(output);
+    }
+
+    await pipeline(outputBlob.stream(), responseStream);
   }
-};
+);
 
 /* exports
   .handler({

services/server/src/server/controllers/verification/verification.common.ts

@@ -23,7 +23,7 @@ import fetch from "node-fetch";
 import { IVerificationService } from "../../services/VerificationService";
 import { ContractMeta, ContractWrapper, getMatchStatus } from "../../common";
 import { ISolidityCompiler } from "@ethereum-sourcify/lib-sourcify";
-import { SolcLambda } from "../../services/compiler/lambda/SolcLambda";
+import { SolcLambdaWithLocalFallback } from "../../services/compiler/lambda-with-fallback/SolcLambdaWithLocalFallback";
 import { SolcLocal } from "../../services/compiler/local/SolcLocal";
 import { StorageService } from "../../services/StorageService";
 import logger from "../../../common/logger";
@@ -32,8 +32,8 @@ import { createHash } from "crypto";
 
 let selectedSolidityCompiler: ISolidityCompiler;
 if (config.get("lambdaCompiler.enabled")) {
-  logger.info("Using lambda solidity compiler");
-  selectedSolidityCompiler = new SolcLambda();
+  logger.info("Using lambda solidity compiler with local fallback");
+  selectedSolidityCompiler = new SolcLambdaWithLocalFallback();
 } else {
   logger.info("Using local solidity compiler");
   selectedSolidityCompiler = new SolcLocal();

services/server/src/server/services/compiler/lambda-with-fallback/SolcLambdaWithLocalFallback.ts

@@ -0,0 +1,42 @@
+import {
+  CompilerOutput,
+  ISolidityCompiler,
+  JsonInput,
+} from "@ethereum-sourcify/lib-sourcify";
+import logger from "../../../../common/logger";
+import { SolcLambda, LambdaResponseLimitExceeded } from "../lambda/SolcLambda";
+import { SolcLocal } from "../local/SolcLocal";
+
+export class SolcLambdaWithLocalFallback implements ISolidityCompiler {
+  private solcLambda = new SolcLambda();
+  private solcLocal = new SolcLocal();
+
+  public async compile(
+    version: string,
+    solcJsonInput: JsonInput,
+    forceEmscripten: boolean = false
+  ): Promise<CompilerOutput> {
+    let compilerOutput: CompilerOutput;
+    try {
+      compilerOutput = await this.solcLambda.compile(
+        version,
+        solcJsonInput,
+        forceEmscripten
+      );
+    } catch (e) {
+      if (e instanceof LambdaResponseLimitExceeded) {
+        logger.error(
+          "Lambda compilation exceeded stream response limit - Falling back to local compilation"
+        );
+        compilerOutput = await this.solcLocal.compile(
+          version,
+          solcJsonInput,
+          forceEmscripten
+        );
+      } else {
+        throw e;
+      }
+    }
+    return compilerOutput;
+  }
+}

services/server/src/server/services/compiler/lambda/SolcLambda.ts

@@ -1,7 +1,7 @@
 import {
   LambdaClient,
-  InvokeCommand,
-  InvokeCommandInput,
+  InvokeWithResponseStreamCommand,
+  InvokeWithResponseStreamCommandInput,
 } from "@aws-sdk/client-lambda";
 import {
   CompilerOutput,
@@ -44,45 +44,75 @@ export class SolcLambda implements ISolidityCompiler {
     logger.debug("Compiling with Lambda", { version });
     const response = await this.invokeLambdaFunction(param);
     logger.debug("Compiled with Lambda", { version });
-    const responseObj = this.parseCompilerOutput(response);
-    logger.silly("Lambda function response", { responseObj });
-    return responseObj;
+    logger.silly("Lambda function response", { response });
+    return response;
   }
 
-  private async invokeLambdaFunction(payload: string): Promise<any> {
-    const params: InvokeCommandInput = {
+  private async invokeLambdaFunction(payload: string): Promise<CompilerOutput> {
+    const params: InvokeWithResponseStreamCommandInput = {
       FunctionName: config.get("lambdaCompiler.functionName") || "compile",
       Payload: payload,
     };
 
-    const command = new InvokeCommand(params);
+    const command = new InvokeWithResponseStreamCommand(params);
     const response = await this.lambdaClient.send(command);
 
-    if (!response.Payload) {
+    if (!response.EventStream) {
       throw new Error(
-        "Error: No response payload received from Lambda function"
+        "Error: No response stream received from Lambda function"
       );
     }
 
-    if (response.FunctionError) {
-      const errorObj: { errorMessage: string; errorType: string } = JSON.parse(
-        Buffer.from(response.Payload).toString("utf8")
-      );
-      logger.error("Error invoking Lambda function", {
-        errorObj,
+    let streamResult = "";
+    for await (const event of response.EventStream) {
+      if (event.InvokeComplete?.ErrorCode) {
+        logger.error("Error invoking Lambda function", {
+          errorCode: event.InvokeComplete.ErrorCode,
+          errorDetails: event.InvokeComplete.ErrorDetails,
+          logResult: event.InvokeComplete.LogResult,
+          lambdaRequestId: response.$metadata.requestId,
+        });
+        throw new Error(
+          `AWS Lambda error: ${event.InvokeComplete.ErrorCode} - ${event.InvokeComplete.ErrorDetails} - lamdbaRequestId: ${response.$metadata.requestId}`
+        );
+      } else if (event.PayloadChunk?.Payload) {
+        streamResult += Buffer.from(event.PayloadChunk.Payload).toString(
+          "utf8"
+        );
+      }
+    }
+    logger.silly("Received stream response", { streamResult });
+
+    let output;
+    try {
+      output = JSON.parse(streamResult);
+    } catch (e) {
+      logger.error("Error parsing Lambda function result", {
+        error: e,
         lambdaRequestId: response.$metadata.requestId,
-        functionError: response.FunctionError,
       });
       throw new Error(
-        `AWS Lambda error: ${errorObj.errorType} - ${errorObj.errorMessage} - lamdbaRequestId: ${response.$metadata.requestId}`
+        `AWS Lambda error: ${e} - lamdbaRequestId: ${response.$metadata.requestId}`
       );
     }
 
-    return response;
-  }
+    if (output.error) {
+      logger.error("Error received from Lambda function", {
+        error: output.error,
+        lambdaRequestId: response.$metadata.requestId,
+      });
+      const errorMessage = `AWS Lambda error: ${output.error} - lamdbaRequestId: ${response.$metadata.requestId}`;
+      if (output.error === "Stream response limit exceeded") {
+        throw new LambdaResponseLimitExceeded(errorMessage);
+      } else {
+        throw new Error(errorMessage);
+      }
+    }
 
-  private parseCompilerOutput(response: any): CompilerOutput {
-    const res = JSON.parse(Buffer.from(response.Payload).toString("utf8"));
-    return res.body as CompilerOutput;
+    return output;
   }
 }
+
+export class LambdaResponseLimitExceeded extends Error {
+  name = "LambdaResponseLimitExceeded";
+}