extend utils & update compilation process (matter-labs#396)

Co-authored-by: Raid Ateir <[email protected]>

.gitignore

@@ -22,7 +22,7 @@ l1-contracts/lcov.info
 l1-contracts/report/*
 l1-contracts/coverage/*
 l1-contracts/out/*
-l1-contracts/broadcast/*
-l1-contracts/script-config/*
-l1-contracts/script-out/*
-!l1-contracts/script-out/.gitkeep
+l1-contracts-foundry/broadcast/*
+l1-contracts-foundry/script-out/*
+!l1-contracts-foundry/script-out/.gitkeep
+*.timestamp

system-contracts/package.json

@@ -63,8 +63,7 @@
     "compile-zasm": "ts-node scripts/compile-zasm.ts",
     "deploy-preimages": "ts-node scripts/deploy-preimages.ts",
     "preprocess:bootloader": "rm -rf ./bootloader/build && yarn ts-node scripts/preprocess-bootloader.ts",
-    "preprocess:system-contracts": "rm -rf ./contracts-preprocessed && ts-node scripts/preprocess-system-contracts.ts",
-    "verify-on-explorer": "hardhat run scripts/verify-on-explorer.ts",
+    "preprocess:system-contracts": "ts-node scripts/preprocess-system-contracts.ts",
     "test": "yarn build:test-system-contracts && hardhat test --network zkSyncTestNode",
     "test-node": "hardhat node-zksync --tag v0.0.1-vm1.5.0",
     "test:bootloader": "cd bootloader/test_infra && cargo run"

system-contracts/scripts/compile-yul.ts

@@ -1,11 +1,23 @@
 // hardhat import should be the first import in the file
 import type { CompilerPaths } from "./utils";
-import { spawn, compilerLocation, prepareCompilerPaths, getSolcLocation } from "./utils";
+import {
+  spawn,
+  compilerLocation,
+  prepareCompilerPaths,
+  getSolcLocation,
+  needsRecompilation,
+  setCompilationTime,
+} from "./utils";
 import * as fs from "fs";
 import { Command } from "commander";
+import * as _path from "path";
 
 const COMPILER_VERSION = "1.3.18";
 const IS_COMPILER_PRE_RELEASE = true;
+const CONTRACTS_DIR = "contracts-preprocessed";
+const BOOTLOADER_DIR = "bootloader";
+const TIMESTAMP_FILE_YUL = "last_compilation_yul.timestamp";
+const TIMESTAMP_FILE_BOOTLOADER = "last_compilation_bootloader.timestamp";
 
 export async function compileYul(paths: CompilerPaths, file: string) {
   const solcCompilerPath = await getSolcLocation();
@@ -32,14 +44,34 @@ async function main() {
   program.version("0.1.0").name("compile yul").description("publish preimages for the L2 contracts");
 
   program.command("compile-bootloader").action(async () => {
-    await compileYulFolder("bootloader/build");
-    await compileYulFolder("bootloader/tests");
+    const timestampFilePath = _path.join(process.cwd(), TIMESTAMP_FILE_BOOTLOADER);
+    const folderToCheck = _path.join(process.cwd(), BOOTLOADER_DIR);
+
+    if (needsRecompilation(folderToCheck, timestampFilePath)) {
+      console.log("Compilation needed.");
+      await compileYulFolder("bootloader/build");
+      await compileYulFolder("bootloader/tests");
+      setCompilationTime(timestampFilePath);
+    } else {
+      console.log("Compilation not needed.");
+      return;
+    }
   });
 
   program.command("compile-precompiles").action(async () => {
-    await compileYulFolder("contracts-preprocessed");
-    await compileYulFolder("contracts-preprocessed/precompiles");
-    await compileYulFolder("contracts-preprocessed/precompiles/test-contracts");
+    const timestampFilePath = _path.join(process.cwd(), TIMESTAMP_FILE_YUL);
+    const folderToCheck = _path.join(process.cwd(), CONTRACTS_DIR);
+
+    if (needsRecompilation(folderToCheck, timestampFilePath)) {
+      console.log("Compilation needed.");
+      await compileYulFolder("contracts-preprocessed");
+      await compileYulFolder("contracts-preprocessed/precompiles");
+      await compileYulFolder("contracts-preprocessed/precompiles/test-contracts");
+      setCompilationTime(timestampFilePath);
+    } else {
+      console.log("Compilation not needed.");
+      return;
+    }
   });
 
   await program.parseAsync(process.argv);

system-contracts/scripts/preprocess-system-contracts.ts

@@ -3,9 +3,11 @@ import path from "path";
 import { renderFile } from "template-file";
 import { glob } from "fast-glob";
 import { Command } from "commander";
+import { needsRecompilation, deleteDir, setCompilationTime, isFolderEmpty } from "./utils";
 
 const CONTRACTS_DIR = "contracts";
 const OUTPUT_DIR = "contracts-preprocessed";
+const TIMESTAMP_FILE = "last_compilation_preprocessing.timestamp"; // File to store the last compilation time
 
 const params = {
   SYSTEM_CONTRACTS_OFFSET: "0x8000",
@@ -17,6 +19,18 @@ async function preprocess(testMode: boolean) {
     params.SYSTEM_CONTRACTS_OFFSET = "0x9000";
   }
 
+  const timestampFilePath = path.join(process.cwd(), TIMESTAMP_FILE);
+  const folderToCheck = path.join(process.cwd(), CONTRACTS_DIR);
+
+  if ((await isFolderEmpty(OUTPUT_DIR)) || needsRecompilation(folderToCheck, timestampFilePath) || testMode) {
+    console.log("Preprocessing needed.");
+    deleteDir(OUTPUT_DIR);
+    setCompilationTime(timestampFilePath);
+  } else {
+    console.log("Preprocessing not needed.");
+    return;
+  }
+
   const contracts = await glob(
     [`${CONTRACTS_DIR}/**/*.sol`, `${CONTRACTS_DIR}/**/*.yul`, `${CONTRACTS_DIR}/**/*.zasm`],
     { onlyFiles: true }

system-contracts/scripts/utils.ts

@@ -7,6 +7,7 @@ import type { Deployer } from "@matterlabs/hardhat-zksync-deploy";
 import type { BigNumberish, BytesLike } from "ethers";
 import { BigNumber, ethers } from "ethers";
 import * as fs from "fs";
+import * as fsPr from "fs/promises";
 import { hashBytecode } from "zksync-web3/build/src/utils";
 import type { YulContractDescription, ZasmContractDescription } from "./constants";
 import { Language, SYSTEM_CONTRACTS } from "./constants";
@@ -257,46 +258,74 @@ export function prepareCompilerPaths(path: string): CompilerPaths {
   return new CompilerPaths(absolutePathSources, absolutePathArtifacts);
 }
 
-/**
- * Performs an API call to the Contract verification API.
- *
- * @param endpoint API endpoint to call.
- * @param queryParams Parameters for a query string.
- * @param requestBody Request body. If provided, a POST request would be met and body would be encoded to JSON.
- * @returns API response parsed as a JSON.
- */
-export async function query(
-  method: HttpMethod,
-  endpoint: string,
-  queryParams?: { [key: string]: string },
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  requestBody?: any
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-): Promise<any> {
-  const url = new URL(endpoint);
-  // Iterate through query params and add them to URL.
-  if (queryParams) {
-    Object.entries(queryParams).forEach(([key, value]) => url.searchParams.set(key, value));
+// Get the latest file modification time in the watched folder
+function getLatestModificationTime(folder: string): Date | null {
+  const files = fs.readdirSync(folder);
+  let latestTime: Date | null = null; // Initialize to null to avoid uninitialized variable
+
+  files.forEach((file) => {
+    const filePath = path.join(folder, file);
+    const stats = fs.statSync(filePath);
+    if (stats.isDirectory()) {
+      const dirLatestTime = getLatestModificationTime(filePath);
+      if (dirLatestTime && (!latestTime || dirLatestTime > latestTime)) {
+        latestTime = dirLatestTime;
+      }
+    } else if (stats.isFile()) {
+      if (!latestTime || stats.mtime > latestTime) {
+        latestTime = stats.mtime;
+      }
+    }
+  });
+
+  return latestTime;
+}
+
+// Read the last compilation timestamp from the file
+export function getLastCompilationTime(timestampFile: string): Date | null {
+  try {
+    if (fs.existsSync(timestampFile)) {
+      const timestamp = fs.readFileSync(timestampFile, "utf-8");
+      return new Date(parseInt(timestamp, 10));
+    }
+  } catch (error) {
+    const err = error as Error; // Cast `error` to `Error`
+    console.error(`Error reading timestamp: ${err.message}`);
   }
+  return null;
+}
 
-  const init = {
-    method,
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify(requestBody),
-  };
-  if (requestBody) {
-    init.body = JSON.stringify(requestBody);
+// Write the current time to the timestamp file
+export function setCompilationTime(timestampFile: string) {
+  fs.writeFileSync(timestampFile, Date.now().toString());
+}
+
+// Determine if recompilation is needed
+export function needsRecompilation(folder: string, timestampFile: string): boolean {
+  const lastCompilationTime = getLastCompilationTime(timestampFile);
+  const latestModificationTime = getLatestModificationTime(folder);
+  if (!lastCompilationTime) {
+    return true; // If there's no history, always recompile
   }
 
-  const response = await fetch(url, init);
+  return latestModificationTime! > lastCompilationTime;
+}
+
+export function deleteDir(path: string): void {
+  try {
+    fs.rmSync(path, { recursive: true, force: true }); // 'recursive: true' deletes all contents, 'force: true' prevents errors if the directory doesn't exist
+    console.log(`Directory '${path}' deleted successfully.`);
+  } catch (error) {
+    console.error(`Error deleting directory '${path}':`, error);
+  }
+}
+
+export async function isFolderEmpty(folderPath: string): Promise<boolean> {
   try {
-    return await response.json();
-  } catch (e) {
-    throw {
-      error: "Could not decode JSON in response",
-      status: `${response.status} ${response.statusText}`,
-    };
+    const files = await fsPr.readdir(folderPath); // Get a list of files in the folder
+    return files.length === 0; // If there are no files, the folder is empty
+  } catch (error) {
+    console.error("No target folder with artifacts.");
+    return true; // Return true if an error, as folder doesn't exist.
   }
 }