From e8d68dc07faa1c8daa59b2a3f1980328b2b49017 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 23 Jan 2019 16:54:27 +0100 Subject: [PATCH 01/12] Implement docker as another backend for sol-compiler --- contracts/examples/compiler.json | 2 + contracts/extensions/compiler.json | 2 + contracts/interfaces/compiler.json | 2 + contracts/libs/compiler.json | 2 + contracts/multisig/compiler.json | 2 + contracts/protocol/compiler.json | 2 + contracts/tokens/compiler.json | 2 + contracts/utils/compiler.json | 2 + packages/ethereum-types/src/index.ts | 6 +- packages/sol-compiler/src/compiler.ts | 76 ++++++++++++++--- .../src/schemas/compiler_options_schema.ts | 2 + packages/sol-compiler/src/utils/compiler.ts | 85 +++++++++++++++++-- .../sol-resolver/src/resolvers/fs_resolver.ts | 4 +- .../src/resolvers/npm_resolver.ts | 13 +-- 14 files changed, 179 insertions(+), 23 deletions(-) diff --git a/contracts/examples/compiler.json b/contracts/examples/compiler.json index 375fa0c553..868a11905b 100644 --- a/contracts/examples/compiler.json +++ b/contracts/examples/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json index 2bb468724b..a5c7bcc21f 100644 --- a/contracts/extensions/compiler.json +++ b/contracts/extensions/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/interfaces/compiler.json b/contracts/interfaces/compiler.json index 38a2325410..496bc23278 100644 --- a/contracts/interfaces/compiler.json +++ b/contracts/interfaces/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json index 349d3063b0..09ce914421 100644 --- a/contracts/libs/compiler.json +++ b/contracts/libs/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json index 5a1f689e2a..9555fbbfd6 100644 --- a/contracts/multisig/compiler.json +++ b/contracts/multisig/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/protocol/compiler.json b/contracts/protocol/compiler.json index 10e5bb0a1c..4fc3712b08 100644 --- a/contracts/protocol/compiler.json +++ b/contracts/protocol/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/tokens/compiler.json b/contracts/tokens/compiler.json index 498c5d8260..12e603cabd 100644 --- a/contracts/tokens/compiler.json +++ b/contracts/tokens/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json index 1524c1eaad..df4ef1e9f9 100644 --- a/contracts/utils/compiler.json +++ b/contracts/utils/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts index a8dcfd68a0..a1f6919a85 100644 --- a/packages/ethereum-types/src/index.ts +++ b/packages/ethereum-types/src/index.ts @@ -503,19 +503,23 @@ export interface Source { /** * Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler - * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories. + * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories. + * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity. * artifactsDir: Directory where you want the generated artifacts.json written to * compilerSettings: Desired settings to pass to the Solidity compiler during compilation. * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description) * contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the * specified directory. + * useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used. * solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all * contracts to compile with the the version specified here. */ export interface CompilerOptions { contractsDir?: string; + workspaceDir?: string; artifactsDir?: string; compilerSettings?: CompilerSettings; contracts?: string[] | '*'; + useDockerisedSolc?: boolean; solcVersion?: string; } // tslint:disable-line:max-file-line-count diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index d38ccbf394..856bcbd480 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -10,6 +10,7 @@ import { URLResolver, } from '@0x/sol-resolver'; import { logUtils } from '@0x/utils'; +import { execSync } from 'child_process'; import * as chokidar from 'chokidar'; import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types'; import * as fs from 'fs'; @@ -23,10 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema'; import { binPaths } from './solc/bin_paths'; import { addHexPrefixToContractBytecode, - compile, + compileDocker, + compileSolcJS, createDirIfDoesNotExistAsync, getContractArtifactIfExistsAsync, - getSolcAsync, + getSolcJSAsync, getSourcesWithDependencies, getSourceTreeHash, parseSolidityVersionRange, @@ -40,6 +42,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*'; const ALL_FILES_IDENTIFIER = '*'; const DEFAULT_CONTRACTS_DIR = path.resolve('contracts'); const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts'); +const DEFAULT_USE_DOCKERISED_SOLC = false; // Solc compiler settings cannot be configured from the commandline. // If you need this configured, please create a `compiler.json` config file // with your desired configurations. @@ -80,10 +83,12 @@ export class Compiler { private readonly _resolver: Resolver; private readonly _nameResolver: NameResolver; private readonly _contractsDir: string; + private readonly _workspaceDir: string; private readonly _compilerSettings: solc.CompilerSettings; private readonly _artifactsDir: string; private readonly _solcVersionIfExists: string | undefined; private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER; + private readonly _useDockerisedSolc: boolean; /** * Instantiates a new instance of the Compiler class. * @param opts Optional compiler options @@ -97,16 +102,23 @@ export class Compiler { : {}; const passedOpts = opts || {}; assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema); - this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR; + this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR); + this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir); + if (!this._contractsDir.includes(this._workspaceDir)) { + throw new Error( + `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`, + ); + } this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion; this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS; this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR; this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER; - this._nameResolver = new NameResolver(path.resolve(this._contractsDir)); + this._useDockerisedSolc = + passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC; + this._nameResolver = new NameResolver(this._contractsDir); const resolver = new FallthroughResolver(); resolver.appendResolver(new URLResolver()); - const packagePath = path.resolve(''); - resolver.appendResolver(new NPMResolver(packagePath)); + resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir)); resolver.appendResolver(new RelativeFSResolver(this._contractsDir)); resolver.appendResolver(new FSResolver()); resolver.appendResolver(this._nameResolver); @@ -205,11 +217,12 @@ export class Compiler { // map contract paths to data about them for later verification and persistence const contractPathToData: ContractPathToData = {}; + const spyResolver = new SpyResolver(this._resolver); for (const contractName of contractNames) { - const contractSource = this._resolver.resolve(contractName); + const contractSource = spyResolver.resolve(contractName); const sourceTreeHashHex = getSourceTreeHash( - this._resolver, + spyResolver, path.join(this._contractsDir, contractSource.path), ).toString('hex'); const contractData = { @@ -242,6 +255,30 @@ export class Compiler { versionToInputs[solcVersion].contractsToCompile.push(contractSource.path); } + const allTouchedFiles = spyResolver.resolvedContractSources.map( + contractSource => `${contractSource.absolutePath}`, + ); + const NODE_MODULES = 'node_modules'; + const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); + const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; + _.map(allTouchedDependencies, dependencyFilePath => { + const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); + const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; + const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); + let packageName; + let packageScopeIfExists; + let dependencyName; + if (_.startsWith(importPath, '@')) { + [packageScopeIfExists, packageName] = importPath.split('/'); + dependencyName = `${packageScopeIfExists}/${packageName}`; + } else { + [packageName] = importPath.split('/'); + dependencyName = `${packageName}`; + } + const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); + dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; + }); + const compilerOutputs: StandardOutput[] = []; const solcVersions = _.keys(versionToInputs); @@ -252,12 +289,31 @@ export class Compiler { input.contractsToCompile }) with Solidity v${solcVersion}...`, ); + let compilerOutput; + let fullSolcVersion; + if (this._useDockerisedSolc) { + const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`; + const versionCommandOutput = execSync(dockerCommand).toString(); + const versionCommandOutputParts = versionCommandOutput.split(' '); + fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim(); + compilerOutput = compileDocker( + this._resolver, + this._contractsDir, + this._workspaceDir, + solcVersion, + dependencyNameToPackagePath, + input.standardInput, + ); + } else { + fullSolcVersion = binPaths[solcVersion]; + const solcInstance = await getSolcJSAsync(solcVersion); + compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput); + } - const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion); - const compilerOutput = compile(this._resolver, solcInstance, input.standardInput); compilerOutputs.push(compilerOutput); for (const contractPath of input.contractsToCompile) { + // console.log('contractsPath', contractPath); const contractName = contractPathToData[contractPath].contractName; const compiledContract = compilerOutput.contracts[contractPath][contractName]; diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts index d4d1b60176..657b801ad6 100644 --- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts +++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts @@ -2,6 +2,7 @@ export const compilerOptionsSchema = { id: '/CompilerOptions', properties: { contractsDir: { type: 'string' }, + workspaceDir: { type: 'string' }, artifactsDir: { type: 'string' }, solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' }, compilerSettings: { type: 'object' }, @@ -19,6 +20,7 @@ export const compilerOptionsSchema = { }, ], }, + useDockerisedSolc: { type: 'boolean' }, }, type: 'object', required: [], diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index db308f2b57..669bdd7a4d 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -1,6 +1,7 @@ import { ContractSource, Resolver } from '@0x/sol-resolver'; import { fetchAsync, logUtils } from '@0x/utils'; import chalk from 'chalk'; +import { execSync } from 'child_process'; import { ContractArtifact } from 'ethereum-types'; import * as ethUtil from 'ethereumjs-util'; import * as _ from 'lodash'; @@ -121,7 +122,7 @@ export function parseDependencies(contractSource: ContractSource): string[] { * @param solcInstance Instance of a solc compiler * @param standardInput Solidity standard JSON input */ -export function compile( +export function compileSolcJS( resolver: Resolver, solcInstance: solc.SolcInstance, standardInput: solc.StandardInput, @@ -137,6 +138,80 @@ export function compile( } return compiled; } + +/** + * Compiles the contracts and prints errors/warnings + * @param resolver Resolver + * @param contractsDir Contracts directory + * @param workspaceDir Workspace directory + * @param solcVersion Version of a solc compiler + * @param dependencyNameToPackagePath Mapping of dependency name to it's package path + * @param standardInput Solidity standard JSON input + */ +export function compileDocker( + resolver: Resolver, + contractsDir: string, + workspaceDir: string, + solcVersion: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, + standardInput: solc.StandardInput, +): solc.StandardOutput { + const standardInputDocker = _.cloneDeep(standardInput); + standardInputDocker.settings.remappings = _.map( + dependencyNameToPackagePath, + (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, + ); + standardInputDocker.sources = _.mapKeys( + standardInputDocker.sources, + (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath, + ); + + const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2); + const dockerCommand = + `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` + + `solc --standard-json --allow-paths ${workspaceDir}`; + const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString(); + const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker); + + if (!_.isUndefined(compiledDocker.errors)) { + printCompilationErrorsAndWarnings(compiledDocker.errors); + } + + compiledDocker.sources = makeContractPathsRelative( + compiledDocker.sources, + contractsDir, + dependencyNameToPackagePath, + ); + compiledDocker.contracts = makeContractPathsRelative( + compiledDocker.contracts, + contractsDir, + dependencyNameToPackagePath, + ); + return compiledDocker; +} + +function makeContractPathRelative( + absolutePath: string, + contractsDir: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, +): string { + let contractPath = absolutePath.replace(`${contractsDir}/`, ''); + _.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => { + contractPath = contractPath.replace(packagePath, dependencyName); + }); + return contractPath; +} + +function makeContractPathsRelative( + absolutePathToSmth: { [absoluteContractPath: string]: any }, + contractsDir: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, +): { [contractPath: string]: any } { + return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) => + makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath), + ); +} + /** * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning). * @param solcErrors The errors field of standard JSON output that contains errors and warnings. @@ -267,13 +342,11 @@ function recursivelyGatherDependencySources( } /** - * Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS, + * Gets the solidity compiler instance. If the compiler is already cached - gets it from FS, * otherwise - fetches it and caches it. * @param solcVersion The compiler version. e.g. 0.5.0 */ -export async function getSolcAsync( - solcVersion: string, -): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> { +export async function getSolcJSAsync(solcVersion: string): Promise { const fullSolcVersion = binPaths[solcVersion]; if (_.isUndefined(fullSolcVersion)) { throw new Error(`${solcVersion} is not a known compiler version`); @@ -297,7 +370,7 @@ export async function getSolcAsync( throw new Error('No compiler available'); } const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename)); - return { solcInstance, fullSolcVersion }; + return solcInstance; } /** diff --git a/packages/sol-resolver/src/resolvers/fs_resolver.ts b/packages/sol-resolver/src/resolvers/fs_resolver.ts index 86128023dc..248fa405e7 100644 --- a/packages/sol-resolver/src/resolvers/fs_resolver.ts +++ b/packages/sol-resolver/src/resolvers/fs_resolver.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as path from 'path'; import { ContractSource } from '../types'; @@ -9,7 +10,8 @@ export class FSResolver extends Resolver { public resolveIfExists(importPath: string): ContractSource | undefined { if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) { const fileContent = fs.readFileSync(importPath).toString(); - return { source: fileContent, path: importPath, absolutePath: importPath }; + const absolutePath = path.resolve(importPath); + return { source: fileContent, path: importPath, absolutePath } as any; } return undefined; } diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index 3c1d095573..1c9e569576 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -8,9 +8,11 @@ import { Resolver } from './resolver'; export class NPMResolver extends Resolver { private readonly _packagePath: string; - constructor(packagePath: string) { + private readonly _workspacePath: string; + constructor(packagePath: string, workspacePath: string) { super(); this._packagePath = packagePath; + this._workspacePath = workspacePath; } public resolveIfExists(importPath: string): ContractSource | undefined { if (!importPath.startsWith('/')) { @@ -23,9 +25,11 @@ export class NPMResolver extends Resolver { [packageName, ...other] = importPath.split('/'); } const pathWithinPackage = path.join(...other); - let currentPath = this._packagePath; - const ROOT_PATH = '/'; - while (currentPath !== ROOT_PATH) { + for ( + let currentPath = this._packagePath; + currentPath.includes(this._workspacePath); + currentPath = path.dirname(currentPath) + ) { const packagePath = _.isUndefined(packageScopeIfExists) ? packageName : path.join(packageScopeIfExists, packageName); @@ -34,7 +38,6 @@ export class NPMResolver extends Resolver { const fileContent = fs.readFileSync(lookupPath).toString(); return { source: fileContent, path: importPath, absolutePath: lookupPath }; } - currentPath = path.dirname(currentPath); } } return undefined; From de36670bdca8f8fb6839f0168bfc5ac49caa28f8 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 23 Jan 2019 17:02:02 +0100 Subject: [PATCH 02/12] Make workspace path in NPMResolver optional --- packages/sol-resolver/src/resolvers/npm_resolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index 1c9e569576..d9377989f9 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -9,7 +9,7 @@ import { Resolver } from './resolver'; export class NPMResolver extends Resolver { private readonly _packagePath: string; private readonly _workspacePath: string; - constructor(packagePath: string, workspacePath: string) { + constructor(packagePath: string, workspacePath: string = '/') { super(); this._packagePath = packagePath; this._workspacePath = workspacePath; From 19064f8cbb21954108c4b8bd4a66879ddb240d7e Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 23 Jan 2019 17:09:54 +0100 Subject: [PATCH 03/12] Add -setup_remote_docker to CI build steps --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f64017b564..90fc2fcc10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,7 @@ jobs: - run: name: yarn command: yarn --frozen-lockfile --ignore-engines install + - setup_remote_docker - run: yarn build:ci:no_website - run: yarn build:ts - save_cache: From 0c12128f64f7d9a8de6088e98c2e638533d6f5bf Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Mon, 28 Jan 2019 16:24:54 +0100 Subject: [PATCH 04/12] Temp CI --- contracts/examples/compiler.json | 1 - contracts/extensions/compiler.json | 1 - contracts/interfaces/compiler.json | 1 - contracts/libs/compiler.json | 1 - contracts/multisig/compiler.json | 1 - contracts/protocol/compiler.json | 1 - contracts/tokens/compiler.json | 1 - contracts/utils/compiler.json | 1 - packages/ethereum-types/src/index.ts | 4 +- packages/sol-compiler/src/compiler.ts | 82 ++++++------- .../src/schemas/compiler_options_schema.ts | 1 - packages/sol-compiler/src/utils/compiler.ts | 109 +++++++++--------- .../src/resolvers/npm_resolver.ts | 13 +-- .../typescript-typings/types/solc/index.d.ts | 2 +- 14 files changed, 93 insertions(+), 126 deletions(-) diff --git a/contracts/examples/compiler.json b/contracts/examples/compiler.json index 868a11905b..7c78fba3bc 100644 --- a/contracts/examples/compiler.json +++ b/contracts/examples/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json index a5c7bcc21f..e95478e348 100644 --- a/contracts/extensions/compiler.json +++ b/contracts/extensions/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/interfaces/compiler.json b/contracts/interfaces/compiler.json index 496bc23278..ab3bc10780 100644 --- a/contracts/interfaces/compiler.json +++ b/contracts/interfaces/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json index 09ce914421..c4f0c6bd97 100644 --- a/contracts/libs/compiler.json +++ b/contracts/libs/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json index 9555fbbfd6..5c4d262dcd 100644 --- a/contracts/multisig/compiler.json +++ b/contracts/multisig/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/protocol/compiler.json b/contracts/protocol/compiler.json index 4fc3712b08..f4d2a5f453 100644 --- a/contracts/protocol/compiler.json +++ b/contracts/protocol/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/tokens/compiler.json b/contracts/tokens/compiler.json index 12e603cabd..eec24a7691 100644 --- a/contracts/tokens/compiler.json +++ b/contracts/tokens/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json index df4ef1e9f9..7473ea6d1e 100644 --- a/contracts/utils/compiler.json +++ b/contracts/utils/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts index a1f6919a85..bbc3682c73 100644 --- a/packages/ethereum-types/src/index.ts +++ b/packages/ethereum-types/src/index.ts @@ -503,8 +503,7 @@ export interface Source { /** * Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler - * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories. - * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity. + * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories. * artifactsDir: Directory where you want the generated artifacts.json written to * compilerSettings: Desired settings to pass to the Solidity compiler during compilation. * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description) @@ -516,7 +515,6 @@ export interface Source { */ export interface CompilerOptions { contractsDir?: string; - workspaceDir?: string; artifactsDir?: string; compilerSettings?: CompilerSettings; contracts?: string[] | '*'; diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 856bcbd480..b5cdf98654 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -28,10 +28,13 @@ import { compileSolcJS, createDirIfDoesNotExistAsync, getContractArtifactIfExistsAsync, + getDependencyNameToPackagePath, getSolcJSAsync, getSourcesWithDependencies, getSourceTreeHash, + makeContractPathsRelative, parseSolidityVersionRange, + printCompilationErrorsAndWarnings, } from './utils/compiler'; import { constants } from './utils/constants'; import { fsWrapper } from './utils/fs_wrapper'; @@ -83,7 +86,6 @@ export class Compiler { private readonly _resolver: Resolver; private readonly _nameResolver: NameResolver; private readonly _contractsDir: string; - private readonly _workspaceDir: string; private readonly _compilerSettings: solc.CompilerSettings; private readonly _artifactsDir: string; private readonly _solcVersionIfExists: string | undefined; @@ -103,12 +105,6 @@ export class Compiler { const passedOpts = opts || {}; assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema); this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR); - this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir); - if (!this._contractsDir.includes(this._workspaceDir)) { - throw new Error( - `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`, - ); - } this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion; this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS; this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR; @@ -118,7 +114,7 @@ export class Compiler { this._nameResolver = new NameResolver(this._contractsDir); const resolver = new FallthroughResolver(); resolver.appendResolver(new URLResolver()); - resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir)); + resolver.appendResolver(new NPMResolver(this._contractsDir)); resolver.appendResolver(new RelativeFSResolver(this._contractsDir)); resolver.appendResolver(new FSResolver()); resolver.appendResolver(this._nameResolver); @@ -217,9 +213,10 @@ export class Compiler { // map contract paths to data about them for later verification and persistence const contractPathToData: ContractPathToData = {}; - const spyResolver = new SpyResolver(this._resolver); + const resolvedContractSources = []; for (const contractName of contractNames) { + const spyResolver = new SpyResolver(this._resolver); const contractSource = spyResolver.resolve(contractName); const sourceTreeHashHex = getSourceTreeHash( spyResolver, @@ -249,40 +246,19 @@ export class Compiler { }; } // add input to the right version batch - versionToInputs[solcVersion].standardInput.sources[contractSource.path] = { - content: contractSource.source, - }; + for (const resolvedContractSource of spyResolver.resolvedContractSources) { + versionToInputs[solcVersion].standardInput.sources[resolvedContractSource.absolutePath] = { + content: resolvedContractSource.source, + }; + } + resolvedContractSources.push(...spyResolver.resolvedContractSources); versionToInputs[solcVersion].contractsToCompile.push(contractSource.path); } - const allTouchedFiles = spyResolver.resolvedContractSources.map( - contractSource => `${contractSource.absolutePath}`, - ); - const NODE_MODULES = 'node_modules'; - const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); - const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; - _.map(allTouchedDependencies, dependencyFilePath => { - const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); - const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; - const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); - let packageName; - let packageScopeIfExists; - let dependencyName; - if (_.startsWith(importPath, '@')) { - [packageScopeIfExists, packageName] = importPath.split('/'); - dependencyName = `${packageScopeIfExists}/${packageName}`; - } else { - [packageName] = importPath.split('/'); - dependencyName = `${packageName}`; - } - const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); - dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; - }); + const dependencyNameToPackagePath = getDependencyNameToPackagePath(resolvedContractSources); const compilerOutputs: StandardOutput[] = []; - - const solcVersions = _.keys(versionToInputs); - for (const solcVersion of solcVersions) { + for (const solcVersion of _.keys(versionToInputs)) { const input = versionToInputs[solcVersion]; logUtils.warn( `Compiling ${input.contractsToCompile.length} contracts (${ @@ -291,29 +267,37 @@ export class Compiler { ); let compilerOutput; let fullSolcVersion; + input.standardInput.settings.remappings = _.map( + dependencyNameToPackagePath, + (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, + ); if (this._useDockerisedSolc) { const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`; const versionCommandOutput = execSync(dockerCommand).toString(); const versionCommandOutputParts = versionCommandOutput.split(' '); fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim(); - compilerOutput = compileDocker( - this._resolver, - this._contractsDir, - this._workspaceDir, - solcVersion, - dependencyNameToPackagePath, - input.standardInput, - ); + compilerOutput = compileDocker(solcVersion, input.standardInput); } else { fullSolcVersion = binPaths[solcVersion]; - const solcInstance = await getSolcJSAsync(solcVersion); - compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput); + compilerOutput = compileSolcJS(solcVersion, input.standardInput); + } + compilerOutput.sources = makeContractPathsRelative( + compilerOutput.sources, + this._contractsDir, + dependencyNameToPackagePath, + ); + compilerOutput.contracts = makeContractPathsRelative( + compilerOutput.contracts, + this._contractsDir, + dependencyNameToPackagePath, + ); + if (!_.isUndefined(compilerOutput.errors)) { + printCompilationErrorsAndWarnings(compilerOutput.errors); } compilerOutputs.push(compilerOutput); for (const contractPath of input.contractsToCompile) { - // console.log('contractsPath', contractPath); const contractName = contractPathToData[contractPath].contractName; const compiledContract = compilerOutput.contracts[contractPath][contractName]; diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts index 657b801ad6..c0766b625c 100644 --- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts +++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts @@ -2,7 +2,6 @@ export const compilerOptionsSchema = { id: '/CompilerOptions', properties: { contractsDir: { type: 'string' }, - workspaceDir: { type: 'string' }, artifactsDir: { type: 'string' }, solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' }, compilerSettings: { type: 'object' }, diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index 669bdd7a4d..6381e525f3 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -118,76 +118,35 @@ export function parseDependencies(contractSource: ContractSource): string[] { /** * Compiles the contracts and prints errors/warnings - * @param resolver Resolver - * @param solcInstance Instance of a solc compiler + * @param solcVersion Version of a solc compiler * @param standardInput Solidity standard JSON input */ -export function compileSolcJS( - resolver: Resolver, - solcInstance: solc.SolcInstance, +export async function compileSolcJSAsync( + solcVersion: string, standardInput: solc.StandardInput, -): solc.StandardOutput { +): Promise { + const solcInstance = await getSolcJSAsync(solcVersion); const standardInputStr = JSON.stringify(standardInput); - const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr, importPath => { - const sourceCodeIfExists = resolver.resolve(importPath); - return { contents: sourceCodeIfExists.source }; - }); + const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr); const compiled: solc.StandardOutput = JSON.parse(standardOutputStr); - if (!_.isUndefined(compiled.errors)) { - printCompilationErrorsAndWarnings(compiled.errors); - } return compiled; } /** * Compiles the contracts and prints errors/warnings - * @param resolver Resolver - * @param contractsDir Contracts directory - * @param workspaceDir Workspace directory * @param solcVersion Version of a solc compiler - * @param dependencyNameToPackagePath Mapping of dependency name to it's package path * @param standardInput Solidity standard JSON input */ -export function compileDocker( - resolver: Resolver, - contractsDir: string, - workspaceDir: string, +export async function compileDockerAsync( solcVersion: string, - dependencyNameToPackagePath: { [dependencyName: string]: string }, standardInput: solc.StandardInput, -): solc.StandardOutput { - const standardInputDocker = _.cloneDeep(standardInput); - standardInputDocker.settings.remappings = _.map( - dependencyNameToPackagePath, - (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, - ); - standardInputDocker.sources = _.mapKeys( - standardInputDocker.sources, - (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath, - ); - - const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2); +): Promise { + const standardInputStr = JSON.stringify(standardInput, null, 2); const dockerCommand = - `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` + - `solc --standard-json --allow-paths ${workspaceDir}`; - const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString(); - const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker); - - if (!_.isUndefined(compiledDocker.errors)) { - printCompilationErrorsAndWarnings(compiledDocker.errors); - } - - compiledDocker.sources = makeContractPathsRelative( - compiledDocker.sources, - contractsDir, - dependencyNameToPackagePath, - ); - compiledDocker.contracts = makeContractPathsRelative( - compiledDocker.contracts, - contractsDir, - dependencyNameToPackagePath, - ); - return compiledDocker; + `docker run -i -a stdin -a stdout -a stderr ethereum/solc:${solcVersion} ` + `solc --standard-json`; + const standardOutputStr = execSync(dockerCommand, { input: standardInputStr }).toString(); + const compiled: solc.StandardOutput = JSON.parse(standardOutputStr); + return compiled; } function makeContractPathRelative( @@ -202,7 +161,13 @@ function makeContractPathRelative( return contractPath; } -function makeContractPathsRelative( +/** + * Makes the path relative removing all system-dependent data. Converts absolute paths to a format suitable for artifacts. + * @param absolutePathToSmth Absolute path to contract or source + * @param contractsDir Current package contracts directory location + * @param dependencyNameToPackagePath Mapping of dependency name to package path + */ +export function makeContractPathsRelative( absolutePathToSmth: { [absoluteContractPath: string]: any }, contractsDir: string, dependencyNameToPackagePath: { [dependencyName: string]: string }, @@ -216,7 +181,7 @@ function makeContractPathsRelative( * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning). * @param solcErrors The errors field of standard JSON output that contains errors and warnings. */ -function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void { +export function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void { const SOLIDITY_WARNING = 'warning'; const errors = _.filter(solcErrors, entry => entry.severity !== SOLIDITY_WARNING); const warnings = _.filter(solcErrors, entry => entry.severity === SOLIDITY_WARNING); @@ -392,3 +357,35 @@ export function addHexPrefixToContractBytecode(compiledContract: solc.StandardCo } } } + +/** + * Takes the list of resolved contract sources from `SpyResolver` and produces a mapping from dependency name + * to package path used in `remappings` later. + * @param contractSources The list of resolved contract sources + */ +export function getDependencyNameToPackagePath( + contractSources: ContractSource[], +): { [dependencyName: string]: string } { + const allTouchedFiles = contractSources.map(contractSource => `${contractSource.absolutePath}`); + const NODE_MODULES = 'node_modules'; + const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); + const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; + _.map(allTouchedDependencies, dependencyFilePath => { + const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); + const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; + const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); + let packageName; + let packageScopeIfExists; + let dependencyName; + if (_.startsWith(importPath, '@')) { + [packageScopeIfExists, packageName] = importPath.split('/'); + dependencyName = `${packageScopeIfExists}/${packageName}`; + } else { + [packageName] = importPath.split('/'); + dependencyName = `${packageName}`; + } + const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); + dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; + }); + return dependencyNameToPackagePath; +} diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index d9377989f9..3c1d095573 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -8,11 +8,9 @@ import { Resolver } from './resolver'; export class NPMResolver extends Resolver { private readonly _packagePath: string; - private readonly _workspacePath: string; - constructor(packagePath: string, workspacePath: string = '/') { + constructor(packagePath: string) { super(); this._packagePath = packagePath; - this._workspacePath = workspacePath; } public resolveIfExists(importPath: string): ContractSource | undefined { if (!importPath.startsWith('/')) { @@ -25,11 +23,9 @@ export class NPMResolver extends Resolver { [packageName, ...other] = importPath.split('/'); } const pathWithinPackage = path.join(...other); - for ( - let currentPath = this._packagePath; - currentPath.includes(this._workspacePath); - currentPath = path.dirname(currentPath) - ) { + let currentPath = this._packagePath; + const ROOT_PATH = '/'; + while (currentPath !== ROOT_PATH) { const packagePath = _.isUndefined(packageScopeIfExists) ? packageName : path.join(packageScopeIfExists, packageName); @@ -38,6 +34,7 @@ export class NPMResolver extends Resolver { const fileContent = fs.readFileSync(lookupPath).toString(); return { source: fileContent, path: importPath, absolutePath: lookupPath }; } + currentPath = path.dirname(currentPath); } } return undefined; diff --git a/packages/typescript-typings/types/solc/index.d.ts b/packages/typescript-typings/types/solc/index.d.ts index f4c05cd7c9..fefad9f6a4 100644 --- a/packages/typescript-typings/types/solc/index.d.ts +++ b/packages/typescript-typings/types/solc/index.d.ts @@ -95,7 +95,7 @@ declare module 'solc' { optimizerEnabled: number, findImports: (importPath: string) => ImportContents, ): CompilationResult; - compileStandardWrapper(input: string, findImports: (importPath: string) => ImportContents): string; + compileStandardWrapper(input: string, findImports?: (importPath: string) => ImportContents): string; } export function loadRemoteVersion( versionName: string, From 3520d2584e94330c388702f445c3a61721027f3a Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Mon, 28 Jan 2019 17:27:53 +0100 Subject: [PATCH 05/12] Add 'useDockerisedSolc' to new packages --- contracts/erc721/compiler.json | 1 + 1 file changed, 1 insertion(+) diff --git a/contracts/erc721/compiler.json b/contracts/erc721/compiler.json index c5f07ddb84..db8589016b 100644 --- a/contracts/erc721/compiler.json +++ b/contracts/erc721/compiler.json @@ -1,6 +1,7 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, From 08f541535b299f47219db7b5219d8551f589dff3 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Mon, 28 Jan 2019 17:32:05 +0100 Subject: [PATCH 06/12] Fix naming --- packages/sol-compiler/src/compiler.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index b5cdf98654..29ddfffaa1 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -24,12 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema'; import { binPaths } from './solc/bin_paths'; import { addHexPrefixToContractBytecode, - compileDocker, - compileSolcJS, + compileDockerAsync, + compileSolcJSAsync, createDirIfDoesNotExistAsync, getContractArtifactIfExistsAsync, getDependencyNameToPackagePath, - getSolcJSAsync, getSourcesWithDependencies, getSourceTreeHash, makeContractPathsRelative, @@ -276,10 +275,10 @@ export class Compiler { const versionCommandOutput = execSync(dockerCommand).toString(); const versionCommandOutputParts = versionCommandOutput.split(' '); fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim(); - compilerOutput = compileDocker(solcVersion, input.standardInput); + compilerOutput = await compileDockerAsync(solcVersion, input.standardInput); } else { fullSolcVersion = binPaths[solcVersion]; - compilerOutput = compileSolcJS(solcVersion, input.standardInput); + compilerOutput = await compileSolcJSAsync(solcVersion, input.standardInput); } compilerOutput.sources = makeContractPathsRelative( compilerOutput.sources, @@ -295,8 +294,6 @@ export class Compiler { printCompilationErrorsAndWarnings(compilerOutput.errors); } - compilerOutputs.push(compilerOutput); - for (const contractPath of input.contractsToCompile) { const contractName = contractPathToData[contractPath].contractName; @@ -320,6 +317,8 @@ export class Compiler { ); } } + + compilerOutputs.push(compilerOutput); } return compilerOutputs; From be7daae91a380bc529fda21fb30fa9138d6f5477 Mon Sep 17 00:00:00 2001 From: Fabio B Date: Wed, 30 Jan 2019 13:03:30 +0100 Subject: [PATCH 07/12] Update packages/ethereum-types/src/index.ts Co-Authored-By: LogvinovLeon --- packages/ethereum-types/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts index bbc3682c73..8339ab5a67 100644 --- a/packages/ethereum-types/src/index.ts +++ b/packages/ethereum-types/src/index.ts @@ -509,7 +509,7 @@ export interface Source { * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description) * contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the * specified directory. - * useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used. + * useDockerisedSolc: If set to true - sol-compiler will try calling a dockerized installations of solc to achieve faster compilation times. Otherwise and by default - solcjs will be used. Defaults to false. * solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all * contracts to compile with the the version specified here. */ From 445a629016136a3f6b361bfb4c732e4a55b52d4c Mon Sep 17 00:00:00 2001 From: Fabio B Date: Wed, 30 Jan 2019 13:08:10 +0100 Subject: [PATCH 08/12] Update packages/sol-compiler/src/utils/compiler.ts Co-Authored-By: LogvinovLeon --- packages/sol-compiler/src/utils/compiler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index 6381e525f3..83c3384cec 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -360,7 +360,7 @@ export function addHexPrefixToContractBytecode(compiledContract: solc.StandardCo /** * Takes the list of resolved contract sources from `SpyResolver` and produces a mapping from dependency name - * to package path used in `remappings` later. + * to package path used in `remappings` later, as well as in generating the "relative" source paths saved to the artifact files. * @param contractSources The list of resolved contract sources */ export function getDependencyNameToPackagePath( From c82a4a5760006056d6f87534605effe150678bf9 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 30 Jan 2019 13:04:53 +0100 Subject: [PATCH 09/12] Fail fast --- packages/sol-compiler/src/compiler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 29ddfffaa1..e0c092bc10 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -280,6 +280,9 @@ export class Compiler { fullSolcVersion = binPaths[solcVersion]; compilerOutput = await compileSolcJSAsync(solcVersion, input.standardInput); } + if (!_.isUndefined(compilerOutput.errors)) { + printCompilationErrorsAndWarnings(compilerOutput.errors); + } compilerOutput.sources = makeContractPathsRelative( compilerOutput.sources, this._contractsDir, @@ -290,9 +293,6 @@ export class Compiler { this._contractsDir, dependencyNameToPackagePath, ); - if (!_.isUndefined(compilerOutput.errors)) { - printCompilationErrorsAndWarnings(compilerOutput.errors); - } for (const contractPath of input.contractsToCompile) { const contractName = contractPathToData[contractPath].contractName; From 33ed6a7c06dad6e96e59892f999a40e90ad93a91 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 30 Jan 2019 13:05:18 +0100 Subject: [PATCH 10/12] Remove + concatenation --- packages/sol-compiler/src/utils/compiler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index 83c3384cec..f02f872f32 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -142,8 +142,7 @@ export async function compileDockerAsync( standardInput: solc.StandardInput, ): Promise { const standardInputStr = JSON.stringify(standardInput, null, 2); - const dockerCommand = - `docker run -i -a stdin -a stdout -a stderr ethereum/solc:${solcVersion} ` + `solc --standard-json`; + const dockerCommand = `docker run -i -a stdin -a stdout -a stderr ethereum/solc:${solcVersion} solc --standard-json`; const standardOutputStr = execSync(dockerCommand, { input: standardInputStr }).toString(); const compiled: solc.StandardOutput = JSON.parse(standardOutputStr); return compiled; From 8d72e253c8bbe371e091b1ea6d365af07fd2ed26 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 30 Jan 2019 13:06:49 +0100 Subject: [PATCH 11/12] Add a comment --- packages/sol-compiler/src/utils/compiler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index f02f872f32..77a969e1c1 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -148,6 +148,11 @@ export async function compileDockerAsync( return compiled; } +/** + * Example "relative" paths: + * /user/leo/0x-monorepo/contracts/extensions/contracts/extension.sol -> extension.sol + * /user/leo/0x-monorepo/node_modules/@0x/contracts-protocol/contracts/exchange.sol -> @0x/contracts-protocol/contracts/exchange.sol + */ function makeContractPathRelative( absolutePath: string, contractsDir: string, From e1244648e06faee44fd030f8a1323ab649cb6443 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 30 Jan 2019 13:08:00 +0100 Subject: [PATCH 12/12] Rename dependencyNameToPackagePath to dependencyNameToPath --- packages/sol-compiler/src/compiler.ts | 8 ++++---- packages/sol-compiler/src/utils/compiler.ts | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index e0c092bc10..efee3eb8ab 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -254,7 +254,7 @@ export class Compiler { versionToInputs[solcVersion].contractsToCompile.push(contractSource.path); } - const dependencyNameToPackagePath = getDependencyNameToPackagePath(resolvedContractSources); + const dependencyNameToPath = getDependencyNameToPackagePath(resolvedContractSources); const compilerOutputs: StandardOutput[] = []; for (const solcVersion of _.keys(versionToInputs)) { @@ -267,7 +267,7 @@ export class Compiler { let compilerOutput; let fullSolcVersion; input.standardInput.settings.remappings = _.map( - dependencyNameToPackagePath, + dependencyNameToPath, (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, ); if (this._useDockerisedSolc) { @@ -286,12 +286,12 @@ export class Compiler { compilerOutput.sources = makeContractPathsRelative( compilerOutput.sources, this._contractsDir, - dependencyNameToPackagePath, + dependencyNameToPath, ); compilerOutput.contracts = makeContractPathsRelative( compilerOutput.contracts, this._contractsDir, - dependencyNameToPackagePath, + dependencyNameToPath, ); for (const contractPath of input.contractsToCompile) { diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index 77a969e1c1..c75f76dac3 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -156,10 +156,10 @@ export async function compileDockerAsync( function makeContractPathRelative( absolutePath: string, contractsDir: string, - dependencyNameToPackagePath: { [dependencyName: string]: string }, + dependencyNameToPath: { [dependencyName: string]: string }, ): string { let contractPath = absolutePath.replace(`${contractsDir}/`, ''); - _.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => { + _.map(dependencyNameToPath, (packagePath: string, dependencyName: string) => { contractPath = contractPath.replace(packagePath, dependencyName); }); return contractPath; @@ -169,15 +169,15 @@ function makeContractPathRelative( * Makes the path relative removing all system-dependent data. Converts absolute paths to a format suitable for artifacts. * @param absolutePathToSmth Absolute path to contract or source * @param contractsDir Current package contracts directory location - * @param dependencyNameToPackagePath Mapping of dependency name to package path + * @param dependencyNameToPath Mapping of dependency name to package path */ export function makeContractPathsRelative( absolutePathToSmth: { [absoluteContractPath: string]: any }, contractsDir: string, - dependencyNameToPackagePath: { [dependencyName: string]: string }, + dependencyNameToPath: { [dependencyName: string]: string }, ): { [contractPath: string]: any } { return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) => - makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath), + makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPath), ); } @@ -373,7 +373,7 @@ export function getDependencyNameToPackagePath( const allTouchedFiles = contractSources.map(contractSource => `${contractSource.absolutePath}`); const NODE_MODULES = 'node_modules'; const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); - const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; + const dependencyNameToPath: { [dependencyName: string]: string } = {}; _.map(allTouchedDependencies, dependencyFilePath => { const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; @@ -389,7 +389,7 @@ export function getDependencyNameToPackagePath( dependencyName = `${packageName}`; } const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); - dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; + dependencyNameToPath[dependencyName] = dependencyPackagePath; }); - return dependencyNameToPackagePath; + return dependencyNameToPath; }