Skip to content
This repository has been archived by the owner on Oct 19, 2024. It is now read-only.

Commit

Permalink
feat(solc): support customized output selection pruning (#1039)
Browse files Browse the repository at this point in the history 
* feat(solc): support customized output selection pruning

* chore(clippy): make clippy happy
  • Loading branch information
@mattsse
mattsse authored Mar 16, 2022
1 parent 5d14198 commit 2d75f9f
Show file tree
Hide file tree
Showing 5 changed files with 327 additions and 165 deletions.
2 changes: 1 addition & 1 deletion ethers-solc/src/artifacts/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pub mod output_selection;
pub mod serde_helpers;
use crate::{
artifacts::output_selection::{ContractOutputSelection, OutputSelection},
cache::FilteredSources,
filter::FilteredSources,
};
pub use serde_helpers::{deserialize_bytes, deserialize_opt_bytes};

Expand Down
155 changes: 2 additions & 153 deletions ethers-solc/src/cache.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
//! Support for compiling contracts
use crate::{
artifacts::{output_selection::OutputSelection, Settings, Sources},
artifacts::Sources,
config::SolcConfig,
error::{Result, SolcError},
filter::{FilteredSource, FilteredSourceInfo, FilteredSources},
resolver::GraphEdges,
utils, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Project, ProjectPathsConfig,
Source,
Expand All @@ -14,7 +15,6 @@ use std::{
btree_map::{BTreeMap, Entry},
hash_map, BTreeSet, HashMap, HashSet,
},
fmt,
fs::{self},
path::{Path, PathBuf},
time::{Duration, UNIX_EPOCH},
Expand Down Expand Up @@ -729,157 +729,6 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> {
}
}

/// Container type for a set of [FilteredSource]
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct FilteredSources(pub BTreeMap<PathBuf, FilteredSource>);

impl FilteredSources {
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}

pub fn len(&self) -> usize {
self.0.len()
}

/// Returns `true` if all files are dirty
pub fn all_dirty(&self) -> bool {
self.0.values().all(|s| s.is_dirty())
}

/// Returns all entries that are dirty
pub fn dirty(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| s.is_dirty())
}

/// Returns all entries that are clean
pub fn clean(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| !s.is_dirty())
}

/// Returns all dirty files
pub fn dirty_files(&self) -> impl Iterator<Item = &PathBuf> + fmt::Debug + '_ {
self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k))
}

/// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to
/// output everything for those dirty files as currently configured in the settings, but output
/// nothing for the other files that are _not_ dirty.
///
/// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the
/// files' output based on their state.
pub fn into_sources(self, settings: &mut Settings) -> Sources {
if !self.all_dirty() {
// settings can be optimized

tracing::trace!(
"Optimizing output selection for {}/{} sources",
self.clean().count(),
self.len()
);

let selection = settings
.output_selection
.as_mut()
.remove("*")
.unwrap_or_else(OutputSelection::default_file_output_selection);

for (file, source) in self.0.iter() {
if source.is_dirty() {
settings
.output_selection
.as_mut()
.insert(format!("{}", file.display()), selection.clone());
} else {
tracing::trace!("Optimizing output for {}", file.display());
settings.output_selection.as_mut().insert(
format!("{}", file.display()),
OutputSelection::empty_file_output_select(),
);
}
}
}
self.into()
}
}

impl From<FilteredSources> for Sources {
fn from(sources: FilteredSources) -> Self {
sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect()
}
}

impl From<Sources> for FilteredSources {
fn from(s: Sources) -> Self {
FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect())
}
}

impl From<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn from(s: BTreeMap<PathBuf, FilteredSource>) -> Self {
FilteredSources(s)
}
}

impl AsRef<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_ref(&self) -> &BTreeMap<PathBuf, FilteredSource> {
&self.0
}
}

impl AsMut<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_mut(&mut self) -> &mut BTreeMap<PathBuf, FilteredSource> {
&mut self.0
}
}

/// Represents the state of a filtered [Source]
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FilteredSource {
/// A source that fits the _dirty_ criteria
Dirty(Source),
/// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set
/// because a _dirty_ file pulls it in, either directly on indirectly.
Clean(Source),
}

impl FilteredSource {
/// Returns the underlying source
pub fn source(&self) -> &Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}

/// Consumes the type and returns the underlying source
pub fn into_source(self) -> Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}

/// Whether this file is actually dirt
pub fn is_dirty(&self) -> bool {
matches!(self, FilteredSource::Dirty(_))
}
}

/// Helper type that determines the state of a source file
struct FilteredSourceInfo {
/// path to the source file
file: PathBuf,
/// contents of the file
source: Source,
/// idx in the [GraphEdges]
idx: usize,
/// whether this file is actually dirty
///
/// See also [ArtifactsCacheInner::is_dirty()]
dirty: bool,
}

/// Abstraction over configured caching which can be either non-existent or an already loaded cache
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]
Expand Down
39 changes: 28 additions & 11 deletions ethers-solc/src/compile/project.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ use crate::{
};
use rayon::prelude::*;

use crate::filter::SparseOutputFileFilter;
use std::{collections::btree_map::BTreeMap, path::PathBuf};

#[derive(Debug)]
Expand All @@ -123,6 +124,8 @@ pub struct ProjectCompiler<'a, T: ArtifactOutput> {
project: &'a Project<T>,
/// how to compile all the sources
sources: CompilerSources,
/// How to select solc [CompilerOutput] for files
sparse_output: SparseOutputFileFilter,
}

impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
Expand Down Expand Up @@ -162,7 +165,7 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
CompilerSources::Sequential(sources_by_version)
};

Ok(Self { edges, project, sources })
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}

/// Compiles the sources with a pinned `Solc` instance
Expand All @@ -176,7 +179,14 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
let sources_by_version = BTreeMap::from([(solc, (version, sources))]);
let sources = CompilerSources::Sequential(sources_by_version);

Ok(Self { edges, project, sources })
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}

/// Applies the specified [SparseOutputFileFilter] to be applied when selecting solc output for
/// specific files to be compiled
pub fn with_sparse_output(mut self, sparse_output: impl Into<SparseOutputFileFilter>) -> Self {
self.sparse_output = sparse_output.into();
self
}

/// Compiles all the sources of the `Project` in the appropriate mode
Expand All @@ -203,13 +213,13 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
/// - sets proper source unit names
/// - check cache
fn preprocess(self) -> Result<PreprocessedState<'a, T>> {
let Self { edges, project, sources } = self;
let Self { edges, project, sources, sparse_output } = self;

let mut cache = ArtifactsCache::new(project, edges)?;
// retain and compile only dirty sources and all their imports
let sources = sources.filtered(&mut cache);

Ok(PreprocessedState { sources, cache })
Ok(PreprocessedState { sources, cache, sparse_output })
}
}

Expand All @@ -222,14 +232,18 @@ struct PreprocessedState<'a, T: ArtifactOutput> {
sources: FilteredCompilerSources,
/// cache that holds [CacheEntry] object if caching is enabled and the project is recompiled
cache: ArtifactsCache<'a, T>,
sparse_output: SparseOutputFileFilter,
}

impl<'a, T: ArtifactOutput> PreprocessedState<'a, T> {
/// advance to the next state by compiling all sources
fn compile(self) -> Result<CompiledState<'a, T>> {
let PreprocessedState { sources, cache } = self;
let output =
sources.compile(&cache.project().solc_config.settings, &cache.project().paths)?;
let PreprocessedState { sources, cache, sparse_output } = self;
let output = sources.compile(
&cache.project().solc_config.settings,
&cache.project().paths,
sparse_output,
)?;

Ok(CompiledState { output, cache })
}
Expand Down Expand Up @@ -351,13 +365,14 @@ impl FilteredCompilerSources {
self,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
match self {
FilteredCompilerSources::Sequential(input) => {
compile_sequential(input, settings, paths)
compile_sequential(input, settings, paths, sparse_output)
}
FilteredCompilerSources::Parallel(input, j) => {
compile_parallel(input, j, settings, paths)
compile_parallel(input, j, settings, paths, sparse_output)
}
}
}
Expand All @@ -377,6 +392,7 @@ fn compile_sequential(
input: VersionedFilteredSources,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
let mut aggregated = AggregatedCompilerOutput::default();
tracing::trace!("compiling {} jobs sequentially", input.len());
Expand All @@ -402,7 +418,7 @@ fn compile_sequential(
// depending on the composition of the filtered sources, the output selection can be
// optimized
let mut opt_settings = settings.clone();
let sources = filtered_sources.into_sources(&mut opt_settings);
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);

for input in CompilerInput::with_sources(sources) {
let actually_dirty = input
Expand Down Expand Up @@ -450,6 +466,7 @@ fn compile_parallel(
num_jobs: usize,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
debug_assert!(num_jobs > 1);
tracing::trace!(
Expand All @@ -475,7 +492,7 @@ fn compile_parallel(
// depending on the composition of the filtered sources, the output selection can be
// optimized
let mut opt_settings = settings.clone();
let sources = filtered_sources.into_sources(&mut opt_settings);
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);

for input in CompilerInput::with_sources(sources) {
let actually_dirty = input
Expand Down
Loading

0 comments on commit 2d75f9f

Please sign in to comment.