From e456f5d4a474350e9af490b6fa699f551432e73d Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 24 Nov 2017 12:29:30 -0800 Subject: [PATCH] Construct a real dependency graph using petgraph --- Cargo.toml | 4 + src/lib.rs | 32 +++++- src/main.rs | 299 +++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 261 insertions(+), 74 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d997a8b..c30e85a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,12 +15,16 @@ chrono = { version = "0.4", features = ["serde"] } failure = "0.1" failure_derive = "0.1" flate2 = "0.2" +fnv = "1.0" gnuplot = "0.0.23" indicatif = "0.8" +isatty = "0.1" palette = "0.2" +petgraph = "0.4" regex = "0.2" reqwest = "0.8" semver = { version = "0.9", features = ["serde"] } +semver-parser = "0.7" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" diff --git a/src/lib.rs b/src/lib.rs index f70968d..f3c494b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,16 +6,19 @@ extern crate failure_derive; extern crate chrono; extern crate failure; +extern crate fnv; extern crate reqwest; extern crate semver; extern crate serde; extern crate serde_json; extern crate url; -use chrono::{DateTime, Utc}; +use chrono::Utc; use failure::Error; +use fnv::FnvHashMap as Map; + use semver::{Version, VersionReq}; use serde::de::DeserializeOwned; @@ -32,6 +35,8 @@ use std::path::{Path, PathBuf}; const PER_PAGE: usize = 100; const RETRIES: usize = 32; +pub type DateTime = chrono::DateTime; + #[derive(Deserialize, Debug)] pub struct IndexPage { pub crates: Vec, @@ -64,13 +69,26 @@ pub struct Dependencies { pub struct Dependency { #[serde(rename = "crate_id")] pub name: String, + pub kind: DependencyKind, pub req: VersionReq, + pub optional: bool, + pub default_features: bool, + pub features: Vec, +} + +#[derive(Deserialize, Clone, Copy, Debug)] +#[serde(rename_all = "lowercase")] +pub enum DependencyKind { + Normal, + Build, + Dev, } #[derive(Deserialize, Debug)] pub struct CrateVersion { pub num: Version, - pub created_at: DateTime, + pub created_at: DateTime, + pub features: Map>, } pub fn cache_index(n: usize) -> Result { @@ -115,6 +133,16 @@ impl Display for FileNotFoundError { } } +impl Display for DependencyKind { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match *self { + DependencyKind::Normal => write!(formatter, "normal"), + DependencyKind::Build => write!(formatter, "build"), + DependencyKind::Dev => write!(formatter, "dev"), + } + } +} + fn cache(endpoint: U, location: P) -> Result where U: AsRef, diff --git a/src/main.rs b/src/main.rs index 632bd85..2761ee4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,12 +5,16 @@ extern crate cargo; extern crate chrono; extern crate failure; extern crate flate2; +extern crate fnv; extern crate gnuplot; extern crate indicatif; +extern crate isatty; extern crate palette; +extern crate petgraph; extern crate regex; extern crate reqwest; extern crate semver; +extern crate semver_parser; extern crate serde; extern crate tar; extern crate unindent; @@ -19,32 +23,42 @@ use cargo::{CliResult, CliError}; use cargo::core::shell::Shell; use cargo::util::{Config, CargoError}; -use chrono::{DateTime, Utc, NaiveDate, NaiveTime, NaiveDateTime}; +use chrono::{Utc, NaiveDate, NaiveTime, NaiveDateTime}; use failure::Error; use flate2::read::GzDecoder; +use fnv::{FnvHashSet as Set, FnvHashMap as Map}; + use gnuplot::{Figure, Fix, Auto, Caption, LineWidth, AxesCommon, Color, MinorScale, Graph, Placement, AlignLeft, AlignTop}; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; +use isatty::stderr_isatty; + use palette::Hue; use palette::pixel::Srgb; +use petgraph::{Incoming, Outgoing}; +use petgraph::graph::NodeIndex; + use regex::Regex; use reqwest::header::ContentLength; -use semver::Version; +use semver::{Version, VersionReq}; +use semver_parser::range::{self, Predicate}; +use semver_parser::range::Op::Compatible; use tar::Archive; use unindent::unindent; -use std::collections::HashSet as Set; use std::env; +use std::fmt::{self, Display}; +use std::mem; use std::path::Path; use std::u64; @@ -148,12 +162,14 @@ fn init() -> Result<(), Error> { let tgz = reqwest::get(snapshot)?.error_for_status()?; let pb = ProgressBar::hidden(); - if let Some(&ContentLength(n)) = tgz.headers().get() { - pb.set_length(n); - pb.set_style(ProgressStyle::default_bar() - .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})") - .progress_chars("#>-")); - pb.set_draw_target(ProgressDrawTarget::stderr()); + if stderr_isatty() { + if let Some(&ContentLength(n)) = tgz.headers().get() { + pb.set_length(n); + pb.set_style(ProgressStyle::default_bar() + .template("[{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})") + .progress_chars("&&.")); + pb.set_draw_target(ProgressDrawTarget::stderr()); + } } let tracker = ProgressRead::new(&pb, tgz); @@ -161,32 +177,81 @@ fn init() -> Result<(), Error> { let mut archive = Archive::new(decoder); archive.unpack(".")?; - pb.finish_with_message("ready to tally!"); + pb.finish_and_clear(); Ok(()) } +#[derive(Debug)] +struct Universe { + graph: petgraph::Graph, + crates: Map>, +} + +#[derive(Debug)] +struct Key { + name: String, + num: Version, + crates_incoming: Set, +} + +impl Display for Key { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}:{}", self.name, self.num) + } +} + +#[derive(Debug)] +enum Edge { + Current { + kind: DependencyKind, + req: VersionReq, + optional: bool, + default_features: bool, + features: Vec, + }, + Obsolete, +} + +impl Display for Edge { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match *self { + Edge::Current { kind, .. } => write!(formatter, "{}", kind), + Edge::Obsolete => write!(formatter, "obsolete"), + } + } +} + #[derive(Debug)] struct Event { name: String, num: Version, - timestamp: DateTime, + timestamp: DateTime, dependencies: Vec, } #[derive(Debug)] struct Matcher { name: String, - num: Option, - crates_using: Set, + req: VersionReq, + matches: Vec, } #[derive(Debug)] struct Row { - timestamp: DateTime, + timestamp: DateTime, counts: Vec, total: usize, } +impl Universe { + fn new() -> Self { + Universe { + graph: petgraph::Graph::new(), + crates: Map::default(), + } + } +} + fn tally(flags: Flags) -> Result<(), Error> { if flags.flag_transitive { return Err(failure::err_msg("--transitive is not implemented")); @@ -195,21 +260,33 @@ fn tally(flags: Flags) -> Result<(), Error> { let mut chronology = load_data(&flags)?; chronology.sort_by(|a, b| a.timestamp.cmp(&b.timestamp)); + let mut universe = Universe::new(); let mut matchers = create_matchers(&flags)?; + let mut table = Vec::::new(); - let mut table = Vec::new(); - let mut all_crates = Set::new(); - for event in chronology { - all_crates.insert(event.name.clone()); - let changed = process_event(&mut matchers, &event)?; - if changed { - table.push(Row { - timestamp: event.timestamp, - counts: matchers.iter().map(|m| m.crates_using.len()).collect(), - total: all_crates.len(), - }); + let n = chronology.len() as u64; + let pb = ProgressBar::hidden(); + if stderr_isatty() { + pb.set_length(n * n); + pb.set_style(ProgressStyle::default_bar() + .template("[{wide_bar:.cyan/blue}] {percent}%") + .progress_chars("&&.")); + pb.set_draw_target(ProgressDrawTarget::stderr()); + } + for (i, event) in chronology.into_iter().enumerate() { + let timestamp = event.timestamp.clone(); + process_event(&mut universe, &mut matchers, event); + let row = compute_counts(&universe, &matchers, timestamp); + let include = match table.last() { + None => row.counts.iter().any(|&count| count != 0), + Some(last) => last.counts != row.counts, + }; + if include { + table.push(row); } + pb.inc(2 * i as u64 + 1); } + pb.finish_and_clear(); if table.is_empty() { return Err(failure::err_msg("nothing found for this crate")); } @@ -260,67 +337,145 @@ fn create_matchers(flags: &Flags) -> Result, Error> { let mut pieces = s.splitn(2, ':'); matchers.push(Matcher { name: pieces.next().unwrap().to_owned(), - num: match pieces.next() { - Some(num) => { - match parse_major_minor(num) { - Ok(num) => Some(num), - Err(_) => { - return Err(failure::err_msg(format!( - "Failed to parse series {:?}, \ - expected something like \"serde:0.9\"", s))); - } - } + req: match pieces.next().unwrap_or("*").parse() { + Ok(req) => req, + Err(err) => { + return Err(failure::err_msg(format!( + "Failed to parse series {}: {}", s, err))); } - None => None, }, - crates_using: Set::new(), + matches: Vec::new(), }); } Ok(matchers) } -fn parse_major_minor(num: &str) -> Result { - let mut pieces = num.splitn(2, '.'); - let major = pieces.next().unwrap().parse()?; - let minor = pieces.next().ok_or(failure::err_msg("missing minor"))?.parse()?; - Ok(Version::new(major, minor, u64::MAX)) -} +fn process_event(universe: &mut Universe, matchers: &mut [Matcher], event: Event) { + // Insert new node in graph + let key = Key { + name: event.name.clone(), + num: event.num.clone(), + crates_incoming: Set::default(), + }; + let new = universe.graph.add_node(key); + + // If there is an older version of this crate, remove its name from + // everything it depends on + if let Some(older) = universe.crates.get(&event.name) { + if let Some(&last) = older.last() { + let mut walk = universe.graph.neighbors_directed(last, Outgoing).detach(); + while let Some(edge) = walk.next_edge(&universe.graph) { + let endpoints = universe.graph.edge_endpoints(edge).unwrap(); + universe.graph[endpoints.1].crates_incoming.remove(&event.name); + mem::replace(&mut universe.graph[edge], Edge::Obsolete); + } + } + } -fn process_event(matchers: &mut [Matcher], event: &Event) -> Result { - let mut changed = false; + // Add edges to all nodes depended on by the new node + for dep in event.dependencies { + if dep.name != event.name { + if let Some(target) = resolve(universe, &dep.name, &dep.req) { + universe.graph.add_edge(new, target, Edge::Current { + kind: dep.kind, + req: dep.req, + optional: dep.optional, + default_features: dep.default_features, + features: dep.features, + }); + universe.graph[target].crates_incoming.insert(event.name.clone()); + } + } + } - for matcher in matchers { - let mut using = false; - for dep in &event.dependencies { - if dep.name == matcher.name { - using = true; - let matches = match matcher.num { - Some(ref version) => { - // Exclude silly wildcard deps - if dep.req.matches(&Version::new(0, u64::MAX, 0)) { - false - } else if dep.req.matches(&Version::new(u64::MAX, 0, 0)) { - false - } else { - dep.req.matches(version) - } + // Find all nodes representing older versions of the same crate + let older = universe.crates.entry(event.name.clone()).or_insert_with(Vec::new); + + // Update edges that previously depended on older version of this crate + for &node in &*older { + if is_compatible(&universe.graph[node].num, &event.num) { + let mut walk = universe.graph.neighbors_directed(node, Incoming).detach(); + while let Some(edge) = walk.next_edge(&universe.graph) { + let mut repoint = false; + if let Edge::Current { ref req, .. } = universe.graph[edge] { + repoint = req.matches(&event.num); + } + if repoint { + let endpoints = universe.graph.edge_endpoints(edge).unwrap(); + let old = mem::replace(&mut universe.graph[edge], Edge::Obsolete); + universe.graph.add_edge(endpoints.0, new, old); + let reverse_dep = universe.graph[endpoints.0].name.clone(); + if universe.graph[endpoints.1].crates_incoming.remove(&reverse_dep) { + universe.graph[new].crates_incoming.insert(reverse_dep); } - None => true, - }; - changed |= if matches { - matcher.crates_using.insert(event.name.clone()) - } else { - matcher.crates_using.remove(&event.name) - }; + } } } - if !using { - changed |= matcher.crates_using.remove(&event.name); + } + + // Update matchers that tally the new node + for matcher in matchers { + if matcher.name == event.name && matcher.req.matches(&event.num) { + matcher.matches.push(new); + } + } + + // Add new node to list of versions of its crate + older.push(new); +} + +fn resolve(universe: &Universe, name: &str, req: &VersionReq) -> Option { + let versions = universe.crates.get(name)?; + let mut max = None::; + for &node in versions { + let key = &universe.graph[node]; + if req.matches(&key.num) { + if max.map(|max| key.num > universe.graph[max].num).unwrap_or(true) { + max = Some(node); + } } } + Some(max.unwrap_or(*versions.last().unwrap())) +} - Ok(changed) +fn is_compatible(older: &Version, newer: &Version) -> bool { + use semver::Identifier as SemverId; + use semver_parser::version::Identifier as ParseId; + let req = range::VersionReq { + predicates: vec![ + Predicate { + op: Compatible, + major: older.major, + minor: Some(older.minor), + patch: Some(older.patch), + pre: older.pre.iter().map(|pre| { + match *pre { + SemverId::Numeric(n) => ParseId::Numeric(n), + SemverId::AlphaNumeric(ref s) => ParseId::AlphaNumeric(s.clone()), + } + }).collect(), + }, + ], + }; + VersionReq::from(req).matches(newer) +} + +fn compute_counts(universe: &Universe, matchers: &[Matcher], timestamp: DateTime) -> Row { + let mut crates = Set::default(); + Row { + timestamp: timestamp, + counts: matchers.iter() + .map(|matcher| { + crates.clear(); + for &node in &matcher.matches { + crates.extend(&universe.graph[node].crates_incoming); + } + crates.len() + }) + .collect(), + total: universe.crates.len(), + } } fn draw_graph(flags: &Flags, table: Vec) { @@ -377,10 +532,10 @@ fn draw_graph(flags: &Flags, table: Vec) { fg.show(); } -fn float_year(dt: &DateTime) -> f64 { +fn float_year(dt: &DateTime) -> f64 { let nd = NaiveDate::from_ymd(2017, 1, 1); let nt = NaiveTime::from_hms_milli(0, 0, 0, 0); - let base = DateTime::::from_utc(NaiveDateTime::new(nd, nt), Utc); + let base = DateTime::from_utc(NaiveDateTime::new(nd, nt), Utc); let offset = dt.signed_duration_since(base.clone()); let year = offset.num_minutes() as f64 / 525960.0 + 2017.0; year