Auto merge of #33476 - nikomatsakis:incr-comp-xcrate, r=mw

track incr. comp. dependencies across crates

This PR refactors the compiler's incremental compilation hashing so that it can track dependencies across crates. The main bits are:

- computing a hash representing the metadata for an item we are emitting
  - we do this by making `MetaData(X)` be the current task while computing metadata for an item
  - this naturally registers reads from any tables and things that we read for that purpose
  - we can then hash all the inputs to those tables
- tracking when we access metadata
  - we do this by registering a read of `MetaData(X)` for each foreign item `X` whose metadata we read
- hashing metadata from foreign items
  - we do this by loading up metadata from a file in the incr. comp. directory
  - if there is no file, we use the SVH for the entire crate

There is one very simple test only at this point. The next PR will be focused on expanding out the tests.

Note that this is based on top of #33228

r? @michaelwoerister

src/librustc/cfg/mod.rs

@@ -64,7 +64,7 @@ impl CFG {
     }
 
     pub fn node_is_reachable(&self, id: ast::NodeId) -> bool {
-        self.graph.depth_traverse(self.entry)
+        self.graph.depth_traverse(self.entry, graph::OUTGOING)
                   .any(|idx| self.graph.node_data(idx).id() == id)
     }
 }

src/librustc/dep_graph/debug.rs

@@ -0,0 +1,69 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Code for debugging the dep-graph.
+
+use super::dep_node::DepNode;
+use std::error::Error;
+use std::fmt::Debug;
+
+/// A dep-node filter goes from a user-defined string to a query over
+/// nodes. Right now the format is like this:
+///
+///     x & y & z
+///
+/// where the format-string of the dep-node must contain `x`, `y`, and
+/// `z`.
+#[derive(Debug)]
+pub struct DepNodeFilter {
+    text: String
+}
+
+impl DepNodeFilter {
+    pub fn new(text: &str) -> Self {
+        DepNodeFilter {
+            text: text.trim().to_string()
+        }
+    }
+
+    /// True if all nodes always pass the filter.
+    pub fn accepts_all(&self) -> bool {
+        self.text.is_empty()
+    }
+
+    /// Tests whether `node` meets the filter, returning true if so.
+    pub fn test<D: Clone + Debug>(&self, node: &DepNode<D>) -> bool {
+        let debug_str = format!("{:?}", node);
+        self.text.split("&")
+                 .map(|s| s.trim())
+                 .all(|f| debug_str.contains(f))
+    }
+}
+
+/// A filter like `F -> G` where `F` and `G` are valid dep-node
+/// filters. This can be used to test the source/target independently.
+pub struct EdgeFilter {
+    pub source: DepNodeFilter,
+    pub target: DepNodeFilter,
+}
+
+impl EdgeFilter {
+    pub fn new(test: &str) -> Result<EdgeFilter, Box<Error>> {
+        let parts: Vec<_> = test.split("->").collect();
+        if parts.len() != 2 {
+            Err(format!("expected a filter like `a&b -> c&d`, not `{}`", test).into())
+        } else {
+            Ok(EdgeFilter {
+                source: DepNodeFilter::new(parts[0]),
+                target: DepNodeFilter::new(parts[1]),
+            })
+        }
+    }
+}

src/librustc/dep_graph/dep_node.rs

@@ -32,6 +32,10 @@ pub enum DepNode<D: Clone + Debug> {
     // Represents the HIR node with the given node-id
     Hir(D),
 
+    // Represents the metadata for a given HIR node, typically found
+    // in an extern crate.
+    MetaData(D),
+
     // Represents different phases in the compiler.
     CrateReader,
     CollectLanguageItems,
@@ -77,6 +81,7 @@ pub enum DepNode<D: Clone + Debug> {
     TransCrateItem(D),
     TransInlinedItem(D),
     TransWriteMetadata,
+    LinkBinary,
 
     // Nodes representing bits of computed IR in the tcx. Each shared
     // table in the tcx (or elsewhere) maps to one of these
@@ -174,7 +179,9 @@ impl<D: Clone + Debug> DepNode<D> {
             LateLintCheck => Some(LateLintCheck),
             TransCrate => Some(TransCrate),
             TransWriteMetadata => Some(TransWriteMetadata),
+            LinkBinary => Some(LinkBinary),
             Hir(ref d) => op(d).map(Hir),
+            MetaData(ref d) => op(d).map(MetaData),
             CollectItem(ref d) => op(d).map(CollectItem),
             CoherenceCheckImpl(ref d) => op(d).map(CoherenceCheckImpl),
             CoherenceOverlapCheck(ref d) => op(d).map(CoherenceOverlapCheck),

src/librustc/dep_graph/mod.rs

@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+pub mod debug;
 mod dep_node;
 mod dep_tracking_map;
 mod edges;
@@ -22,3 +23,4 @@ pub use self::dep_node::DepNode;
 pub use self::graph::DepGraph;
 pub use self::query::DepGraphQuery;
 pub use self::visit::visit_all_items_in_krate;
+pub use self::raii::DepTask;

src/librustc/dep_graph/query.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use rustc_data_structures::fnv::FnvHashMap;
-use rustc_data_structures::graph::{Graph, NodeIndex};
+use rustc_data_structures::graph::{Direction, INCOMING, Graph, NodeIndex, OUTGOING};
 use std::fmt::Debug;
 use std::hash::Hash;
 
@@ -63,20 +63,29 @@ impl<D: Clone + Debug + Hash + Eq> DepGraphQuery<D> {
                   .collect()
     }
 
-    /// All nodes reachable from `node`. In other words, things that
-    /// will have to be recomputed if `node` changes.
-    pub fn transitive_dependents(&self, node: DepNode<D>) -> Vec<DepNode<D>> {
+    fn reachable_nodes(&self, node: DepNode<D>, direction: Direction) -> Vec<DepNode<D>> {
         if let Some(&index) = self.indices.get(&node) {
-            self.graph.depth_traverse(index)
+            self.graph.depth_traverse(index, direction)
                       .map(|s| self.graph.node_data(s).clone())
                       .collect()
         } else {
             vec![]
         }
     }
 
+    /// All nodes reachable from `node`. In other words, things that
+    /// will have to be recomputed if `node` changes.
+    pub fn transitive_successors(&self, node: DepNode<D>) -> Vec<DepNode<D>> {
+        self.reachable_nodes(node, OUTGOING)
+    }
+
+    /// All nodes that can reach `node`.
+    pub fn transitive_predecessors(&self, node: DepNode<D>) -> Vec<DepNode<D>> {
+        self.reachable_nodes(node, INCOMING)
+    }
+
     /// Just the outgoing edges from `node`.
-    pub fn immediate_dependents(&self, node: DepNode<D>) -> Vec<DepNode<D>> {
+    pub fn immediate_successors(&self, node: DepNode<D>) -> Vec<DepNode<D>> {
         if let Some(&index) = self.indices.get(&node) {
             self.graph.successor_nodes(index)
                       .map(|s| self.graph.node_data(s).clone())

src/librustc/dep_graph/thread.rs

@@ -28,6 +28,7 @@ use super::DepGraphQuery;
 use super::DepNode;
 use super::edges::DepGraphEdges;
 
+#[derive(Debug)]
 pub enum DepMessage {
     Read(DepNode<DefId>),
     Write(DepNode<DefId>),
@@ -117,6 +118,8 @@ impl DepGraphThreadData {
     /// the buffer is full, this may swap.)
     #[inline]
     pub fn enqueue(&self, message: DepMessage) {
+        debug!("enqueue: {:?} tasks_pushed={}", message, self.tasks_pushed.get());
+
         // Regardless of whether dep graph construction is enabled, we
         // still want to check that we always have a valid task on the
         // stack when a read/write/etc event occurs.

src/librustc/dep_graph/visit.rs

@@ -42,7 +42,8 @@ pub fn visit_all_items_in_krate<'a, 'tcx, V, F>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
             let _task = self.tcx.dep_graph.in_task(task_id);
             debug!("Started task {:?}", task_id);
             self.tcx.dep_graph.read(DepNode::Hir(item_def_id));
-            self.visitor.visit_item(i)
+            self.visitor.visit_item(i);
+            debug!("Ended task {:?}", task_id);
         }
     }
 

src/librustc/hir/lowering.rs

@@ -45,6 +45,7 @@ use hir::map::Definitions;
 use hir::map::definitions::DefPathData;
 use hir::def_id::{DefIndex, DefId};
 use hir::def::{Def, PathResolution};
+use session::Session;
 
 use std::collections::BTreeMap;
 use std::iter;
@@ -97,8 +98,16 @@ impl Resolver for DummyResolver {
     }
 }
 
-pub fn lower_crate(krate: &Crate, id_assigner: &NodeIdAssigner, resolver: &mut Resolver)
+pub fn lower_crate(sess: &Session,
+                   krate: &Crate,
+                   id_assigner: &NodeIdAssigner,
+                   resolver: &mut Resolver)
                    -> hir::Crate {
+    // We're constructing the HIR here; we don't care what we will
+    // read, since we haven't even constructed the *input* to
+    // incr. comp. yet.
+    let _ignore = sess.dep_graph.in_ignore();
+
     LoweringContext {
         crate_root: if std_inject::no_core(krate) {
             None

src/librustc/hir/map/definitions.rs

@@ -82,8 +82,10 @@ impl DefPath {
         let mut data = vec![];
         let mut index = Some(start_index);
         loop {
+            debug!("DefPath::make: krate={:?} index={:?}", krate, index);
             let p = index.unwrap();
             let key = get_key(p);
+            debug!("DefPath::make: key={:?}", key);
             match key.disambiguated_data.data {
                 DefPathData::CrateRoot => {
                     assert!(key.parent.is_none());
@@ -178,6 +180,10 @@ impl Definitions {
         self.data[index.as_usize()].key.clone()
     }
 
+    pub fn def_index_for_def_key(&self, key: DefKey) -> Option<DefIndex> {
+        self.key_map.get(&key).cloned()
+    }
+
     /// Returns the path from the crate root to `index`. The root
     /// nodes are not included in the path (i.e., this will be an
     /// empty vector for the crate root). For an inlined item, this
@@ -208,37 +214,6 @@ impl Definitions {
         }
     }
 
-    pub fn retrace_path(&self, path: &DefPath) -> Option<DefIndex> {
-        debug!("retrace_path(path={:?})", path);
-
-        // we assume that we only want to retrace paths relative to
-        // the crate root
-        assert!(path.is_local());
-
-        let root_key = DefKey {
-            parent: None,
-            disambiguated_data: DisambiguatedDefPathData {
-                data: DefPathData::CrateRoot,
-                disambiguator: 0,
-            },
-        };
-        let root_id = self.key_map[&root_key];
-
-        debug!("retrace_path: root_id={:?}", root_id);
-
-        let mut id = root_id;
-        for data in &path.data {
-            let key = DefKey { parent: Some(id), disambiguated_data: data.clone() };
-            debug!("key = {:?}", key);
-            id = match self.key_map.get(&key) {
-                Some(&id) => id,
-                None => return None
-            };
-        }
-
-        Some(id)
-    }
-
     pub fn create_def_with_parent(&mut self,
                                   parent: Option<DefIndex>,
                                   node_id: ast::NodeId,

src/librustc/hir/map/mod.rs

@@ -19,7 +19,7 @@ use dep_graph::{DepGraph, DepNode};
 
 use middle::cstore::InlinedItem;
 use middle::cstore::InlinedItem as II;
-use hir::def_id::{CRATE_DEF_INDEX, DefId};
+use hir::def_id::{CRATE_DEF_INDEX, DefId, DefIndex};
 
 use syntax::abi::Abi;
 use syntax::ast::{self, Name, NodeId, DUMMY_NODE_ID, };
@@ -160,10 +160,10 @@ pub struct Forest {
 }
 
 impl Forest {
-    pub fn new(krate: Crate, dep_graph: DepGraph) -> Forest {
+    pub fn new(krate: Crate, dep_graph: &DepGraph) -> Forest {
         Forest {
             krate: krate,
-            dep_graph: dep_graph,
+            dep_graph: dep_graph.clone(),
             inlined_items: TypedArena::new()
         }
     }
@@ -285,9 +285,8 @@ impl<'ast> Map<'ast> {
         self.definitions.borrow().def_path(def_id.index)
     }
 
-    pub fn retrace_path(&self, path: &DefPath) -> Option<DefId> {
-        self.definitions.borrow().retrace_path(path)
-                                 .map(DefId::local)
+    pub fn def_index_for_def_key(&self, def_key: DefKey) -> Option<DefIndex> {
+        self.definitions.borrow().def_index_for_def_key(def_key)
     }
 
     pub fn local_def_id(&self, node: NodeId) -> DefId {