Generate region values directly to reduce memory usage.

Also modify `SparseBitMatrix` so that it does not require knowing the
dimensions in advance, but instead grows on demand.

Author: davidtwco

Diff for: src/librustc_data_structures/bitvec.rs

@@ -281,10 +281,10 @@ where
 }
 
 impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
-    /// Create a new `rows x columns` matrix, initially empty.
-    pub fn new(rows: R, _columns: C) -> SparseBitMatrix<R, C> {
-        SparseBitMatrix {
-            vector: IndexVec::from_elem_n(SparseBitSet::new(), rows.index()),
+    /// Create a new empty sparse bit matrix with no rows or columns.
+    pub fn new() -> Self {
+        Self {
+            vector: IndexVec::new(),
         }
     }
 
@@ -293,6 +293,14 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
     ///
     /// Returns true if this changed the matrix, and false otherwise.
     pub fn add(&mut self, row: R, column: C) -> bool {
+        debug!(
+            "add(row={:?}, column={:?}, current_len={})",
+            row,
+            column,
+            self.vector.len()
+        );
+        self.vector
+            .ensure_contains_elem(row, || SparseBitSet::new());
         self.vector[row].insert(column)
     }
 
@@ -301,7 +309,7 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
     /// if the matrix represents (transitive) reachability, can
     /// `row` reach `column`?
     pub fn contains(&self, row: R, column: C) -> bool {
-        self.vector[row].contains(column)
+        self.vector.get(row).map_or(false, |r| r.contains(column))
     }
 
     /// Add the bits from row `read` to the bits from row `write`,
@@ -315,16 +323,27 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
         let mut changed = false;
 
         if read != write {
-            let (bit_set_read, bit_set_write) = self.vector.pick2_mut(read, write);
+            if self.vector.get(read).is_some() {
+                self.vector
+                    .ensure_contains_elem(write, || SparseBitSet::new());
+                let (bit_set_read, bit_set_write) = self.vector.pick2_mut(read, write);
 
-            for read_chunk in bit_set_read.chunks() {
-                changed = changed | bit_set_write.insert_chunk(read_chunk).any();
+                for read_chunk in bit_set_read.chunks() {
+                    changed = changed | bit_set_write.insert_chunk(read_chunk).any();
+                }
             }
         }
 
         changed
     }
 
+    /// Merge a row, `from`, into the `into` row.
+    pub fn merge_into(&mut self, into: R, from: &SparseBitSet<C>) -> bool {
+        self.vector
+            .ensure_contains_elem(into, || SparseBitSet::new());
+        self.vector[into].insert_from(from)
+    }
+
     /// True if `sub` is a subset of `sup`
     pub fn is_subset(&self, sub: R, sup: R) -> bool {
         sub == sup || {
@@ -336,10 +355,20 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
         }
     }
 
+    /// Number of elements in the matrix.
+    pub fn len(&self) -> usize {
+        self.vector.len()
+    }
+
     /// Iterates through all the columns set to true in a given row of
     /// the matrix.
     pub fn iter<'a>(&'a self, row: R) -> impl Iterator<Item = C> + 'a {
-        self.vector[row].iter()
+        self.vector.get(row).into_iter().flat_map(|r| r.iter())
+    }
+
+    /// Iterates through each row and the accompanying bit set.
+    pub fn iter_enumerated<'a>(&'a self) -> impl Iterator<Item = (R, &'a SparseBitSet<C>)> + 'a {
+        self.vector.iter_enumerated()
     }
 }
 
@@ -445,6 +474,15 @@ impl<I: Idx> SparseBitSet<I> {
         }
     }
 
+    /// Insert into bit set from another bit set.
+    pub fn insert_from(&mut self, from: &SparseBitSet<I>) -> bool {
+        let mut changed = false;
+        for read_chunk in from.chunks() {
+            changed = changed | self.insert_chunk(read_chunk).any();
+        }
+        changed
+    }
+
     pub fn remove_chunk(&mut self, chunk: SparseChunk<I>) -> SparseChunk<I> {
         if chunk.bits == 0 {
             return chunk;

Diff for: src/librustc_data_structures/indexed_vec.rs

@@ -518,10 +518,28 @@ impl<I: Idx, T> IndexVec<I, T> {
 }
 
 impl<I: Idx, T: Clone> IndexVec<I, T> {
+    /// Grows the index vector so that it contains an entry for
+    /// `elem`; if that is already true, then has no
+    /// effect. Otherwise, inserts new values as needed by invoking
+    /// `fill_value`.
+    #[inline]
+    pub fn ensure_contains_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
+        let min_new_len = elem.index() + 1;
+        if self.len() < min_new_len {
+            self.raw.resize_with(min_new_len, fill_value);
+        }
+    }
+
     #[inline]
     pub fn resize(&mut self, new_len: usize, value: T) {
         self.raw.resize(new_len, value)
     }
+
+    #[inline]
+    pub fn resize_to_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
+        let min_new_len = elem.index() + 1;
+        self.raw.resize_with(min_new_len, fill_value);
+    }
 }
 
 impl<I: Idx, T: Ord> IndexVec<I, T> {

Diff for: src/librustc_data_structures/lib.rs

@@ -30,6 +30,7 @@
 #![feature(optin_builtin_traits)]
 #![feature(macro_vis_matcher)]
 #![feature(allow_internal_unstable)]
+#![feature(vec_resize_with)]
 
 #![cfg_attr(unix, feature(libc))]
 #![cfg_attr(test, feature(test))]

Diff for: src/librustc_mir/borrow_check/nll/constraint_generation.rs

@@ -21,7 +21,6 @@ use rustc::mir::{Local, Statement, Terminator};
 use rustc::ty::fold::TypeFoldable;
 use rustc::ty::subst::Substs;
 use rustc::ty::{self, CanonicalTy, ClosureSubsts, GeneratorSubsts};
-use std::iter;
 
 pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
     infcx: &InferCtxt<'cx, 'gcx, 'tcx>,
@@ -30,7 +29,6 @@ pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
     location_table: &LocationTable,
     mir: &Mir<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
-    liveness_set_from_typeck: &[(ty::Region<'tcx>, Location)],
 ) {
     let mut cg = ConstraintGeneration {
         borrow_set,
@@ -40,8 +38,6 @@ pub(super) fn generate_constraints<'cx, 'gcx, 'tcx>(
         all_facts,
     };
 
-    cg.add_region_liveness_constraints_from_type_check(liveness_set_from_typeck);
-
     for (bb, data) in mir.basic_blocks().iter_enumerated() {
         cg.visit_basic_block_data(bb, data);
     }
@@ -189,42 +185,6 @@ impl<'cg, 'cx, 'gcx, 'tcx> Visitor<'tcx> for ConstraintGeneration<'cg, 'cx, 'gcx
 }
 
 impl<'cx, 'cg, 'gcx, 'tcx> ConstraintGeneration<'cx, 'cg, 'gcx, 'tcx> {
-    /// The MIR type checker generates region liveness constraints
-    /// that we also have to respect.
-    fn add_region_liveness_constraints_from_type_check(
-        &mut self,
-        liveness_set: &[(ty::Region<'tcx>, Location)],
-    ) {
-        debug!(
-            "add_region_liveness_constraints_from_type_check(liveness_set={} items)",
-            liveness_set.len(),
-        );
-
-        let ConstraintGeneration {
-            regioncx,
-            location_table,
-            all_facts,
-            ..
-        } = self;
-
-        for (region, location) in liveness_set {
-            debug!("generate: {:#?} is live at {:#?}", region, location);
-            let region_vid = regioncx.to_region_vid(region);
-            regioncx.add_live_element(region_vid, *location);
-        }
-
-        if let Some(all_facts) = all_facts {
-            all_facts
-                .region_live_at
-                .extend(liveness_set.into_iter().flat_map(|(region, location)| {
-                    let r = regioncx.to_region_vid(region);
-                    let p1 = location_table.start_index(*location);
-                    let p2 = location_table.mid_index(*location);
-                    iter::once((r, p1)).chain(iter::once((r, p2)))
-                }));
-        }
-    }
-
     /// Some variable with type `live_ty` is "regular live" at
     /// `location` -- i.e., it may be used later. This means that all
     /// regions appearing in the type `live_ty` must be live at

Diff for: src/librustc_mir/borrow_check/nll/mod.rs

@@ -12,6 +12,7 @@ use borrow_check::borrow_set::BorrowSet;
 use borrow_check::location::{LocationIndex, LocationTable};
 use borrow_check::nll::facts::AllFactsExt;
 use borrow_check::nll::type_check::MirTypeckRegionConstraints;
+use borrow_check::nll::region_infer::values::RegionValueElements;
 use dataflow::indexes::BorrowIndex;
 use dataflow::move_paths::MoveData;
 use dataflow::FlowAtLocation;
@@ -99,6 +100,8 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
         None
     };
 
+    let elements = &Rc::new(RegionValueElements::new(mir, universal_regions.len()));
+
     // Run the MIR type-checker.
     let liveness = &LivenessResults::compute(mir);
     let constraint_sets = type_check::type_check(
@@ -113,6 +116,7 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
         &mut all_facts,
         flow_inits,
         move_data,
+        elements,
     );
 
     if let Some(all_facts) = &mut all_facts {
@@ -126,7 +130,7 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
     // base constraints generated by the type-check.
     let var_origins = infcx.take_region_var_origins();
     let MirTypeckRegionConstraints {
-        liveness_set,
+        liveness_constraints,
         outlives_constraints,
         type_tests,
     } = constraint_sets;
@@ -136,6 +140,8 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
         mir,
         outlives_constraints,
         type_tests,
+        liveness_constraints,
+        elements,
     );
 
     // Generate various additional constraints.
@@ -146,7 +152,6 @@ pub(in borrow_check) fn compute_regions<'cx, 'gcx, 'tcx>(
         location_table,
         &mir,
         borrow_set,
-        &liveness_set,
     );
     invalidation::generate_invalidates(
         infcx,

Diff for: src/librustc_mir/borrow_check/nll/region_infer/mod.rs

@@ -37,7 +37,7 @@ mod annotation;
 mod dump_mir;
 mod error_reporting;
 mod graphviz;
-mod values;
+pub mod values;
 use self::values::{RegionValueElements, RegionValues};
 
 use super::ToRegionVid;
@@ -66,8 +66,8 @@ pub struct RegionInferenceContext<'tcx> {
     /// the SCC (see `constraint_sccs`) and for error reporting.
     constraint_graph: Rc<ConstraintGraph>,
 
-    /// The SCC computed from `constraints` and
-    /// `constraint_graph`. Used to compute the values of each region.
+    /// The SCC computed from `constraints` and the constraint graph. Used to compute the values
+    /// of each region.
     constraint_sccs: Rc<Sccs<RegionVid, ConstraintSccIndex>>,
 
     /// The final inferred values of the region variables; we compute
@@ -207,15 +207,13 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     pub(crate) fn new(
         var_infos: VarInfos,
         universal_regions: UniversalRegions<'tcx>,
-        mir: &Mir<'tcx>,
+        _mir: &Mir<'tcx>,
         outlives_constraints: ConstraintSet,
         type_tests: Vec<TypeTest<'tcx>>,
+        liveness_constraints: RegionValues<RegionVid>,
+        elements: &Rc<RegionValueElements>,
     ) -> Self {
         let universal_regions = Rc::new(universal_regions);
-        let num_region_variables = var_infos.len();
-        let num_universal_regions = universal_regions.len();
-
-        let elements = &Rc::new(RegionValueElements::new(mir, num_universal_regions));
 
         // Create a RegionDefinition for each inference variable.
         let definitions: IndexVec<_, _> = var_infos
@@ -227,15 +225,20 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         let constraint_graph = Rc::new(constraints.graph(definitions.len()));
         let constraint_sccs = Rc::new(constraints.compute_sccs(&constraint_graph));
 
-        let scc_values = RegionValues::new(elements, constraint_sccs.num_sccs());
+        let mut scc_values = RegionValues::new(elements);
+
+        for (region, location_set) in liveness_constraints.iter_enumerated() {
+            let scc = constraint_sccs.scc(region);
+            scc_values.merge_into(scc, location_set);
+        }
 
         let mut result = Self {
             definitions,
             elements: elements.clone(),
-            liveness_constraints: RegionValues::new(elements, num_region_variables),
+            liveness_constraints,
             constraints,
-            constraint_sccs,
             constraint_graph,
+            constraint_sccs,
             scc_values,
             type_tests,
             universal_regions,
@@ -414,7 +417,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             constraints
         });
 
-        // To propagate constriants, we walk the DAG induced by the
+        // To propagate constraints, we walk the DAG induced by the
         // SCC. For each SCC, we visit its successors and compute
         // their values, then we union all those values to get our
         // own.

Diff for: src/librustc_mir/borrow_check/nll/region_infer/values.rs

@@ -10,7 +10,7 @@
 
 use rustc::mir::{BasicBlock, Location, Mir};
 use rustc::ty::RegionVid;
-use rustc_data_structures::bitvec::SparseBitMatrix;
+use rustc_data_structures::bitvec::{SparseBitMatrix, SparseBitSet};
 use rustc_data_structures::indexed_vec::Idx;
 use rustc_data_structures::indexed_vec::IndexVec;
 use std::fmt::Debug;
@@ -55,11 +55,6 @@ impl RegionValueElements {
         }
     }
 
-    /// Total number of element indices that exist.
-    crate fn num_elements(&self) -> usize {
-        self.num_points + self.num_universal_regions
-    }
-
     /// Converts an element of a region value into a `RegionElementIndex`.
     crate fn index<T: ToElementIndex>(&self, elem: T) -> RegionElementIndex {
         elem.to_element_index(self)
@@ -188,18 +183,10 @@ impl<N: Idx> RegionValues<N> {
     /// Creates a new set of "region values" that tracks causal information.
     /// Each of the regions in num_region_variables will be initialized with an
     /// empty set of points and no causal information.
-    crate fn new(elements: &Rc<RegionValueElements>, num_region_variables: usize) -> Self {
-        assert!(
-            elements.num_universal_regions <= num_region_variables,
-            "universal regions are a subset of the region variables"
-        );
-
+    crate fn new(elements: &Rc<RegionValueElements>) -> Self {
         Self {
             elements: elements.clone(),
-            matrix: SparseBitMatrix::new(
-                N::new(num_region_variables),
-                RegionElementIndex::new(elements.num_elements()),
-            ),
+            matrix: SparseBitMatrix::new(),
         }
     }
 
@@ -227,6 +214,18 @@ impl<N: Idx> RegionValues<N> {
         self.matrix.contains(r, i)
     }
 
+    /// Iterates through each row and the accompanying bit set.
+    pub fn iter_enumerated<'a>(
+        &'a self
+    ) -> impl Iterator<Item = (N, &'a SparseBitSet<RegionElementIndex>)> + 'a {
+        self.matrix.iter_enumerated()
+    }
+
+    /// Merge a row, `from`, originating in another `RegionValues` into the `into` row.
+    pub fn merge_into(&mut self, into: N, from: &SparseBitSet<RegionElementIndex>) -> bool {
+        self.matrix.merge_into(into, from)
+    }
+
     /// True if `sup_region` contains all the CFG points that
     /// `sub_region` contains. Ignores universal regions.
     crate fn contains_points(&self, sup_region: N, sub_region: N) -> bool {