From 85bbd7d608c82b8f2b88d4d5b3ade0b3e0b2f761 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 1 Feb 2020 19:36:45 +0100 Subject: [PATCH 01/25] perf: Avoid allocating a new Vec on each stall --- compiler/rustc_trait_selection/src/traits/fulfill.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 9a8b5534dfe83..3fb56de4a8835 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -678,10 +678,10 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { } /// Returns the set of inference variables contained in a trait ref. -fn trait_ref_infer_vars<'a, 'tcx>( - selcx: &mut SelectionContext<'a, 'tcx>, +fn trait_ref_infer_vars<'a, 'tcx, 'b>( + selcx: &'b mut SelectionContext<'a, 'tcx>, trait_ref: ty::PolyTraitRef<'tcx>, -) -> Vec> { +) -> impl Iterator> + 'b + Captures<'a> + Captures<'tcx> { selcx .infcx() .resolve_vars_if_possible(&trait_ref) From 4cf514a064984b6992ca042bc926395a9d67e457 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 3 Feb 2020 23:22:53 +0100 Subject: [PATCH 02/25] perf: Only process changed obligations in ObligationForest This rewrites most of ObligationForest forest so to avoid iterating through the entire set of obligations on each `select` attempt. Instead only the obligations that can actually make progress are processed. This gives great speedups in benchmarks such as `inflate` which create a large number of pending obligations. To support this the unification tables where extended to to keep track of which type inference variables that has actually changed at each step. Which then lets `ObligationForest` get a list of only the changed variables at each step which it can map back to its obligations. In addition to this primary change, many of the other iterations in `ObligationForest` were refactored to only process lists of the nodes that they actually are interested in. The extra bookkeeping needed for this was possible without the primary change but were a performance regressions there as they slowed down the main loop. As the main loop is no longer the main issue these optimizations could be re-applied. --- Cargo.lock | 476 ++++++++++---- compiler/rustc_data_structures/src/lib.rs | 3 + .../src/logged_unification_table.rs | 172 +++++ .../rustc_data_structures/src/modified_set.rs | 139 +++++ .../src/obligation_forest/mod.rs | 585 ++++++++++++------ .../src/obligation_forest/tests.rs | 19 +- .../rustc_data_structures/src/unify_log.rs | 56 ++ compiler/rustc_index/src/bit_set.rs | 7 + compiler/rustc_infer/src/infer/combine.rs | 1 + compiler/rustc_infer/src/infer/mod.rs | 14 +- .../rustc_infer/src/infer/type_variable.rs | 44 +- compiler/rustc_infer/src/traits/engine.rs | 11 + compiler/rustc_middle/src/ty/sty.rs | 48 +- .../rustc_trait_selection/src/autoderef.rs | 2 +- .../src/traits/fulfill.rs | 175 +++--- .../rustc_typeck/src/check/fn_ctxt/_impl.rs | 4 +- compiler/rustc_typeck/src/check/inherited.rs | 8 +- compiler/rustc_typeck/src/check/mod.rs | 7 - 18 files changed, 1321 insertions(+), 450 deletions(-) create mode 100644 compiler/rustc_data_structures/src/logged_unification_table.rs create mode 100644 compiler/rustc_data_structures/src/modified_set.rs create mode 100644 compiler/rustc_data_structures/src/unify_log.rs diff --git a/Cargo.lock b/Cargo.lock index 65d20190c0db5..9d9ffb49b230c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,8 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -295,7 +297,7 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" [[package]] name = "cargo" -version = "0.50.0" +version = "0.49.0" dependencies = [ "anyhow", "atty", @@ -306,11 +308,11 @@ dependencies = [ "clap", "core-foundation", "crates-io", - "crossbeam-utils 0.8.0", + "crossbeam-utils 0.7.2", "crypto-hash", "curl", "curl-sys", - "env_logger 0.8.1", + "env_logger 0.7.1", "filetime", "flate2", "fwdansi", @@ -686,12 +688,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "const_fn" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" - [[package]] name = "constant_time_eq" version = "0.1.5" @@ -825,18 +821,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5" -dependencies = [ - "autocfg", - "cfg-if 1.0.0", - "const_fn", - "lazy_static", -] - [[package]] name = "crypto-hash" version = "0.3.4" @@ -1045,19 +1029,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "env_logger" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54532e3223c5af90a6a757c90b5c5521564b07e5e7a958681bcd2afad421cdcd" -dependencies = [ - "atty", - "humantime 2.0.1", - "log", - "regex", - "termcolor", -] - [[package]] name = "error_index_generator" version = "0.0.0" @@ -2706,13 +2677,13 @@ dependencies = [ "lazy_static", "log", "rls-span", - "rustc-ap-rustc_ast", - "rustc-ap-rustc_ast_pretty", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_parse", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", + "rustc-ap-rustc_ast 679.0.0", + "rustc-ap-rustc_ast_pretty 679.0.0", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_errors 679.0.0", + "rustc-ap-rustc_parse 679.0.0", + "rustc-ap-rustc_session 679.0.0", + "rustc-ap-rustc_span 679.0.0", ] [[package]] @@ -3020,16 +2991,43 @@ dependencies = [ "mdbook", ] +[[package]] +name = "rustc-ap-rustc_arena" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2958af0d6e0458434a25cd3a96f6e19f24f71bf50b900add520dec52e212866b" +dependencies = [ + "rustc-ap-rustc_data_structures 677.0.0", + "smallvec 1.4.2", +] + [[package]] name = "rustc-ap-rustc_arena" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8e941a8fc3878a111d2bbfe78e39522d884136f0b412b12592195f26f653476" dependencies = [ - "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_data_structures 679.0.0", "smallvec 1.4.2", ] +[[package]] +name = "rustc-ap-rustc_ast" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c82c2510460f2133548e62399e5acd30c25ae6ece30245baab3d1e00c2fefac" +dependencies = [ + "bitflags", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_index 677.0.0", + "rustc-ap-rustc_lexer 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "smallvec 1.4.2", + "tracing", +] + [[package]] name = "rustc-ap-rustc_ast" version = "679.0.0" @@ -3037,32 +3035,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b58b6b035710df7f339a2bf86f6dafa876efd95439540970e24609e33598ca6" dependencies = [ "bitflags", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_index", - "rustc-ap-rustc_lexer", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_index 679.0.0", + "rustc-ap-rustc_lexer 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_span 679.0.0", "smallvec 1.4.2", "tracing", ] [[package]] name = "rustc-ap-rustc_ast_passes" -version = "679.0.0" +version = "677.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d379a900d6a1f098490d92ab83e87487dcee2e4ec3f04c3ac4512b5117b64e2" +checksum = "83977da57f81c6edd89bad47e49136680eaa33288de4abb702e95358c2a0fc6c" dependencies = [ - "itertools 0.9.0", - "rustc-ap-rustc_ast", - "rustc-ap-rustc_ast_pretty", + "itertools 0.8.2", + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_ast_pretty 677.0.0", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_feature", - "rustc-ap-rustc_parse", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_feature 677.0.0", + "rustc-ap-rustc_parse 677.0.0", + "rustc-ap-rustc_session 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "tracing", +] + +[[package]] +name = "rustc-ap-rustc_ast_pretty" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "becf4ca1638b214694c71a8752192683048ab8bd47947cc481f57bd48157eeb9" +dependencies = [ + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "rustc-ap-rustc_target 677.0.0", "tracing", ] @@ -3072,31 +3082,62 @@ version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658d925c0da9e3c5cddc5e54f4fa8c03b41aff1fc6dc5e41837c1118ad010ac0" dependencies = [ - "rustc-ap-rustc_ast", - "rustc-ap-rustc_span", - "rustc-ap-rustc_target", + "rustc-ap-rustc_ast 679.0.0", + "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_target 679.0.0", "tracing", ] [[package]] name = "rustc-ap-rustc_attr" -version = "679.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f387037534f34c148aed753622677500e42d190a095670e7ac3fffc09811a59" -dependencies = [ - "rustc-ap-rustc_ast", - "rustc-ap-rustc_ast_pretty", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_feature", - "rustc-ap-rustc_lexer", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f21ca5dadce8a40d75a2756b77eab75b4c2d827f645c622dd93ee2285599640" +dependencies = [ + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_ast_pretty 677.0.0", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_feature 677.0.0", + "rustc-ap-rustc_lexer 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_session 677.0.0", + "rustc-ap-rustc_span 677.0.0", "version_check", ] +[[package]] +name = "rustc-ap-rustc_data_structures" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4cd204764727fde9abf75333eb661f058bfc7242062d91019440fe1b240688b" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "ena", + "indexmap", + "jobserver", + "lazy_static", + "libc", + "measureme 0.7.1", + "parking_lot 0.10.2", + "rustc-ap-rustc_graphviz 677.0.0", + "rustc-ap-rustc_index 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-hash", + "rustc-rayon", + "rustc-rayon-core", + "smallvec 1.4.2", + "stable_deref_trait", + "stacker", + "tempfile", + "tracing", + "winapi 0.3.9", +] + [[package]] name = "rustc-ap-rustc_data_structures" version = "679.0.0" @@ -3112,10 +3153,10 @@ dependencies = [ "libc", "measureme 0.7.1", "parking_lot 0.11.0", - "rustc-ap-rustc_graphviz", - "rustc-ap-rustc_index", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", + "rustc-ap-rustc_graphviz 679.0.0", + "rustc-ap-rustc_index 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", "rustc-hash", "rustc-rayon", "rustc-rayon-core", @@ -3127,6 +3168,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rustc-ap-rustc_errors" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58116f119e37f14c029f99077b347069621118e048a69df74695b98204e7c136" +dependencies = [ + "annotate-snippets 0.8.0", + "atty", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "termcolor", + "termize", + "tracing", + "unicode-width", + "winapi 0.3.9", +] + [[package]] name = "rustc-ap-rustc_errors" version = "679.0.0" @@ -3135,10 +3195,10 @@ checksum = "2b3263ddcfa9eb911e54a4e8088878dd9fd10e00d8b99b01033ba4a2733fe91d" dependencies = [ "annotate-snippets 0.8.0", "atty", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_span 679.0.0", "termcolor", "termize", "tracing", @@ -3148,49 +3208,83 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_expand" -version = "679.0.0" +version = "677.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ab7e68cede8a2273fd8b8623002ce9dc832e061dfc3330e9bcc1fc2a722d73" +checksum = "48e3c4bda9b64b92805bebe7431fdb8e24fd112b35a8c6d2174827441f10a6b2" dependencies = [ - "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast 677.0.0", "rustc-ap-rustc_ast_passes", - "rustc-ap-rustc_ast_pretty", + "rustc-ap-rustc_ast_pretty 677.0.0", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_feature", - "rustc-ap-rustc_lexer", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_parse", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_feature 677.0.0", + "rustc-ap-rustc_lexer 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_parse 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_session 677.0.0", + "rustc-ap-rustc_span 677.0.0", "smallvec 1.4.2", "tracing", ] +[[package]] +name = "rustc-ap-rustc_feature" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b612bb67d3fc49f395b03fc4ea4384a0145b05afbadab725803074ec827632b" +dependencies = [ + "lazy_static", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_span 677.0.0", +] + [[package]] name = "rustc-ap-rustc_feature" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea2dc95421bc19bbd4d939399833a882c46b684283b4267ad1fcf982fc043d9" dependencies = [ - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_span 679.0.0", ] +[[package]] +name = "rustc-ap-rustc_fs_util" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7630ad1a73a8434ee920676148cb5440ac57509bd20e94ec41087fb0b1d11c28" + [[package]] name = "rustc-ap-rustc_fs_util" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e44c1804f09635f83f6cf1e04c2e92f8aeb7b4e850ac6c53d373dab02c13053" +[[package]] +name = "rustc-ap-rustc_graphviz" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a603fca4817062eb4fb23ff129d475bd66a69fb32f34ed4362ae950cf814b49d" + [[package]] name = "rustc-ap-rustc_graphviz" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc491f2b9be6e928f6df6b287549b8d50c48e8eff8638345155f40fa2cfb785d" +[[package]] +name = "rustc-ap-rustc_index" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9850c4a5d7c341513e10802bca9588bf8f452ceea2d5cfa87b934246a52622bc" +dependencies = [ + "arrayvec", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", +] + [[package]] name = "rustc-ap-rustc_index" version = "679.0.0" @@ -3198,8 +3292,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa73f3fed413cdb6290738a10267da17b9ae8e02087334778b9a8c9491c5efc0" dependencies = [ "arrayvec", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", +] + +[[package]] +name = "rustc-ap-rustc_lexer" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d86722e5a1a615b198327d0d794cd9cbc8b9db4542276fc51fe078924de68ea" +dependencies = [ + "unicode-xid", ] [[package]] @@ -3211,6 +3314,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "rustc-ap-rustc_macros" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3fc8482e44cabdda7ac9a8e224aef62ebdf95274d629dac8db3b42321025fea" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "rustc-ap-rustc_macros" version = "679.0.0" @@ -3223,6 +3338,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "rustc-ap-rustc_parse" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3716cdcd978a91dbd4a2788400e90e809527f841426fbeb92f882f9b8582f3ab" +dependencies = [ + "bitflags", + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_ast_pretty 677.0.0", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_feature 677.0.0", + "rustc-ap-rustc_lexer 677.0.0", + "rustc-ap-rustc_session 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "smallvec 1.4.2", + "tracing", + "unicode-normalization", +] + [[package]] name = "rustc-ap-rustc_parse" version = "679.0.0" @@ -3230,19 +3365,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0342675835251571471d3dca9ea1576a853a8dfa1f4b0084db283c861223cb60" dependencies = [ "bitflags", - "rustc-ap-rustc_ast", - "rustc-ap-rustc_ast_pretty", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_feature", - "rustc-ap-rustc_lexer", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", + "rustc-ap-rustc_ast 679.0.0", + "rustc-ap-rustc_ast_pretty 679.0.0", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_errors 679.0.0", + "rustc-ap-rustc_feature 679.0.0", + "rustc-ap-rustc_lexer 679.0.0", + "rustc-ap-rustc_session 679.0.0", + "rustc-ap-rustc_span 679.0.0", "smallvec 1.4.2", "tracing", "unicode-normalization", ] +[[package]] +name = "rustc-ap-rustc_serialize" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68046d07988b349b2e1c8bc1c9664a1d06519354aa677b9df358c5c5c058da0" +dependencies = [ + "indexmap", + "smallvec 1.4.2", +] + [[package]] name = "rustc-ap-rustc_serialize" version = "679.0.0" @@ -3253,6 +3398,27 @@ dependencies = [ "smallvec 1.4.2", ] +[[package]] +name = "rustc-ap-rustc_session" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85735553501a4de0c8904e37b7ccef79cc1c585a7d7f2cfa02cc38e0d149f982" +dependencies = [ + "bitflags", + "getopts", + "num_cpus", + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_feature 677.0.0", + "rustc-ap-rustc_fs_util 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "rustc-ap-rustc_target 677.0.0", + "tracing", +] + [[package]] name = "rustc-ap-rustc_session" version = "679.0.0" @@ -3262,18 +3428,37 @@ dependencies = [ "bitflags", "getopts", "num_cpus", - "rustc-ap-rustc_ast", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", - "rustc-ap-rustc_feature", - "rustc-ap-rustc_fs_util", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_span", - "rustc-ap-rustc_target", + "rustc-ap-rustc_ast 679.0.0", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_errors 679.0.0", + "rustc-ap-rustc_feature 679.0.0", + "rustc-ap-rustc_fs_util 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_target 679.0.0", "tracing", ] +[[package]] +name = "rustc-ap-rustc_span" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c49ae8a0d3b9e27c6ffe8febeaa30f899294fff012de70625f9ee81c54fda85" +dependencies = [ + "cfg-if 0.1.10", + "md-5", + "rustc-ap-rustc_arena 677.0.0", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_index 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "scoped-tls", + "sha-1", + "tracing", + "unicode-width", +] + [[package]] name = "rustc-ap-rustc_span" version = "679.0.0" @@ -3282,17 +3467,32 @@ checksum = "1c267f15c3cfc82a8a441d2bf86bcccf299d1eb625822468e3d8ee6f7c5a1c89" dependencies = [ "cfg-if 0.1.10", "md-5", - "rustc-ap-rustc_arena", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_index", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", + "rustc-ap-rustc_arena 679.0.0", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_index 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", "scoped-tls", "sha-1", "tracing", "unicode-width", ] +[[package]] +name = "rustc-ap-rustc_target" +version = "677.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1765f447594740c501c7b666b87639aa7c1dae2bf8c3166d5d2dca16646fd034" +dependencies = [ + "bitflags", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_index 677.0.0", + "rustc-ap-rustc_macros 677.0.0", + "rustc-ap-rustc_serialize 677.0.0", + "rustc-ap-rustc_span 677.0.0", + "tracing", +] + [[package]] name = "rustc-ap-rustc_target" version = "679.0.0" @@ -3300,11 +3500,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1b4b266c4d44aac0f7f83b6741d8f0545b03d1ce32f3b5254f2014225cb96c" dependencies = [ "bitflags", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_index", - "rustc-ap-rustc_macros", - "rustc-ap-rustc_serialize", - "rustc-ap-rustc_span", + "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_index 679.0.0", + "rustc-ap-rustc_macros 679.0.0", + "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_span 679.0.0", "tracing", ] @@ -4313,7 +4513,7 @@ dependencies = [ [[package]] name = "rustfmt-nightly" -version = "1.4.22" +version = "1.4.21" dependencies = [ "annotate-snippets 0.6.1", "anyhow", @@ -4329,15 +4529,15 @@ dependencies = [ "lazy_static", "log", "regex", - "rustc-ap-rustc_ast", - "rustc-ap-rustc_ast_pretty", + "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_ast_pretty 677.0.0", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures", - "rustc-ap-rustc_errors", + "rustc-ap-rustc_data_structures 677.0.0", + "rustc-ap-rustc_errors 677.0.0", "rustc-ap-rustc_expand", - "rustc-ap-rustc_parse", - "rustc-ap-rustc_session", - "rustc-ap-rustc_span", + "rustc-ap-rustc_parse 677.0.0", + "rustc-ap-rustc_session 677.0.0", + "rustc-ap-rustc_span 677.0.0", "rustc-workspace-hack", "rustfmt-config_proc_macro", "serde", @@ -5572,3 +5772,7 @@ checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" dependencies = [ "linked-hash-map", ] + +[[patch.unused]] +name = "backtrace" +version = "0.3.50" diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 7669b78834c3f..4bbf31122e3b2 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -73,6 +73,7 @@ pub mod flock; pub mod fx; pub mod graph; pub mod jobserver; +pub mod logged_unification_table; pub mod macros; pub mod map_in_place; pub mod obligation_forest; @@ -83,6 +84,7 @@ pub mod small_c_str; pub mod snapshot_map; pub mod stable_map; pub mod svh; +pub mod unify_log; pub use ena::snapshot_vec; pub mod sorted_map; pub mod stable_set; @@ -90,6 +92,7 @@ pub mod stable_set; pub mod stable_hasher; mod atomic_ref; pub mod fingerprint; +pub mod modified_set; pub mod profiling; pub mod sharded; pub mod stack; diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs new file mode 100644 index 0000000000000..eb23d62550ac9 --- /dev/null +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -0,0 +1,172 @@ +use std::ops::Range; + +use rustc_index::vec::{Idx, IndexVec}; + +use crate::modified_set as ms; +use crate::unify as ut; +use crate::unify_log as ul; + +pub struct LoggedUnificationTable { + relations: ut::UnificationTable>, + unify_log: ul::UnifyLog, + modified_set: ms::ModifiedSet, + reference_counts: IndexVec, +} + +impl LoggedUnificationTable +where + K: ut::UnifyKey + From, + I: Idx + From, +{ + pub fn new() -> Self { + Self { + relations: ut::UnificationTable::new(), + unify_log: ul::UnifyLog::new(), + modified_set: ms::ModifiedSet::new(), + reference_counts: IndexVec::new(), + } + } + + pub fn unify(&mut self, a: I, b: I) + where + K::Value: ut::UnifyValue, + { + self.unify_var_var(a, b).unwrap(); + } + + pub fn instantiate(&mut self, vid: I, ty: K::Value) -> K + where + K::Value: ut::UnifyValue, + { + if !self.unify_log.get(vid).is_empty() + || self.reference_counts.get(vid).map_or(false, |c| *c != 0) + { + self.modified_set.set(vid); + } + let vid = vid.into(); + debug_assert!(self.relations.find(vid) == vid); + self.relations.union_value(vid, ty); + + vid + } + + pub fn find(&mut self, vid: I) -> K { + self.relations.find(vid) + } + + pub fn unify_var_value( + &mut self, + vid: I, + value: K::Value, + ) -> Result<(), ::Error> { + let vid = self.find(vid).into(); + if !self.unify_log.get(vid).is_empty() + || self.reference_counts.get(vid).map_or(false, |c| *c != 0) + { + self.modified_set.set(vid); + } + self.relations.unify_var_value(vid, value) + } + + pub fn unify_var_var(&mut self, a: I, b: I) -> Result<(), ::Error> { + let a_root = self.relations.find(a); + let b_root = self.relations.find(b); + if a_root == b_root { + return Ok(()); + } + + self.relations.unify_var_var(a_root, b_root)?; + + if a_root == self.relations.find(a_root) { + self.unify_log.unify(a_root.into(), b_root.into()); + } else { + self.unify_log.unify(b_root.into(), a_root.into()); + } + Ok(()) + } + + pub fn union_value(&mut self, vid: I, value: K::Value) + where + K::Value: ut::UnifyValue, + { + let vid = self.find(vid).into(); + self.instantiate(vid, value); + } + + pub fn probe_value(&mut self, vid: I) -> K::Value { + self.relations.probe_value(vid) + } + + #[inline(always)] + pub fn inlined_probe_value(&mut self, vid: I) -> K::Value { + self.relations.inlined_probe_value(vid) + } + + pub fn new_key(&mut self, value: K::Value) -> K { + self.relations.new_key(value) + } + + pub fn len(&self) -> usize { + self.relations.len() + } + + pub fn snapshot(&mut self) -> Snapshot { + Snapshot { + snapshot: self.relations.snapshot(), + unify_log_snapshot: self.unify_log.snapshot(), + modified_snapshot: self.modified_set.snapshot(), + } + } + + pub fn rollback_to(&mut self, s: Snapshot) { + let Snapshot { snapshot, unify_log_snapshot, modified_snapshot } = s; + self.relations.rollback_to(snapshot); + self.unify_log.rollback_to(unify_log_snapshot); + self.modified_set.rollback_to(modified_snapshot); + } + + pub fn commit(&mut self, s: Snapshot) { + let Snapshot { snapshot, unify_log_snapshot, modified_snapshot } = s; + self.relations.commit(snapshot); + self.unify_log.commit(unify_log_snapshot); + self.modified_set.commit(modified_snapshot); + } + + pub fn vars_since_snapshot(&mut self, s: &Snapshot) -> Range { + self.relations.vars_since_snapshot(&s.snapshot) + } + + pub fn register(&mut self) -> ms::Offset { + self.modified_set.register() + } + + pub fn deregister(&mut self, offset: ms::Offset) { + self.modified_set.deregister(offset); + } + + pub fn watch_variable(&mut self, index: I) { + self.reference_counts.ensure_contains_elem(index, || 0); + self.reference_counts[index] += 1; + } + + pub fn unwatch_variable(&mut self, index: I) { + self.reference_counts[index] -= 1; + } + + pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I) -> bool) { + let unify_log = &self.unify_log; + self.modified_set.drain(offset, |vid| { + for &unified_vid in unify_log.get(vid) { + f(unified_vid); + } + + f(vid) + }) + } +} + +pub struct Snapshot { + snapshot: ut::Snapshot>, + unify_log_snapshot: ul::Snapshot, + modified_snapshot: ms::Snapshot, +} diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs new file mode 100644 index 0000000000000..e20042f082971 --- /dev/null +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -0,0 +1,139 @@ +use std::{collections::VecDeque, marker::PhantomData}; + +use rustc_index::{bit_set::BitSet, vec::Idx}; + +#[derive(Clone, Debug)] +pub struct ModifiedSet { + modified: VecDeque, + snapshots: usize, + modified_set: BitSet, + undo_offsets: Vec, + offsets: Vec, +} + +impl Default for ModifiedSet { + fn default() -> Self { + Self { + modified: Default::default(), + snapshots: 0, + modified_set: BitSet::new_empty(0), + offsets: Vec::new(), + undo_offsets: Vec::new(), + } + } +} + +impl ModifiedSet { + pub fn new() -> Self { + Self::default() + } + + pub fn set(&mut self, index: T) { + if index.index() >= self.modified_set.domain_size() { + self.modified_set.resize(index.index() + 1); + } + if self.modified_set.insert(index) { + self.modified.push_back(index); + } + } + + pub fn drain(&mut self, offset: &Offset, mut f: impl FnMut(T) -> bool) { + let offset = &mut self.offsets[offset.index]; + for &index in self.modified.iter().skip(*offset) { + if f(index) {} + } + *offset = self.modified.len(); + } + + pub fn snapshot(&mut self) -> Snapshot { + self.snapshots += 1; + let offsets_start = self.undo_offsets.len(); + self.undo_offsets.extend_from_slice(&self.offsets); + Snapshot { + modified_len: self.modified.len(), + offsets_start, + offsets_len: self.offsets.len(), + _marker: PhantomData, + } + } + + pub fn rollback_to(&mut self, snapshot: Snapshot) { + self.snapshots -= 1; + for &index in self.modified.iter().skip(snapshot.modified_len) { + self.modified_set.remove(index); + } + self.modified.truncate(snapshot.modified_len); + let mut offsets = self.offsets.iter_mut(); + for (offset, &saved_offset) in offsets.by_ref().zip( + &self.undo_offsets + [snapshot.offsets_start..snapshot.offsets_start + snapshot.offsets_len], + ) { + *offset = saved_offset; + } + for offset in offsets { + *offset = self.modified.len().min(*offset); + } + self.undo_offsets.truncate(snapshot.offsets_start); + + if self.snapshots == 0 { + let min = self.offsets.iter().copied().min().unwrap_or(0); + // Any indices still in `modified` may not have been instantiated, so if we observe them again + // we need to notify any listeners again + for index in self.modified.drain(..min) { + self.modified_set.remove(index); + } + for offset in &mut self.offsets { + *offset -= min; + } + } + } + + pub fn commit(&mut self, snapshot: Snapshot) { + self.snapshots -= 1; + self.undo_offsets.truncate(snapshot.offsets_start); + if self.snapshots == 0 { + // Everything up until this point is committed, so we can forget anything before the + // current offsets + let min = self.offsets.iter().copied().min().unwrap_or(0); + self.modified.drain(..min); + for offset in &mut self.offsets { + *offset -= min; + } + } + } + + pub fn register(&mut self) -> Offset { + let index = self.offsets.len(); + self.offsets.push(0); + Offset { index, _marker: PhantomData } + } + + pub fn deregister(&mut self, offset: Offset) { + assert_eq!(offset.index, self.offsets.len() - 1); + self.offsets.pop(); + std::mem::forget(offset); + } +} + +#[must_use] +pub struct Offset { + index: usize, + _marker: PhantomData, +} + +impl Drop for Offset { + fn drop(&mut self) { + if !std::thread::panicking() { + panic!("Offsets should be deregistered") + } + } +} + +#[must_use] +#[derive(Debug)] +pub struct Snapshot { + modified_len: usize, + offsets_start: usize, + offsets_len: usize, + _marker: PhantomData, +} diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index a5b2df1da5d6d..509aa4adf4ac2 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -74,11 +74,12 @@ use crate::fx::{FxHashMap, FxHashSet}; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash; use std::marker::PhantomData; +use std::mem; mod graphviz; @@ -87,12 +88,16 @@ mod tests; pub trait ForestObligation: Clone + Debug { type CacheKey: Clone + hash::Hash + Eq + Debug; + type Variable: Clone + hash::Hash + Eq + Debug; + type WatcherOffset; /// Converts this `ForestObligation` suitable for use as a cache key. /// If two distinct `ForestObligations`s return the same cache key, /// then it must be sound to use the result of processing one obligation /// (e.g. success for error) for the other obligation fn as_cache_key(&self) -> Self::CacheKey; + + fn stalled_on(&self) -> &[Self::Variable]; } pub trait ObligationProcessor { @@ -115,6 +120,16 @@ pub trait ObligationProcessor { fn process_backedge<'c, I>(&mut self, cycle: I, _marker: PhantomData<&'c Self::Obligation>) where I: Clone + Iterator; + + fn unblocked( + &self, + offset: &::WatcherOffset, + f: impl FnMut(::Variable) -> bool, + ); + fn register(&self) -> ::WatcherOffset; + fn deregister(&self, offset: ::WatcherOffset); + fn watch_variable(&self, var: ::Variable); + fn unwatch_variable(&self, var: ::Variable); } /// The result type used by `process_obligation`. @@ -131,28 +146,32 @@ struct ObligationTreeId(usize); type ObligationTreeIdGenerator = std::iter::Map, fn(usize) -> ObligationTreeId>; +/// `usize` indices are used here and throughout this module, rather than +/// `rustc_index::newtype_index!` indices, because this code is hot enough +/// that the `u32`-to-`usize` conversions that would be required are +/// significant, and space considerations are not important. +type NodeIndex = usize; + pub struct ObligationForest { /// The list of obligations. In between calls to `process_obligations`, /// this list only contains nodes in the `Pending` or `Waiting` state. - /// - /// `usize` indices are used here and throughout this module, rather than - /// `rustc_index::newtype_index!` indices, because this code is hot enough - /// that the `u32`-to-`usize` conversions that would be required are - /// significant, and space considerations are not important. nodes: Vec>, - /// A cache of predicates that have been successfully completed. - done_cache: FxHashSet, + /// Nodes must be processed in the order that they were added which this list keeps track of. + pending_nodes: Vec, + success_or_waiting_nodes: Vec, + error_or_done_nodes: RefCell>, + + /// Nodes that have been removed and are ready to be reused + dead_nodes: Vec, /// A cache of the nodes in `nodes`, indexed by predicate. Unfortunately, /// its contents are not guaranteed to match those of `nodes`. See the /// comments in `process_obligation` for details. - active_cache: FxHashMap, - - /// A vector reused in compress() and find_cycles_from_node(), to avoid allocating new vectors. - reused_node_vec: Vec, + active_cache: FxHashMap>, obligation_tree_id_generator: ObligationTreeIdGenerator, + node_number: u32, /// Per tree error cache. This is used to deduplicate errors, /// which is necessary to avoid trait resolution overflow in @@ -162,37 +181,105 @@ pub struct ObligationForest { /// /// [details]: https://github.com/rust-lang/rust/pull/53255#issuecomment-421184780 error_cache: FxHashMap>, + + stalled_on: FxHashMap>, + unblocked: std::collections::BinaryHeap, + check_next: Vec, + offset: Option, +} + +struct Unblocked { + index: usize, + order: u32, +} + +use std::cmp::Ordering; +impl PartialEq for Unblocked { + fn eq(&self, other: &Self) -> bool { + self.order == other.order + } +} +impl Eq for Unblocked {} +impl PartialOrd for Unblocked { + fn partial_cmp(&self, other: &Self) -> Option { + other.order.partial_cmp(&self.order) + } +} +impl Ord for Unblocked { + fn cmp(&self, other: &Self) -> Ordering { + other.order.cmp(&self.order) + } } #[derive(Debug)] -struct Node { +struct Node { obligation: O, state: Cell, + alternative_predicates: Vec, + /// Obligations that depend on this obligation for their completion. They /// must all be in a non-pending state. - dependents: Vec, + dependents: Vec, + reverse_dependents: Vec, /// If true, `dependents[0]` points to a "parent" node, which requires /// special treatment upon error but is otherwise treated the same. /// (It would be more idiomatic to store the parent node in a separate - /// `Option` field, but that slows down the common case of + /// `Option` field, but that slows down the common case of /// iterating over the parent and other descendants together.) has_parent: bool, /// Identifier of the obligation tree to which this node belongs. obligation_tree_id: ObligationTreeId, + node_number: u32, } -impl Node { - fn new(parent: Option, obligation: O, obligation_tree_id: ObligationTreeId) -> Node { +impl Node +where + O: ForestObligation, +{ + fn new( + parent: Option, + obligation: O, + obligation_tree_id: ObligationTreeId, + node_number: u32, + ) -> Node { Node { obligation, state: Cell::new(NodeState::Pending), + alternative_predicates: vec![], dependents: if let Some(parent_index) = parent { vec![parent_index] } else { vec![] }, + reverse_dependents: vec![], has_parent: parent.is_some(), obligation_tree_id, + node_number, + } + } + + fn init( + &mut self, + parent: Option, + obligation: O, + obligation_tree_id: ObligationTreeId, + node_number: u32, + ) { + self.obligation = obligation; + debug_assert!( + self.state.get() == NodeState::Done || self.state.get() == NodeState::Error, + "{:?}", + self.state + ); + self.state.set(NodeState::Pending); + self.alternative_predicates.clear(); + self.dependents.clear(); + self.reverse_dependents.clear(); + if let Some(parent_index) = parent { + self.dependents.push(parent_index); } + self.has_parent = parent.is_some(); + self.obligation_tree_id = obligation_tree_id; + self.node_number = node_number; } } @@ -269,14 +356,6 @@ pub trait OutcomeTrait { pub struct Outcome { /// Backtrace of obligations that were found to be in error. pub errors: Vec>, - - /// If true, then we saw no successful obligations, which means - /// there is no point in further iteration. This is based on the - /// assumption that when trait matching returns `Error` or - /// `Unchanged`, those results do not affect environmental - /// inference state. (Note that if we invoke `process_obligations` - /// with no pending obligations, stalled will be true.) - pub stalled: bool, } impl OutcomeTrait for Outcome { @@ -314,14 +393,29 @@ impl ObligationForest { pub fn new() -> ObligationForest { ObligationForest { nodes: vec![], - done_cache: Default::default(), + pending_nodes: vec![], + success_or_waiting_nodes: vec![], + error_or_done_nodes: RefCell::new(vec![]), + dead_nodes: vec![], active_cache: Default::default(), - reused_node_vec: vec![], obligation_tree_id_generator: (0..).map(ObligationTreeId), + node_number: 0, error_cache: Default::default(), + stalled_on: Default::default(), + unblocked: Default::default(), + check_next: Default::default(), + offset: None, } } + pub fn offset(&self) -> Option<&O::WatcherOffset> { + self.offset.as_ref() + } + + pub fn take_offset(&mut self) -> Option { + self.offset.take() + } + /// Returns the total number of nodes in the forest that have not /// yet been fully resolved. pub fn len(&self) -> usize { @@ -335,15 +429,26 @@ impl ObligationForest { } // Returns Err(()) if we already know this obligation failed. - fn register_obligation_at(&mut self, obligation: O, parent: Option) -> Result<(), ()> { - if self.done_cache.contains(&obligation.as_cache_key()) { - debug!("register_obligation_at: ignoring already done obligation: {:?}", obligation); - return Ok(()); - } - - match self.active_cache.entry(obligation.as_cache_key()) { + fn register_obligation_at( + &mut self, + obligation: O, + parent: Option, + ) -> Result<(), ()> { + debug_assert!(obligation.stalled_on().is_empty()); + match self.active_cache.entry(obligation.as_cache_key().clone()) { Entry::Occupied(o) => { - let node = &mut self.nodes[*o.get()]; + let index = match o.get() { + Some(index) => *index, + None => { + debug!( + "register_obligation_at: ignoring already done obligation: {:?}", + obligation + ); + return Ok(()); + } + }; + let node = &mut self.nodes[index]; + let state = node.state.get(); if let Some(parent_index) = parent { // If the node is already in `active_cache`, it has already // had its chance to be marked with a parent. So if it's @@ -351,9 +456,10 @@ impl ObligationForest { // dependents as a non-parent. if !node.dependents.contains(&parent_index) { node.dependents.push(parent_index); + self.nodes[parent_index].reverse_dependents.push(index); } } - if let NodeState::Error = node.state.get() { Err(()) } else { Ok(()) } + if let NodeState::Error = state { Err(()) } else { Ok(()) } } Entry::Vacant(v) => { let obligation_tree_id = match parent { @@ -367,13 +473,32 @@ impl ObligationForest { .get(&obligation_tree_id) .map(|errors| errors.contains(&obligation.as_cache_key())) .unwrap_or(false); + let node_number = self.node_number; + self.node_number += 1; if already_failed { Err(()) } else { - let new_index = self.nodes.len(); - v.insert(new_index); - self.nodes.push(Node::new(parent, obligation, obligation_tree_id)); + let new_index = if let Some(new_index) = self.dead_nodes.pop() { + let node = &mut self.nodes[new_index]; + node.init(parent, obligation, obligation_tree_id, node_number); + new_index + } else { + let new_index = self.nodes.len(); + self.nodes.push(Node::new( + parent, + obligation, + obligation_tree_id, + node_number, + )); + new_index + }; + if let Some(parent_index) = parent { + self.nodes[parent_index].reverse_dependents.push(new_index); + } + self.pending_nodes.push(new_index); + self.unblocked.push(Unblocked { index: new_index, order: node_number }); + v.insert(Some(new_index)); Ok(()) } } @@ -383,11 +508,9 @@ impl ObligationForest { /// Converts all remaining obligations to the given error. pub fn to_errors(&mut self, error: E) -> Vec> { let errors = self - .nodes + .pending_nodes .iter() - .enumerate() - .filter(|(_index, node)| node.state.get() == NodeState::Pending) - .map(|(index, _node)| Error { error: error.clone(), backtrace: self.error_at(index) }) + .map(|&index| Error { error: error.clone(), backtrace: self.error_at(index) }) .collect(); self.compress(|_| assert!(false)); @@ -399,14 +522,14 @@ impl ObligationForest { where F: Fn(&O) -> P, { - self.nodes + self.pending_nodes .iter() - .filter(|node| node.state.get() == NodeState::Pending) + .map(|&index| &self.nodes[index]) .map(|node| f(&node.obligation)) .collect() } - fn insert_into_error_cache(&mut self, index: usize) { + fn insert_into_error_cache(&mut self, index: NodeIndex) { let node = &self.nodes[index]; self.error_cache .entry(node.obligation_tree_id) @@ -423,73 +546,171 @@ impl ObligationForest { P: ObligationProcessor, OUT: OutcomeTrait>, { - let mut outcome = OUT::new(); - - // Note that the loop body can append new nodes, and those new nodes - // will then be processed by subsequent iterations of the loop. - // - // We can't use an iterator for the loop because `self.nodes` is - // appended to and the borrow checker would complain. We also can't use - // `for index in 0..self.nodes.len() { ... }` because the range would - // be computed with the initial length, and we would miss the appended - // nodes. Therefore we use a `while` loop. - let mut index = 0; - while let Some(node) = self.nodes.get_mut(index) { - // `processor.process_obligation` can modify the predicate within - // `node.obligation`, and that predicate is the key used for - // `self.active_cache`. This means that `self.active_cache` can get - // out of sync with `nodes`. It's not very common, but it does - // happen, and code in `compress` has to allow for it. - if node.state.get() != NodeState::Pending { - index += 1; - continue; - } + if self.offset.is_none() { + self.offset = Some(processor.register()); + } + warn!( + "Begin process {}, pending: {}", + self.nodes.len(), + self.nodes.iter().filter(|n| n.state.get() == NodeState::Pending).count() + ); + let mut errors = vec![]; + let mut stalled = true; + + self.unblock_nodes(processor); + + let mut run_again = true; + while !self.unblocked.is_empty() || (!self.check_next.is_empty() && run_again) { + run_again = false; + let nodes = &self.nodes; + self.unblocked.extend( + self.check_next + .drain(..) + .map(|index| Unblocked { index, order: nodes[index].node_number }), + ); + while let Some(Unblocked { index, .. }) = self.unblocked.pop() { + if self.unblocked.peek().map(|u| u.index) == Some(index) { + continue; + } + let node = &mut self.nodes[index]; - match processor.process_obligation(&mut node.obligation) { - ProcessResult::Unchanged => { - // No change in state. + if node.state.get() != NodeState::Pending { + continue; } - ProcessResult::Changed(children) => { - // We are not (yet) stalled. - outcome.mark_not_stalled(); - node.state.set(NodeState::Success); - - for child in children { - let st = self.register_obligation_at(child, Some(index)); - if let Err(()) = st { - // Error already reported - propagate it - // to our node. - self.error_at(index); + for var in node.obligation.stalled_on() { + match self.stalled_on.entry(var.clone()) { + Entry::Vacant(_) => (), + Entry::Occupied(mut entry) => { + let nodes = entry.get_mut(); + nodes.retain(|node_index| *node_index != index); + if nodes.is_empty() { + processor.unwatch_variable(var.clone()); + entry.remove(); + } } } } - ProcessResult::Error(err) => { - outcome.mark_not_stalled(); - outcome.record_error(Error { error: err, backtrace: self.error_at(index) }); + + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + let before = node.obligation.as_cache_key(); + let result = processor.process_obligation(&mut node.obligation); + let after = node.obligation.as_cache_key(); + if before != after { + node.alternative_predicates.push(before); + } + + self.unblock_nodes(processor); + let node = &mut self.nodes[index]; + match result { + ProcessResult::Unchanged => { + for var in node.obligation.stalled_on() { + self.stalled_on + .entry(var.clone()) + .or_insert_with(|| { + processor.watch_variable(var.clone()); + Vec::new() + }) + .push(index); + } + + if node.obligation.stalled_on().is_empty() { + self.check_next.push(index); + } + // No change in state. + } + ProcessResult::Changed(children) => { + // We are not (yet) stalled. + stalled = false; + node.state.set(NodeState::Success); + self.success_or_waiting_nodes.push(index); + + for child in children { + let st = self.register_obligation_at(child, Some(index)); + if let Err(()) = st { + // Error already reported - propagate it + // to our node. + self.error_at(index); + } + } + } + ProcessResult::Error(err) => { + stalled = false; + errors.push(Error { error: err, backtrace: self.error_at(index) }); + } } } - index += 1; } - // There's no need to perform marking, cycle processing and compression when nothing - // changed. - if !outcome.is_stalled() { - self.mark_successes(); - self.process_cycles(processor); - self.compress(|obl| outcome.record_completed(obl)); + if stalled { + warn!("Stalled {}", self.nodes.len()); + // There's no need to perform marking, cycle processing and compression when nothing + // changed. + return Outcome { + completed: if do_completed == DoCompleted::Yes { Some(vec![]) } else { None }, + errors, + }; } - outcome + warn!("Compressing {}", self.nodes.len()); + self.mark_successes(); + self.process_cycles(processor); + let completed = self.compress(do_completed); + warn!("Compressed {}", self.nodes.len()); + + warn!("Stalled on: {:?}", self.stalled_on.keys().collect::>()); + + Outcome { completed, errors } + } + + #[inline(never)] + fn unblock_nodes

(&mut self, processor: &mut P) + where + P: ObligationProcessor, + { + let nodes = &mut self.nodes; + let stalled_on = &mut self.stalled_on; + let unblocked = &mut self.unblocked; + let mut temp = Vec::new(); + processor.unblocked(self.offset.as_ref().unwrap(), |var| { + if let Some(unblocked_nodes) = stalled_on.remove(&var) { + for node_index in unblocked_nodes { + debug_assert!( + nodes[node_index].state.get() == NodeState::Pending, + "Unblocking non-pending2: {:?}", + nodes[node_index].obligation + ); + unblocked.push(Unblocked { + index: node_index, + order: nodes[node_index].node_number, + }); + } + temp.push(var); + } + true + }); + for var in temp { + processor.unwatch_variable(var); + } } /// Returns a vector of obligations for `p` and all of its /// ancestors, putting them into the error state in the process. - fn error_at(&self, mut index: usize) -> Vec { - let mut error_stack: Vec = vec![]; + fn error_at(&self, mut index: NodeIndex) -> Vec { + let mut error_stack: Vec = vec![]; let mut trace = vec![]; + let mut error_or_done_nodes = self.error_or_done_nodes.borrow_mut(); + loop { let node = &self.nodes[index]; + match node.state.get() { + NodeState::Error | NodeState::Done => (), // Already added to `error_or_done_nodes` + _ => error_or_done_nodes.push(index), + } node.state.set(NodeState::Error); trace.push(node.obligation.clone()); if node.has_parent { @@ -517,9 +738,10 @@ impl ObligationForest { /// Mark all `Waiting` nodes as `Success`, except those that depend on a /// pending node. - fn mark_successes(&self) { + fn mark_successes(&mut self) { // Convert all `Waiting` nodes to `Success`. - for node in &self.nodes { + for &index in &self.success_or_waiting_nodes { + let node = &self.nodes[index]; if node.state.get() == NodeState::Waiting { node.state.set(NodeState::Success); } @@ -527,12 +749,18 @@ impl ObligationForest { // Convert `Success` nodes that depend on a pending node back to // `Waiting`. - for node in &self.nodes { + let mut pending_nodes = mem::take(&mut self.pending_nodes); + pending_nodes.retain(|&index| { + let node = &self.nodes[index]; if node.state.get() == NodeState::Pending { // This call site is hot. self.inlined_mark_dependents_as_waiting(node); + true + } else { + false } - } + }); + self.pending_nodes = pending_nodes; } // This always-inlined function is for the hot call site. @@ -564,8 +792,11 @@ impl ObligationForest { where P: ObligationProcessor, { - let mut stack = std::mem::take(&mut self.reused_node_vec); - for (index, node) in self.nodes.iter().enumerate() { + let mut stack = vec![]; + + let success_or_waiting_nodes = mem::take(&mut self.success_or_waiting_nodes); + for &index in &success_or_waiting_nodes { + let node = &self.nodes[index]; // For some benchmarks this state test is extremely hot. It's a win // to handle the no-op cases immediately to avoid the cost of the // function call. @@ -573,13 +804,18 @@ impl ObligationForest { self.find_cycles_from_node(&mut stack, processor, index); } } + self.success_or_waiting_nodes = success_or_waiting_nodes; debug_assert!(stack.is_empty()); self.reused_node_vec = stack; } - fn find_cycles_from_node

(&self, stack: &mut Vec, processor: &mut P, index: usize) - where + fn find_cycles_from_node

( + &self, + stack: &mut Vec, + processor: &mut P, + index: NodeIndex, + ) where P: ObligationProcessor, { let node = &self.nodes[index]; @@ -592,11 +828,12 @@ impl ObligationForest { } stack.pop(); node.state.set(NodeState::Done); + self.error_or_done_nodes.borrow_mut().push(index); } Some(rpos) => { // Cycle detected. processor.process_backedge( - stack[rpos..].iter().map(GetObligation(&self.nodes)), + stack[rpos..].iter().map(|i| &self.nodes[*i].obligation), PhantomData, ); } @@ -608,116 +845,66 @@ impl ObligationForest { /// indices and hence invalidates any outstanding indices. `process_cycles` /// must be run beforehand to remove any cycles on `Success` nodes. #[inline(never)] - fn compress(&mut self, mut outcome_cb: impl FnMut(&O)) { - let orig_nodes_len = self.nodes.len(); - let mut node_rewrites: Vec<_> = std::mem::take(&mut self.reused_node_vec); - debug_assert!(node_rewrites.is_empty()); - node_rewrites.extend(0..orig_nodes_len); - let mut dead_nodes = 0; - - // Move removable nodes to the end, preserving the order of the - // remaining nodes. - // - // LOOP INVARIANT: - // self.nodes[0..index - dead_nodes] are the first remaining nodes - // self.nodes[index - dead_nodes..index] are all dead - // self.nodes[index..] are unchanged - for index in 0..orig_nodes_len { - let node = &self.nodes[index]; - match node.state.get() { - NodeState::Pending | NodeState::Waiting => { - if dead_nodes > 0 { - self.nodes.swap(index, index - dead_nodes); - node_rewrites[index] -= dead_nodes; - } + fn compress(&mut self, do_completed: DoCompleted) -> Option> { + let mut removed_done_obligations: Vec = vec![]; + + let mut error_or_done_nodes = mem::take(self.error_or_done_nodes.get_mut()); + for &index in &error_or_done_nodes { + let node = &mut self.nodes[index]; + let reverse_dependents = mem::take(&mut node.reverse_dependents); + for &reverse_index in &reverse_dependents { + let reverse_node = &mut self.nodes[reverse_index]; + if reverse_node.dependents.first() == Some(&index) { + reverse_node.has_parent = false; } + reverse_node.dependents.retain(|&i| i != index); + } + let node = &mut self.nodes[index]; + node.reverse_dependents = reverse_dependents; + + match node.state.get() { NodeState::Done => { - // This lookup can fail because the contents of - // `self.active_cache` are not guaranteed to match those of - // `self.nodes`. See the comment in `process_obligation` - // for more details. - if let Some((predicate, _)) = - self.active_cache.remove_entry(&node.obligation.as_cache_key()) - { - self.done_cache.insert(predicate); - } else { - self.done_cache.insert(node.obligation.as_cache_key().clone()); + // Mark as done + if let Some(opt) = self.active_cache.get_mut(&node.obligation.as_cache_key()) { + *opt = None; } - // Extract the success stories. - outcome_cb(&node.obligation); - node_rewrites[index] = orig_nodes_len; - dead_nodes += 1; + for alt in &node.alternative_predicates { + if let Some(opt) = self.active_cache.get_mut(alt) { + *opt = None + } + } + + if do_completed == DoCompleted::Yes { + // Extract the success stories. + removed_done_obligations.push(node.obligation.clone()); + } + + self.dead_nodes.push(index); } NodeState::Error => { // We *intentionally* remove the node from the cache at this point. Otherwise // tests must come up with a different type on every type error they // check against. self.active_cache.remove(&node.obligation.as_cache_key()); - self.insert_into_error_cache(index); - node_rewrites[index] = orig_nodes_len; - dead_nodes += 1; - } - NodeState::Success => unreachable!(), - } - } - - if dead_nodes > 0 { - // Remove the dead nodes and rewrite indices. - self.nodes.truncate(orig_nodes_len - dead_nodes); - self.apply_rewrites(&node_rewrites); - } - - node_rewrites.truncate(0); - self.reused_node_vec = node_rewrites; - } - - fn apply_rewrites(&mut self, node_rewrites: &[usize]) { - let orig_nodes_len = node_rewrites.len(); - - for node in &mut self.nodes { - let mut i = 0; - while let Some(dependent) = node.dependents.get_mut(i) { - let new_index = node_rewrites[*dependent]; - if new_index >= orig_nodes_len { - node.dependents.swap_remove(i); - if i == 0 && node.has_parent { - // We just removed the parent. - node.has_parent = false; + for alt in &node.alternative_predicates { + self.active_cache.remove(alt); } - } else { - *dependent = new_index; - i += 1; + self.insert_into_error_cache(index); + self.dead_nodes.push(index); } + NodeState::Pending | NodeState::Waiting | NodeState::Success => unreachable!(), } } - - // This updating of `self.active_cache` is necessary because the - // removal of nodes within `compress` can fail. See above. - self.active_cache.retain(|_predicate, index| { - let new_index = node_rewrites[*index]; - if new_index >= orig_nodes_len { - false - } else { - *index = new_index; - true - } + error_or_done_nodes.clear(); + *self.error_or_done_nodes.get_mut() = error_or_done_nodes; + + let nodes = &self.nodes; + self.success_or_waiting_nodes.retain(|&index| match nodes[index].state.get() { + NodeState::Waiting | NodeState::Success => true, + NodeState::Done | NodeState::Error => false, + NodeState::Pending => unreachable!(), }); - } -} - -// I need a Clone closure. -#[derive(Clone)] -struct GetObligation<'a, O>(&'a [Node]); - -impl<'a, 'b, O> FnOnce<(&'b usize,)> for GetObligation<'a, O> { - type Output = &'a O; - extern "rust-call" fn call_once(self, args: (&'b usize,)) -> &'a O { - &self.0[*args.0].obligation - } -} -impl<'a, 'b, O> FnMut<(&'b usize,)> for GetObligation<'a, O> { - extern "rust-call" fn call_mut(&mut self, args: (&'b usize,)) -> &'a O { - &self.0[*args.0].obligation + if do_completed == DoCompleted::Yes { Some(removed_done_obligations) } else { None } } } diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index 371c62c063fa7..61cc9a2506a94 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -5,10 +5,16 @@ use std::marker::PhantomData; impl<'a> super::ForestObligation for &'a str { type CacheKey = &'a str; + type Variable = (); + type WatcherOffset = (); fn as_cache_key(&self) -> Self::CacheKey { self } + + fn stalled_on(&self) -> &[Self::Variable] { + &[] + } } struct ClosureObligationProcessor { @@ -66,7 +72,7 @@ where impl ObligationProcessor for ClosureObligationProcessor where - O: super::ForestObligation + fmt::Debug, + O: super::ForestObligation + fmt::Debug, E: fmt::Debug, OF: FnMut(&mut O) -> ProcessResult, BF: FnMut(&[O]), @@ -86,6 +92,17 @@ where I: Clone + Iterator, { } + + fn unblocked( + &self, + _offset: &::WatcherOffset, + _f: impl FnMut(::Variable) -> bool, + ) { + } + fn register(&self) -> ::WatcherOffset {} + fn deregister(&self, _offset: ::WatcherOffset) {} + fn watch_variable(&self, _var: ::Variable) {} + fn unwatch_variable(&self, _var: ::Variable) {} } #[test] diff --git a/compiler/rustc_data_structures/src/unify_log.rs b/compiler/rustc_data_structures/src/unify_log.rs new file mode 100644 index 0000000000000..a5d1c32807b68 --- /dev/null +++ b/compiler/rustc_data_structures/src/unify_log.rs @@ -0,0 +1,56 @@ +use std::marker::PhantomData; + +use rustc_index::vec::{Idx, IndexVec}; + +pub struct UnifyLog { + unified_vars: IndexVec>, + undo_log: Vec<(T, u32)>, + snapshots: usize, +} + +impl UnifyLog { + pub fn new() -> Self { + UnifyLog { unified_vars: IndexVec::new(), undo_log: Vec::new(), snapshots: 0 } + } + + pub fn unify(&mut self, root: T, other: T) { + self.unified_vars.ensure_contains_elem(root, Vec::new); + self.unified_vars.ensure_contains_elem(other, Vec::new); + let (root_ids, other_ids) = self.unified_vars.pick2_mut(root, other); + self.undo_log.push((root, root_ids.len() as u32)); + for &other in &*other_ids { + if !root_ids.contains(&other) { + root_ids.push(other); + } + } + root_ids.push(other); + } + + pub fn get(&self, root: T) -> &[T] { + self.unified_vars.get(root).map(|v| &v[..]).unwrap_or(&[][..]) + } + + pub fn snapshot(&mut self) -> Snapshot { + self.snapshots += 1; + Snapshot { undo_log_len: self.undo_log.len() as u32, _marker: PhantomData } + } + + pub fn commit(&mut self, _snapshot: Snapshot) { + self.snapshots -= 1; + if self.snapshots == 0 { + self.undo_log.clear(); + } + } + + pub fn rollback_to(&mut self, snapshot: Snapshot) { + self.snapshots -= 1; + for (index, len) in self.undo_log.drain(snapshot.undo_log_len as usize..) { + self.unified_vars[index].truncate(len as usize); + } + } +} + +pub struct Snapshot { + undo_log_len: u32, + _marker: PhantomData, +} diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 8e00e54650df0..3fd3b25beb974 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -59,6 +59,13 @@ impl BitSet { result } + pub fn resize(&mut self, domain_size: usize) { + let num_words = num_words(domain_size); + self.domain_size = domain_size; + self.words.resize(num_words, 0); + self.clear_excess_bits(); + } + /// Clear all elements. #[inline] pub fn clear(&mut self) { diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs index 6a1715ef81899..aa740a8d27422 100644 --- a/compiler/rustc_infer/src/infer/combine.rs +++ b/compiler/rustc_infer/src/infer/combine.rs @@ -76,6 +76,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> { match (a.kind(), b.kind()) { // Relate integral variables to other types (&ty::Infer(ty::IntVar(a_id)), &ty::Infer(ty::IntVar(b_id))) => { + warn!("Unify int: {:?} {:?}", a_id, b_id); self.inner .borrow_mut() .int_unification_table() diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index ff7bbf0562f60..5342b13bcb74c 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -11,6 +11,8 @@ pub(crate) use self::undo_log::{InferCtxtUndoLogs, Snapshot, UndoLog}; use crate::traits::{self, ObligationCause, PredicateObligations, TraitEngine}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::logged_unification_table as lut; +use rustc_data_structures::modified_set as ms; use rustc_data_structures::sync::Lrc; use rustc_data_structures::undo_log::Rollback; use rustc_data_structures::unify as ut; @@ -148,10 +150,10 @@ pub struct InferCtxtInner<'tcx> { const_unification_storage: ut::UnificationTableStorage>, /// Map from integral variable to the kind of integer it represents. - int_unification_storage: ut::UnificationTableStorage, + int_unification_table: lut::LoggedUnificationTable, /// Map from floating variable to the kind of float it represents. - float_unification_storage: ut::UnificationTableStorage, + float_unification_table: lut::LoggedUnificationTable, /// Tracks the set of region variables and the constraints between them. /// This is initially `Some(_)` but when @@ -202,10 +204,10 @@ impl<'tcx> InferCtxtInner<'tcx> { projection_cache: Default::default(), type_variable_storage: type_variable::TypeVariableStorage::new(), undo_log: InferCtxtUndoLogs::default(), - const_unification_storage: ut::UnificationTableStorage::new(), - int_unification_storage: ut::UnificationTableStorage::new(), - float_unification_storage: ut::UnificationTableStorage::new(), - region_constraint_storage: Some(RegionConstraintStorage::new()), + const_unification_table: ut::UnificationTableStorage::new(), + int_unification_table: lut::LoggedUnificationTable::new(), + float_unification_table: lut::LoggedUnificationTable::new(), + region_constraints: Some(RegionConstraintStorage::new()), region_obligations: vec![], } } diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 35b97fff3da1f..cb2145ecbc58c 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -5,6 +5,8 @@ use rustc_span::Span; use crate::infer::InferCtxtUndoLogs; +use rustc_data_structures::logged_unification_table as lut; +use rustc_data_structures::modified_set as ms; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::unify as ut; use std::cmp; @@ -64,7 +66,7 @@ pub struct TypeVariableStorage<'tcx> { /// Two variables are unified in `eq_relations` when we have a /// constraint `?X == ?Y`. This table also stores, for each key, /// the known value. - eq_relations: ut::UnificationTableStorage>, + eq_relations: lut::LoggedUnificationTable, TyVid>, /// Two variables are unified in `sub_relations` when we have a /// constraint `?X <: ?Y` *or* a constraint `?Y <: ?X`. This second @@ -156,7 +158,7 @@ impl<'tcx> TypeVariableStorage<'tcx> { pub fn new() -> TypeVariableStorage<'tcx> { TypeVariableStorage { values: sv::SnapshotVecStorage::new(), - eq_relations: ut::UnificationTableStorage::new(), + eq_relations: lut::LoggedUnificationTable::new(), sub_relations: ut::UnificationTableStorage::new(), } } @@ -203,6 +205,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { pub fn sub(&mut self, a: ty::TyVid, b: ty::TyVid) { debug_assert!(self.probe(a).is_unknown()); debug_assert!(self.probe(b).is_unknown()); + self.sub_relations().union(a, b); } @@ -409,6 +412,30 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { }) .collect() } + + pub fn drain_modified_set( + &mut self, + offset: &ms::Offset, + f: impl FnMut(ty::TyVid) -> bool, + ) { + self.eq_relations.drain_modified_set(offset, f) + } + + pub fn register_unify_watcher(&mut self) -> ms::Offset { + self.eq_relations.register() + } + + pub fn deregister_unify_watcher(&mut self, offset: ms::Offset) { + self.eq_relations.deregister(offset); + } + + pub fn watch_variable(&mut self, vid: ty::TyVid) { + self.eq_relations.watch_variable(vid); + } + + pub fn unwatch_variable(&mut self, vid: ty::TyVid) { + self.eq_relations.unwatch_variable(vid); + } } impl sv::SnapshotVecDelegate for Delegate { @@ -431,6 +458,13 @@ impl sv::SnapshotVecDelegate for Delegate { } } +impl sv::SnapshotVecDelegate for UnifiedVarsDelegate { + type Value = Vec; + type Undo = (); + + fn reverse(_values: &mut Vec, _action: ()) {} +} + /////////////////////////////////////////////////////////////////////////// /// These structs (a newtyped TyVid) are used as the unification key @@ -450,6 +484,12 @@ impl<'tcx> From for TyVidEqKey<'tcx> { } } +impl<'tcx> From> for ty::TyVid { + fn from(vid: TyVidEqKey<'tcx>) -> Self { + vid.vid + } +} + impl<'tcx> ut::UnifyKey for TyVidEqKey<'tcx> { type Value = TypeVariableValue<'tcx>; fn index(&self) -> u32 { diff --git a/compiler/rustc_infer/src/traits/engine.rs b/compiler/rustc_infer/src/traits/engine.rs index 2710debea9478..15c61587f6f10 100644 --- a/compiler/rustc_infer/src/traits/engine.rs +++ b/compiler/rustc_infer/src/traits/engine.rs @@ -54,6 +54,17 @@ pub trait TraitEngine<'tcx>: 'tcx { infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>>; + fn select_all_where_possible( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> { + let result = self.select_where_possible(infcx); + self.deregister(infcx); + result + } + + fn deregister(&mut self, _infcx: &InferCtxt<'_, 'tcx>) {} + fn pending_obligations(&self) -> Vec>; } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 0fd48d0928257..cdad15401bd9e 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1041,7 +1041,11 @@ impl Binder { where T: TypeFoldable<'tcx>, { - if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) } + if self.0.has_escaping_bound_vars() { + None + } else { + Some(self.skip_binder()) + } } /// Given two things that have the same binder level, @@ -1463,6 +1467,18 @@ pub struct TyVid { pub index: u32, } +impl Idx for TyVid { + #[inline] + fn new(idx: usize) -> Self { + assert!(idx <= u32::max_value() as usize); + TyVid { index: idx as u32 } + } + #[inline] + fn index(self) -> usize { + self.index as usize + } +} + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] pub struct ConstVid<'tcx> { pub index: u32, @@ -1474,11 +1490,35 @@ pub struct IntVid { pub index: u32, } +impl Idx for IntVid { + #[inline] + fn new(idx: usize) -> Self { + assert!(idx <= u32::max_value() as usize); + IntVid { index: idx as u32 } + } + #[inline] + fn index(self) -> usize { + self.index as usize + } +} + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] pub struct FloatVid { pub index: u32, } +impl Idx for FloatVid { + #[inline] + fn new(idx: usize) -> Self { + assert!(idx <= u32::max_value() as usize); + FloatVid { index: idx as u32 } + } + #[inline] + fn index(self) -> usize { + self.index as usize + } +} + rustc_index::newtype_index! { pub struct RegionVid { DEBUG_FORMAT = custom, @@ -1877,7 +1917,11 @@ impl<'tcx> TyS<'tcx> { #[inline] pub fn is_phantom_data(&self) -> bool { - if let Adt(def, _) = self.kind() { def.is_phantom_data() } else { false } + if let Adt(def, _) = self.kind() { + def.is_phantom_data() + } else { + false + } } #[inline] diff --git a/compiler/rustc_trait_selection/src/autoderef.rs b/compiler/rustc_trait_selection/src/autoderef.rs index b9c5123e49a0e..2a89b32769c67 100644 --- a/compiler/rustc_trait_selection/src/autoderef.rs +++ b/compiler/rustc_trait_selection/src/autoderef.rs @@ -153,7 +153,7 @@ impl<'a, 'tcx> Autoderef<'a, 'tcx> { ), cause, ); - if let Err(e) = fulfillcx.select_where_possible(&self.infcx) { + if let Err(e) = fulfillcx.select_all_where_possible(&self.infcx) { // This shouldn't happen, except for evaluate/fulfill mismatches, // but that's not a reason for an ICE (`predicate_may_hold` is conservative // by design). diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 3fb56de4a8835..121e3bdc85eec 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -30,10 +30,16 @@ impl<'tcx> ForestObligation for PendingPredicateObligation<'tcx> { /// as the `ParamEnv` can influence whether fulfillment succeeds /// or fails. type CacheKey = ty::ParamEnvAnd<'tcx, ty::Predicate<'tcx>>; + type Variable = ty::InferTy; + type WatcherOffset = WatcherOffset; fn as_cache_key(&self) -> Self::CacheKey { self.obligation.param_env.and(self.obligation.predicate) } + + fn stalled_on(&self) -> &[Self::Variable] { + &self.stalled_on + } } /// The fulfillment context is used to drive trait resolution. It @@ -125,27 +131,22 @@ impl<'a, 'tcx> FulfillmentContext<'tcx> { let mut errors = Vec::new(); - loop { - debug!("select: starting another iteration"); + debug!("select: starting another iteration"); - // Process pending obligations. - let outcome: Outcome<_, _> = - self.predicates.process_obligations(&mut FulfillProcessor { - selcx, - register_region_obligations: self.register_region_obligations, - }); - debug!("select: outcome={:#?}", outcome); + // Process pending obligations. + let outcome = self.predicates.process_obligations( + &mut FulfillProcessor { + selcx, + register_region_obligations: self.register_region_obligations, + }, + DoCompleted::No, + ); + debug!("select: outcome={:#?}", outcome); - // FIXME: if we kept the original cache key, we could mark projection - // obligations as complete for the projection cache here. + // FIXME: if we kept the original cache key, we could mark projection + // obligations as complete for the projection cache here. - errors.extend(outcome.errors.into_iter().map(to_fulfillment_error)); - - // If nothing new was added, no need to keep looping. - if outcome.stalled { - break; - } - } + errors.extend(outcome.errors.into_iter().map(to_fulfillment_error)); debug!( "select({} predicates remaining, {} errors) done", @@ -216,15 +217,22 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>> { - self.select_where_possible(infcx)?; - - let errors: Vec<_> = self - .predicates - .to_errors(CodeAmbiguity) - .into_iter() - .map(to_fulfillment_error) - .collect(); - if errors.is_empty() { Ok(()) } else { Err(errors) } + let result = (|| { + self.select_where_possible(infcx)?; + + let errors: Vec<_> = self + .predicates + .to_errors(CodeAmbiguity) + .into_iter() + .map(to_fulfillment_error) + .collect(); + + if errors.is_empty() { Ok(()) } else { Err(errors) } + })(); + if let Some(offset) = self.predicates.take_offset() { + infcx.deregister_unify_watcher(offset); + } + result } fn select_where_possible( @@ -235,6 +243,12 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { self.select(&mut selcx) } + fn deregister(&mut self, infcx: &InferCtxt<'_, 'tcx>) { + if let Some(offset) = self.predicates.take_offset() { + infcx.deregister_unify_watcher(offset); + } + } + fn pending_obligations(&self) -> Vec> { self.predicates.map_pending_obligations(|o| o.obligation.clone()) } @@ -267,72 +281,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { &mut self, pending_obligation: &mut Self::Obligation, ) -> ProcessResult { - // If we were stalled on some unresolved variables, first check whether - // any of them have been resolved; if not, don't bother doing more work - // yet. - let change = match pending_obligation.stalled_on.len() { - // Match arms are in order of frequency, which matters because this - // code is so hot. 1 and 0 dominate; 2+ is fairly rare. - 1 => { - let infer_var = pending_obligation.stalled_on[0]; - self.selcx.infcx().ty_or_const_infer_var_changed(infer_var) - } - 0 => { - // In this case we haven't changed, but wish to make a change. - true - } - _ => { - // This `for` loop was once a call to `all()`, but this lower-level - // form was a perf win. See #64545 for details. - (|| { - for &infer_var in &pending_obligation.stalled_on { - if self.selcx.infcx().ty_or_const_infer_var_changed(infer_var) { - return true; - } - } - false - })() - } - }; - - if !change { - debug!( - "process_predicate: pending obligation {:?} still stalled on {:?}", - self.selcx.infcx().resolve_vars_if_possible(&pending_obligation.obligation), - pending_obligation.stalled_on - ); - return ProcessResult::Unchanged; - } - - self.progress_changed_obligations(pending_obligation) - } - - fn process_backedge<'c, I>( - &mut self, - cycle: I, - _marker: PhantomData<&'c PendingPredicateObligation<'tcx>>, - ) where - I: Clone + Iterator>, - { - if self.selcx.coinductive_match(cycle.clone().map(|s| s.obligation.predicate)) { - debug!("process_child_obligations: coinductive match"); - } else { - let cycle: Vec<_> = cycle.map(|c| c.obligation.clone()).collect(); - self.selcx.infcx().report_overflow_error_cycle(&cycle); - } - } -} - -impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { - // The code calling this method is extremely hot and only rarely - // actually uses this, so move this part of the code - // out of that loop. - #[inline(never)] - fn progress_changed_obligations( - &mut self, - pending_obligation: &mut PendingPredicateObligation<'tcx>, - ) -> ProcessResult, FulfillmentErrorCode<'tcx>> { - pending_obligation.stalled_on.truncate(0); + pending_obligation.stalled_on.clear(); let obligation = &mut pending_obligation.obligation; @@ -600,6 +549,46 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { } } + fn process_backedge<'c, I>( + &mut self, + cycle: I, + _marker: PhantomData<&'c PendingPredicateObligation<'tcx>>, + ) where + I: Clone + Iterator>, + { + if self.selcx.coinductive_match(cycle.clone().map(|s| s.obligation.predicate)) { + debug!("process_child_obligations: coinductive match"); + } else { + let cycle: Vec<_> = cycle.map(|c| c.obligation.clone()).collect(); + self.selcx.infcx().report_overflow_error_cycle(&cycle); + } + } + + fn unblocked( + &self, + offset: &WatcherOffset, + f: impl FnMut(::Variable) -> bool, + ) { + self.selcx.infcx().drain_modifications(offset, f); + } + + fn register(&self) -> WatcherOffset { + self.selcx.infcx().register_unify_watcher() + } + + fn deregister(&self, offset: WatcherOffset) { + self.selcx.infcx().deregister_unify_watcher(offset); + } + + fn watch_variable(&self, var: ::Variable) { + self.selcx.infcx().watch_variable(var); + } + fn unwatch_variable(&self, var: ::Variable) { + self.selcx.infcx().unwatch_variable(var); + } +} + +impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { #[instrument(level = "debug", skip(self, obligation, stalled_on))] fn process_trait_obligation( &mut self, diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs index f87e6b607d46e..44ca6ed4f7965 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs @@ -691,7 +691,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { pub(in super::super) fn select_all_obligations_or_error(&self) { debug!("select_all_obligations_or_error"); - if let Err(errors) = self.fulfillment_cx.borrow_mut().select_all_or_error(&self) { + if let Err(errors) = self.fulfillment_cx.borrow_mut().select_or_error(&self) { self.report_fulfillment_errors(&errors, self.inh.body_id, false); } } @@ -702,7 +702,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { fallback_has_occurred: bool, mutate_fullfillment_errors: impl Fn(&mut Vec>), ) { - let result = self.fulfillment_cx.borrow_mut().select_where_possible(self); + let result = self.fulfillment_cx.borrow_mut().select_all_where_possible(self); if let Err(mut errors) = result { mutate_fullfillment_errors(&mut errors); self.report_fulfillment_errors(&errors, self.inh.body_id, fallback_has_occurred); diff --git a/compiler/rustc_typeck/src/check/inherited.rs b/compiler/rustc_typeck/src/check/inherited.rs index 7e580485c3de4..a6e81be98f21a 100644 --- a/compiler/rustc_typeck/src/check/inherited.rs +++ b/compiler/rustc_typeck/src/check/inherited.rs @@ -33,7 +33,7 @@ pub struct Inherited<'a, 'tcx> { pub(super) locals: RefCell>>, - pub(super) fulfillment_cx: RefCell>>, + pub(super) fulfillment_cx: RefCell + 'tcx>>, // Some additional `Sized` obligations badly affect type inference. // These obligations are added in a later stage of typeck. @@ -70,6 +70,12 @@ pub struct Inherited<'a, 'tcx> { pub(super) body_id: Option, } +impl<'a, 'tcx> Drop for Inherited<'a, 'tcx> { + fn drop(&mut self) { + self.fulfillment_cx.get_mut().deregister(&self.infcx); + } +} + impl<'a, 'tcx> Deref for Inherited<'a, 'tcx> { type Target = InferCtxt<'a, 'tcx>; fn deref(&self) -> &Self::Target { diff --git a/compiler/rustc_typeck/src/check/mod.rs b/compiler/rustc_typeck/src/check/mod.rs index 169ad0df3a5c9..02b52bbd355fa 100644 --- a/compiler/rustc_typeck/src/check/mod.rs +++ b/compiler/rustc_typeck/src/check/mod.rs @@ -1136,13 +1136,6 @@ impl ItemLikeVisitor<'tcx> for CheckItemTypesVisitor<'tcx> { fn visit_impl_item(&mut self, _: &'tcx hir::ImplItem<'tcx>) {} } -fn typeck_item_bodies(tcx: TyCtxt<'_>, crate_num: CrateNum) { - debug_assert!(crate_num == LOCAL_CRATE); - tcx.par_body_owners(|body_owner_def_id| { - tcx.ensure().typeck(body_owner_def_id); - }); -} - fn fatally_break_rust(sess: &Session) { let handler = sess.diagnostic(); handler.span_bug_no_panic( From 64b2bbd55032c413a221c16071796caab6224389 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 17 Feb 2020 12:00:54 +0100 Subject: [PATCH 03/25] perf: Avoid logging unifications that ObligationForest do not watch --- .../src/logged_unification_table.rs | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index eb23d62550ac9..cb6b21ebe5e6d 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -38,9 +38,8 @@ where where K::Value: ut::UnifyValue, { - if !self.unify_log.get(vid).is_empty() - || self.reference_counts.get(vid).map_or(false, |c| *c != 0) - { + if self.needs_log(vid) { + warn!("ModifiedSet {:?} => {:?}", vid, ty); self.modified_set.set(vid); } let vid = vid.into(); @@ -60,31 +59,37 @@ where value: K::Value, ) -> Result<(), ::Error> { let vid = self.find(vid).into(); - if !self.unify_log.get(vid).is_empty() - || self.reference_counts.get(vid).map_or(false, |c| *c != 0) - { + if self.needs_log(vid) { self.modified_set.set(vid); } self.relations.unify_var_value(vid, value) } pub fn unify_var_var(&mut self, a: I, b: I) -> Result<(), ::Error> { - let a_root = self.relations.find(a); - let b_root = self.relations.find(b); - if a_root == b_root { + let a = self.relations.find(a); + let b = self.relations.find(b); + if a == b { return Ok(()); } - self.relations.unify_var_var(a_root, b_root)?; + self.relations.unify_var_var(a, b)?; - if a_root == self.relations.find(a_root) { - self.unify_log.unify(a_root.into(), b_root.into()); - } else { - self.unify_log.unify(b_root.into(), a_root.into()); + if self.needs_log(a.into()) || self.needs_log(b.into()) { + warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); + if a == self.relations.find(a) { + self.unify_log.unify(a.into(), b.into()); + } else { + self.unify_log.unify(b.into(), a.into()); + } } Ok(()) } + fn needs_log(&self, vid: I) -> bool { + !self.unify_log.get(vid).is_empty() + || self.reference_counts.get(vid).map_or(false, |c| *c != 0) + } + pub fn union_value(&mut self, vid: I, value: K::Value) where K::Value: ut::UnifyValue, @@ -145,6 +150,7 @@ where } pub fn watch_variable(&mut self, index: I) { + debug_assert!(index == self.relations.find(index).into()); self.reference_counts.ensure_contains_elem(index, || 0); self.reference_counts[index] += 1; } From bb70251337e3a18c8af9395a815c150b00a5f9bb Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 17 Feb 2020 15:18:53 +0100 Subject: [PATCH 04/25] refactor: Prefer copy_from_slice instead of explicit loop --- compiler/rustc_data_structures/src/modified_set.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index e20042f082971..6219301e14902 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -63,14 +63,12 @@ impl ModifiedSet { self.modified_set.remove(index); } self.modified.truncate(snapshot.modified_len); - let mut offsets = self.offsets.iter_mut(); - for (offset, &saved_offset) in offsets.by_ref().zip( + let (offsets, offsets_rest) = self.offsets.split_at_mut(snapshot.offsets_len); + offsets.copy_from_slice( &self.undo_offsets [snapshot.offsets_start..snapshot.offsets_start + snapshot.offsets_len], - ) { - *offset = saved_offset; - } - for offset in offsets { + ); + for offset in offsets_rest { *offset = self.modified.len().min(*offset); } self.undo_offsets.truncate(snapshot.offsets_start); From 65014d8306442463eadb455df963156edf89895e Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 17 Feb 2020 21:51:56 +0100 Subject: [PATCH 05/25] Optimize ModifiedSet for faster snapshot/rollback Snapshot/rollback is done a lot in certain workloads so optimizing for it makes sense as there is very little additional overhead for the set/drain case (which are called relatively rarely) --- .../src/logged_unification_table.rs | 11 ++- .../rustc_data_structures/src/modified_set.rs | 76 +++++++++---------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index cb6b21ebe5e6d..e351f4169c3a9 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -74,11 +74,14 @@ where self.relations.unify_var_var(a, b)?; - if self.needs_log(a.into()) || self.needs_log(b.into()) { - warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); - if a == self.relations.find(a) { + if a == self.relations.find(a) { + if self.needs_log(b.into()) { + warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); self.unify_log.unify(a.into(), b.into()); - } else { + } + } else { + if self.needs_log(a.into()) { + warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); self.unify_log.unify(b.into(), a.into()); } } diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index 6219301e14902..0bc612b2680a3 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -2,12 +2,17 @@ use std::{collections::VecDeque, marker::PhantomData}; use rustc_index::{bit_set::BitSet, vec::Idx}; +#[derive(Copy, Clone, Debug)] +enum Undo { + Add(T), + Drain { index: usize, offset: usize }, +} + #[derive(Clone, Debug)] pub struct ModifiedSet { - modified: VecDeque, + modified: VecDeque>, snapshots: usize, modified_set: BitSet, - undo_offsets: Vec, offsets: Vec, } @@ -18,7 +23,6 @@ impl Default for ModifiedSet { snapshots: 0, modified_set: BitSet::new_empty(0), offsets: Vec::new(), - undo_offsets: Vec::new(), } } } @@ -33,62 +37,59 @@ impl ModifiedSet { self.modified_set.resize(index.index() + 1); } if self.modified_set.insert(index) { - self.modified.push_back(index); + self.modified.push_back(Undo::Add(index)); } } - pub fn drain(&mut self, offset: &Offset, mut f: impl FnMut(T) -> bool) { - let offset = &mut self.offsets[offset.index]; - for &index in self.modified.iter().skip(*offset) { - if f(index) {} + pub fn drain(&mut self, index: &Offset, mut f: impl FnMut(T) -> bool) { + let offset = &mut self.offsets[index.index]; + if *offset < self.modified.len() { + for &undo in self.modified.iter().skip(*offset) { + if let Undo::Add(index) = undo { + f(index); + } + } + self.modified.push_back(Undo::Drain { index: index.index, offset: *offset }); + *offset = self.modified.len(); } - *offset = self.modified.len(); } pub fn snapshot(&mut self) -> Snapshot { self.snapshots += 1; - let offsets_start = self.undo_offsets.len(); - self.undo_offsets.extend_from_slice(&self.offsets); - Snapshot { - modified_len: self.modified.len(), - offsets_start, - offsets_len: self.offsets.len(), - _marker: PhantomData, - } + Snapshot { modified_len: self.modified.len(), _marker: PhantomData } } pub fn rollback_to(&mut self, snapshot: Snapshot) { self.snapshots -= 1; - for &index in self.modified.iter().skip(snapshot.modified_len) { - self.modified_set.remove(index); - } - self.modified.truncate(snapshot.modified_len); - let (offsets, offsets_rest) = self.offsets.split_at_mut(snapshot.offsets_len); - offsets.copy_from_slice( - &self.undo_offsets - [snapshot.offsets_start..snapshot.offsets_start + snapshot.offsets_len], - ); - for offset in offsets_rest { - *offset = self.modified.len().min(*offset); + if snapshot.modified_len < self.modified.len() { + for &undo in + self.modified.iter().rev().take(self.modified.len() - snapshot.modified_len) + { + match undo { + Undo::Add(index) => { + self.modified_set.remove(index); + } + Undo::Drain { index, offset } => { + if let Some(o) = self.offsets.get_mut(index) { + *o = offset; + } + } + } + } + self.modified.truncate(snapshot.modified_len); } - self.undo_offsets.truncate(snapshot.offsets_start); if self.snapshots == 0 { let min = self.offsets.iter().copied().min().unwrap_or(0); - // Any indices still in `modified` may not have been instantiated, so if we observe them again - // we need to notify any listeners again - for index in self.modified.drain(..min) { - self.modified_set.remove(index); - } + self.modified.drain(..min); for offset in &mut self.offsets { *offset -= min; } } } - pub fn commit(&mut self, snapshot: Snapshot) { + pub fn commit(&mut self, _snapshot: Snapshot) { self.snapshots -= 1; - self.undo_offsets.truncate(snapshot.offsets_start); if self.snapshots == 0 { // Everything up until this point is committed, so we can forget anything before the // current offsets @@ -102,6 +103,7 @@ impl ModifiedSet { pub fn register(&mut self) -> Offset { let index = self.offsets.len(); + self.modified.push_back(Undo::Drain { index, offset: 0 }); self.offsets.push(0); Offset { index, _marker: PhantomData } } @@ -131,7 +133,5 @@ impl Drop for Offset { #[derive(Debug)] pub struct Snapshot { modified_len: usize, - offsets_start: usize, - offsets_len: usize, _marker: PhantomData, } From 6eeea051a95270089726640d31101c6374540090 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Tue, 18 Feb 2020 08:12:39 +0100 Subject: [PATCH 06/25] refactor: Make UnifyLog have lower overhead snapshots --- .../src/logged_unification_table.rs | 28 ++--- .../rustc_data_structures/src/unify_log.rs | 110 +++++++++++++++--- 2 files changed, 103 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index e351f4169c3a9..f9c7b82fbbdfa 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -1,6 +1,6 @@ use std::ops::Range; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::Idx; use crate::modified_set as ms; use crate::unify as ut; @@ -10,7 +10,6 @@ pub struct LoggedUnificationTable { relations: ut::UnificationTable>, unify_log: ul::UnifyLog, modified_set: ms::ModifiedSet, - reference_counts: IndexVec, } impl LoggedUnificationTable @@ -23,7 +22,6 @@ where relations: ut::UnificationTable::new(), unify_log: ul::UnifyLog::new(), modified_set: ms::ModifiedSet::new(), - reference_counts: IndexVec::new(), } } @@ -38,7 +36,7 @@ where where K::Value: ut::UnifyValue, { - if self.needs_log(vid) { + if self.unify_log.needs_log(vid) { warn!("ModifiedSet {:?} => {:?}", vid, ty); self.modified_set.set(vid); } @@ -59,7 +57,7 @@ where value: K::Value, ) -> Result<(), ::Error> { let vid = self.find(vid).into(); - if self.needs_log(vid) { + if self.unify_log.needs_log(vid) { self.modified_set.set(vid); } self.relations.unify_var_value(vid, value) @@ -75,24 +73,13 @@ where self.relations.unify_var_var(a, b)?; if a == self.relations.find(a) { - if self.needs_log(b.into()) { - warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); - self.unify_log.unify(a.into(), b.into()); - } + self.unify_log.unify(a.into(), b.into()); } else { - if self.needs_log(a.into()) { - warn!("Log: {:?} {:?} => {:?}", a, b, I::from(self.relations.find(a))); - self.unify_log.unify(b.into(), a.into()); - } + self.unify_log.unify(b.into(), a.into()); } Ok(()) } - fn needs_log(&self, vid: I) -> bool { - !self.unify_log.get(vid).is_empty() - || self.reference_counts.get(vid).map_or(false, |c| *c != 0) - } - pub fn union_value(&mut self, vid: I, value: K::Value) where K::Value: ut::UnifyValue, @@ -154,12 +141,11 @@ where pub fn watch_variable(&mut self, index: I) { debug_assert!(index == self.relations.find(index).into()); - self.reference_counts.ensure_contains_elem(index, || 0); - self.reference_counts[index] += 1; + self.unify_log.watch_variable(index) } pub fn unwatch_variable(&mut self, index: I) { - self.reference_counts[index] -= 1; + self.unify_log.unwatch_variable(index) } pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I) -> bool) { diff --git a/compiler/rustc_data_structures/src/unify_log.rs b/compiler/rustc_data_structures/src/unify_log.rs index a5d1c32807b68..72b4371f5ea4b 100644 --- a/compiler/rustc_data_structures/src/unify_log.rs +++ b/compiler/rustc_data_structures/src/unify_log.rs @@ -2,32 +2,105 @@ use std::marker::PhantomData; use rustc_index::vec::{Idx, IndexVec}; +enum Undo { + Move { index: T, old: usize }, + Extend { group_index: usize, len: usize }, + NewGroup { index: T }, +} + pub struct UnifyLog { - unified_vars: IndexVec>, - undo_log: Vec<(T, u32)>, + unified_vars: IndexVec, + groups: Vec>, + undo_log: Vec>, + reference_counts: IndexVec, snapshots: usize, } +fn pick2_mut(self_: &mut [T], a: I, b: I) -> (&mut T, &mut T) { + let (ai, bi) = (a.index(), b.index()); + assert!(ai != bi); + + if ai < bi { + let (c1, c2) = self_.split_at_mut(bi); + (&mut c1[ai], &mut c2[0]) + } else { + let (c2, c1) = pick2_mut(self_, b, a); + (c1, c2) + } +} + impl UnifyLog { pub fn new() -> Self { - UnifyLog { unified_vars: IndexVec::new(), undo_log: Vec::new(), snapshots: 0 } + UnifyLog { + unified_vars: IndexVec::new(), + groups: Vec::new(), + undo_log: Vec::new(), + reference_counts: IndexVec::new(), + snapshots: 0, + } } pub fn unify(&mut self, root: T, other: T) { - self.unified_vars.ensure_contains_elem(root, Vec::new); - self.unified_vars.ensure_contains_elem(other, Vec::new); - let (root_ids, other_ids) = self.unified_vars.pick2_mut(root, other); - self.undo_log.push((root, root_ids.len() as u32)); - for &other in &*other_ids { - if !root_ids.contains(&other) { - root_ids.push(other); + if !self.needs_log(other) { + return; + } + self.unified_vars.ensure_contains_elem(root, usize::max_value); + self.unified_vars.ensure_contains_elem(other, usize::max_value); + let mut root_group = self.unified_vars[root]; + let other_group = self.unified_vars[other]; + + if other_group == usize::max_value() { + let root_vec = if root_group == usize::max_value() { + root_group = self.groups.len(); + self.unified_vars[root] = root_group; + self.groups.push(Vec::new()); + self.undo_log.push(Undo::NewGroup { index: root }); + self.groups.last_mut().unwrap() + } else { + let root_vec = &mut self.groups[root_group]; + self.undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); + root_vec + }; + root_vec.push(other); + } else { + if root_group == usize::max_value() { + let group = &mut self.unified_vars[root]; + self.undo_log.push(Undo::Move { index: root, old: *group }); + *group = other_group; + self.groups[other_group].push(other); + } else { + let (root_vec, other_vec) = pick2_mut(&mut self.groups, root_group, other_group); + self.undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); + root_vec.extend_from_slice(other_vec); + + if self.reference_counts.get(other).map_or(false, |c| *c != 0) { + root_vec.push(other); + } } } - root_ids.push(other); } pub fn get(&self, root: T) -> &[T] { - self.unified_vars.get(root).map(|v| &v[..]).unwrap_or(&[][..]) + match self.unified_vars.get(root) { + Some(group) => match self.groups.get(*group) { + Some(v) => v, + None => &[], + }, + None => &[], + } + } + + pub fn needs_log(&self, vid: T) -> bool { + !self.get(vid).is_empty() || self.reference_counts.get(vid).map_or(false, |c| *c != 0) + } + + pub fn watch_variable(&mut self, index: T) { + self.reference_counts.ensure_contains_elem(index, || 0); + self.reference_counts[index] += 1; + } + + pub fn unwatch_variable(&mut self, index: T) { + self.reference_counts[index] -= 1; } pub fn snapshot(&mut self) -> Snapshot { @@ -44,8 +117,17 @@ impl UnifyLog { pub fn rollback_to(&mut self, snapshot: Snapshot) { self.snapshots -= 1; - for (index, len) in self.undo_log.drain(snapshot.undo_log_len as usize..) { - self.unified_vars[index].truncate(len as usize); + for undo in self.undo_log.drain(snapshot.undo_log_len as usize..).rev() { + match undo { + Undo::Extend { group_index, len } => { + self.groups[group_index].truncate(len as usize) + } + Undo::Move { index, old } => self.unified_vars[index] = old, + Undo::NewGroup { index } => { + self.groups.pop(); + self.unified_vars[index] = usize::max_value(); + } + } } } } From aa4595adb608ac54e25dedcee1565e719c9ad759 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Wed, 19 Feb 2020 17:45:54 +0100 Subject: [PATCH 07/25] perf: No need to track modified variables in a bitset Each variable will only be set once anyway due since only variable instantiation calls it. --- .../rustc_data_structures/src/modified_set.rs | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index 0bc612b2680a3..91f2f73c2097a 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -1,6 +1,6 @@ use std::{collections::VecDeque, marker::PhantomData}; -use rustc_index::{bit_set::BitSet, vec::Idx}; +use rustc_index::vec::Idx; #[derive(Copy, Clone, Debug)] enum Undo { @@ -12,18 +12,12 @@ enum Undo { pub struct ModifiedSet { modified: VecDeque>, snapshots: usize, - modified_set: BitSet, offsets: Vec, } impl Default for ModifiedSet { fn default() -> Self { - Self { - modified: Default::default(), - snapshots: 0, - modified_set: BitSet::new_empty(0), - offsets: Vec::new(), - } + Self { modified: Default::default(), snapshots: 0, offsets: Vec::new() } } } @@ -33,12 +27,7 @@ impl ModifiedSet { } pub fn set(&mut self, index: T) { - if index.index() >= self.modified_set.domain_size() { - self.modified_set.resize(index.index() + 1); - } - if self.modified_set.insert(index) { - self.modified.push_back(Undo::Add(index)); - } + self.modified.push_back(Undo::Add(index)); } pub fn drain(&mut self, index: &Offset, mut f: impl FnMut(T) -> bool) { @@ -66,9 +55,7 @@ impl ModifiedSet { self.modified.iter().rev().take(self.modified.len() - snapshot.modified_len) { match undo { - Undo::Add(index) => { - self.modified_set.remove(index); - } + Undo::Add(_) => {} Undo::Drain { index, offset } => { if let Some(o) = self.offsets.get_mut(index) { *o = offset; From 31fb673fa52e2eef154353b424d3d17f23983deb Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Wed, 19 Feb 2020 17:47:05 +0100 Subject: [PATCH 08/25] Start reading where ModifiedSet currently is --- .../src/logged_unification_table.rs | 223 +++++++++++++++--- .../rustc_data_structures/src/modified_set.rs | 99 ++++---- .../rustc_data_structures/src/unify_log.rs | 58 ++--- compiler/rustc_infer/src/infer/mod.rs | 1 - 4 files changed, 242 insertions(+), 139 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index f9c7b82fbbdfa..8c1239aab779b 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -1,15 +1,145 @@ -use std::ops::Range; +use std::{marker::PhantomData, ops::Range}; use rustc_index::vec::Idx; use crate::modified_set as ms; +use crate::snapshot_vec as sv; use crate::unify as ut; use crate::unify_log as ul; +use ena::undo_log::{Rollback, Snapshots, UndoLogs}; + +enum UndoLog { + Relation(sv::UndoLog>), + UnifyLog(ul::Undo), + ModifiedSet(ms::Undo), +} + +impl From>> for UndoLog { + fn from(l: sv::UndoLog>) -> Self { + UndoLog::Relation(l) + } +} + +impl From> for UndoLog { + fn from(l: ul::Undo) -> Self { + UndoLog::UnifyLog(l) + } +} + +impl From for UndoLog { + fn from(l: ms::Undo) -> Self { + UndoLog::ModifiedSet(l) + } +} + +struct Logs { + logs: Vec>, + num_open_snapshots: usize, +} + +impl Default for Logs { + fn default() -> Self { + Self { logs: Default::default(), num_open_snapshots: Default::default() } + } +} + +impl UndoLogs for Logs +where + UndoLog: From, +{ + fn num_open_snapshots(&self) -> usize { + self.num_open_snapshots + } + fn push(&mut self, undo: T) { + if self.in_snapshot() { + self.logs.push(undo.into()) + } + } + fn extend(&mut self, undos: J) + where + Self: Sized, + J: IntoIterator, + { + if self.in_snapshot() { + self.logs.extend(undos.into_iter().map(UndoLog::from)) + } + } +} + +struct RollbackView<'a, K: ut::UnifyKey, I: Idx> { + relations: &'a mut ut::UnificationStorage, + unify_log: &'a mut ul::UnifyLog, + modified_set: &'a mut ms::ModifiedSet, +} + +impl Rollback> for RollbackView<'_, K, I> { + fn reverse(&mut self, undo: UndoLog) { + match undo { + UndoLog::Relation(undo) => self.relations.reverse(undo), + UndoLog::UnifyLog(undo) => self.unify_log.reverse(undo), + UndoLog::ModifiedSet(undo) => self.modified_set.reverse(undo), + } + } +} + +impl Snapshots> for Logs { + type Snapshot = Snapshot; + fn actions_since_snapshot(&self, snapshot: &Self::Snapshot) -> &[UndoLog] { + &self.logs[snapshot.undo_len..] + } + + fn start_snapshot(&mut self) -> Self::Snapshot { + unreachable!() + } + + fn rollback_to(&mut self, values: &mut impl Rollback>, snapshot: Self::Snapshot) { + debug!("rollback_to({})", snapshot.undo_len); + self.assert_open_snapshot(&snapshot); + + while self.logs.len() > snapshot.undo_len { + values.reverse(self.logs.pop().unwrap()); + } + + if self.num_open_snapshots == 1 { + // The root snapshot. It's safe to clear the undo log because + // there's no snapshot further out that we might need to roll back + // to. + assert!(snapshot.undo_len == 0); + self.logs.clear(); + } + + self.num_open_snapshots -= 1; + } + + fn commit(&mut self, snapshot: Self::Snapshot) { + debug!("commit({})", snapshot.undo_len); + + if self.num_open_snapshots == 1 { + // The root snapshot. It's safe to clear the undo log because + // there's no snapshot further out that we might need to roll back + // to. + assert!(snapshot.undo_len == 0); + self.logs.clear(); + } + + self.num_open_snapshots -= 1; + } +} + +impl Logs { + fn assert_open_snapshot(&self, snapshot: &Snapshot) { + // Failures here may indicate a failure to follow a stack discipline. + assert!(self.logs.len() >= snapshot.undo_len); + assert!(self.num_open_snapshots > 0); + } +} + pub struct LoggedUnificationTable { - relations: ut::UnificationTable>, + relations: ut::UnificationStorage, unify_log: ul::UnifyLog, modified_set: ms::ModifiedSet, + undo_log: Logs, } impl LoggedUnificationTable @@ -19,12 +149,19 @@ where { pub fn new() -> Self { Self { - relations: ut::UnificationTable::new(), + relations: Default::default(), unify_log: ul::UnifyLog::new(), modified_set: ms::ModifiedSet::new(), + undo_log: Logs::default(), } } + fn relations( + &mut self, + ) -> ut::UnificationTable, &mut Logs>> { + ut::UnificationTable::with_log(&mut self.relations, &mut self.undo_log) + } + pub fn unify(&mut self, a: I, b: I) where K::Value: ut::UnifyValue, @@ -38,17 +175,18 @@ where { if self.unify_log.needs_log(vid) { warn!("ModifiedSet {:?} => {:?}", vid, ty); - self.modified_set.set(vid); + self.modified_set.set(&mut self.undo_log, vid); } let vid = vid.into(); - debug_assert!(self.relations.find(vid) == vid); - self.relations.union_value(vid, ty); + let mut relations = self.relations(); + debug_assert!(relations.find(vid) == vid); + relations.union_value(vid, ty); vid } pub fn find(&mut self, vid: I) -> K { - self.relations.find(vid) + self.relations().find(vid) } pub fn unify_var_value( @@ -58,24 +196,25 @@ where ) -> Result<(), ::Error> { let vid = self.find(vid).into(); if self.unify_log.needs_log(vid) { - self.modified_set.set(vid); + self.modified_set.set(&mut self.undo_log, vid); } - self.relations.unify_var_value(vid, value) + self.relations().unify_var_value(vid, value) } pub fn unify_var_var(&mut self, a: I, b: I) -> Result<(), ::Error> { - let a = self.relations.find(a); - let b = self.relations.find(b); + let mut relations = self.relations(); + let a = relations.find(a); + let b = relations.find(b); if a == b { return Ok(()); } - self.relations.unify_var_var(a, b)?; + relations.unify_var_var(a, b)?; - if a == self.relations.find(a) { - self.unify_log.unify(a.into(), b.into()); + if a == relations.find(a) { + self.unify_log.unify(&mut self.undo_log, a.into(), b.into()); } else { - self.unify_log.unify(b.into(), a.into()); + self.unify_log.unify(&mut self.undo_log, b.into(), a.into()); } Ok(()) } @@ -89,46 +228,52 @@ where } pub fn probe_value(&mut self, vid: I) -> K::Value { - self.relations.probe_value(vid) + self.relations().probe_value(vid) } #[inline(always)] pub fn inlined_probe_value(&mut self, vid: I) -> K::Value { - self.relations.inlined_probe_value(vid) + self.relations().inlined_probe_value(vid) } pub fn new_key(&mut self, value: K::Value) -> K { - self.relations.new_key(value) + self.relations().new_key(value) } pub fn len(&self) -> usize { self.relations.len() } + pub fn vars_since_snapshot(&mut self, s: &Snapshot) -> Range { + K::from(I::new(s.value_count))..K::from(I::new(self.relations().len())) + } + pub fn snapshot(&mut self) -> Snapshot { + self.undo_log.num_open_snapshots += 1; Snapshot { - snapshot: self.relations.snapshot(), - unify_log_snapshot: self.unify_log.snapshot(), - modified_snapshot: self.modified_set.snapshot(), + undo_len: self.undo_log.logs.len(), + value_count: self.relations().len(), + _marker: PhantomData, } } - pub fn rollback_to(&mut self, s: Snapshot) { - let Snapshot { snapshot, unify_log_snapshot, modified_snapshot } = s; - self.relations.rollback_to(snapshot); - self.unify_log.rollback_to(unify_log_snapshot); - self.modified_set.rollback_to(modified_snapshot); - } + pub fn rollback_to(&mut self, snapshot: Snapshot) { + let Self { relations, unify_log, modified_set, .. } = self; + + self.undo_log + .rollback_to(&mut RollbackView { relations, unify_log, modified_set }, snapshot); - pub fn commit(&mut self, s: Snapshot) { - let Snapshot { snapshot, unify_log_snapshot, modified_snapshot } = s; - self.relations.commit(snapshot); - self.unify_log.commit(unify_log_snapshot); - self.modified_set.commit(modified_snapshot); + if self.undo_log.num_open_snapshots == 0 { + self.modified_set.clear(); + } } - pub fn vars_since_snapshot(&mut self, s: &Snapshot) -> Range { - self.relations.vars_since_snapshot(&s.snapshot) + pub fn commit(&mut self, snapshot: Snapshot) { + self.undo_log.commit(snapshot); + + if self.undo_log.num_open_snapshots == 0 { + self.modified_set.clear(); + } } pub fn register(&mut self) -> ms::Offset { @@ -140,7 +285,7 @@ where } pub fn watch_variable(&mut self, index: I) { - debug_assert!(index == self.relations.find(index).into()); + debug_assert!(index == self.relations().find(index).into()); self.unify_log.watch_variable(index) } @@ -150,7 +295,7 @@ where pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I) -> bool) { let unify_log = &self.unify_log; - self.modified_set.drain(offset, |vid| { + self.modified_set.drain(&mut self.undo_log, offset, |vid| { for &unified_vid in unify_log.get(vid) { f(unified_vid); } @@ -161,7 +306,7 @@ where } pub struct Snapshot { - snapshot: ut::Snapshot>, - unify_log_snapshot: ul::Snapshot, - modified_snapshot: ms::Snapshot, + undo_len: usize, + value_count: usize, + _marker: PhantomData<(K, I)>, } diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index 91f2f73c2097a..6823d15f7f87b 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -1,23 +1,24 @@ -use std::{collections::VecDeque, marker::PhantomData}; +use std::marker::PhantomData; use rustc_index::vec::Idx; +use ena::undo_log::{Rollback, UndoLogs}; + #[derive(Copy, Clone, Debug)] -enum Undo { - Add(T), +pub enum Undo { + Add, Drain { index: usize, offset: usize }, } #[derive(Clone, Debug)] pub struct ModifiedSet { - modified: VecDeque>, - snapshots: usize, + modified: Vec, offsets: Vec, } impl Default for ModifiedSet { fn default() -> Self { - Self { modified: Default::default(), snapshots: 0, offsets: Vec::new() } + Self { modified: Default::default(), offsets: Vec::new() } } } @@ -26,72 +27,38 @@ impl ModifiedSet { Self::default() } - pub fn set(&mut self, index: T) { - self.modified.push_back(Undo::Add(index)); + pub fn set(&mut self, undo_log: &mut impl UndoLogs, index: T) { + self.modified.push(index); + undo_log.push(Undo::Add); } - pub fn drain(&mut self, index: &Offset, mut f: impl FnMut(T) -> bool) { + pub fn drain( + &mut self, + undo_log: &mut impl UndoLogs, + index: &Offset, + mut f: impl FnMut(T) -> bool, + ) { let offset = &mut self.offsets[index.index]; if *offset < self.modified.len() { - for &undo in self.modified.iter().skip(*offset) { - if let Undo::Add(index) = undo { - f(index); - } + for &index in &self.modified[*offset..] { + f(index); } - self.modified.push_back(Undo::Drain { index: index.index, offset: *offset }); + undo_log.push(Undo::Drain { index: index.index, offset: *offset }); *offset = self.modified.len(); } } - pub fn snapshot(&mut self) -> Snapshot { - self.snapshots += 1; - Snapshot { modified_len: self.modified.len(), _marker: PhantomData } - } - - pub fn rollback_to(&mut self, snapshot: Snapshot) { - self.snapshots -= 1; - if snapshot.modified_len < self.modified.len() { - for &undo in - self.modified.iter().rev().take(self.modified.len() - snapshot.modified_len) - { - match undo { - Undo::Add(_) => {} - Undo::Drain { index, offset } => { - if let Some(o) = self.offsets.get_mut(index) { - *o = offset; - } - } - } - } - self.modified.truncate(snapshot.modified_len); - } - - if self.snapshots == 0 { - let min = self.offsets.iter().copied().min().unwrap_or(0); - self.modified.drain(..min); - for offset in &mut self.offsets { - *offset -= min; - } - } - } - - pub fn commit(&mut self, _snapshot: Snapshot) { - self.snapshots -= 1; - if self.snapshots == 0 { - // Everything up until this point is committed, so we can forget anything before the - // current offsets - let min = self.offsets.iter().copied().min().unwrap_or(0); - self.modified.drain(..min); - for offset in &mut self.offsets { - *offset -= min; - } + pub fn clear(&mut self) { + let min = self.offsets.iter().copied().min().unwrap_or_else(|| self.modified.len()); + self.modified.drain(..min); + for offset in &mut self.offsets { + *offset -= min; } } pub fn register(&mut self) -> Offset { let index = self.offsets.len(); - self.modified.push_back(Undo::Drain { index, offset: 0 }); - self.offsets.push(0); + self.offsets.push(self.modified.len()); Offset { index, _marker: PhantomData } } @@ -102,6 +69,21 @@ impl ModifiedSet { } } +impl Rollback for ModifiedSet { + fn reverse(&mut self, undo: Undo) { + match undo { + Undo::Add => { + self.modified.pop(); + } + Undo::Drain { index, offset } => { + if let Some(o) = self.offsets.get_mut(index) { + *o = offset; + } + } + } + } +} + #[must_use] pub struct Offset { index: usize, @@ -120,5 +102,6 @@ impl Drop for Offset { #[derive(Debug)] pub struct Snapshot { modified_len: usize, + undo_log_len: usize, _marker: PhantomData, } diff --git a/compiler/rustc_data_structures/src/unify_log.rs b/compiler/rustc_data_structures/src/unify_log.rs index 72b4371f5ea4b..538aee3484eb0 100644 --- a/compiler/rustc_data_structures/src/unify_log.rs +++ b/compiler/rustc_data_structures/src/unify_log.rs @@ -1,8 +1,8 @@ -use std::marker::PhantomData; - use rustc_index::vec::{Idx, IndexVec}; -enum Undo { +use ena::undo_log::{Rollback, UndoLogs}; + +pub enum Undo { Move { index: T, old: usize }, Extend { group_index: usize, len: usize }, NewGroup { index: T }, @@ -11,9 +11,7 @@ enum Undo { pub struct UnifyLog { unified_vars: IndexVec, groups: Vec>, - undo_log: Vec>, reference_counts: IndexVec, - snapshots: usize, } fn pick2_mut(self_: &mut [T], a: I, b: I) -> (&mut T, &mut T) { @@ -34,13 +32,11 @@ impl UnifyLog { UnifyLog { unified_vars: IndexVec::new(), groups: Vec::new(), - undo_log: Vec::new(), reference_counts: IndexVec::new(), - snapshots: 0, } } - pub fn unify(&mut self, root: T, other: T) { + pub fn unify(&mut self, undo_log: &mut impl UndoLogs>, root: T, other: T) { if !self.needs_log(other) { return; } @@ -54,23 +50,23 @@ impl UnifyLog { root_group = self.groups.len(); self.unified_vars[root] = root_group; self.groups.push(Vec::new()); - self.undo_log.push(Undo::NewGroup { index: root }); + undo_log.push(Undo::NewGroup { index: root }); self.groups.last_mut().unwrap() } else { let root_vec = &mut self.groups[root_group]; - self.undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); + undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); root_vec }; root_vec.push(other); } else { if root_group == usize::max_value() { let group = &mut self.unified_vars[root]; - self.undo_log.push(Undo::Move { index: root, old: *group }); + undo_log.push(Undo::Move { index: root, old: *group }); *group = other_group; self.groups[other_group].push(other); } else { let (root_vec, other_vec) = pick2_mut(&mut self.groups, root_group, other_group); - self.undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); + undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); root_vec.extend_from_slice(other_vec); if self.reference_counts.get(other).map_or(false, |c| *c != 0) { @@ -102,37 +98,17 @@ impl UnifyLog { pub fn unwatch_variable(&mut self, index: T) { self.reference_counts[index] -= 1; } +} - pub fn snapshot(&mut self) -> Snapshot { - self.snapshots += 1; - Snapshot { undo_log_len: self.undo_log.len() as u32, _marker: PhantomData } - } - - pub fn commit(&mut self, _snapshot: Snapshot) { - self.snapshots -= 1; - if self.snapshots == 0 { - self.undo_log.clear(); - } - } - - pub fn rollback_to(&mut self, snapshot: Snapshot) { - self.snapshots -= 1; - for undo in self.undo_log.drain(snapshot.undo_log_len as usize..).rev() { - match undo { - Undo::Extend { group_index, len } => { - self.groups[group_index].truncate(len as usize) - } - Undo::Move { index, old } => self.unified_vars[index] = old, - Undo::NewGroup { index } => { - self.groups.pop(); - self.unified_vars[index] = usize::max_value(); - } +impl Rollback> for UnifyLog { + fn reverse(&mut self, undo: Undo) { + match undo { + Undo::Extend { group_index, len } => self.groups[group_index].truncate(len as usize), + Undo::Move { index, old } => self.unified_vars[index] = old, + Undo::NewGroup { index } => { + self.groups.pop(); + self.unified_vars[index] = usize::max_value(); } } } } - -pub struct Snapshot { - undo_log_len: u32, - _marker: PhantomData, -} diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 5342b13bcb74c..c4a201dd9cd25 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -12,7 +12,6 @@ use crate::traits::{self, ObligationCause, PredicateObligations, TraitEngine}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::logged_unification_table as lut; -use rustc_data_structures::modified_set as ms; use rustc_data_structures::sync::Lrc; use rustc_data_structures::undo_log::Rollback; use rustc_data_structures::unify as ut; From cd86ce146ae99798802b7b2a11abee91f12979fa Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 11 Apr 2020 17:29:34 +0200 Subject: [PATCH 09/25] Rebase on top of undo_log unification --- .../src/logged_unification_table.rs | 111 ++++++---- .../rustc_data_structures/src/modified_set.rs | 24 ++- compiler/rustc_infer/src/infer/combine.rs | 16 +- compiler/rustc_infer/src/infer/freshen.rs | 6 +- compiler/rustc_infer/src/infer/fudge.rs | 19 +- compiler/rustc_infer/src/infer/mod.rs | 201 +++++++++++++----- .../rustc_infer/src/infer/nll_relate/mod.rs | 10 +- compiler/rustc_infer/src/infer/resolve.rs | 4 +- .../rustc_infer/src/infer/type_variable.rs | 52 +++-- compiler/rustc_infer/src/infer/undo_log.rs | 23 +- compiler/rustc_middle/src/ich/impls_ty.rs | 2 +- compiler/rustc_middle/src/infer/unify_key.rs | 53 ++++- compiler/rustc_middle/src/ty/consts/kind.rs | 6 +- compiler/rustc_middle/src/ty/context.rs | 4 +- .../rustc_middle/src/ty/structural_impls.rs | 4 +- compiler/rustc_middle/src/ty/sty.rs | 16 +- compiler/rustc_trait_selection/src/infer.rs | 6 +- .../src/traits/fulfill.rs | 71 ++++--- 18 files changed, 421 insertions(+), 207 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index 8c1239aab779b..d525cc325c18a 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -9,10 +9,10 @@ use crate::unify_log as ul; use ena::undo_log::{Rollback, Snapshots, UndoLogs}; -enum UndoLog { +pub enum UndoLog { Relation(sv::UndoLog>), UnifyLog(ul::Undo), - ModifiedSet(ms::Undo), + ModifiedSet(ms::Undo), } impl From>> for UndoLog { @@ -27,8 +27,8 @@ impl From> for UndoLog { } } -impl From for UndoLog { - fn from(l: ms::Undo) -> Self { +impl From> for UndoLog { + fn from(l: ms::Undo) -> Self { UndoLog::ModifiedSet(l) } } @@ -56,6 +56,10 @@ where self.logs.push(undo.into()) } } + fn clear(&mut self) { + self.logs.clear(); + self.num_open_snapshots = 0; + } fn extend(&mut self, undos: J) where Self: Sized, @@ -67,13 +71,7 @@ where } } -struct RollbackView<'a, K: ut::UnifyKey, I: Idx> { - relations: &'a mut ut::UnificationStorage, - unify_log: &'a mut ul::UnifyLog, - modified_set: &'a mut ms::ModifiedSet, -} - -impl Rollback> for RollbackView<'_, K, I> { +impl Rollback> for LoggedUnificationTableStorage { fn reverse(&mut self, undo: UndoLog) { match undo { UndoLog::Relation(undo) => self.relations.reverse(undo), @@ -93,12 +91,18 @@ impl Snapshots> for Logs { unreachable!() } - fn rollback_to(&mut self, values: &mut impl Rollback>, snapshot: Self::Snapshot) { + fn rollback_to(&mut self, values: impl FnOnce() -> R, snapshot: Self::Snapshot) + where + R: Rollback>, + { debug!("rollback_to({})", snapshot.undo_len); self.assert_open_snapshot(&snapshot); - while self.logs.len() > snapshot.undo_len { - values.reverse(self.logs.pop().unwrap()); + if self.logs.len() > snapshot.undo_len { + let mut values = values(); + while self.logs.len() > snapshot.undo_len { + values.reverse(self.logs.pop().unwrap()); + } } if self.num_open_snapshots == 1 { @@ -135,14 +139,18 @@ impl Logs { } } -pub struct LoggedUnificationTable { - relations: ut::UnificationStorage, +pub struct LoggedUnificationTableStorage { + relations: ut::UnificationTableStorage, unify_log: ul::UnifyLog, modified_set: ms::ModifiedSet, - undo_log: Logs, } -impl LoggedUnificationTable +pub struct LoggedUnificationTable<'a, K: ut::UnifyKey, I: Idx, L> { + storage: &'a mut LoggedUnificationTableStorage, + undo_log: L, +} + +impl LoggedUnificationTableStorage where K: ut::UnifyKey + From, I: Idx + From, @@ -152,14 +160,34 @@ where relations: Default::default(), unify_log: ul::UnifyLog::new(), modified_set: ms::ModifiedSet::new(), - undo_log: Logs::default(), } } + pub fn with_log(&mut self, undo_log: L) -> LoggedUnificationTable<'_, K, I, L> { + LoggedUnificationTable { storage: self, undo_log } + } +} + +impl LoggedUnificationTable<'_, K, I, L> +where + K: ut::UnifyKey, + I: Idx, +{ + pub fn len(&self) -> usize { + self.storage.relations.len() + } +} + +impl LoggedUnificationTable<'_, K, I, L> +where + K: ut::UnifyKey + From, + I: Idx + From, + L: UndoLogs> + UndoLogs> + UndoLogs>>, +{ fn relations( &mut self, - ) -> ut::UnificationTable, &mut Logs>> { - ut::UnificationTable::with_log(&mut self.relations, &mut self.undo_log) + ) -> ut::UnificationTable, &mut L>> { + ut::UnificationTable::with_log(&mut self.storage.relations, &mut self.undo_log) } pub fn unify(&mut self, a: I, b: I) @@ -173,9 +201,9 @@ where where K::Value: ut::UnifyValue, { - if self.unify_log.needs_log(vid) { + if self.storage.unify_log.needs_log(vid) { warn!("ModifiedSet {:?} => {:?}", vid, ty); - self.modified_set.set(&mut self.undo_log, vid); + self.storage.modified_set.set(&mut self.undo_log, vid); } let vid = vid.into(); let mut relations = self.relations(); @@ -195,8 +223,8 @@ where value: K::Value, ) -> Result<(), ::Error> { let vid = self.find(vid).into(); - if self.unify_log.needs_log(vid) { - self.modified_set.set(&mut self.undo_log, vid); + if self.storage.unify_log.needs_log(vid) { + self.storage.modified_set.set(&mut self.undo_log, vid); } self.relations().unify_var_value(vid, value) } @@ -212,9 +240,9 @@ where relations.unify_var_var(a, b)?; if a == relations.find(a) { - self.unify_log.unify(&mut self.undo_log, a.into(), b.into()); + self.storage.unify_log.unify(&mut self.undo_log, a.into(), b.into()); } else { - self.unify_log.unify(&mut self.undo_log, b.into(), a.into()); + self.storage.unify_log.unify(&mut self.undo_log, b.into(), a.into()); } Ok(()) } @@ -240,14 +268,11 @@ where self.relations().new_key(value) } - pub fn len(&self) -> usize { - self.relations.len() - } - pub fn vars_since_snapshot(&mut self, s: &Snapshot) -> Range { K::from(I::new(s.value_count))..K::from(I::new(self.relations().len())) } + /* FIXME pub fn snapshot(&mut self) -> Snapshot { self.undo_log.num_open_snapshots += 1; Snapshot { @@ -258,44 +283,44 @@ where } pub fn rollback_to(&mut self, snapshot: Snapshot) { - let Self { relations, unify_log, modified_set, .. } = self; + let UnificationTableStorage { relations, unify_log, modified_set, .. } = self.storage; - self.undo_log - .rollback_to(&mut RollbackView { relations, unify_log, modified_set }, snapshot); + self.undo_log.rollback_to(|| RollbackView { relations, unify_log, modified_set }, snapshot); if self.undo_log.num_open_snapshots == 0 { - self.modified_set.clear(); + self.storage.modified_set.clear(); } } pub fn commit(&mut self, snapshot: Snapshot) { self.undo_log.commit(snapshot); - if self.undo_log.num_open_snapshots == 0 { - self.modified_set.clear(); + if self.undo_log.num_open_snapshots() == 0 { + self.storage.modified_set.clear(); } } + */ pub fn register(&mut self) -> ms::Offset { - self.modified_set.register() + self.storage.modified_set.register() } pub fn deregister(&mut self, offset: ms::Offset) { - self.modified_set.deregister(offset); + self.storage.modified_set.deregister(offset); } pub fn watch_variable(&mut self, index: I) { debug_assert!(index == self.relations().find(index).into()); - self.unify_log.watch_variable(index) + self.storage.unify_log.watch_variable(index) } pub fn unwatch_variable(&mut self, index: I) { - self.unify_log.unwatch_variable(index) + self.storage.unify_log.unwatch_variable(index) } pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I) -> bool) { - let unify_log = &self.unify_log; - self.modified_set.drain(&mut self.undo_log, offset, |vid| { + let unify_log = &self.storage.unify_log; + self.storage.modified_set.drain(&mut self.undo_log, offset, |vid| { for &unified_vid in unify_log.get(vid) { f(unified_vid); } diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index 6823d15f7f87b..5d1f636e09f83 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -5,11 +5,14 @@ use rustc_index::vec::Idx; use ena::undo_log::{Rollback, UndoLogs}; #[derive(Copy, Clone, Debug)] -pub enum Undo { +enum UndoInner { Add, Drain { index: usize, offset: usize }, } +#[derive(Copy, Clone, Debug)] +pub struct Undo(UndoInner, PhantomData); + #[derive(Clone, Debug)] pub struct ModifiedSet { modified: Vec, @@ -27,14 +30,14 @@ impl ModifiedSet { Self::default() } - pub fn set(&mut self, undo_log: &mut impl UndoLogs, index: T) { + pub fn set(&mut self, undo_log: &mut impl UndoLogs>, index: T) { self.modified.push(index); - undo_log.push(Undo::Add); + undo_log.push(Undo(UndoInner::Add, PhantomData)); } pub fn drain( &mut self, - undo_log: &mut impl UndoLogs, + undo_log: &mut impl UndoLogs>, index: &Offset, mut f: impl FnMut(T) -> bool, ) { @@ -43,7 +46,8 @@ impl ModifiedSet { for &index in &self.modified[*offset..] { f(index); } - undo_log.push(Undo::Drain { index: index.index, offset: *offset }); + undo_log + .push(Undo(UndoInner::Drain { index: index.index, offset: *offset }, PhantomData)); *offset = self.modified.len(); } } @@ -69,13 +73,13 @@ impl ModifiedSet { } } -impl Rollback for ModifiedSet { - fn reverse(&mut self, undo: Undo) { - match undo { - Undo::Add => { +impl Rollback> for ModifiedSet { + fn reverse(&mut self, undo: Undo) { + match undo.0 { + UndoInner::Add => { self.modified.pop(); } - Undo::Drain { index, offset } => { + UndoInner::Drain { index, offset } => { if let Some(o) = self.offsets.get_mut(index) { *o = offset; } diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs index aa740a8d27422..9277615f12dd2 100644 --- a/compiler/rustc_infer/src/infer/combine.rs +++ b/compiler/rustc_infer/src/infer/combine.rs @@ -226,9 +226,11 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> { fn unify_const_variable( &self, param_env: ty::ParamEnv<'tcx>, - target_vid: ty::ConstVid<'tcx>, + target_vid: ty::ConstVid, ct: &'tcx ty::Const<'tcx>, vid_is_expected: bool, + vid: ty::ConstVid, + value: &'tcx ty::Const<'tcx>, ) -> RelateResult<'tcx, &'tcx ty::Const<'tcx>> { let (for_universe, span) = { let mut inner = self.inner.borrow_mut(); @@ -722,10 +724,14 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> { if self.for_universe.can_name(universe) { Ok(c) } else { - let new_var_id = variable_table.new_key(ConstVarValue { - origin: var_value.origin, - val: ConstVariableValue::Unknown { universe: self.for_universe }, - }); + let new_var_id = variable_table + .new_key(ConstVarValue { + origin: var_value.origin, + val: ConstVariableValue::Unknown { + universe: self.for_universe, + }, + }) + .vid; Ok(self.tcx().mk_const_var(new_var_id, c.ty)) } } diff --git a/compiler/rustc_infer/src/infer/freshen.rs b/compiler/rustc_infer/src/infer/freshen.rs index b3d7876c6e819..4082e673edc82 100644 --- a/compiler/rustc_infer/src/infer/freshen.rs +++ b/compiler/rustc_infer/src/infer/freshen.rs @@ -46,7 +46,7 @@ pub struct TypeFreshener<'a, 'tcx> { ty_freshen_count: u32, const_freshen_count: u32, ty_freshen_map: FxHashMap>, - const_freshen_map: FxHashMap, &'tcx ty::Const<'tcx>>, + const_freshen_map: FxHashMap>, } impl<'a, 'tcx> TypeFreshener<'a, 'tcx> { @@ -88,12 +88,12 @@ impl<'a, 'tcx> TypeFreshener<'a, 'tcx> { fn freshen_const( &mut self, opt_ct: Option<&'tcx ty::Const<'tcx>>, - key: ty::InferConst<'tcx>, + key: ty::InferConst, freshener: F, ty: Ty<'tcx>, ) -> &'tcx ty::Const<'tcx> where - F: FnOnce(u32) -> ty::InferConst<'tcx>, + F: FnOnce(u32) -> ty::InferConst, { if let Some(ct) = opt_ct { return ct.fold_with(self); diff --git a/compiler/rustc_infer/src/infer/fudge.rs b/compiler/rustc_infer/src/infer/fudge.rs index d7bc636db8f8f..7aa2a67dc6649 100644 --- a/compiler/rustc_infer/src/infer/fudge.rs +++ b/compiler/rustc_infer/src/infer/fudge.rs @@ -1,9 +1,11 @@ +use rustc_index::vec::Idx; +use rustc_middle::infer::unify_key::ConstVidEqKey; use rustc_middle::ty::fold::{TypeFoldable, TypeFolder}; use rustc_middle::ty::{self, ConstVid, FloatVid, IntVid, RegionVid, Ty, TyCtxt, TyVid}; use super::type_variable::TypeVariableOrigin; use super::InferCtxt; -use super::{ConstVariableOrigin, RegionVariableOrigin, UnificationTable}; +use super::{ConstVariableOrigin, LoggedUnificationTable, RegionVariableOrigin}; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::unify as ut; @@ -11,25 +13,26 @@ use ut::UnifyKey; use std::ops::Range; -fn vars_since_snapshot<'tcx, T>( - table: &mut UnificationTable<'_, 'tcx, T>, +fn vars_since_snapshot<'tcx, T, K>( + table: &mut LoggedUnificationTable<'_, 'tcx, T, K>, snapshot_var_len: usize, ) -> Range where T: UnifyKey, + K: Idx, super::UndoLog<'tcx>: From>>, { T::from_index(snapshot_var_len as u32)..T::from_index(table.len() as u32) } fn const_vars_since_snapshot<'tcx>( - table: &mut UnificationTable<'_, 'tcx, ConstVid<'tcx>>, + table: &mut LoggedUnificationTable<'_, 'tcx, ConstVidEqKey<'tcx>, ConstVid>, snapshot_var_len: usize, -) -> (Range>, Vec) { +) -> (Range, Vec) { let range = vars_since_snapshot(table, snapshot_var_len); ( - range.start..range.end, - (range.start.index..range.end.index) + range.start.into()..range.end.into(), + (range.start.vid.index..range.end.vid.index) .map(|index| table.probe_value(ConstVid::from_index(index)).origin) .collect(), ) @@ -173,7 +176,7 @@ pub struct InferenceFudger<'a, 'tcx> { int_vars: Range, float_vars: Range, region_vars: (Range, Vec), - const_vars: (Range>, Vec), + const_vars: (Range, Vec), } impl<'a, 'tcx> TypeFolder<'tcx> for InferenceFudger<'a, 'tcx> { diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index c4a201dd9cd25..dd16d74f0c6a1 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -12,6 +12,7 @@ use crate::traits::{self, ObligationCause, PredicateObligations, TraitEngine}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::logged_unification_table as lut; +use rustc_data_structures::modified_set as ms; use rustc_data_structures::sync::Lrc; use rustc_data_structures::undo_log::Rollback; use rustc_data_structures::unify as ut; @@ -20,7 +21,9 @@ use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::infer::canonical::{Canonical, CanonicalVarValues}; use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue}; -use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind, ToType}; +use rustc_middle::infer::unify_key::{ + ConstVariableOrigin, ConstVariableOriginKind, ConstVidEqKey, ToType, +}; use rustc_middle::mir; use rustc_middle::mir::interpret::EvalToConstValueResult; use rustc_middle::traits::select; @@ -83,12 +86,15 @@ pub type InferResult<'tcx, T> = Result, TypeError<'tcx>>; pub type Bound = Option; pub type UnitResult<'tcx> = RelateResult<'tcx, ()>; // "unify result" -pub type FixupResult<'tcx, T> = Result>; // "fixup result" +pub type FixupResult = Result; // "fixup result" pub(crate) type UnificationTable<'a, 'tcx, T> = ut::UnificationTable< ut::InPlace, &'a mut InferCtxtUndoLogs<'tcx>>, >; +pub(crate) type LoggedUnificationTable<'a, 'tcx, T, K = T> = + lut::LoggedUnificationTable<'a, T, K, &'a mut InferCtxtUndoLogs<'tcx>>; + /// How we should handle region solving. /// /// This is used so that the region values inferred by HIR region solving are @@ -146,13 +152,14 @@ pub struct InferCtxtInner<'tcx> { type_variable_storage: type_variable::TypeVariableStorage<'tcx>, /// Map from const parameter variable to the kind of const it represents. - const_unification_storage: ut::UnificationTableStorage>, + const_unification_storage: + lut::LoggedUnificationTableStorage, ty::ConstVid>, /// Map from integral variable to the kind of integer it represents. - int_unification_table: lut::LoggedUnificationTable, + int_unification_storage: lut::LoggedUnificationTableStorage, /// Map from floating variable to the kind of float it represents. - float_unification_table: lut::LoggedUnificationTable, + float_unification_storage: lut::LoggedUnificationTableStorage, /// Tracks the set of region variables and the constraints between them. /// This is initially `Some(_)` but when @@ -203,10 +210,10 @@ impl<'tcx> InferCtxtInner<'tcx> { projection_cache: Default::default(), type_variable_storage: type_variable::TypeVariableStorage::new(), undo_log: InferCtxtUndoLogs::default(), - const_unification_table: ut::UnificationTableStorage::new(), - int_unification_table: lut::LoggedUnificationTable::new(), - float_unification_table: lut::LoggedUnificationTable::new(), - region_constraints: Some(RegionConstraintStorage::new()), + const_unification_storage: lut::LoggedUnificationTableStorage::new(), + int_unification_storage: lut::LoggedUnificationTableStorage::new(), + float_unification_storage: lut::LoggedUnificationTableStorage::new(), + region_constraint_storage: Some(RegionConstraintStorage::new()), region_obligations: vec![], } } @@ -227,41 +234,19 @@ impl<'tcx> InferCtxtInner<'tcx> { } #[inline] - fn int_unification_table( - &mut self, - ) -> ut::UnificationTable< - ut::InPlace< - ty::IntVid, - &mut ut::UnificationStorage, - &mut InferCtxtUndoLogs<'tcx>, - >, - > { + fn int_unification_table(&mut self) -> LoggedUnificationTable<'_, 'tcx, ty::IntVid> { self.int_unification_storage.with_log(&mut self.undo_log) } #[inline] - fn float_unification_table( - &mut self, - ) -> ut::UnificationTable< - ut::InPlace< - ty::FloatVid, - &mut ut::UnificationStorage, - &mut InferCtxtUndoLogs<'tcx>, - >, - > { + fn float_unification_table(&mut self) -> LoggedUnificationTable<'_, 'tcx, ty::FloatVid> { self.float_unification_storage.with_log(&mut self.undo_log) } #[inline] fn const_unification_table( &mut self, - ) -> ut::UnificationTable< - ut::InPlace< - ty::ConstVid<'tcx>, - &mut ut::UnificationStorage>, - &mut InferCtxtUndoLogs<'tcx>, - >, - > { + ) -> LoggedUnificationTable<'_, 'tcx, ConstVidEqKey<'tcx>, ty::ConstVid> { self.const_unification_storage.with_log(&mut self.undo_log) } @@ -495,11 +480,11 @@ pub enum NLLRegionVariableOrigin { // FIXME(eddyb) investigate overlap between this and `TyOrConstInferVar`. #[derive(Copy, Clone, Debug)] -pub enum FixupError<'tcx> { +pub enum FixupError { UnresolvedIntTy(IntVid), UnresolvedFloatTy(FloatVid), UnresolvedTy(TyVid), - UnresolvedConst(ConstVid<'tcx>), + UnresolvedConst(ConstVid), } /// See the `region_obligations` field for more information. @@ -510,7 +495,7 @@ pub struct RegionObligation<'tcx> { pub origin: SubregionOrigin<'tcx>, } -impl<'tcx> fmt::Display for FixupError<'tcx> { +impl fmt::Display for FixupError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use self::FixupError::*; @@ -1021,15 +1006,20 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { .inner .borrow_mut() .const_unification_table() - .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe } }); + .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe } }) + .vid; self.tcx.mk_const_var(vid, ty) } - pub fn next_const_var_id(&self, origin: ConstVariableOrigin) -> ConstVid<'tcx> { - self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue { - origin, - val: ConstVariableValue::Unknown { universe: self.universe() }, - }) + pub fn next_const_var_id(&self, origin: ConstVariableOrigin) -> ConstVid { + self.inner + .borrow_mut() + .const_unification_table() + .new_key(ConstVarValue { + origin, + val: ConstVariableValue::Unknown { universe: self.universe() }, + }) + .vid } fn next_int_var_id(&self) -> IntVid { @@ -1134,11 +1124,15 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { ), span, }; - let const_var_id = - self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue { + let const_var_id = self + .inner + .borrow_mut() + .const_unification_table() + .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe: self.universe() }, - }); + }) + .vid; self.tcx.mk_const_var(const_var_id, self.tcx.type_of(param.def_id)).into() } } @@ -1344,7 +1338,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { pub fn probe_const_var( &self, - vid: ty::ConstVid<'tcx>, + vid: ty::ConstVid, ) -> Result<&'tcx ty::Const<'tcx>, ty::UniverseIndex> { match self.inner.borrow_mut().const_unification_table().probe_value(vid).val { ConstVariableValue::Known { value } => Ok(value), @@ -1352,7 +1346,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { } } - pub fn fully_resolve>(&self, value: &T) -> FixupResult<'tcx, T> { + pub fn fully_resolve>(&self, value: &T) -> FixupResult { /*! * Attempts to resolve all type/region/const variables in * `value`. Region inference must have been run already (e.g., @@ -1572,7 +1566,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { /// inference variables), and it handles both `Ty` and `ty::Const` without /// having to resort to storing full `GenericArg`s in `stalled_on`. #[inline(always)] - pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) -> bool { + pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar) -> bool { match infer_var { TyOrConstInferVar::Ty(v) => { use self::type_variable::TypeVariableValue; @@ -1612,12 +1606,113 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { } } } + + pub fn drain_modifications( + &self, + offset: &WatcherOffset, + mut f: impl FnMut(TyOrConstInferVar) -> bool, + ) { + let mut inner = self.inner.borrow_mut(); + + inner + .type_variables() + .drain_modified_set(&offset.ty_offset, |v| f(TyOrConstInferVar::Ty(v))); + + inner + .int_unification_table() + .drain_modified_set(&offset.int_offset, |vid| f(TyOrConstInferVar::TyInt(vid))); + + inner + .float_unification_table() + .drain_modified_set(&offset.float_offset, |vid| f(TyOrConstInferVar::TyFloat(vid))); + + inner + .const_unification_table() + .drain_modified_set(&offset.const_offset, |vid| f(TyOrConstInferVar::Const(vid))); + } + + pub fn register_unify_watcher(&self) -> WatcherOffset { + let mut inner = self.inner.borrow_mut(); + WatcherOffset { + ty_offset: inner.type_variables().register_unify_watcher(), + + int_offset: inner.int_unification_table().register(), + + float_offset: inner.float_unification_table().register(), + + const_offset: inner.const_unification_table().register(), + } + } + + pub fn deregister_unify_watcher(&self, offset: WatcherOffset) { + let mut inner = self.inner.borrow_mut(); + + inner.type_variables().deregister_unify_watcher(offset.ty_offset); + + inner.int_unification_table().deregister(offset.int_offset); + + inner.float_unification_table().deregister(offset.float_offset); + + inner.const_unification_table().deregister(offset.const_offset); + } + + pub fn watch_variable(&self, infer: TyOrConstInferVar) { + let mut inner = self.inner.borrow_mut(); + match infer { + TyOrConstInferVar::Ty(v) => inner.type_variables().watch_variable(v), + + TyOrConstInferVar::TyInt(v) => inner.int_unification_table().watch_variable(v), + + TyOrConstInferVar::TyFloat(v) => inner.float_unification_table().watch_variable(v), + + TyOrConstInferVar::Const(v) => inner.const_unification_table().watch_variable(v), + } + } + + pub fn unwatch_variable(&self, infer: TyOrConstInferVar) { + let mut inner = self.inner.borrow_mut(); + match infer { + TyOrConstInferVar::Ty(v) => inner.type_variables().unwatch_variable(v), + + TyOrConstInferVar::TyInt(v) => inner.int_unification_table().unwatch_variable(v), + + TyOrConstInferVar::TyFloat(v) => inner.float_unification_table().unwatch_variable(v), + + TyOrConstInferVar::Const(v) => inner.const_unification_table().unwatch_variable(v), + } + } + + pub fn root_ty_or_const(&self, infer: TyOrConstInferVar) -> TyOrConstInferVar { + let mut inner = self.inner.borrow_mut(); + match infer { + TyOrConstInferVar::Ty(v) => TyOrConstInferVar::Ty(inner.type_variables().root_var(v)), + + TyOrConstInferVar::TyInt(v) => { + TyOrConstInferVar::TyInt(inner.int_unification_table().find(v)) + } + + TyOrConstInferVar::TyFloat(v) => { + TyOrConstInferVar::TyFloat(inner.float_unification_table().find(v)) + } + + TyOrConstInferVar::Const(v) => { + TyOrConstInferVar::Const(inner.const_unification_table().find(v).vid) + } + } + } +} + +pub struct WatcherOffset { + ty_offset: ms::Offset, + int_offset: ms::Offset, + float_offset: ms::Offset, + const_offset: ms::Offset, } /// Helper for `ty_or_const_infer_var_changed` (see comment on that), currently /// used only for `traits::fulfill`'s list of `stalled_on` inference variables. -#[derive(Copy, Clone, Debug)] -pub enum TyOrConstInferVar<'tcx> { +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum TyOrConstInferVar { /// Equivalent to `ty::Infer(ty::TyVar(_))`. Ty(TyVid), /// Equivalent to `ty::Infer(ty::IntVar(_))`. @@ -1626,10 +1721,10 @@ pub enum TyOrConstInferVar<'tcx> { TyFloat(FloatVid), /// Equivalent to `ty::ConstKind::Infer(ty::InferConst::Var(_))`. - Const(ConstVid<'tcx>), + Const(ConstVid), } -impl TyOrConstInferVar<'tcx> { +impl TyOrConstInferVar { /// Tries to extract an inference variable from a type or a constant, returns `None` /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs index abdd6edea9024..9af9fb286ae8c 100644 --- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs +++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs @@ -978,10 +978,12 @@ where match var_value.val.known() { Some(u) => self.relate(u, u), None => { - let new_var_id = variable_table.new_key(ConstVarValue { - origin: var_value.origin, - val: ConstVariableValue::Unknown { universe: self.universe }, - }); + let new_var_id = variable_table + .new_key(ConstVarValue { + origin: var_value.origin, + val: ConstVariableValue::Unknown { universe: self.universe }, + }) + .vid; Ok(self.tcx().mk_const_var(new_var_id, a.ty)) } } diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs index 337772d70b823..fff2d11dd4db3 100644 --- a/compiler/rustc_infer/src/infer/resolve.rs +++ b/compiler/rustc_infer/src/infer/resolve.rs @@ -162,7 +162,7 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeFinder<'a, 'tcx> { /// Full type resolution replaces all type and region variables with /// their concrete results. If any variable cannot be replaced (never unified, etc) /// then an `Err` result is returned. -pub fn fully_resolve<'a, 'tcx, T>(infcx: &InferCtxt<'a, 'tcx>, value: &T) -> FixupResult<'tcx, T> +pub fn fully_resolve<'a, 'tcx, T>(infcx: &InferCtxt<'a, 'tcx>, value: &T) -> FixupResult where T: TypeFoldable<'tcx>, { @@ -178,7 +178,7 @@ where // `err` field is not enforceable otherwise. struct FullTypeResolver<'a, 'tcx> { infcx: &'a InferCtxt<'a, 'tcx>, - err: Option>, + err: Option, } impl<'a, 'tcx> TypeFolder<'tcx> for FullTypeResolver<'a, 'tcx> { diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index cb2145ecbc58c..4f1fb5e5d4067 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -9,6 +9,7 @@ use rustc_data_structures::logged_unification_table as lut; use rustc_data_structures::modified_set as ms; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::unify as ut; +use rustc_data_structures::unify_log as ul; use std::cmp; use std::marker::PhantomData; use std::ops::Range; @@ -17,14 +18,14 @@ use rustc_data_structures::undo_log::{Rollback, UndoLogs}; /// Represents a single undo-able action that affects a type inference variable. pub(crate) enum UndoLog<'tcx> { - EqRelation(sv::UndoLog>>), + EqRelation(lut::UndoLog, ty::TyVid>), SubRelation(sv::UndoLog>), Values(sv::UndoLog), } /// Convert from a specific kind of undo to the more general UndoLog -impl<'tcx> From>>> for UndoLog<'tcx> { - fn from(l: sv::UndoLog>>) -> Self { +impl<'tcx> From, ty::TyVid>> for UndoLog<'tcx> { + fn from(l: lut::UndoLog, ty::TyVid>) -> Self { UndoLog::EqRelation(l) } } @@ -50,6 +51,24 @@ impl<'tcx> From for UndoLog<'tcx> { } } +impl<'tcx> From>>> for UndoLog<'tcx> { + fn from(l: sv::UndoLog>>) -> Self { + UndoLog::EqRelation(l.into()) + } +} + +impl From> for UndoLog<'_> { + fn from(l: ul::Undo) -> Self { + UndoLog::EqRelation(l.into()) + } +} + +impl From> for UndoLog<'_> { + fn from(l: ms::Undo) -> Self { + UndoLog::EqRelation(l.into()) + } +} + impl<'tcx> Rollback> for TypeVariableStorage<'tcx> { fn reverse(&mut self, undo: UndoLog<'tcx>) { match undo { @@ -66,7 +85,7 @@ pub struct TypeVariableStorage<'tcx> { /// Two variables are unified in `eq_relations` when we have a /// constraint `?X == ?Y`. This table also stores, for each key, /// the known value. - eq_relations: lut::LoggedUnificationTable, TyVid>, + eq_relations: lut::LoggedUnificationTableStorage, TyVid>, /// Two variables are unified in `sub_relations` when we have a /// constraint `?X <: ?Y` *or* a constraint `?Y <: ?X`. This second @@ -153,12 +172,13 @@ pub(crate) struct Instantiate { } pub(crate) struct Delegate; +pub(crate) struct UnifiedVarsDelegate; impl<'tcx> TypeVariableStorage<'tcx> { pub fn new() -> TypeVariableStorage<'tcx> { TypeVariableStorage { values: sv::SnapshotVecStorage::new(), - eq_relations: lut::LoggedUnificationTable::new(), + eq_relations: lut::LoggedUnificationTableStorage::new(), sub_relations: ut::UnificationTableStorage::new(), } } @@ -195,7 +215,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { pub fn equate(&mut self, a: ty::TyVid, b: ty::TyVid) { debug_assert!(self.probe(a).is_unknown()); debug_assert!(self.probe(b).is_unknown()); - self.eq_relations().union(a, b); + self.eq_relations().unify(a, b); self.sub_relations().union(a, b); } @@ -326,7 +346,9 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { } #[inline] - fn eq_relations(&mut self) -> super::UnificationTable<'_, 'tcx, TyVidEqKey<'tcx>> { + fn eq_relations( + &mut self, + ) -> super::LoggedUnificationTable<'_, 'tcx, TyVidEqKey<'tcx>, ty::TyVid> { self.storage.eq_relations.with_log(self.undo_log) } @@ -379,10 +401,8 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { if vid.index < new_elem_threshold { // quick check to see if this variable was // created since the snapshot started or not. - let mut eq_relations = ut::UnificationTable::with_log( - &mut self.storage.eq_relations, - &mut *self.undo_log, - ); + let mut eq_relations = + self.storage.eq_relations.with_log(&mut *self.undo_log); let escaping_type = match eq_relations.probe_value(vid) { TypeVariableValue::Unknown { .. } => bug!(), TypeVariableValue::Known { value } => value, @@ -418,23 +438,23 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { offset: &ms::Offset, f: impl FnMut(ty::TyVid) -> bool, ) { - self.eq_relations.drain_modified_set(offset, f) + self.eq_relations().drain_modified_set(offset, f) } pub fn register_unify_watcher(&mut self) -> ms::Offset { - self.eq_relations.register() + self.eq_relations().register() } pub fn deregister_unify_watcher(&mut self, offset: ms::Offset) { - self.eq_relations.deregister(offset); + self.eq_relations().deregister(offset); } pub fn watch_variable(&mut self, vid: ty::TyVid) { - self.eq_relations.watch_variable(vid); + self.eq_relations().watch_variable(vid); } pub fn unwatch_variable(&mut self, vid: ty::TyVid) { - self.eq_relations.unwatch_variable(vid); + self.eq_relations().unwatch_variable(vid); } } diff --git a/compiler/rustc_infer/src/infer/undo_log.rs b/compiler/rustc_infer/src/infer/undo_log.rs index 2cfd6bb904c41..b344816694746 100644 --- a/compiler/rustc_infer/src/infer/undo_log.rs +++ b/compiler/rustc_infer/src/infer/undo_log.rs @@ -1,8 +1,12 @@ use std::marker::PhantomData; +use rustc_data_structures::logged_unification_table as lut; +use rustc_data_structures::modified_set as ms; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::undo_log::{Rollback, UndoLogs}; use rustc_data_structures::unify as ut; +use rustc_data_structures::unify_log as ul; +use rustc_middle::infer::unify_key; use rustc_middle::ty; use crate::{ @@ -18,9 +22,9 @@ pub struct Snapshot<'tcx> { /// Records the 'undo' data fora single operation that affects some form of inference variable. pub(crate) enum UndoLog<'tcx> { TypeVariables(type_variable::UndoLog<'tcx>), - ConstUnificationTable(sv::UndoLog>>), - IntUnificationTable(sv::UndoLog>), - FloatUnificationTable(sv::UndoLog>), + ConstUnificationTable(lut::UndoLog, ty::ConstVid>), + IntUnificationTable(lut::UndoLog), + FloatUnificationTable(lut::UndoLog), RegionConstraintCollector(region_constraints::UndoLog<'tcx>), RegionUnificationTable(sv::UndoLog>), ProjectionCache(traits::UndoLog<'tcx>), @@ -40,26 +44,35 @@ macro_rules! impl_from { } // Upcast from a single kind of "undoable action" to the general enum + impl_from! { RegionConstraintCollector(region_constraints::UndoLog<'tcx>), TypeVariables(type_variable::UndoLog<'tcx>), + TypeVariables(lut::UndoLog, ty::TyVid>), TypeVariables(sv::UndoLog>>), TypeVariables(sv::UndoLog>), TypeVariables(sv::UndoLog), TypeVariables(type_variable::Instantiate), + TypeVariables(ms::Undo), + TypeVariables(ul::Undo), IntUnificationTable(sv::UndoLog>), + IntUnificationTable(ms::Undo), + IntUnificationTable(ul::Undo), FloatUnificationTable(sv::UndoLog>), + FloatUnificationTable(ms::Undo), + FloatUnificationTable(ul::Undo), - ConstUnificationTable(sv::UndoLog>>), + ConstUnificationTable(sv::UndoLog>>), + ConstUnificationTable(ms::Undo), + ConstUnificationTable(ul::Undo), RegionUnificationTable(sv::UndoLog>), ProjectionCache(traits::UndoLog<'tcx>), } -/// The Rollback trait defines how to rollback a particular action. impl<'tcx> Rollback> for InferCtxtInner<'tcx> { fn reverse(&mut self, undo: UndoLog<'tcx>) { match undo { diff --git a/compiler/rustc_middle/src/ich/impls_ty.rs b/compiler/rustc_middle/src/ich/impls_ty.rs index 8f15c99f951fe..bebb394939466 100644 --- a/compiler/rustc_middle/src/ich/impls_ty.rs +++ b/compiler/rustc_middle/src/ich/impls_ty.rs @@ -104,7 +104,7 @@ impl<'a> HashStable> for ty::RegionVid { } } -impl<'a, 'tcx> HashStable> for ty::ConstVid<'tcx> { +impl<'a> HashStable> for ty::ConstVid { #[inline] fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { self.index.hash_stable(hcx, hasher); diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs index 16e9aafb25a54..416677745c1bd 100644 --- a/compiler/rustc_middle/src/infer/unify_key.rs +++ b/compiler/rustc_middle/src/infer/unify_key.rs @@ -1,9 +1,10 @@ use crate::ty::{self, FloatVarValue, InferConst, IntVarValue, Ty, TyCtxt}; +use rustc_data_structures::logged_unification_table::LoggedUnificationTable; +use rustc_data_structures::modified_set as ms; use rustc_data_structures::snapshot_vec; use rustc_data_structures::undo_log::UndoLogs; -use rustc_data_structures::unify::{ - self, EqUnifyValue, InPlace, NoError, UnificationTable, UnifyKey, UnifyValue, -}; +use rustc_data_structures::unify::{self, EqUnifyValue, NoError, UnifyKey, UnifyValue}; +use rustc_data_structures::unify_log as ul; use rustc_span::def_id::DefId; use rustc_span::symbol::Symbol; use rustc_span::Span; @@ -159,13 +160,44 @@ pub struct ConstVarValue<'tcx> { pub val: ConstVariableValue<'tcx>, } -impl<'tcx> UnifyKey for ty::ConstVid<'tcx> { - type Value = ConstVarValue<'tcx>; +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)] +pub struct ConstVidEqKey<'tcx> { + pub vid: ty::ConstVid, + pub phantom: PhantomData<&'tcx ()>, +} + +impl From for ConstVidEqKey<'_> { + fn from(vid: ty::ConstVid) -> Self { + Self { vid, phantom: PhantomData } + } +} + +impl<'tcx> From> for ty::ConstVid { + fn from(vid: ConstVidEqKey<'tcx>) -> Self { + vid.vid + } +} + +impl UnifyKey for ty::ConstVid { + type Value = (); fn index(&self) -> u32 { self.index } fn from_index(i: u32) -> Self { - ty::ConstVid { index: i, phantom: PhantomData } + ty::ConstVid { index: i } + } + fn tag() -> &'static str { + "ConstVid" + } +} + +impl<'tcx> UnifyKey for ConstVidEqKey<'tcx> { + type Value = ConstVarValue<'tcx>; + fn index(&self) -> u32 { + self.vid.index + } + fn from_index(i: u32) -> Self { + ConstVidEqKey { vid: ty::ConstVid { index: i }, phantom: PhantomData } } fn tag() -> &'static str { "ConstVid" @@ -207,13 +239,14 @@ impl<'tcx> UnifyValue for ConstVarValue<'tcx> { impl<'tcx> EqUnifyValue for &'tcx ty::Const<'tcx> {} -pub fn replace_if_possible( - table: &mut UnificationTable, V, L>>, +pub fn replace_if_possible( + table: &mut LoggedUnificationTable<'_, ConstVidEqKey<'tcx>, ty::ConstVid, L>, c: &'tcx ty::Const<'tcx>, ) -> &'tcx ty::Const<'tcx> where - V: snapshot_vec::VecLike>>, - L: UndoLogs>>>, + L: UndoLogs>>> + + UndoLogs> + + UndoLogs>, { if let ty::Const { val: ty::ConstKind::Infer(InferConst::Var(vid)), .. } = c { match table.probe_value(*vid).val.known() { diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs index ede28522000af..a9253b568708e 100644 --- a/compiler/rustc_middle/src/ty/consts/kind.rs +++ b/compiler/rustc_middle/src/ty/consts/kind.rs @@ -17,7 +17,7 @@ pub enum ConstKind<'tcx> { Param(ty::ParamConst), /// Infer the value of the const. - Infer(InferConst<'tcx>), + Infer(InferConst), /// Bound const variable, used only when preparing a trait query. Bound(ty::DebruijnIndex, ty::BoundVar), @@ -70,9 +70,9 @@ impl<'tcx> ConstKind<'tcx> { /// An inference variable for a const, for use in const generics. #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)] #[derive(HashStable)] -pub enum InferConst<'tcx> { +pub enum InferConst { /// Infer the value of the const. - Var(ty::ConstVid<'tcx>), + Var(ty::ConstVid), /// A fresh const variable. See `infer::freshen` for more details. Fresh(u32), } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f6ea6743a0e04..44ce8ff77216c 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -2318,7 +2318,7 @@ impl<'tcx> TyCtxt<'tcx> { } #[inline] - pub fn mk_const_var(self, v: ConstVid<'tcx>, ty: Ty<'tcx>) -> &'tcx Const<'tcx> { + pub fn mk_const_var(self, v: ConstVid, ty: Ty<'tcx>) -> &'tcx Const<'tcx> { self.mk_const(ty::Const { val: ty::ConstKind::Infer(InferConst::Var(v)), ty }) } @@ -2338,7 +2338,7 @@ impl<'tcx> TyCtxt<'tcx> { } #[inline] - pub fn mk_const_infer(self, ic: InferConst<'tcx>, ty: Ty<'tcx>) -> &'tcx ty::Const<'tcx> { + pub fn mk_const_infer(self, ic: InferConst, ty: Ty<'tcx>) -> &'tcx ty::Const<'tcx> { self.mk_const(ty::Const { val: ty::ConstKind::Infer(ic), ty }) } diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 53521d0e9f332..c81533fbfa5b0 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -133,7 +133,7 @@ impl fmt::Debug for ty::TyVid { } } -impl<'tcx> fmt::Debug for ty::ConstVid<'tcx> { +impl fmt::Debug for ty::ConstVid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_#{}c", self.index) } @@ -1112,7 +1112,7 @@ impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> { } } -impl<'tcx> TypeFoldable<'tcx> for InferConst<'tcx> { +impl<'tcx> TypeFoldable<'tcx> for InferConst { fn super_fold_with>(&self, _folder: &mut F) -> Self { *self } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index cdad15401bd9e..802e5d137ae91 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -23,7 +23,6 @@ use rustc_target::abi::VariantIdx; use rustc_target::spec::abi; use std::borrow::Cow; use std::cmp::Ordering; -use std::marker::PhantomData; use std::ops::Range; use ty::util::IntTypeExt; @@ -1480,9 +1479,20 @@ impl Idx for TyVid { } #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] -pub struct ConstVid<'tcx> { +pub struct ConstVid { pub index: u32, - pub phantom: PhantomData<&'tcx ()>, +} + +impl Idx for ConstVid { + #[inline] + fn new(idx: usize) -> Self { + assert!(idx <= u32::max_value() as usize); + Self { index: idx as u32 } + } + #[inline] + fn index(self) -> usize { + self.index as usize + } } #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs index 4ec1b29bca4f1..6252e026d1ca7 100644 --- a/compiler/rustc_trait_selection/src/infer.rs +++ b/compiler/rustc_trait_selection/src/infer.rs @@ -126,11 +126,13 @@ impl<'tcx> InferCtxtBuilderExt<'tcx> for InferCtxtBuilder<'tcx> { |ref infcx, key, canonical_inference_vars| { let mut fulfill_cx = TraitEngine::new(infcx.tcx); let value = operation(infcx, &mut *fulfill_cx, key)?; - infcx.make_canonicalized_query_response( + let x = infcx.make_canonicalized_query_response( canonical_inference_vars, value, &mut *fulfill_cx, - ) + ); + fulfill_cx.deregister(infcx); + x }, ) } diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 121e3bdc85eec..bbf75feb48e06 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -1,4 +1,5 @@ -use crate::infer::{InferCtxt, TyOrConstInferVar}; +use crate::infer::{InferCtxt, TyOrConstInferVar, WatcherOffset}; +use rustc_data_structures::captures::Captures; use rustc_data_structures::obligation_forest::ProcessResult; use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome}; use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor}; @@ -30,7 +31,7 @@ impl<'tcx> ForestObligation for PendingPredicateObligation<'tcx> { /// as the `ParamEnv` can influence whether fulfillment succeeds /// or fails. type CacheKey = ty::ParamEnvAnd<'tcx, ty::Predicate<'tcx>>; - type Variable = ty::InferTy; + type Variable = TyOrConstInferVar; type WatcherOffset = WatcherOffset; fn as_cache_key(&self) -> Self::CacheKey { @@ -88,7 +89,7 @@ pub struct PendingPredicateObligation<'tcx> { // should mostly optimize for reading speed, while modifying is not as relevant. // // For whatever reason using a boxed slice is slower than using a `Vec` here. - pub stalled_on: Vec>, + pub stalled_on: Vec, } // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -259,9 +260,15 @@ struct FulfillProcessor<'a, 'b, 'tcx> { register_region_obligations: bool, } -fn mk_pending(os: Vec>) -> Vec> { +fn mk_pending( + infcx: &InferCtxt<'_, 'tcx>, + os: Vec>, +) -> Vec> { os.into_iter() - .map(|o| PendingPredicateObligation { obligation: o, stalled_on: vec![] }) + .map(|mut o| { + o.predicate = infcx.resolve_vars_if_possible(&o.predicate); + PendingPredicateObligation { obligation: o, stalled_on: vec![] } + }) .collect() } @@ -293,6 +300,8 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { debug!(?obligation, ?obligation.cause, "process_obligation"); let infcx = self.selcx.infcx(); + let ty_or_const_var = + |v| infcx.root_ty_or_const(TyOrConstInferVar::maybe_from_ty(v).unwrap()); match obligation.predicate.kind() { ty::PredicateKind::ForAll(binder) => match binder.skip_binder() { @@ -402,38 +411,30 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { obligation.cause.span, ) { None => { - pending_obligation.stalled_on = - vec![TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap()]; + pending_obligation.stalled_on.clear(); + pending_obligation.stalled_on.push(TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap()); ProcessResult::Unchanged } Some(os) => ProcessResult::Changed(mk_pending(os)), } } - ty::PredicateAtom::Subtype(subtype) => { - match self.selcx.infcx().subtype_predicate( - &obligation.cause, - obligation.param_env, - Binder::dummy(subtype), - ) { - None => { - // None means that both are unresolved. - pending_obligation.stalled_on = vec![ - TyOrConstInferVar::maybe_from_ty(subtype.a).unwrap(), - TyOrConstInferVar::maybe_from_ty(subtype.b).unwrap(), - ]; - ProcessResult::Unchanged - } - Some(Ok(ok)) => ProcessResult::Changed(mk_pending(ok.obligations)), - Some(Err(err)) => { - let expected_found = - ExpectedFound::new(subtype.a_is_expected, subtype.a, subtype.b); - ProcessResult::Error(FulfillmentErrorCode::CodeSubtypeError( - expected_found, - err, - )) - } + &ty::PredicateKind::WellFormed(arg) => { + match wf::obligations( + self.selcx.infcx(), + obligation.param_env, + obligation.cause.body_id, + arg, + obligation.cause.span, + ) { + None => { + pending_obligation.stalled_on.clear(); + pending_obligation + .stalled_on + .push(TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap()); + ProcessResult::Unchanged } + Some(os) => ProcessResult::Changed(mk_pending(infcx, os)), } ty::PredicateAtom::ConstEvaluatable(def_id, substs) => { @@ -670,11 +671,11 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { fn trait_ref_infer_vars<'a, 'tcx, 'b>( selcx: &'b mut SelectionContext<'a, 'tcx>, trait_ref: ty::PolyTraitRef<'tcx>, -) -> impl Iterator> + 'b + Captures<'a> + Captures<'tcx> { - selcx - .infcx() +) -> impl Iterator + 'b + Captures<'a> + Captures<'tcx> { + let infcx = selcx.infcx(); + infcx .resolve_vars_if_possible(&trait_ref) - .skip_binder() + .skip_binder() // ok b/c this check doesn't care about regions .substs .iter() // FIXME(eddyb) try using `skip_current_subtree` to skip everything that @@ -682,7 +683,7 @@ fn trait_ref_infer_vars<'a, 'tcx, 'b>( .filter(|arg| arg.has_infer_types_or_consts()) .flat_map(|arg| arg.walk()) .filter_map(TyOrConstInferVar::maybe_from_generic_arg) - .collect() + .map(move |var| infcx.root_ty_or_const(var)) } fn to_fulfillment_error<'tcx>( From 0f08a69749eacb98f19c43f990a2cceb7a605fb6 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 11 Apr 2020 17:47:44 +0200 Subject: [PATCH 10/25] Clear the modified set once no snapshots exist --- .../src/logged_unification_table.rs | 139 +----------------- compiler/rustc_infer/src/infer/mod.rs | 9 +- .../rustc_infer/src/infer/type_variable.rs | 4 + 3 files changed, 15 insertions(+), 137 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index d525cc325c18a..5b37a7f927ab4 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -1,5 +1,3 @@ -use std::{marker::PhantomData, ops::Range}; - use rustc_index::vec::Idx; use crate::modified_set as ms; @@ -7,7 +5,7 @@ use crate::snapshot_vec as sv; use crate::unify as ut; use crate::unify_log as ul; -use ena::undo_log::{Rollback, Snapshots, UndoLogs}; +use ena::undo_log::{Rollback, UndoLogs}; pub enum UndoLog { Relation(sv::UndoLog>), @@ -33,44 +31,6 @@ impl From> for UndoLog { } } -struct Logs { - logs: Vec>, - num_open_snapshots: usize, -} - -impl Default for Logs { - fn default() -> Self { - Self { logs: Default::default(), num_open_snapshots: Default::default() } - } -} - -impl UndoLogs for Logs -where - UndoLog: From, -{ - fn num_open_snapshots(&self) -> usize { - self.num_open_snapshots - } - fn push(&mut self, undo: T) { - if self.in_snapshot() { - self.logs.push(undo.into()) - } - } - fn clear(&mut self) { - self.logs.clear(); - self.num_open_snapshots = 0; - } - fn extend(&mut self, undos: J) - where - Self: Sized, - J: IntoIterator, - { - if self.in_snapshot() { - self.logs.extend(undos.into_iter().map(UndoLog::from)) - } - } -} - impl Rollback> for LoggedUnificationTableStorage { fn reverse(&mut self, undo: UndoLog) { match undo { @@ -81,64 +41,6 @@ impl Rollback> for LoggedUnificationTable } } -impl Snapshots> for Logs { - type Snapshot = Snapshot; - fn actions_since_snapshot(&self, snapshot: &Self::Snapshot) -> &[UndoLog] { - &self.logs[snapshot.undo_len..] - } - - fn start_snapshot(&mut self) -> Self::Snapshot { - unreachable!() - } - - fn rollback_to(&mut self, values: impl FnOnce() -> R, snapshot: Self::Snapshot) - where - R: Rollback>, - { - debug!("rollback_to({})", snapshot.undo_len); - self.assert_open_snapshot(&snapshot); - - if self.logs.len() > snapshot.undo_len { - let mut values = values(); - while self.logs.len() > snapshot.undo_len { - values.reverse(self.logs.pop().unwrap()); - } - } - - if self.num_open_snapshots == 1 { - // The root snapshot. It's safe to clear the undo log because - // there's no snapshot further out that we might need to roll back - // to. - assert!(snapshot.undo_len == 0); - self.logs.clear(); - } - - self.num_open_snapshots -= 1; - } - - fn commit(&mut self, snapshot: Self::Snapshot) { - debug!("commit({})", snapshot.undo_len); - - if self.num_open_snapshots == 1 { - // The root snapshot. It's safe to clear the undo log because - // there's no snapshot further out that we might need to roll back - // to. - assert!(snapshot.undo_len == 0); - self.logs.clear(); - } - - self.num_open_snapshots -= 1; - } -} - -impl Logs { - fn assert_open_snapshot(&self, snapshot: &Snapshot) { - // Failures here may indicate a failure to follow a stack discipline. - assert!(self.logs.len() >= snapshot.undo_len); - assert!(self.num_open_snapshots > 0); - } -} - pub struct LoggedUnificationTableStorage { relations: ut::UnificationTableStorage, unify_log: ul::UnifyLog, @@ -268,39 +170,10 @@ where self.relations().new_key(value) } - pub fn vars_since_snapshot(&mut self, s: &Snapshot) -> Range { - K::from(I::new(s.value_count))..K::from(I::new(self.relations().len())) + pub fn clear_modified_set(&mut self) { + self.storage.modified_set.clear(); } - /* FIXME - pub fn snapshot(&mut self) -> Snapshot { - self.undo_log.num_open_snapshots += 1; - Snapshot { - undo_len: self.undo_log.logs.len(), - value_count: self.relations().len(), - _marker: PhantomData, - } - } - - pub fn rollback_to(&mut self, snapshot: Snapshot) { - let UnificationTableStorage { relations, unify_log, modified_set, .. } = self.storage; - - self.undo_log.rollback_to(|| RollbackView { relations, unify_log, modified_set }, snapshot); - - if self.undo_log.num_open_snapshots == 0 { - self.storage.modified_set.clear(); - } - } - - pub fn commit(&mut self, snapshot: Snapshot) { - self.undo_log.commit(snapshot); - - if self.undo_log.num_open_snapshots() == 0 { - self.storage.modified_set.clear(); - } - } - */ - pub fn register(&mut self) -> ms::Offset { self.storage.modified_set.register() } @@ -329,9 +202,3 @@ where }) } } - -pub struct Snapshot { - undo_len: usize, - value_count: usize, - _marker: PhantomData<(K, I)>, -} diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index dd16d74f0c6a1..ac241ec0aad11 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -14,7 +14,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::logged_unification_table as lut; use rustc_data_structures::modified_set as ms; use rustc_data_structures::sync::Lrc; -use rustc_data_structures::undo_log::Rollback; +use rustc_data_structures::undo_log::{Rollback, UndoLogs}; use rustc_data_structures::unify as ut; use rustc_errors::DiagnosticBuilder; use rustc_hir as hir; @@ -763,6 +763,13 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { let mut inner = self.inner.borrow_mut(); inner.rollback_to(undo_snapshot); inner.unwrap_region_constraints().rollback_to(region_constraints_snapshot); + + if UndoLogs::>::num_open_snapshots(&inner.undo_log) == 0 { + inner.type_variables().clear_modified_set(); + inner.int_unification_table().clear_modified_set(); + inner.float_unification_table().clear_modified_set(); + inner.const_unification_table().clear_modified_set(); + } } fn commit_from(&self, snapshot: CombinedSnapshot<'a, 'tcx>) { diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 4f1fb5e5d4067..36d95f549328c 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -433,6 +433,10 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { .collect() } + pub fn clear_modified_set(&mut self) { + self.eq_relations().clear_modified_set(); + } + pub fn drain_modified_set( &mut self, offset: &ms::Offset, From f5dbb04753e184347ecff771616bfbb3e2b7e7b6 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 4 May 2020 16:59:29 +0200 Subject: [PATCH 11/25] Document the additions --- .../src/logged_unification_table.rs | 18 +++- .../rustc_data_structures/src/modified_set.rs | 2 +- .../src/obligation_forest/mod.rs | 90 ++++++++++++------- .../src/obligation_forest/tests.rs | 2 +- compiler/rustc_infer/src/infer/mod.rs | 14 +-- .../rustc_infer/src/infer/type_variable.rs | 10 +-- .../src/traits/fulfill.rs | 2 +- 7 files changed, 84 insertions(+), 54 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index 5b37a7f927ab4..2a63b401f2745 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -41,12 +41,15 @@ impl Rollback> for LoggedUnificationTable } } +/// Storage for `LoggedUnificationTable` pub struct LoggedUnificationTableStorage { relations: ut::UnificationTableStorage, unify_log: ul::UnifyLog, modified_set: ms::ModifiedSet, } +/// UnificationTableStorage which logs which variables has been unfified with a value, allowing watchers +/// to only iterate over the changed variables instead of all variables pub struct LoggedUnificationTable<'a, K: ut::UnifyKey, I: Idx, L> { storage: &'a mut LoggedUnificationTableStorage, undo_log: L, @@ -170,28 +173,37 @@ where self.relations().new_key(value) } + /// Clears any modifications currently tracked. Usually this can only be done once there are no + /// snapshots active as the modifications may otherwise be needed after a rollback pub fn clear_modified_set(&mut self) { self.storage.modified_set.clear(); } - pub fn register(&mut self) -> ms::Offset { + /// Registers a watcher on the unifications done in this table + pub fn register_watcher(&mut self) -> ms::Offset { self.storage.modified_set.register() } - pub fn deregister(&mut self, offset: ms::Offset) { + /// Deregisters a watcher previously registered in this table + pub fn deregister_watcher(&mut self, offset: ms::Offset) { self.storage.modified_set.deregister(offset); } + /// Watches the variable at `index` allowing any watchers to be notified to unifications with + /// `index` pub fn watch_variable(&mut self, index: I) { debug_assert!(index == self.relations().find(index).into()); self.storage.unify_log.watch_variable(index) } + /// Unwatches a previous watch at `index` pub fn unwatch_variable(&mut self, index: I) { self.storage.unify_log.unwatch_variable(index) } - pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I) -> bool) { + /// Iterates through all unified variables since the last call to `drain_modified_set` + /// passing the unified variable to `f` + pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I)) { let unify_log = &self.storage.unify_log; self.storage.modified_set.drain(&mut self.undo_log, offset, |vid| { for &unified_vid in unify_log.get(vid) { diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index 5d1f636e09f83..b7187ffdf1aeb 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -39,7 +39,7 @@ impl ModifiedSet { &mut self, undo_log: &mut impl UndoLogs>, index: &Offset, - mut f: impl FnMut(T) -> bool, + mut f: impl FnMut(T), ) { let offset = &mut self.offsets[index.index]; if *offset < self.modified.len() { diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 509aa4adf4ac2..e184d7b492b64 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -75,7 +75,9 @@ use crate::fx::{FxHashMap, FxHashSet}; use std::cell::{Cell, RefCell}; +use std::cmp::Ordering; use std::collections::hash_map::Entry; +use std::collections::BinaryHeap; use std::fmt::Debug; use std::hash; use std::marker::PhantomData; @@ -87,8 +89,11 @@ mod graphviz; mod tests; pub trait ForestObligation: Clone + Debug { + /// A key used to avoid evaluating the same obligation twice type CacheKey: Clone + hash::Hash + Eq + Debug; + /// The variable type used in the obligation when it could not yet be fulfilled type Variable: Clone + hash::Hash + Eq + Debug; + /// A type which tracks which variables has been unified type WatcherOffset; /// Converts this `ForestObligation` suitable for use as a cache key. @@ -97,6 +102,8 @@ pub trait ForestObligation: Clone + Debug { /// (e.g. success for error) for the other obligation fn as_cache_key(&self) -> Self::CacheKey; + /// Returns which variables this obligation is currently stalled on. If the slice is empty then + /// the variables stalled on are unknown. fn stalled_on(&self) -> &[Self::Variable]; } @@ -124,7 +131,7 @@ pub trait ObligationProcessor { fn unblocked( &self, offset: &::WatcherOffset, - f: impl FnMut(::Variable) -> bool, + f: impl FnMut(::Variable), ); fn register(&self) -> ::WatcherOffset; fn deregister(&self, offset: ::WatcherOffset); @@ -157,9 +164,16 @@ pub struct ObligationForest { /// this list only contains nodes in the `Pending` or `Waiting` state. nodes: Vec>, - /// Nodes must be processed in the order that they were added which this list keeps track of. + /// Nodes must be processed in the order that they were added so we give each node a unique, + /// number allowing them to be ordered when processing them. + node_number: u32, + + /// Stores the indices of the nodes currently in the pending state pending_nodes: Vec, + + /// Stores the indices of the nodes currently in the success or waiting states success_or_waiting_nodes: Vec, + /// Stores the indices of the nodes currently in the error or done states error_or_done_nodes: RefCell>, /// Nodes that have been removed and are ready to be reused @@ -171,7 +185,6 @@ pub struct ObligationForest { active_cache: FxHashMap>, obligation_tree_id_generator: ObligationTreeIdGenerator, - node_number: u32, /// Per tree error cache. This is used to deduplicate errors, /// which is necessary to avoid trait resolution overflow in @@ -182,18 +195,25 @@ pub struct ObligationForest { /// [details]: https://github.com/rust-lang/rust/pull/53255#issuecomment-421184780 error_cache: FxHashMap>, + /// Stores which nodes would be unblocked once `O::Variable` is unified stalled_on: FxHashMap>, - unblocked: std::collections::BinaryHeap, + /// Stores the node indices that are unblocked and should be processed at the next opportunity + unblocked: BinaryHeap, + /// Stores nodes which should be processed on the next iteration since the variables they are + /// actually blocked on are unknown check_next: Vec, + /// The offset that this `ObligationForest` has registered. Should be de-registered before + /// dropping this forest. offset: Option, } +/// Helper struct for use with `BinaryHeap` to process nodes in the order that they were added to +/// the forest struct Unblocked { - index: usize, + index: NodeIndex, order: u32, } -use std::cmp::Ordering; impl PartialEq for Unblocked { fn eq(&self, other: &Self) -> bool { self.order == other.order @@ -216,11 +236,14 @@ struct Node { obligation: O, state: Cell, + /// A predicate (and its key) can changed during processing. If it does we need to register the + /// old predicate so that we can remove or mark it as done if this node errors or is done. alternative_predicates: Vec, /// Obligations that depend on this obligation for their completion. They /// must all be in a non-pending state. dependents: Vec, + /// Obligations that this obligation depends on for their completion. reverse_dependents: Vec, /// If true, `dependents[0]` points to a "parent" node, which requires @@ -257,6 +280,7 @@ where } } + /// Initializes a node, reusing the existing allocations fn init( &mut self, parent: Option, @@ -473,12 +497,16 @@ impl ObligationForest { .get(&obligation_tree_id) .map(|errors| errors.contains(&obligation.as_cache_key())) .unwrap_or(false); + // Retrieves a fresh number for the new node so that each node are processed in the + // order that they were created let node_number = self.node_number; self.node_number += 1; if already_failed { Err(()) } else { + // If we have a dead node we can reuse it and it's associated allocations, + // otherwise allocate a new node let new_index = if let Some(new_index) = self.dead_nodes.pop() { let node = &mut self.nodes[new_index]; node.init(parent, obligation, obligation_tree_id, node_number); @@ -549,11 +577,6 @@ impl ObligationForest { if self.offset.is_none() { self.offset = Some(processor.register()); } - warn!( - "Begin process {}, pending: {}", - self.nodes.len(), - self.nodes.iter().filter(|n| n.state.get() == NodeState::Pending).count() - ); let mut errors = vec![]; let mut stalled = true; @@ -577,6 +600,8 @@ impl ObligationForest { if node.state.get() != NodeState::Pending { continue; } + + // Any variables we were stalled on are now resolved so remove the watches for var in node.obligation.stalled_on() { match self.stalled_on.entry(var.clone()) { Entry::Vacant(_) => (), @@ -607,18 +632,22 @@ impl ObligationForest { let node = &mut self.nodes[index]; match result { ProcessResult::Unchanged => { - for var in node.obligation.stalled_on() { - self.stalled_on - .entry(var.clone()) - .or_insert_with(|| { - processor.watch_variable(var.clone()); - Vec::new() - }) - .push(index); - } - - if node.obligation.stalled_on().is_empty() { + // We stalled but the variables that caused it are unknown so we run + // `index` again at the next opportunity + let stalled_on = node.obligation.stalled_on(); + if stalled_on.is_empty() { self.check_next.push(index); + } else { + // Register every variable that we stal + for var in stalled_on { + self.stalled_on + .entry(var.clone()) + .or_insert_with(|| { + processor.watch_variable(var.clone()); + Vec::new() + }) + .push(index); + } } // No change in state. } @@ -646,7 +675,6 @@ impl ObligationForest { } if stalled { - warn!("Stalled {}", self.nodes.len()); // There's no need to perform marking, cycle processing and compression when nothing // changed. return Outcome { @@ -655,13 +683,9 @@ impl ObligationForest { }; } - warn!("Compressing {}", self.nodes.len()); self.mark_successes(); self.process_cycles(processor); let completed = self.compress(do_completed); - warn!("Compressed {}", self.nodes.len()); - - warn!("Stalled on: {:?}", self.stalled_on.keys().collect::>()); Outcome { completed, errors } } @@ -678,19 +702,16 @@ impl ObligationForest { processor.unblocked(self.offset.as_ref().unwrap(), |var| { if let Some(unblocked_nodes) = stalled_on.remove(&var) { for node_index in unblocked_nodes { + let node = &nodes[node_index]; debug_assert!( - nodes[node_index].state.get() == NodeState::Pending, + node.state.get() == NodeState::Pending, "Unblocking non-pending2: {:?}", - nodes[node_index].obligation + node.obligation ); - unblocked.push(Unblocked { - index: node_index, - order: nodes[node_index].node_number, - }); + unblocked.push(Unblocked { index: node_index, order: node.node_number }); } temp.push(var); } - true }); for var in temp { processor.unwatch_variable(var); @@ -848,6 +869,7 @@ impl ObligationForest { fn compress(&mut self, do_completed: DoCompleted) -> Option> { let mut removed_done_obligations: Vec = vec![]; + // Compress the forest by removing any nodes marked as error or done let mut error_or_done_nodes = mem::take(self.error_or_done_nodes.get_mut()); for &index in &error_or_done_nodes { let node = &mut self.nodes[index]; diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index 61cc9a2506a94..eeefd23515e80 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -96,7 +96,7 @@ where fn unblocked( &self, _offset: &::WatcherOffset, - _f: impl FnMut(::Variable) -> bool, + _f: impl FnMut(::Variable), ) { } fn register(&self) -> ::WatcherOffset {} diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index ac241ec0aad11..6e25fb30edc0e 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -1617,7 +1617,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { pub fn drain_modifications( &self, offset: &WatcherOffset, - mut f: impl FnMut(TyOrConstInferVar) -> bool, + mut f: impl FnMut(TyOrConstInferVar), ) { let mut inner = self.inner.borrow_mut(); @@ -1643,11 +1643,11 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { WatcherOffset { ty_offset: inner.type_variables().register_unify_watcher(), - int_offset: inner.int_unification_table().register(), + int_offset: inner.int_unification_table().register_watcher(), - float_offset: inner.float_unification_table().register(), + float_offset: inner.float_unification_table().register_watcher(), - const_offset: inner.const_unification_table().register(), + const_offset: inner.const_unification_table().register_watcher(), } } @@ -1656,11 +1656,11 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { inner.type_variables().deregister_unify_watcher(offset.ty_offset); - inner.int_unification_table().deregister(offset.int_offset); + inner.int_unification_table().deregister_watcher(offset.int_offset); - inner.float_unification_table().deregister(offset.float_offset); + inner.float_unification_table().deregister_watcher(offset.float_offset); - inner.const_unification_table().deregister(offset.const_offset); + inner.const_unification_table().deregister_watcher(offset.const_offset); } pub fn watch_variable(&self, infer: TyOrConstInferVar) { diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 36d95f549328c..91a3ed58f0533 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -437,20 +437,16 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { self.eq_relations().clear_modified_set(); } - pub fn drain_modified_set( - &mut self, - offset: &ms::Offset, - f: impl FnMut(ty::TyVid) -> bool, - ) { + pub fn drain_modified_set(&mut self, offset: &ms::Offset, f: impl FnMut(ty::TyVid)) { self.eq_relations().drain_modified_set(offset, f) } pub fn register_unify_watcher(&mut self) -> ms::Offset { - self.eq_relations().register() + self.eq_relations().register_watcher() } pub fn deregister_unify_watcher(&mut self, offset: ms::Offset) { - self.eq_relations().deregister(offset); + self.eq_relations().deregister_watcher(offset); } pub fn watch_variable(&mut self, vid: ty::TyVid) { diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index bbf75feb48e06..bcfc521251922 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -568,7 +568,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { fn unblocked( &self, offset: &WatcherOffset, - f: impl FnMut(::Variable) -> bool, + f: impl FnMut(::Variable), ) { self.selcx.infcx().drain_modifications(offset, f); } From 22abfb69bf1e0789420ebcb1446b5ac1af77074c Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 30 May 2020 20:01:20 +0200 Subject: [PATCH 12/25] refactor: Simplify obligation_forest --- .../src/logged_unification_table.rs | 1 - .../src/obligation_forest/mod.rs | 206 +++++++++--------- .../src/obligation_forest/tests.rs | 8 +- .../rustc_data_structures/src/unify_log.rs | 42 ++-- .../src/traits/fulfill.rs | 7 +- 5 files changed, 139 insertions(+), 125 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index 2a63b401f2745..c5bdf6f2207bd 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -107,7 +107,6 @@ where K::Value: ut::UnifyValue, { if self.storage.unify_log.needs_log(vid) { - warn!("ModifiedSet {:?} => {:?}", vid, ty); self.storage.modified_set.set(&mut self.undo_log, vid); } let vid = vid.into(); diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index e184d7b492b64..7ae34ccf61a2d 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -1,4 +1,3 @@ -//! The `ObligationForest` is a utility data structure used in trait //! matching to track the set of outstanding obligations (those not yet //! resolved to success or error). It also tracks the "backtrace" of each //! pending obligation (why we are trying to figure this out in the first @@ -133,8 +132,11 @@ pub trait ObligationProcessor { offset: &::WatcherOffset, f: impl FnMut(::Variable), ); - fn register(&self) -> ::WatcherOffset; - fn deregister(&self, offset: ::WatcherOffset); + fn register_variable_watcher(&self) -> ::WatcherOffset; + fn deregister_variable_watcher( + &self, + offset: ::WatcherOffset, + ); fn watch_variable(&self, var: ::Variable); fn unwatch_variable(&self, var: ::Variable); } @@ -159,6 +161,11 @@ type ObligationTreeIdGenerator = /// significant, and space considerations are not important. type NodeIndex = usize; +enum CacheState { + Active(NodeIndex), + Done, +} + pub struct ObligationForest { /// The list of obligations. In between calls to `process_obligations`, /// this list only contains nodes in the `Pending` or `Waiting` state. @@ -171,18 +178,21 @@ pub struct ObligationForest { /// Stores the indices of the nodes currently in the pending state pending_nodes: Vec, - /// Stores the indices of the nodes currently in the success or waiting states + /// Stores the indices of the nodes currently in the success or waiting states. + /// Can also contain `Done` or `Error` nodes as `process_cycles` does not remove a node + /// immediately but instead upon the next time that node is processed. success_or_waiting_nodes: Vec, /// Stores the indices of the nodes currently in the error or done states error_or_done_nodes: RefCell>, - /// Nodes that have been removed and are ready to be reused + /// Nodes that have been removed and are ready to be reused (pure optimization to reuse + /// allocations) dead_nodes: Vec, /// A cache of the nodes in `nodes`, indexed by predicate. Unfortunately, /// its contents are not guaranteed to match those of `nodes`. See the /// comments in `process_obligation` for details. - active_cache: FxHashMap>, + active_cache: FxHashMap, obligation_tree_id_generator: ObligationTreeIdGenerator, @@ -201,10 +211,12 @@ pub struct ObligationForest { unblocked: BinaryHeap, /// Stores nodes which should be processed on the next iteration since the variables they are /// actually blocked on are unknown - check_next: Vec, + stalled_on_unknown: Vec, /// The offset that this `ObligationForest` has registered. Should be de-registered before /// dropping this forest. offset: Option, + /// Reusable vector for storing unblocked nodes whose watch should be removed + temp_unblocked_nodes: Vec, } /// Helper struct for use with `BinaryHeap` to process nodes in the order that they were added to @@ -427,7 +439,8 @@ impl ObligationForest { error_cache: Default::default(), stalled_on: Default::default(), unblocked: Default::default(), - check_next: Default::default(), + stalled_on_unknown: Default::default(), + temp_unblocked_nodes: Default::default(), offset: None, } } @@ -462,8 +475,8 @@ impl ObligationForest { match self.active_cache.entry(obligation.as_cache_key().clone()) { Entry::Occupied(o) => { let index = match o.get() { - Some(index) => *index, - None => { + CacheState::Active(index) => *index, + CacheState::Done => { debug!( "register_obligation_at: ignoring already done obligation: {:?}", obligation @@ -497,14 +510,15 @@ impl ObligationForest { .get(&obligation_tree_id) .map(|errors| errors.contains(&obligation.as_cache_key())) .unwrap_or(false); - // Retrieves a fresh number for the new node so that each node are processed in the - // order that they were created - let node_number = self.node_number; - self.node_number += 1; if already_failed { Err(()) } else { + // Retrieves a fresh number for the new node so that each node are processed in the + // order that they were created + let node_number = self.node_number; + self.node_number += 1; + // If we have a dead node we can reuse it and it's associated allocations, // otherwise allocate a new node let new_index = if let Some(new_index) = self.dead_nodes.pop() { @@ -524,9 +538,10 @@ impl ObligationForest { if let Some(parent_index) = parent { self.nodes[parent_index].reverse_dependents.push(new_index); } + self.pending_nodes.push(new_index); self.unblocked.push(Unblocked { index: new_index, order: node_number }); - v.insert(Some(new_index)); + v.insert(CacheState::Active(new_index)); Ok(()) } } @@ -575,101 +590,84 @@ impl ObligationForest { OUT: OutcomeTrait>, { if self.offset.is_none() { - self.offset = Some(processor.register()); + self.offset = Some(processor.register_variable_watcher()); } let mut errors = vec![]; let mut stalled = true; self.unblock_nodes(processor); - let mut run_again = true; - while !self.unblocked.is_empty() || (!self.check_next.is_empty() && run_again) { - run_again = false; - let nodes = &self.nodes; - self.unblocked.extend( - self.check_next - .drain(..) - .map(|index| Unblocked { index, order: nodes[index].node_number }), - ); - while let Some(Unblocked { index, .. }) = self.unblocked.pop() { - if self.unblocked.peek().map(|u| u.index) == Some(index) { - continue; - } - let node = &mut self.nodes[index]; + let nodes = &self.nodes; + self.unblocked.extend( + self.stalled_on_unknown + .drain(..) + .map(|index| Unblocked { index, order: nodes[index].node_number }), + ); + while let Some(Unblocked { index, .. }) = self.unblocked.pop() { + // Skip any duplicates since we only need to processes the node once + if self.unblocked.peek().map(|u| u.index) == Some(index) { + continue; + } - if node.state.get() != NodeState::Pending { - continue; - } + let node = &mut self.nodes[index]; - // Any variables we were stalled on are now resolved so remove the watches - for var in node.obligation.stalled_on() { - match self.stalled_on.entry(var.clone()) { - Entry::Vacant(_) => (), - Entry::Occupied(mut entry) => { - let nodes = entry.get_mut(); - nodes.retain(|node_index| *node_index != index); - if nodes.is_empty() { - processor.unwatch_variable(var.clone()); - entry.remove(); - } - } - } - } + if node.state.get() != NodeState::Pending { + continue; + } - // `processor.process_obligation` can modify the predicate within - // `node.obligation`, and that predicate is the key used for - // `self.active_cache`. This means that `self.active_cache` can get - // out of sync with `nodes`. It's not very common, but it does - // happen, and code in `compress` has to allow for it. - let before = node.obligation.as_cache_key(); - let result = processor.process_obligation(&mut node.obligation); - let after = node.obligation.as_cache_key(); - if before != after { - node.alternative_predicates.push(before); - } + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + let before = node.obligation.as_cache_key(); + let result = processor.process_obligation(&mut node.obligation); + let after = node.obligation.as_cache_key(); + if before != after { + node.alternative_predicates.push(before); + } - self.unblock_nodes(processor); - let node = &mut self.nodes[index]; - match result { - ProcessResult::Unchanged => { + self.unblock_nodes(processor); + let node = &mut self.nodes[index]; + match result { + ProcessResult::Unchanged => { + let stalled_on = node.obligation.stalled_on(); + if stalled_on.is_empty() { // We stalled but the variables that caused it are unknown so we run // `index` again at the next opportunity - let stalled_on = node.obligation.stalled_on(); - if stalled_on.is_empty() { - self.check_next.push(index); - } else { - // Register every variable that we stal - for var in stalled_on { - self.stalled_on - .entry(var.clone()) - .or_insert_with(|| { - processor.watch_variable(var.clone()); - Vec::new() - }) - .push(index); - } + self.stalled_on_unknown.push(index); + } else { + // Register every variable that we stalled on + for var in stalled_on { + self.stalled_on + .entry(var.clone()) + .or_insert_with(|| { + processor.watch_variable(var.clone()); + Vec::new() + }) + .push(index); } - // No change in state. } - ProcessResult::Changed(children) => { - // We are not (yet) stalled. - stalled = false; - node.state.set(NodeState::Success); - self.success_or_waiting_nodes.push(index); - - for child in children { - let st = self.register_obligation_at(child, Some(index)); - if let Err(()) = st { - // Error already reported - propagate it - // to our node. - self.error_at(index); - } + // No change in state. + } + ProcessResult::Changed(children) => { + // We are not (yet) stalled. + stalled = false; + node.state.set(NodeState::Success); + self.success_or_waiting_nodes.push(index); + + for child in children { + let st = self.register_obligation_at(child, Some(index)); + if let Err(()) = st { + // Error already reported - propagate it + // to our node. + self.error_at(index); } } - ProcessResult::Error(err) => { - stalled = false; - errors.push(Error { error: err, backtrace: self.error_at(index) }); - } + } + ProcessResult::Error(err) => { + stalled = false; + errors.push(Error { error: err, backtrace: self.error_at(index) }); } } } @@ -698,7 +696,8 @@ impl ObligationForest { let nodes = &mut self.nodes; let stalled_on = &mut self.stalled_on; let unblocked = &mut self.unblocked; - let mut temp = Vec::new(); + let temp_unblocked_nodes = &mut self.temp_unblocked_nodes; + temp_unblocked_nodes.clear(); processor.unblocked(self.offset.as_ref().unwrap(), |var| { if let Some(unblocked_nodes) = stalled_on.remove(&var) { for node_index in unblocked_nodes { @@ -710,10 +709,10 @@ impl ObligationForest { ); unblocked.push(Unblocked { index: node_index, order: node.node_number }); } - temp.push(var); + temp_unblocked_nodes.push(var); } }); - for var in temp { + for var in temp_unblocked_nodes.drain(..) { processor.unwatch_variable(var); } } @@ -876,10 +875,13 @@ impl ObligationForest { let reverse_dependents = mem::take(&mut node.reverse_dependents); for &reverse_index in &reverse_dependents { let reverse_node = &mut self.nodes[reverse_index]; - if reverse_node.dependents.first() == Some(&index) { - reverse_node.has_parent = false; + + if let Some(i) = reverse_node.dependents.iter().position(|x| *x == index) { + reverse_node.dependents.swap_remove(i); + if i == 0 { + reverse_node.has_parent = false; + } } - reverse_node.dependents.retain(|&i| i != index); } let node = &mut self.nodes[index]; node.reverse_dependents = reverse_dependents; @@ -888,11 +890,11 @@ impl ObligationForest { NodeState::Done => { // Mark as done if let Some(opt) = self.active_cache.get_mut(&node.obligation.as_cache_key()) { - *opt = None; + *opt = CacheState::Done; } for alt in &node.alternative_predicates { if let Some(opt) = self.active_cache.get_mut(alt) { - *opt = None + *opt = CacheState::Done; } } diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index eeefd23515e80..f28b35b9f070c 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -99,8 +99,12 @@ where _f: impl FnMut(::Variable), ) { } - fn register(&self) -> ::WatcherOffset {} - fn deregister(&self, _offset: ::WatcherOffset) {} + fn register_variable_watcher(&self) -> ::WatcherOffset {} + fn deregister_variable_watcher( + &self, + _offset: ::WatcherOffset, + ) { + } fn watch_variable(&self, _var: ::Variable) {} fn unwatch_variable(&self, _var: ::Variable) {} } diff --git a/compiler/rustc_data_structures/src/unify_log.rs b/compiler/rustc_data_structures/src/unify_log.rs index 538aee3484eb0..e98628ad4a199 100644 --- a/compiler/rustc_data_structures/src/unify_log.rs +++ b/compiler/rustc_data_structures/src/unify_log.rs @@ -8,6 +8,9 @@ pub enum Undo { NewGroup { index: T }, } +/// Tracks which variables (represented by indices) that has been unified with eachother. +/// Since there is often only a few variables that are interesting one must call `watch_variable` +/// before this records any variables unified with that variable. pub struct UnifyLog { unified_vars: IndexVec, groups: Vec>, @@ -22,8 +25,8 @@ fn pick2_mut(self_: &mut [T], a: I, b: I) -> (&mut T, &mut T) { let (c1, c2) = self_.split_at_mut(bi); (&mut c1[ai], &mut c2[0]) } else { - let (c2, c1) = pick2_mut(self_, b, a); - (c1, c2) + let (c1, c2) = self_.split_at_mut(ai); + (&mut c2[0], &mut c1[bi]) } } @@ -40,35 +43,40 @@ impl UnifyLog { if !self.needs_log(other) { return; } - self.unified_vars.ensure_contains_elem(root, usize::max_value); - self.unified_vars.ensure_contains_elem(other, usize::max_value); + self.unified_vars.ensure_contains_elem(root.max(other), usize::max_value); let mut root_group = self.unified_vars[root]; let other_group = self.unified_vars[other]; - if other_group == usize::max_value() { - let root_vec = if root_group == usize::max_value() { + match (root_group, other_group) { + (usize::MAX, usize::MAX) => { + // Neither variable is part of a group, create a new one at the root and associate + // other root_group = self.groups.len(); self.unified_vars[root] = root_group; - self.groups.push(Vec::new()); + self.groups.push(vec![other]); undo_log.push(Undo::NewGroup { index: root }); - self.groups.last_mut().unwrap() - } else { - let root_vec = &mut self.groups[root_group]; - undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); - root_vec - }; - root_vec.push(other); - } else { - if root_group == usize::max_value() { + } + (usize::MAX, _) => { + // `other` has a group, point `root` to it and associate other let group = &mut self.unified_vars[root]; undo_log.push(Undo::Move { index: root, old: *group }); *group = other_group; self.groups[other_group].push(other); - } else { + } + (_, usize::MAX) => { + // `root` hasa group, just associate `other` + let root_vec = &mut self.groups[root_group]; + undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); + root_vec.push(other); + } + _ => { + // Both variables has their own groups, associate all of `other` to root let (root_vec, other_vec) = pick2_mut(&mut self.groups, root_group, other_group); undo_log.push(Undo::Extend { group_index: root_group, len: root_vec.len() }); root_vec.extend_from_slice(other_vec); + // We only need to add `other` if there is a watcher for it (there might only be + // watchers for the other variables in its group) if self.reference_counts.get(other).map_or(false, |c| *c != 0) { root_vec.push(other); } diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index bcfc521251922..c8860a624058b 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -570,14 +570,15 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { offset: &WatcherOffset, f: impl FnMut(::Variable), ) { - self.selcx.infcx().drain_modifications(offset, f); + let infcx = self.selcx.infcx(); + infcx.drain_modifications(offset, f); } - fn register(&self) -> WatcherOffset { + fn register_variable_watcher(&self) -> WatcherOffset { self.selcx.infcx().register_unify_watcher() } - fn deregister(&self, offset: WatcherOffset) { + fn deregister_variable_watcher(&self, offset: WatcherOffset) { self.selcx.infcx().deregister_unify_watcher(offset); } From f771696fb291dd2e3dc63b6d7e7ba8f07e3103ef Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Fri, 5 Jun 2020 19:09:15 +0200 Subject: [PATCH 13/25] perf: No need to resolve variables in fulfillment --- .../src/traits/fulfill.rs | 81 ++++++++++--------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index c8860a624058b..f40c63f8e7b3c 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -260,15 +260,9 @@ struct FulfillProcessor<'a, 'b, 'tcx> { register_region_obligations: bool, } -fn mk_pending( - infcx: &InferCtxt<'_, 'tcx>, - os: Vec>, -) -> Vec> { +fn mk_pending(os: Vec>) -> Vec> { os.into_iter() - .map(|mut o| { - o.predicate = infcx.resolve_vars_if_possible(&o.predicate); - PendingPredicateObligation { obligation: o, stalled_on: vec![] } - }) + .map(|o| PendingPredicateObligation { obligation: o, stalled_on: vec![] }) .collect() } @@ -412,29 +406,38 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { ) { None => { pending_obligation.stalled_on.clear(); - pending_obligation.stalled_on.push(TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap()); + pending_obligation.stalled_on.push(infcx.root_ty_or_const( + TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap(), + )); ProcessResult::Unchanged } Some(os) => ProcessResult::Changed(mk_pending(os)), } } - &ty::PredicateKind::WellFormed(arg) => { - match wf::obligations( - self.selcx.infcx(), - obligation.param_env, - obligation.cause.body_id, - arg, - obligation.cause.span, - ) { - None => { - pending_obligation.stalled_on.clear(); - pending_obligation - .stalled_on - .push(TyOrConstInferVar::maybe_from_generic_arg(arg).unwrap()); - ProcessResult::Unchanged + ty::PredicateAtom::Subtype(subtype) => { + match self.selcx.infcx().subtype_predicate( + &obligation.cause, + obligation.param_env, + Binder::dummy(subtype), + ) { + None => { + // None means that both are unresolved. + pending_obligation.stalled_on.clear(); + pending_obligation.stalled_on.push(root_ty_or_const_var(subtype.a)); + pending_obligation.stalled_on.push(root_ty_or_const_var(subtype.b)); + ProcessResult::Unchanged + } + Some(Ok(ok)) => ProcessResult::Changed(mk_pending(ok.obligations)), + Some(Err(err)) => { + let expected_found = + ExpectedFound::new(subtype.a_is_expected, subtype.a, subtype.b); + ProcessResult::Error(FulfillmentErrorCode::CodeSubtypeError( + expected_found, + err, + )) + } } - Some(os) => ProcessResult::Changed(mk_pending(infcx, os)), } ty::PredicateAtom::ConstEvaluatable(def_id, substs) => { @@ -447,10 +450,12 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { ) { Ok(()) => ProcessResult::Changed(vec![]), Err(ErrorHandled::TooGeneric) => { - pending_obligation.stalled_on = substs - .iter() - .filter_map(|ty| TyOrConstInferVar::maybe_from_generic_arg(ty)) - .collect(); + pending_obligation.stalled_on.extend( + substs + .iter() + .filter_map(|ty| TyOrConstInferVar::maybe_from_generic_arg(ty)) + .map(|ty| infcx.root_ty_or_const(ty)), + ); ProcessResult::Unchanged } Err(e) => ProcessResult::Error(CodeSelectionError(ConstEvalFailure(e))), @@ -492,13 +497,13 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { ) { Ok(val) => Ok(Const::from_value(self.selcx.tcx(), val, c.ty)), Err(ErrorHandled::TooGeneric) => { - stalled_on.append( - &mut substs - .iter() + stalled_on.extend( + substs + .types() .filter_map(|arg| { TyOrConstInferVar::maybe_from_generic_arg(arg) }) - .collect(), + .map(|ty| infcx.root_ty_or_const(ty)), ); Err(ErrorHandled::TooGeneric) } @@ -596,7 +601,7 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { &mut self, obligation: &PredicateObligation<'tcx>, trait_obligation: TraitObligation<'tcx>, - stalled_on: &mut Vec>, + stalled_on: &mut Vec, ) -> ProcessResult, FulfillmentErrorCode<'tcx>> { let infcx = self.selcx.infcx(); if obligation.predicate.is_global() { @@ -623,10 +628,10 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { // only reason we can fail to make progress on // trait selection is because we don't have enough // information about the types in the trait. - *stalled_on = trait_ref_infer_vars( + stalled_on.extend(trait_ref_infer_vars( self.selcx, trait_obligation.predicate.map_bound(|pred| pred.trait_ref), - ); + )); debug!( "process_predicate: pending obligation {:?} now stalled on {:?}", @@ -647,16 +652,16 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { fn process_projection_obligation( &mut self, project_obligation: PolyProjectionObligation<'tcx>, - stalled_on: &mut Vec>, + stalled_on: &mut Vec, ) -> ProcessResult, FulfillmentErrorCode<'tcx>> { let tcx = self.selcx.tcx(); match project::poly_project_and_unify_type(self.selcx, &project_obligation) { Ok(Ok(Some(os))) => ProcessResult::Changed(mk_pending(os)), Ok(Ok(None)) => { - *stalled_on = trait_ref_infer_vars( + stalled_on.extend(trait_ref_infer_vars( self.selcx, project_obligation.predicate.to_poly_trait_ref(tcx), - ); + )); ProcessResult::Unchanged } // Let the caller handle the recursion From 34f3303b9ab205a88a281c7b0d9fe70ef7c6da64 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 6 Jun 2020 13:31:33 +0200 Subject: [PATCH 14/25] Document obligation forest changes --- .../src/logged_unification_table.rs | 8 +- .../rustc_data_structures/src/modified_set.rs | 34 +++++-- .../src/obligation_forest/mod.rs | 99 +++++++++++-------- .../src/obligation_forest/tests.rs | 2 +- .../rustc_data_structures/src/unify_log.rs | 15 ++- compiler/rustc_infer/src/infer/mod.rs | 16 +-- .../rustc_infer/src/infer/type_variable.rs | 4 +- .../src/traits/fulfill.rs | 8 +- 8 files changed, 115 insertions(+), 71 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index c5bdf6f2207bd..531fb705a06e2 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -48,7 +48,7 @@ pub struct LoggedUnificationTableStorage { modified_set: ms::ModifiedSet, } -/// UnificationTableStorage which logs which variables has been unfified with a value, allowing watchers +/// UnificationTableStorage which logs which variables has been unified with a value, allowing watchers /// to only iterate over the changed variables instead of all variables pub struct LoggedUnificationTable<'a, K: ut::UnifyKey, I: Idx, L> { storage: &'a mut LoggedUnificationTableStorage, @@ -200,11 +200,11 @@ where self.storage.unify_log.unwatch_variable(index) } - /// Iterates through all unified variables since the last call to `drain_modified_set` + /// Iterates through all unified variables since the last call to `notify_watcher` /// passing the unified variable to `f` - pub fn drain_modified_set(&mut self, offset: &ms::Offset, mut f: impl FnMut(I)) { + pub fn notify_watcher(&mut self, offset: &ms::Offset, mut f: impl FnMut(I)) { let unify_log = &self.storage.unify_log; - self.storage.modified_set.drain(&mut self.undo_log, offset, |vid| { + self.storage.modified_set.notify_watcher(&mut self.undo_log, offset, |vid| { for &unified_vid in unify_log.get(vid) { f(unified_vid); } diff --git a/compiler/rustc_data_structures/src/modified_set.rs b/compiler/rustc_data_structures/src/modified_set.rs index b7187ffdf1aeb..dcea3b40751cb 100644 --- a/compiler/rustc_data_structures/src/modified_set.rs +++ b/compiler/rustc_data_structures/src/modified_set.rs @@ -13,6 +13,8 @@ enum UndoInner { #[derive(Copy, Clone, Debug)] pub struct Undo(UndoInner, PhantomData); +/// Tracks which indices have been modified and allows watchers to registered and notified of these +/// changes. #[derive(Clone, Debug)] pub struct ModifiedSet { modified: Vec, @@ -26,32 +28,40 @@ impl Default for ModifiedSet { } impl ModifiedSet { + /// Creates a new `ModifiedSet` pub fn new() -> Self { Self::default() } + /// Marks `index` as "modified". A subsequent call to `drain` will notify the callback with + /// `index` pub fn set(&mut self, undo_log: &mut impl UndoLogs>, index: T) { self.modified.push(index); undo_log.push(Undo(UndoInner::Add, PhantomData)); } - pub fn drain( + /// Calls `f` with all the indices that have been modified since the last call to + /// `notify_watcher` + pub fn notify_watcher( &mut self, undo_log: &mut impl UndoLogs>, - index: &Offset, + watcher_offset: &Offset, mut f: impl FnMut(T), ) { - let offset = &mut self.offsets[index.index]; + let offset = &mut self.offsets[watcher_offset.index]; if *offset < self.modified.len() { for &index in &self.modified[*offset..] { f(index); } - undo_log - .push(Undo(UndoInner::Drain { index: index.index, offset: *offset }, PhantomData)); + undo_log.push(Undo( + UndoInner::Drain { index: watcher_offset.index, offset: *offset }, + PhantomData, + )); *offset = self.modified.len(); } } + /// Clears the set of all modifications that have been drained by all watchers pub fn clear(&mut self) { let min = self.offsets.iter().copied().min().unwrap_or_else(|| self.modified.len()); self.modified.drain(..min); @@ -60,14 +70,24 @@ impl ModifiedSet { } } + /// Registers a new watcher on this set. + /// + /// NOTE: Watchers must be removed in the reverse order that they were registered pub fn register(&mut self) -> Offset { let index = self.offsets.len(); self.offsets.push(self.modified.len()); Offset { index, _marker: PhantomData } } + /// De-registers a watcher on this set. + /// + /// NOTE: Watchers must be removed in the reverse order that they were registered pub fn deregister(&mut self, offset: Offset) { - assert_eq!(offset.index, self.offsets.len() - 1); + assert_eq!( + offset.index, + self.offsets.len() - 1, + "Watchers must be removed in the reverse order that they were registered" + ); self.offsets.pop(); std::mem::forget(offset); } @@ -88,6 +108,8 @@ impl Rollback> for ModifiedSet { } } +/// A registered offset into a `ModifiedSet`. Tracks how much a watcher has seen so far to avoid +/// being notified of the same event twice. #[must_use] pub struct Offset { index: usize, diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 7ae34ccf61a2d..66fe3f5a37932 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -1,3 +1,4 @@ +//! The `ObligationForest` is a utility data structure used in trait //! matching to track the set of outstanding obligations (those not yet //! resolved to success or error). It also tracks the "backtrace" of each //! pending obligation (why we are trying to figure this out in the first @@ -41,7 +42,7 @@ //! now considered to be in error. //! //! When the call to `process_obligations` completes, you get back an `Outcome`, -//! which includes three bits of information: +//! which includes two bits of information: //! //! - `completed`: a list of obligations where processing was fully //! completed without error (meaning that all transitive subobligations @@ -52,13 +53,6 @@ //! all the obligations in `C` have been found completed. //! - `errors`: a list of errors that occurred and associated backtraces //! at the time of error, which can be used to give context to the user. -//! - `stalled`: if true, then none of the existing obligations were -//! *shallowly successful* (that is, no callback returned `Changed(_)`). -//! This implies that all obligations were either errors or returned an -//! ambiguous result, which means that any further calls to -//! `process_obligations` would simply yield back further ambiguous -//! results. This is used by the `FulfillmentContext` to decide when it -//! has reached a steady state. //! //! ### Implementation details //! @@ -88,17 +82,17 @@ mod graphviz; mod tests; pub trait ForestObligation: Clone + Debug { - /// A key used to avoid evaluating the same obligation twice + /// A key used to avoid evaluating the same obligation twice. type CacheKey: Clone + hash::Hash + Eq + Debug; - /// The variable type used in the obligation when it could not yet be fulfilled + /// The variable type used in the obligation when it could not yet be fulfilled. type Variable: Clone + hash::Hash + Eq + Debug; - /// A type which tracks which variables has been unified + /// A type which tracks which variables has been unified. type WatcherOffset; /// Converts this `ForestObligation` suitable for use as a cache key. /// If two distinct `ForestObligations`s return the same cache key, /// then it must be sound to use the result of processing one obligation - /// (e.g. success for error) for the other obligation + /// (e.g. success for error) for the other obligation. fn as_cache_key(&self) -> Self::CacheKey; /// Returns which variables this obligation is currently stalled on. If the slice is empty then @@ -127,7 +121,9 @@ pub trait ObligationProcessor { where I: Clone + Iterator; - fn unblocked( + /// Calls `f` with all the variables that have been unblocked (instantiated) since the last call + /// to `notify_unblocked`. + fn notify_unblocked( &self, offset: &::WatcherOffset, f: impl FnMut(::Variable), @@ -175,14 +171,15 @@ pub struct ObligationForest { /// number allowing them to be ordered when processing them. node_number: u32, - /// Stores the indices of the nodes currently in the pending state + /// Stores the indices of the nodes currently in the pending state. pending_nodes: Vec, /// Stores the indices of the nodes currently in the success or waiting states. /// Can also contain `Done` or `Error` nodes as `process_cycles` does not remove a node /// immediately but instead upon the next time that node is processed. success_or_waiting_nodes: Vec, - /// Stores the indices of the nodes currently in the error or done states + + /// Stores the indices of the nodes currently in the error or done states. error_or_done_nodes: RefCell>, /// Nodes that have been removed and are ready to be reused (pure optimization to reuse @@ -205,22 +202,26 @@ pub struct ObligationForest { /// [details]: https://github.com/rust-lang/rust/pull/53255#issuecomment-421184780 error_cache: FxHashMap>, - /// Stores which nodes would be unblocked once `O::Variable` is unified + /// Stores which nodes would be unblocked once `O::Variable` is unified. stalled_on: FxHashMap>, - /// Stores the node indices that are unblocked and should be processed at the next opportunity + + /// Stores the node indices that are unblocked and should be processed at the next opportunity. unblocked: BinaryHeap, + /// Stores nodes which should be processed on the next iteration since the variables they are - /// actually blocked on are unknown + /// actually blocked on are unknown. stalled_on_unknown: Vec, + /// The offset that this `ObligationForest` has registered. Should be de-registered before /// dropping this forest. - offset: Option, - /// Reusable vector for storing unblocked nodes whose watch should be removed + /// + watcher_offset: Option, + /// Reusable vector for storing unblocked nodes whose watch should be removed. temp_unblocked_nodes: Vec, } /// Helper struct for use with `BinaryHeap` to process nodes in the order that they were added to -/// the forest +/// the forest. struct Unblocked { index: NodeIndex, order: u32, @@ -248,13 +249,14 @@ struct Node { obligation: O, state: Cell, - /// A predicate (and its key) can changed during processing. If it does we need to register the + /// A predicate (and its key) can change during processing. If it does we need to register the /// old predicate so that we can remove or mark it as done if this node errors or is done. alternative_predicates: Vec, /// Obligations that depend on this obligation for their completion. They /// must all be in a non-pending state. dependents: Vec, + /// Obligations that this obligation depends on for their completion. reverse_dependents: Vec, @@ -267,6 +269,9 @@ struct Node { /// Identifier of the obligation tree to which this node belongs. obligation_tree_id: ObligationTreeId, + + /// Nodes must be processed in the order that they were added so we give each node a unique + /// number allowing them to be ordered when processing them. node_number: u32, } @@ -292,8 +297,9 @@ where } } - /// Initializes a node, reusing the existing allocations - fn init( + /// Initializes a node, reusing the existing allocations. Used when removing a node from the + /// dead_nodes list + fn reinit( &mut self, parent: Option, obligation: O, @@ -441,16 +447,19 @@ impl ObligationForest { unblocked: Default::default(), stalled_on_unknown: Default::default(), temp_unblocked_nodes: Default::default(), - offset: None, + watcher_offset: None, } } - pub fn offset(&self) -> Option<&O::WatcherOffset> { - self.offset.as_ref() + /// Returns the `WatcherOffset` regsitered with the notification table. See the field + /// `ObligationForest::watcher_offset`. + pub fn watcher_offset(&self) -> Option<&O::WatcherOffset> { + self.watcher_offset.as_ref() } - pub fn take_offset(&mut self) -> Option { - self.offset.take() + /// Removes the watcher_offset, allowing it to be deregistered + pub fn take_watcher_offset(&mut self) -> Option { + self.watcher_offset.take() } /// Returns the total number of nodes in the forest that have not @@ -523,7 +532,7 @@ impl ObligationForest { // otherwise allocate a new node let new_index = if let Some(new_index) = self.dead_nodes.pop() { let node = &mut self.nodes[new_index]; - node.init(parent, obligation, obligation_tree_id, node_number); + node.reinit(parent, obligation, obligation_tree_id, node_number); new_index } else { let new_index = self.nodes.len(); @@ -565,11 +574,7 @@ impl ObligationForest { where F: Fn(&O) -> P, { - self.pending_nodes - .iter() - .map(|&index| &self.nodes[index]) - .map(|node| f(&node.obligation)) - .collect() + self.pending_nodes.iter().map(|&index| f(&self.nodes[index].obligation)).collect() } fn insert_into_error_cache(&mut self, index: NodeIndex) { @@ -589,8 +594,8 @@ impl ObligationForest { P: ObligationProcessor, OUT: OutcomeTrait>, { - if self.offset.is_none() { - self.offset = Some(processor.register_variable_watcher()); + if self.watcher_offset.is_none() { + self.watcher_offset = Some(processor.register_variable_watcher()); } let mut errors = vec![]; let mut stalled = true; @@ -688,7 +693,10 @@ impl ObligationForest { Outcome { completed, errors } } - #[inline(never)] + /// Checks which nodes have been unblocked since the last time this was called. All nodes that + /// were unblocked are added to the `unblocked` queue and all watches associated with the + /// variables blocking those nodes are deregistered (since they are now instantiated, they will + /// neither block a node, nor be instantiated again) fn unblock_nodes

(&mut self, processor: &mut P) where P: ObligationProcessor, @@ -698,7 +706,7 @@ impl ObligationForest { let unblocked = &mut self.unblocked; let temp_unblocked_nodes = &mut self.temp_unblocked_nodes; temp_unblocked_nodes.clear(); - processor.unblocked(self.offset.as_ref().unwrap(), |var| { + processor.notify_unblocked(self.watcher_offset.as_ref().unwrap(), |var| { if let Some(unblocked_nodes) = stalled_on.remove(&var) { for node_index in unblocked_nodes { let node = &nodes[node_index]; @@ -861,9 +869,8 @@ impl ObligationForest { } } - /// Compresses the vector, removing all popped nodes. This adjusts the - /// indices and hence invalidates any outstanding indices. `process_cycles` - /// must be run beforehand to remove any cycles on `Success` nodes. + /// Compresses the forest, moving all nodes marked `Done` or `Error` into `dead_nodes` for later reuse + /// `process_cycles` must be run beforehand to remove any cycles on `Success` nodes. #[inline(never)] fn compress(&mut self, do_completed: DoCompleted) -> Option> { let mut removed_done_obligations: Vec = vec![]; @@ -872,6 +879,8 @@ impl ObligationForest { let mut error_or_done_nodes = mem::take(self.error_or_done_nodes.get_mut()); for &index in &error_or_done_nodes { let node = &mut self.nodes[index]; + + // Remove this node from all the nodes that depends on it let reverse_dependents = mem::take(&mut node.reverse_dependents); for &reverse_index in &reverse_dependents { let reverse_node = &mut self.nodes[reverse_index]; @@ -892,6 +901,8 @@ impl ObligationForest { if let Some(opt) = self.active_cache.get_mut(&node.obligation.as_cache_key()) { *opt = CacheState::Done; } + // If the node's predicate changed at some point we mark all its alternate + // predicates as done as well for alt in &node.alternative_predicates { if let Some(opt) = self.active_cache.get_mut(alt) { *opt = CacheState::Done; @@ -903,6 +914,8 @@ impl ObligationForest { removed_done_obligations.push(node.obligation.clone()); } + // Store the node so it and its allocations can be used when another node is + // allocated self.dead_nodes.push(index); } NodeState::Error => { @@ -910,6 +923,8 @@ impl ObligationForest { // tests must come up with a different type on every type error they // check against. self.active_cache.remove(&node.obligation.as_cache_key()); + // If the node's predicate changed at some point we remove all its alternate + // predicates as well for alt in &node.alternative_predicates { self.active_cache.remove(alt); } diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index f28b35b9f070c..f9b3c4dc7930e 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -93,7 +93,7 @@ where { } - fn unblocked( + fn notify_unblocked( &self, _offset: &::WatcherOffset, _f: impl FnMut(::Variable), diff --git a/compiler/rustc_data_structures/src/unify_log.rs b/compiler/rustc_data_structures/src/unify_log.rs index e98628ad4a199..1384dc5395213 100644 --- a/compiler/rustc_data_structures/src/unify_log.rs +++ b/compiler/rustc_data_structures/src/unify_log.rs @@ -10,7 +10,11 @@ pub enum Undo { /// Tracks which variables (represented by indices) that has been unified with eachother. /// Since there is often only a few variables that are interesting one must call `watch_variable` -/// before this records any variables unified with that variable. +/// on any variable record unifications with. Used in conjuction with a `ModifiedSet` to accurately +/// track which variables has been instantiated. +/// +/// NOTE: The methods on this expect and only work correctly if a root variable from +/// an `UnificationTable` is provided. pub struct UnifyLog { unified_vars: IndexVec, groups: Vec>, @@ -31,6 +35,7 @@ fn pick2_mut(self_: &mut [T], a: I, b: I) -> (&mut T, &mut T) { } impl UnifyLog { + /// Returns a new `UnifyLog` pub fn new() -> Self { UnifyLog { unified_vars: IndexVec::new(), @@ -39,6 +44,8 @@ impl UnifyLog { } } + /// Logs that `root` were unified with `other`. Allowing all variables that were unified with + /// root to be returned by `get` (if those variables are watched) pub fn unify(&mut self, undo_log: &mut impl UndoLogs>, root: T, other: T) { if !self.needs_log(other) { return; @@ -84,6 +91,8 @@ impl UnifyLog { } } + /// Returns the variables that `root` were unified with. The returned list may or may not + /// contain `root` itself. pub fn get(&self, root: T) -> &[T] { match self.unified_vars.get(root) { Some(group) => match self.groups.get(*group) { @@ -94,15 +103,19 @@ impl UnifyLog { } } + /// Returns true if `vid` is something that needs to be logged to a watcher pub fn needs_log(&self, vid: T) -> bool { !self.get(vid).is_empty() || self.reference_counts.get(vid).map_or(false, |c| *c != 0) } + /// Starts a watch on `index`. Any calls to `watch_variable` should be matched by call to + /// `unwatch_variable` when the watch is no longer needed pub fn watch_variable(&mut self, index: T) { self.reference_counts.ensure_contains_elem(index, || 0); self.reference_counts[index] += 1; } + /// Removes a watch on `index` pub fn unwatch_variable(&mut self, index: T) { self.reference_counts[index] -= 1; } diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 6e25fb30edc0e..95dda45c17285 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -1614,28 +1614,22 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { } } - pub fn drain_modifications( - &self, - offset: &WatcherOffset, - mut f: impl FnMut(TyOrConstInferVar), - ) { + pub fn notify_watcher(&self, offset: &WatcherOffset, mut f: impl FnMut(TyOrConstInferVar)) { let mut inner = self.inner.borrow_mut(); - inner - .type_variables() - .drain_modified_set(&offset.ty_offset, |v| f(TyOrConstInferVar::Ty(v))); + inner.type_variables().notify_watcher(&offset.ty_offset, |v| f(TyOrConstInferVar::Ty(v))); inner .int_unification_table() - .drain_modified_set(&offset.int_offset, |vid| f(TyOrConstInferVar::TyInt(vid))); + .notify_watcher(&offset.int_offset, |vid| f(TyOrConstInferVar::TyInt(vid))); inner .float_unification_table() - .drain_modified_set(&offset.float_offset, |vid| f(TyOrConstInferVar::TyFloat(vid))); + .notify_watcher(&offset.float_offset, |vid| f(TyOrConstInferVar::TyFloat(vid))); inner .const_unification_table() - .drain_modified_set(&offset.const_offset, |vid| f(TyOrConstInferVar::Const(vid))); + .notify_watcher(&offset.const_offset, |vid| f(TyOrConstInferVar::Const(vid))); } pub fn register_unify_watcher(&self) -> WatcherOffset { diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 91a3ed58f0533..0eef669fcf396 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -437,8 +437,8 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { self.eq_relations().clear_modified_set(); } - pub fn drain_modified_set(&mut self, offset: &ms::Offset, f: impl FnMut(ty::TyVid)) { - self.eq_relations().drain_modified_set(offset, f) + pub fn notify_watcher(&mut self, offset: &ms::Offset, f: impl FnMut(ty::TyVid)) { + self.eq_relations().notify_watcher(offset, f) } pub fn register_unify_watcher(&mut self) -> ms::Offset { diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index f40c63f8e7b3c..02b40718bd4ae 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -230,7 +230,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { if errors.is_empty() { Ok(()) } else { Err(errors) } })(); - if let Some(offset) = self.predicates.take_offset() { + if let Some(offset) = self.predicates.take_watcher_offset() { infcx.deregister_unify_watcher(offset); } result @@ -245,7 +245,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { } fn deregister(&mut self, infcx: &InferCtxt<'_, 'tcx>) { - if let Some(offset) = self.predicates.take_offset() { + if let Some(offset) = self.predicates.take_watcher_offset() { infcx.deregister_unify_watcher(offset); } } @@ -570,13 +570,13 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { } } - fn unblocked( + fn notify_unblocked( &self, offset: &WatcherOffset, f: impl FnMut(::Variable), ) { let infcx = self.selcx.infcx(); - infcx.drain_modifications(offset, f); + infcx.notify_watcher(offset, f); } fn register_variable_watcher(&self) -> WatcherOffset { From 795c8106a1b6add148aa909589fa4babfb570b53 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 6 Jun 2020 14:14:12 +0200 Subject: [PATCH 15/25] Restore clearing of stalled_on before process_obligation --- .../src/obligation_forest/mod.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 66fe3f5a37932..e0944f8f02eae 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -620,6 +620,29 @@ impl ObligationForest { continue; } + // One of the variables we stalled on unblocked us. If the node were blocked on other + // variables as well then remove those stalls. If the node is still stalled on one of + // those variables after `process_obligation` it will simply be added back to + // `self.stalled_on` + let stalled_on = node.obligation.stalled_on(); + if stalled_on.len() > 1 { + for var in stalled_on { + match self.stalled_on.entry(var.clone()) { + Entry::Vacant(_) => (), + Entry::Occupied(mut entry) => { + let nodes = entry.get_mut(); + if let Some(i) = nodes.iter().position(|x| *x == index) { + nodes.swap_remove(i); + } + if nodes.is_empty() { + processor.unwatch_variable(var.clone()); + entry.remove(); + } + } + } + } + } + // `processor.process_obligation` can modify the predicate within // `node.obligation`, and that predicate is the key used for // `self.active_cache`. This means that `self.active_cache` can get From 4dcb97343cba89142217fbbf09d61c2675025fdf Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sat, 18 Jul 2020 12:55:53 +0200 Subject: [PATCH 16/25] fix: Avoid dropping and re-registering offsets in the obligation forest This could in theory cause modified variables to be lost, causing typecheck failures. To make the deregistering easier a helper type to deregister on drop were added --- .../src/obligation_forest/mod.rs | 183 ++++++++++-------- .../src/infer/canonical/query_response.rs | 2 +- compiler/rustc_infer/src/traits/engine.rs | 103 +++++++++- .../rustc_trait_selection/src/autoderef.rs | 4 +- .../src/traits/chalk_fulfill.rs | 2 +- .../src/traits/codegen.rs | 4 +- .../src/traits/engine.rs | 50 ++++- .../src/traits/fulfill.rs | 6 +- compiler/rustc_traits/src/dropck_outlives.rs | 2 +- .../rustc_typeck/src/check/compare_method.rs | 8 +- src/test/ui/obligation-forest-bug.rs | 66 +++++++ 11 files changed, 329 insertions(+), 101 deletions(-) create mode 100644 src/test/ui/obligation-forest-bug.rs diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index e0944f8f02eae..114e59f75c964 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -216,6 +216,9 @@ pub struct ObligationForest { /// dropping this forest. /// watcher_offset: Option, + /// We do not want to process any further obligations after the offset has been deregistered as that could mean unified variables are lost, leading to typecheck failures. + /// So we mark this as done and panic if a caller tries to resume processing. + done: bool, /// Reusable vector for storing unblocked nodes whose watch should be removed. temp_unblocked_nodes: Vec, } @@ -448,6 +451,7 @@ impl ObligationForest { stalled_on_unknown: Default::default(), temp_unblocked_nodes: Default::default(), watcher_offset: None, + done: false, } } @@ -459,6 +463,7 @@ impl ObligationForest { /// Removes the watcher_offset, allowing it to be deregistered pub fn take_watcher_offset(&mut self) -> Option { + self.done = true; self.watcher_offset.take() } @@ -562,6 +567,7 @@ impl ObligationForest { let errors = self .pending_nodes .iter() + .filter(|&&index| self.nodes[index].state.get() == NodeState::Pending) .map(|&index| Error { error: error.clone(), backtrace: self.error_at(index) }) .collect(); @@ -574,7 +580,14 @@ impl ObligationForest { where F: Fn(&O) -> P, { - self.pending_nodes.iter().map(|&index| f(&self.nodes[index].obligation)).collect() + self.pending_nodes + .iter() + .filter_map(|&index| { + let node = &self.nodes[index]; + if node.state.get() == NodeState::Pending { Some(node) } else { None } + }) + .map(|node| f(&node.obligation)) + .collect() } fn insert_into_error_cache(&mut self, index: NodeIndex) { @@ -595,6 +608,7 @@ impl ObligationForest { OUT: OutcomeTrait>, { if self.watcher_offset.is_none() { + assert!(!self.done); self.watcher_offset = Some(processor.register_variable_watcher()); } let mut errors = vec![]; @@ -602,100 +616,105 @@ impl ObligationForest { self.unblock_nodes(processor); - let nodes = &self.nodes; - self.unblocked.extend( - self.stalled_on_unknown - .drain(..) - .map(|index| Unblocked { index, order: nodes[index].node_number }), - ); - while let Some(Unblocked { index, .. }) = self.unblocked.pop() { - // Skip any duplicates since we only need to processes the node once - if self.unblocked.peek().map(|u| u.index) == Some(index) { - continue; - } + let mut made_progress_this_iteration = true; + while made_progress_this_iteration { + made_progress_this_iteration = false; + let nodes = &self.nodes; + self.unblocked.extend( + self.stalled_on_unknown + .drain(..) + .map(|index| Unblocked { index, order: nodes[index].node_number }), + ); + while let Some(Unblocked { index, .. }) = self.unblocked.pop() { + // Skip any duplicates since we only need to processes the node once + if self.unblocked.peek().map(|u| u.index) == Some(index) { + continue; + } - let node = &mut self.nodes[index]; + let node = &mut self.nodes[index]; - if node.state.get() != NodeState::Pending { - continue; - } + if node.state.get() != NodeState::Pending { + continue; + } - // One of the variables we stalled on unblocked us. If the node were blocked on other - // variables as well then remove those stalls. If the node is still stalled on one of - // those variables after `process_obligation` it will simply be added back to - // `self.stalled_on` - let stalled_on = node.obligation.stalled_on(); - if stalled_on.len() > 1 { - for var in stalled_on { - match self.stalled_on.entry(var.clone()) { - Entry::Vacant(_) => (), - Entry::Occupied(mut entry) => { - let nodes = entry.get_mut(); - if let Some(i) = nodes.iter().position(|x| *x == index) { - nodes.swap_remove(i); - } - if nodes.is_empty() { - processor.unwatch_variable(var.clone()); - entry.remove(); + // One of the variables we stalled on unblocked us. If the node were blocked on other + // variables as well then remove those stalls. If the node is still stalled on one of + // those variables after `process_obligation` it will simply be added back to + // `self.stalled_on` + let stalled_on = node.obligation.stalled_on(); + if stalled_on.len() > 1 { + for var in stalled_on { + match self.stalled_on.entry(var.clone()) { + Entry::Vacant(_) => (), + Entry::Occupied(mut entry) => { + let nodes = entry.get_mut(); + if let Some(i) = nodes.iter().position(|x| *x == index) { + nodes.swap_remove(i); + } + if nodes.is_empty() { + processor.unwatch_variable(var.clone()); + entry.remove(); + } } } } } - } - // `processor.process_obligation` can modify the predicate within - // `node.obligation`, and that predicate is the key used for - // `self.active_cache`. This means that `self.active_cache` can get - // out of sync with `nodes`. It's not very common, but it does - // happen, and code in `compress` has to allow for it. - let before = node.obligation.as_cache_key(); - let result = processor.process_obligation(&mut node.obligation); - let after = node.obligation.as_cache_key(); - if before != after { - node.alternative_predicates.push(before); - } + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + let before = node.obligation.as_cache_key(); + let result = processor.process_obligation(&mut node.obligation); + let after = node.obligation.as_cache_key(); + if before != after { + node.alternative_predicates.push(before); + } - self.unblock_nodes(processor); - let node = &mut self.nodes[index]; - match result { - ProcessResult::Unchanged => { - let stalled_on = node.obligation.stalled_on(); - if stalled_on.is_empty() { - // We stalled but the variables that caused it are unknown so we run - // `index` again at the next opportunity - self.stalled_on_unknown.push(index); - } else { - // Register every variable that we stalled on - for var in stalled_on { - self.stalled_on - .entry(var.clone()) - .or_insert_with(|| { - processor.watch_variable(var.clone()); - Vec::new() - }) - .push(index); + self.unblock_nodes(processor); + let node = &mut self.nodes[index]; + match result { + ProcessResult::Unchanged => { + let stalled_on = node.obligation.stalled_on(); + if stalled_on.is_empty() { + // We stalled but the variables that caused it are unknown so we run + // `index` again at the next opportunity + self.stalled_on_unknown.push(index); + } else { + // Register every variable that we stalled on + for var in stalled_on { + self.stalled_on + .entry(var.clone()) + .or_insert_with(|| { + processor.watch_variable(var.clone()); + Vec::new() + }) + .push(index); + } } + // No change in state. } - // No change in state. - } - ProcessResult::Changed(children) => { - // We are not (yet) stalled. - stalled = false; - node.state.set(NodeState::Success); - self.success_or_waiting_nodes.push(index); - - for child in children { - let st = self.register_obligation_at(child, Some(index)); - if let Err(()) = st { - // Error already reported - propagate it - // to our node. - self.error_at(index); + ProcessResult::Changed(children) => { + made_progress_this_iteration = true; + // We are not (yet) stalled. + stalled = false; + node.state.set(NodeState::Success); + self.success_or_waiting_nodes.push(index); + + for child in children { + let st = self.register_obligation_at(child, Some(index)); + if let Err(()) = st { + // Error already reported - propagate it + // to our node. + self.error_at(index); + } } } - } - ProcessResult::Error(err) => { - stalled = false; - errors.push(Error { error: err, backtrace: self.error_at(index) }); + ProcessResult::Error(err) => { + stalled = false; + errors.push(Error { error: err, backtrace: self.error_at(index) }); + } } } } diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs index 93e19521893ef..67b89bbb06a2f 100644 --- a/compiler/rustc_infer/src/infer/canonical/query_response.rs +++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs @@ -122,7 +122,7 @@ impl<'cx, 'tcx> InferCtxt<'cx, 'tcx> { } // Anything left unselected *now* must be an ambiguity. - let ambig_errors = fulfill_cx.select_all_or_error(self).err().unwrap_or_else(Vec::new); + let ambig_errors = fulfill_cx.select_or_error(self).err().unwrap_or_else(Vec::new); debug!("ambig_errors = {:#?}", ambig_errors); let region_obligations = self.take_registered_region_obligations(); diff --git a/compiler/rustc_infer/src/traits/engine.rs b/compiler/rustc_infer/src/traits/engine.rs index 15c61587f6f10..2ce64335e7920 100644 --- a/compiler/rustc_infer/src/traits/engine.rs +++ b/compiler/rustc_infer/src/traits/engine.rs @@ -6,7 +6,7 @@ use rustc_middle::ty::{self, ToPredicate, Ty, WithConstness}; use super::FulfillmentError; use super::{ObligationCause, PredicateObligation}; -pub trait TraitEngine<'tcx>: 'tcx { +pub trait TraitEngine<'tcx> { fn normalize_projection_type( &mut self, infcx: &InferCtxt<'_, 'tcx>, @@ -44,20 +44,35 @@ pub trait TraitEngine<'tcx>: 'tcx { obligation: PredicateObligation<'tcx>, ); - fn select_all_or_error( + fn select_or_error( &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>>; + fn select_all_or_error( + mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> + where + Self: Sized, + { + let result = self.select_or_error(infcx); + self.deregister(infcx); + result + } + fn select_where_possible( &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>>; fn select_all_where_possible( - &mut self, + mut self, infcx: &InferCtxt<'_, 'tcx>, - ) -> Result<(), Vec>> { + ) -> Result<(), Vec>> + where + Self: Sized, + { let result = self.select_where_possible(infcx); self.deregister(infcx); result @@ -68,6 +83,86 @@ pub trait TraitEngine<'tcx>: 'tcx { fn pending_obligations(&self) -> Vec>; } +impl TraitEngine<'tcx> for Box +where + T: ?Sized + TraitEngine<'tcx>, +{ + fn normalize_projection_type( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + param_env: ty::ParamEnv<'tcx>, + projection_ty: ty::ProjectionTy<'tcx>, + cause: ObligationCause<'tcx>, + ) -> Ty<'tcx> { + T::normalize_projection_type(self, infcx, param_env, projection_ty, cause) + } + + fn register_bound( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + param_env: ty::ParamEnv<'tcx>, + ty: Ty<'tcx>, + def_id: DefId, + cause: ObligationCause<'tcx>, + ) { + T::register_bound(self, infcx, param_env, ty, def_id, cause) + } + + fn register_predicate_obligation( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + obligation: PredicateObligation<'tcx>, + ) { + T::register_predicate_obligation(self, infcx, obligation) + } + + fn select_or_error( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> { + T::select_or_error(self, infcx) + } + + fn select_all_or_error( + mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> + where + Self: Sized, + { + let result = self.select_or_error(infcx); + self.deregister(infcx); + result + } + + fn select_where_possible( + &mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> { + T::select_where_possible(self, infcx) + } + + fn select_all_where_possible( + mut self, + infcx: &InferCtxt<'_, 'tcx>, + ) -> Result<(), Vec>> + where + Self: Sized, + { + let result = self.select_where_possible(infcx); + self.deregister(infcx); + result + } + + fn deregister(&mut self, infcx: &InferCtxt<'_, 'tcx>) { + T::deregister(self, infcx) + } + + fn pending_obligations(&self) -> Vec> { + T::pending_obligations(self) + } +} + pub trait TraitEngineExt<'tcx> { fn register_predicate_obligations( &mut self, diff --git a/compiler/rustc_trait_selection/src/autoderef.rs b/compiler/rustc_trait_selection/src/autoderef.rs index 2a89b32769c67..4f6fadcdd78f3 100644 --- a/compiler/rustc_trait_selection/src/autoderef.rs +++ b/compiler/rustc_trait_selection/src/autoderef.rs @@ -153,14 +153,16 @@ impl<'a, 'tcx> Autoderef<'a, 'tcx> { ), cause, ); - if let Err(e) = fulfillcx.select_all_where_possible(&self.infcx) { + if let Err(e) = fulfillcx.select_where_possible(&self.infcx) { // This shouldn't happen, except for evaluate/fulfill mismatches, // but that's not a reason for an ICE (`predicate_may_hold` is conservative // by design). debug!("overloaded_deref_ty: encountered errors {:?} while fulfilling", e); + fulfillcx.deregister(&self.infcx); return None; } let obligations = fulfillcx.pending_obligations(); + fulfillcx.deregister(&self.infcx); debug!("overloaded_deref_ty({:?}) = ({:?}, {:?})", ty, normalized_ty, obligations); self.state.obligations.extend(obligations); diff --git a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs index adc8ae5908656..992aa33210337 100644 --- a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs @@ -42,7 +42,7 @@ impl TraitEngine<'tcx> for FulfillmentContext<'tcx> { self.obligations.insert(obligation); } - fn select_all_or_error( + fn select_or_error( &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>> { diff --git a/compiler/rustc_trait_selection/src/traits/codegen.rs b/compiler/rustc_trait_selection/src/traits/codegen.rs index 3cb6ec8626186..8af576d4ea3fd 100644 --- a/compiler/rustc_trait_selection/src/traits/codegen.rs +++ b/compiler/rustc_trait_selection/src/traits/codegen.rs @@ -89,7 +89,7 @@ pub fn codegen_fulfill_obligation<'tcx>( debug!("fulfill_obligation: register_predicate_obligation {:?}", predicate); fulfill_cx.register_predicate_obligation(&infcx, predicate); }); - let impl_source = drain_fulfillment_cx_or_panic(&infcx, &mut fulfill_cx, &impl_source); + let impl_source = drain_fulfillment_cx_or_panic(&infcx, fulfill_cx, &impl_source); info!("Cache miss: {:?} => {:?}", trait_ref, impl_source); Ok(impl_source) @@ -109,7 +109,7 @@ pub fn codegen_fulfill_obligation<'tcx>( /// the complete picture of the type. fn drain_fulfillment_cx_or_panic( infcx: &InferCtxt<'_, 'tcx>, - fulfill_cx: &mut FulfillmentContext<'tcx>, + fulfill_cx: FulfillmentContext<'tcx>, result: &T, ) -> T where diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs index 4d4778869794b..18fe0cd4ebc88 100644 --- a/compiler/rustc_trait_selection/src/traits/engine.rs +++ b/compiler/rustc_trait_selection/src/traits/engine.rs @@ -1,13 +1,17 @@ +use rustc_infer::infer::InferCtxt; use rustc_middle::ty::TyCtxt; use super::TraitEngine; use super::{ChalkFulfillmentContext, FulfillmentContext}; -pub trait TraitEngineExt<'tcx> { +pub trait TraitEngineExt<'tcx>: TraitEngine<'tcx> { fn new(tcx: TyCtxt<'tcx>) -> Box; + fn new_with_deregister<'cx>( + infcx: &'cx InferCtxt<'cx, 'tcx>, + ) -> DeregisterOnDropEngine<'cx, 'tcx, Box>; } -impl<'tcx> TraitEngineExt<'tcx> for dyn TraitEngine<'tcx> { +impl<'tcx> TraitEngineExt<'tcx> for dyn TraitEngine<'tcx> + 'tcx { fn new(tcx: TyCtxt<'tcx>) -> Box { if tcx.sess.opts.debugging_opts.chalk { Box::new(ChalkFulfillmentContext::new()) @@ -15,4 +19,46 @@ impl<'tcx> TraitEngineExt<'tcx> for dyn TraitEngine<'tcx> { Box::new(FulfillmentContext::new()) } } + fn new_with_deregister<'cx>( + infcx: &'cx InferCtxt<'cx, 'tcx>, + ) -> DeregisterOnDropEngine<'cx, 'tcx, Box> { + DeregisterOnDropEngine { engine: Self::new(infcx.tcx), infcx } + } +} + +/// Deregisters any variable watches on drop automatically +pub struct DeregisterOnDropEngine<'cx, 'tcx, T> +where + T: TraitEngine<'tcx>, +{ + infcx: &'cx InferCtxt<'cx, 'tcx>, + engine: T, +} + +impl<'tcx, T> Drop for DeregisterOnDropEngine<'_, 'tcx, T> +where + T: TraitEngine<'tcx>, +{ + fn drop(&mut self) { + self.engine.deregister(self.infcx) + } +} + +impl<'tcx, T> std::ops::Deref for DeregisterOnDropEngine<'_, 'tcx, T> +where + T: TraitEngine<'tcx>, +{ + type Target = T; + fn deref(&self) -> &Self::Target { + &self.engine + } +} + +impl<'tcx, T> std::ops::DerefMut for DeregisterOnDropEngine<'_, 'tcx, T> +where + T: TraitEngine<'tcx>, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.engine + } } diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 02b40718bd4ae..3305d912c6c3b 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -214,7 +214,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { .register_obligation(PendingPredicateObligation { obligation, stalled_on: vec![] }); } - fn select_all_or_error( + fn select_or_error( &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>> { @@ -424,8 +424,8 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { None => { // None means that both are unresolved. pending_obligation.stalled_on.clear(); - pending_obligation.stalled_on.push(root_ty_or_const_var(subtype.a)); - pending_obligation.stalled_on.push(root_ty_or_const_var(subtype.b)); + pending_obligation.stalled_on.push(ty_or_const_var(subtype.a)); + pending_obligation.stalled_on.push(ty_or_const_var(subtype.b)); ProcessResult::Unchanged } Some(Ok(ok)) => ProcessResult::Changed(mk_pending(ok.obligations)), diff --git a/compiler/rustc_traits/src/dropck_outlives.rs b/compiler/rustc_traits/src/dropck_outlives.rs index 6cffa6d02a4e3..2d988cbcf1e54 100644 --- a/compiler/rustc_traits/src/dropck_outlives.rs +++ b/compiler/rustc_traits/src/dropck_outlives.rs @@ -75,7 +75,7 @@ fn dropck_outlives<'tcx>( // Set used to detect infinite recursion. let mut ty_set = FxHashSet::default(); - let mut fulfill_cx = TraitEngine::new(infcx.tcx); + let mut fulfill_cx = TraitEngine::new_with_deregister(infcx); let cause = ObligationCause::dummy(); let mut constraints = DtorckConstraint::empty(); diff --git a/compiler/rustc_typeck/src/check/compare_method.rs b/compiler/rustc_typeck/src/check/compare_method.rs index 4acc7451a2131..7141c9a4fb183 100644 --- a/compiler/rustc_typeck/src/check/compare_method.rs +++ b/compiler/rustc_typeck/src/check/compare_method.rs @@ -320,7 +320,7 @@ fn compare_predicate_entailment<'tcx>( // Check that all obligations are satisfied by the implementation's // version. - if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_all_or_error(&infcx) { + if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_or_error(&infcx) { infcx.report_fulfillment_errors(errors, None, false); return Err(ErrorReported); } @@ -1028,7 +1028,7 @@ crate fn compare_const_impl<'tcx>( // Check that all obligations are satisfied by the implementation's // version. - if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_all_or_error(&infcx) { + if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_or_error(&infcx) { infcx.report_fulfillment_errors(errors, None, false); return; } @@ -1144,7 +1144,7 @@ fn compare_type_predicate_entailment<'tcx>( // Check that all obligations are satisfied by the implementation's // version. - if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_all_or_error(&infcx) { + if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_or_error(&infcx) { infcx.report_fulfillment_errors(errors, None, false); return Err(ErrorReported); } @@ -1272,7 +1272,7 @@ pub fn check_type_bounds<'tcx>( // Check that all obligations are satisfied by the implementation's // version. - if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_all_or_error(&infcx) { + if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_or_error(&infcx) { infcx.report_fulfillment_errors(errors, None, false); return Err(ErrorReported); } diff --git a/src/test/ui/obligation-forest-bug.rs b/src/test/ui/obligation-forest-bug.rs new file mode 100644 index 0000000000000..f8603d63b0514 --- /dev/null +++ b/src/test/ui/obligation-forest-bug.rs @@ -0,0 +1,66 @@ +// run-pass +#![allow(dead_code)] + +use std::marker::PhantomData; + +pub trait Consumer { + type Result; +} + +pub trait IndexedParallelIterator: ExactSizeIterator { + type Item; +} + +pub struct CollectConsumer<'c, T: Send> { + target: &'c mut [T], +} + +impl<'c, T: Send + 'c> Consumer for CollectConsumer<'c, T> { + type Result = CollectResult<'c, T>; +} + +pub struct CollectResult<'c, T> { + start: *mut T, + len: usize, + invariant_lifetime: PhantomData<&'c mut &'c mut [T]>, +} + +unsafe impl<'c, T> Send for CollectResult<'c, T> where T: Send {} + +pub fn unzip_indexed(_: I, _: CA) -> CA::Result +where + I: IndexedParallelIterator, + CA: Consumer, +{ + unimplemented!() +} + +struct Collect<'c, T: Send> { + vec: &'c mut Vec, + len: usize, +} + +pub fn unzip_into_vecs(pi: I, left: &mut Vec, _: &mut Vec) +where + I: IndexedParallelIterator, + A: Send, + B: Send, +{ + let len = pi.len(); + Collect::new(left, len).with_consumer(|left_consumer| unzip_indexed(pi, left_consumer)); +} + +impl<'c, T: Send + 'c> Collect<'c, T> { + fn new(vec: &'c mut Vec, len: usize) -> Self { + Collect { vec, len } + } + + fn with_consumer(self, _: F) + where + F: FnOnce(CollectConsumer) -> CollectResult, + { + unimplemented!() + } +} + +fn main() {} From c49182b71a46a1120f27c3d675e3926ee48367ff Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sun, 9 Aug 2020 14:48:55 +0200 Subject: [PATCH 17/25] Ensure that predicates that change are marked as done The alternative_predicates does not appear necessary, as long as the predicate is marked done. --- .../src/obligation_forest/mod.rs | 31 +++---------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 114e59f75c964..3636a445e726e 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -252,10 +252,6 @@ struct Node { obligation: O, state: Cell, - /// A predicate (and its key) can change during processing. If it does we need to register the - /// old predicate so that we can remove or mark it as done if this node errors or is done. - alternative_predicates: Vec, - /// Obligations that depend on this obligation for their completion. They /// must all be in a non-pending state. dependents: Vec, @@ -291,7 +287,6 @@ where Node { obligation, state: Cell::new(NodeState::Pending), - alternative_predicates: vec![], dependents: if let Some(parent_index) = parent { vec![parent_index] } else { vec![] }, reverse_dependents: vec![], has_parent: parent.is_some(), @@ -316,7 +311,6 @@ where self.state ); self.state.set(NodeState::Pending); - self.alternative_predicates.clear(); self.dependents.clear(); self.reverse_dependents.clear(); if let Some(parent_index) = parent { @@ -625,6 +619,7 @@ impl ObligationForest { .drain(..) .map(|index| Unblocked { index, order: nodes[index].node_number }), ); + while let Some(Unblocked { index, .. }) = self.unblocked.pop() { // Skip any duplicates since we only need to processes the node once if self.unblocked.peek().map(|u| u.index) == Some(index) { @@ -665,12 +660,7 @@ impl ObligationForest { // `self.active_cache`. This means that `self.active_cache` can get // out of sync with `nodes`. It's not very common, but it does // happen, and code in `compress` has to allow for it. - let before = node.obligation.as_cache_key(); let result = processor.process_obligation(&mut node.obligation); - let after = node.obligation.as_cache_key(); - if before != after { - node.alternative_predicates.push(before); - } self.unblock_nodes(processor); let node = &mut self.nodes[index]; @@ -940,16 +930,10 @@ impl ObligationForest { match node.state.get() { NodeState::Done => { // Mark as done - if let Some(opt) = self.active_cache.get_mut(&node.obligation.as_cache_key()) { - *opt = CacheState::Done; - } - // If the node's predicate changed at some point we mark all its alternate - // predicates as done as well - for alt in &node.alternative_predicates { - if let Some(opt) = self.active_cache.get_mut(alt) { - *opt = CacheState::Done; - } - } + *self + .active_cache + .entry(node.obligation.as_cache_key()) + .or_insert(CacheState::Done) = CacheState::Done; if do_completed == DoCompleted::Yes { // Extract the success stories. @@ -965,11 +949,6 @@ impl ObligationForest { // tests must come up with a different type on every type error they // check against. self.active_cache.remove(&node.obligation.as_cache_key()); - // If the node's predicate changed at some point we remove all its alternate - // predicates as well - for alt in &node.alternative_predicates { - self.active_cache.remove(alt); - } self.insert_into_error_cache(index); self.dead_nodes.push(index); } From 8e6ca044794c68fd9d3215c968ebe323d1094903 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Tue, 11 Aug 2020 12:50:21 +0200 Subject: [PATCH 18/25] Restore alternative predicates for obligation forest Extracted a test from the object crate found while running on CI (did not exist the in the actual test suite). --- .../src/obligation_forest/mod.rs | 22 +++++++++++++++ compiler/rustc_middle/src/infer/unify_key.rs | 2 +- compiler/rustc_middle/src/ty/sty.rs | 12 ++------- src/test/ui/obligation-forest-bug-69218-2.rs | 27 +++++++++++++++++++ ...-bug.rs => obligation-forest-bug-69218.rs} | 0 5 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 src/test/ui/obligation-forest-bug-69218-2.rs rename src/test/ui/{obligation-forest-bug.rs => obligation-forest-bug-69218.rs} (100%) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 3636a445e726e..c5d188d068c94 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -252,6 +252,10 @@ struct Node { obligation: O, state: Cell, + /// A predicate (and its key) can change during processing. If it does we need to register the + /// old predicate so that we can remove or mark it as done if this node errors or is done. + alternative_predicates: Vec, + /// Obligations that depend on this obligation for their completion. They /// must all be in a non-pending state. dependents: Vec, @@ -287,6 +291,7 @@ where Node { obligation, state: Cell::new(NodeState::Pending), + alternative_predicates: vec![], dependents: if let Some(parent_index) = parent { vec![parent_index] } else { vec![] }, reverse_dependents: vec![], has_parent: parent.is_some(), @@ -311,6 +316,7 @@ where self.state ); self.state.set(NodeState::Pending); + self.alternative_predicates.clear(); self.dependents.clear(); self.reverse_dependents.clear(); if let Some(parent_index) = parent { @@ -660,7 +666,12 @@ impl ObligationForest { // `self.active_cache`. This means that `self.active_cache` can get // out of sync with `nodes`. It's not very common, but it does // happen, and code in `compress` has to allow for it. + let before = node.obligation.as_cache_key(); let result = processor.process_obligation(&mut node.obligation); + let after = node.obligation.as_cache_key(); + if before != after { + node.alternative_predicates.push(before); + } self.unblock_nodes(processor); let node = &mut self.nodes[index]; @@ -934,6 +945,12 @@ impl ObligationForest { .active_cache .entry(node.obligation.as_cache_key()) .or_insert(CacheState::Done) = CacheState::Done; + // If the node's predicate changed at some point we mark all its alternate + // predicates as done as well + for alt in node.alternative_predicates.drain(..) { + *self.active_cache.entry(alt).or_insert(CacheState::Done) = + CacheState::Done; + } if do_completed == DoCompleted::Yes { // Extract the success stories. @@ -949,6 +966,11 @@ impl ObligationForest { // tests must come up with a different type on every type error they // check against. self.active_cache.remove(&node.obligation.as_cache_key()); + // If the node's predicate changed at some point we remove all its alternate + // predicates as well + for alt in &node.alternative_predicates { + self.active_cache.remove(alt); + } self.insert_into_error_cache(index); self.dead_nodes.push(index); } diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs index 416677745c1bd..431651dc0ea80 100644 --- a/compiler/rustc_middle/src/infer/unify_key.rs +++ b/compiler/rustc_middle/src/infer/unify_key.rs @@ -160,7 +160,7 @@ pub struct ConstVarValue<'tcx> { pub val: ConstVariableValue<'tcx>, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] pub struct ConstVidEqKey<'tcx> { pub vid: ty::ConstVid, pub phantom: PhantomData<&'tcx ()>, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 802e5d137ae91..f791696bf16d9 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1040,11 +1040,7 @@ impl Binder { where T: TypeFoldable<'tcx>, { - if self.0.has_escaping_bound_vars() { - None - } else { - Some(self.skip_binder()) - } + if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) } } /// Given two things that have the same binder level, @@ -1927,11 +1923,7 @@ impl<'tcx> TyS<'tcx> { #[inline] pub fn is_phantom_data(&self) -> bool { - if let Adt(def, _) = self.kind() { - def.is_phantom_data() - } else { - false - } + if let Adt(def, _) = self.kind() { def.is_phantom_data() } else { false } } #[inline] diff --git a/src/test/ui/obligation-forest-bug-69218-2.rs b/src/test/ui/obligation-forest-bug-69218-2.rs new file mode 100644 index 0000000000000..3ae2fdce930a4 --- /dev/null +++ b/src/test/ui/obligation-forest-bug-69218-2.rs @@ -0,0 +1,27 @@ +// run-pass +#![allow(dead_code)] + +use std::borrow::Cow; + +pub type Result = std::result::Result; + +pub struct CompressedData<'data> { + pub format: CompressionFormat, + pub data: &'data [u8], +} + +pub enum CompressionFormat { + None, + Unknown, +} + +impl<'data> CompressedData<'data> { + pub fn decompress(self) -> Result> { + match self.format { + CompressionFormat::None => Ok(Cow::Borrowed(self.data)), + _ => Err("Unsupported compressed data."), + } + } +} + +fn main() {} diff --git a/src/test/ui/obligation-forest-bug.rs b/src/test/ui/obligation-forest-bug-69218.rs similarity index 100% rename from src/test/ui/obligation-forest-bug.rs rename to src/test/ui/obligation-forest-bug-69218.rs From dee05217da6f70360417f33ac84558e5904ba48f Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sun, 27 Sep 2020 02:10:12 +0200 Subject: [PATCH 19/25] Fix rebase --- .../rustc_data_structures/src/logged_unification_table.rs | 5 +++++ compiler/rustc_infer/src/infer/combine.rs | 8 +++----- compiler/rustc_typeck/src/check/mod.rs | 7 +++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index 531fb705a06e2..717763a69c149 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -121,6 +121,11 @@ where self.relations().find(vid) } + pub fn unioned(&mut self, l: I, r: I) -> bool { + let mut relations = self.relations(); + relations.find(l) == relations.find(r) + } + pub fn unify_var_value( &mut self, vid: I, diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs index 9277615f12dd2..06aeab874efbb 100644 --- a/compiler/rustc_infer/src/infer/combine.rs +++ b/compiler/rustc_infer/src/infer/combine.rs @@ -229,8 +229,6 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> { target_vid: ty::ConstVid, ct: &'tcx ty::Const<'tcx>, vid_is_expected: bool, - vid: ty::ConstVid, - value: &'tcx ty::Const<'tcx>, ) -> RelateResult<'tcx, &'tcx ty::Const<'tcx>> { let (for_universe, span) = { let mut inner = self.inner.borrow_mut(); @@ -799,7 +797,7 @@ struct ConstInferUnifier<'cx, 'tcx> { /// The vid of the const variable that is in the process of being /// instantiated; if we find this within the const we are folding, /// that means we would have created a cyclic const. - target_vid: ty::ConstVid<'tcx>, + target_vid: ty::ConstVid, } // We use `TypeRelation` here to propagate `RelateResult` upwards. @@ -934,7 +932,7 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> { // an inference variable which is unioned with `target_vid`. // // Not doing so can easily result in stack overflows. - if variable_table.unioned(self.target_vid, vid) { + if variable_table.unioned(self.target_vid.into(), vid) { return Err(TypeError::CyclicConst(c)); } @@ -949,7 +947,7 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> { origin: var_value.origin, val: ConstVariableValue::Unknown { universe: self.for_universe }, }); - Ok(self.tcx().mk_const_var(new_var_id, c.ty)) + Ok(self.tcx().mk_const_var(new_var_id.vid, c.ty)) } } } diff --git a/compiler/rustc_typeck/src/check/mod.rs b/compiler/rustc_typeck/src/check/mod.rs index 02b52bbd355fa..169ad0df3a5c9 100644 --- a/compiler/rustc_typeck/src/check/mod.rs +++ b/compiler/rustc_typeck/src/check/mod.rs @@ -1136,6 +1136,13 @@ impl ItemLikeVisitor<'tcx> for CheckItemTypesVisitor<'tcx> { fn visit_impl_item(&mut self, _: &'tcx hir::ImplItem<'tcx>) {} } +fn typeck_item_bodies(tcx: TyCtxt<'_>, crate_num: CrateNum) { + debug_assert!(crate_num == LOCAL_CRATE); + tcx.par_body_owners(|body_owner_def_id| { + tcx.ensure().typeck(body_owner_def_id); + }); +} + fn fatally_break_rust(sess: &Session) { let handler = sess.diagnostic(); handler.span_bug_no_panic( From a5c43707b99801d384377b4d42deeec51f3bdd2b Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Fri, 16 Oct 2020 18:54:45 +0200 Subject: [PATCH 20/25] test --- .../src/obligation_forest/mod.rs | 178 ++++++++++++++---- .../src/traits/fulfill.rs | 43 +++++ 2 files changed, 180 insertions(+), 41 deletions(-) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index c5d188d068c94..b97b48b6536cf 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -104,6 +104,11 @@ pub trait ObligationProcessor { type Obligation: ForestObligation; type Error: Debug; + fn checked_process_obligation( + &mut self, + obligation: &mut Self::Obligation, + ) -> ProcessResult; + fn process_obligation( &mut self, obligation: &mut Self::Obligation, @@ -383,49 +388,18 @@ enum NodeState { Error, } -/// This trait allows us to have two different Outcome types: -/// - the normal one that does as little as possible -/// - one for tests that does some additional work and checking -pub trait OutcomeTrait { - type Error; - type Obligation; - - fn new() -> Self; - fn mark_not_stalled(&mut self); - fn is_stalled(&self) -> bool; - fn record_completed(&mut self, outcome: &Self::Obligation); - fn record_error(&mut self, error: Self::Error); -} - #[derive(Debug)] pub struct Outcome { /// Backtrace of obligations that were found to be in error. pub errors: Vec>, } -impl OutcomeTrait for Outcome { - type Error = Error; - type Obligation = O; - - fn new() -> Self { - Self { stalled: true, errors: vec![] } - } - - fn mark_not_stalled(&mut self) { - self.stalled = false; - } - - fn is_stalled(&self) -> bool { - self.stalled - } - - fn record_completed(&mut self, _outcome: &Self::Obligation) { - // do nothing - } - - fn record_error(&mut self, error: Self::Error) { - self.errors.push(error) - } +/// Should `process_obligations` compute the `Outcome::completed` field of its +/// result? +#[derive(PartialEq, Copy, Clone)] +pub enum DoCompleted { + No, + Yes, } #[derive(Debug, PartialEq, Eq)] @@ -602,14 +576,18 @@ impl ObligationForest { /// be called in a loop until `outcome.stalled` is false. /// /// This _cannot_ be unrolled (presently, at least). - pub fn process_obligations(&mut self, processor: &mut P) -> OUT + pub fn process_obligations

(&mut self, processor: &mut P) -> Outcome where P: ObligationProcessor, - OUT: OutcomeTrait>, { if self.watcher_offset.is_none() { assert!(!self.done); - self.watcher_offset = Some(processor.register_variable_watcher()); + if self.nodes.len() > 100 { + self.watcher_offset = Some(processor.register_variable_watcher()); + } + if let Some(outcome) = self.process_obligations_simple(processor, do_completed) { + return outcome; + } } let mut errors = vec![]; let mut stalled = true; @@ -713,6 +691,7 @@ impl ObligationForest { } } ProcessResult::Error(err) => { + made_progress_this_iteration = true; stalled = false; errors.push(Error { error: err, backtrace: self.error_at(index) }); } @@ -732,10 +711,127 @@ impl ObligationForest { self.mark_successes(); self.process_cycles(processor); let completed = self.compress(do_completed); - Outcome { completed, errors } } + fn process_obligations_simple

( + &mut self, + processor: &mut P, + do_completed: DoCompleted, + ) -> Option> + where + P: ObligationProcessor, + { + let mut errors = vec![]; + let mut stalled = true; + + // Note that the loop body can append new nodes, and those new nodes + // will then be processed by subsequent iterations of the loop. + // + // We can't use an iterator for the loop because `self.nodes` is + // appended to and the borrow checker would complain. We also can't use + // `for index in 0..self.nodes.len() { ... }` because the range would + // be computed with the initial length, and we would miss the appended + // nodes. Therefore we use a `while` loop. + loop { + let mut i = 0; + let mut made_progress_this_iteration = false; + while let Some(&index) = self.pending_nodes.get(i) { + let node = &mut self.nodes[index]; + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + if node.state.get() != NodeState::Pending { + i += 1; + continue; + } + + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + let before = node.obligation.as_cache_key(); + let result = processor.checked_process_obligation(&mut node.obligation); + let after = node.obligation.as_cache_key(); + if before != after { + node.alternative_predicates.push(before); + } + + match result { + ProcessResult::Unchanged => { + // No change in state. + if self.watcher_offset.is_some() { + let stalled_on = node.obligation.stalled_on(); + if stalled_on.is_empty() { + // We stalled but the variables that caused it are unknown so we run + // `index` again at the next opportunity + self.stalled_on_unknown.push(index); + } else { + // Register every variable that we stalled on + for var in stalled_on { + self.stalled_on + .entry(var.clone()) + .or_insert_with(|| { + processor.watch_variable(var.clone()); + Vec::new() + }) + .push(index); + } + } + } + } + ProcessResult::Changed(children) => { + // We are not (yet) stalled. + stalled = false; + node.state.set(NodeState::Success); + made_progress_this_iteration = true; + self.success_or_waiting_nodes.push(index); + + for child in children { + let st = self.register_obligation_at(child, Some(index)); + if let Err(()) = st { + // Error already reported - propagate it + // to our node. + self.error_at(index); + } + } + } + ProcessResult::Error(err) => { + stalled = false; + errors.push(Error { error: err, backtrace: self.error_at(index) }); + } + } + i += 1; + } + + if stalled { + // There's no need to perform marking, cycle processing and compression when nothing + // changed. + return Some(Outcome { + completed: if do_completed == DoCompleted::Yes { Some(vec![]) } else { None }, + errors, + }); + } + + if !made_progress_this_iteration { + break; + } + + if self.watcher_offset.is_some() { + return None; + } + + self.mark_successes(); + self.process_cycles(processor); + self.compress(do_completed); + } + + Some(Outcome { completed: None, errors }) + } + /// Checks which nodes have been unblocked since the last time this was called. All nodes that /// were unblocked are added to the `unblocked` queue and all watches associated with the /// variables blocking those nodes are deregistered (since they are now instantiated, they will diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 3305d912c6c3b..eb952fcdbab70 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -278,6 +278,49 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { /// This is always inlined, despite its size, because it has a single /// callsite and it is called *very* frequently. #[inline(always)] + fn checked_process_obligation( + &mut self, + pending_obligation: &mut Self::Obligation, + ) -> ProcessResult { + // If we were stalled on some unresolved variables, first check whether + // any of them have been resolved; if not, don't bother doing more work + // yet. + let change = match pending_obligation.stalled_on.len() { + // Match arms are in order of frequency, which matters because this + // code is so hot. 1 and 0 dominate; 2+ is fairly rare. + 1 => { + let infer_var = pending_obligation.stalled_on[0]; + self.selcx.infcx().ty_or_const_infer_var_changed(infer_var) + } + 0 => { + // In this case we haven't changed, but wish to make a change. + true + } + _ => { + // This `for` loop was once a call to `all()`, but this lower-level + // form was a perf win. See #64545 for details. + (|| { + for &infer_var in &pending_obligation.stalled_on { + if self.selcx.infcx().ty_or_const_infer_var_changed(infer_var) { + return true; + } + } + false + })() + } + }; + if !change { + debug!( + "process_predicate: pending obligation {:?} still stalled on {:?}", + self.selcx.infcx().resolve_vars_if_possible(&pending_obligation.obligation), + pending_obligation.stalled_on + ); + return ProcessResult::Unchanged; + } + + self.process_obligation(pending_obligation) + } + fn process_obligation( &mut self, pending_obligation: &mut Self::Obligation, From cca4b127a64e8ba7c053b7bf2f536c4f27fef668 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Fri, 16 Oct 2020 19:41:38 +0200 Subject: [PATCH 21/25] a --- .../src/traits/fulfill.rs | 30 +++++++------------ compiler/rustc_typeck/src/check/check.rs | 3 +- .../rustc_typeck/src/check/fn_ctxt/_impl.rs | 2 +- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index eb952fcdbab70..ae5539c5e143c 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -218,22 +218,16 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> { &mut self, infcx: &InferCtxt<'_, 'tcx>, ) -> Result<(), Vec>> { - let result = (|| { - self.select_where_possible(infcx)?; - - let errors: Vec<_> = self - .predicates - .to_errors(CodeAmbiguity) - .into_iter() - .map(to_fulfillment_error) - .collect(); - - if errors.is_empty() { Ok(()) } else { Err(errors) } - })(); - if let Some(offset) = self.predicates.take_watcher_offset() { - infcx.deregister_unify_watcher(offset); - } - result + self.select_where_possible(infcx)?; + + let errors: Vec<_> = self + .predicates + .to_errors(CodeAmbiguity) + .into_iter() + .map(to_fulfillment_error) + .collect(); + + if errors.is_empty() { Ok(()) } else { Err(errors) } } fn select_where_possible( @@ -543,9 +537,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { stalled_on.extend( substs .types() - .filter_map(|arg| { - TyOrConstInferVar::maybe_from_generic_arg(arg) - }) + .filter_map(|ty| TyOrConstInferVar::maybe_from_ty(ty)) .map(|ty| infcx.root_ty_or_const(ty)), ); Err(ErrorHandled::TooGeneric) diff --git a/compiler/rustc_typeck/src/check/check.rs b/compiler/rustc_typeck/src/check/check.rs index 8f2537404c5cc..03a160383474d 100644 --- a/compiler/rustc_typeck/src/check/check.rs +++ b/compiler/rustc_typeck/src/check/check.rs @@ -11,6 +11,7 @@ use rustc_hir::lang_items::LangItem; use rustc_hir::{ItemKind, Node}; use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt}; +use rustc_infer::traits::TraitEngine; use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::util::{Discr, IntTypeExt, Representability}; @@ -616,7 +617,7 @@ fn check_opaque_meets_bounds<'tcx>( // Check that all obligations are satisfied by the implementation's // version. - if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_all_or_error(&infcx) { + if let Err(ref errors) = inh.fulfillment_cx.borrow_mut().select_or_error(&infcx) { infcx.report_fulfillment_errors(errors, None, false); } diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs index 44ca6ed4f7965..2ca1ed2a9d91d 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs @@ -702,7 +702,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { fallback_has_occurred: bool, mutate_fullfillment_errors: impl Fn(&mut Vec>), ) { - let result = self.fulfillment_cx.borrow_mut().select_all_where_possible(self); + let result = self.fulfillment_cx.borrow_mut().select_where_possible(self); if let Err(mut errors) = result { mutate_fullfillment_errors(&mut errors); self.report_fulfillment_errors(&errors, self.inh.body_id, fallback_has_occurred); From 929b4a96a0dda2aab9b92953713ad7e5f8d166c7 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sun, 18 Oct 2020 09:51:46 +0200 Subject: [PATCH 22/25] a --- .../rustc_data_structures/src/obligation_forest/mod.rs | 10 ++++++++-- .../src/obligation_forest/tests.rs | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index b97b48b6536cf..36d6a831287d6 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -733,6 +733,7 @@ impl ObligationForest { // `for index in 0..self.nodes.len() { ... }` because the range would // be computed with the initial length, and we would miss the appended // nodes. Therefore we use a `while` loop. + let mut completed = vec![]; loop { let mut i = 0; let mut made_progress_this_iteration = false; @@ -826,10 +827,15 @@ impl ObligationForest { self.mark_successes(); self.process_cycles(processor); - self.compress(do_completed); + if let Some(mut c) = self.compress(do_completed) { + completed.append(&mut c); + } } - Some(Outcome { completed: None, errors }) + Some(Outcome { + completed: if do_completed == DoCompleted::Yes { Some(completed) } else { None }, + errors, + }) } /// Checks which nodes have been unblocked since the last time this was called. All nodes that diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index f9b3c4dc7930e..0a0a8a4b11296 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -80,6 +80,13 @@ where type Obligation = O; type Error = E; + fn checked_process_obligation( + &mut self, + obligation: &mut Self::Obligation, + ) -> ProcessResult { + (self.process_obligation)(obligation) + } + fn process_obligation( &mut self, obligation: &mut Self::Obligation, From 22c5b9403a75a7526bff23ba06a61d4129f21e2c Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Mon, 26 Oct 2020 15:43:10 +0100 Subject: [PATCH 23/25] a --- Cargo.lock | 476 +++++------------- .../src/obligation_forest/mod.rs | 18 +- .../src/traits/fulfill.rs | 2 +- 3 files changed, 152 insertions(+), 344 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d9ffb49b230c..65d20190c0db5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,8 +133,6 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" version = "0.3.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -297,7 +295,7 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" [[package]] name = "cargo" -version = "0.49.0" +version = "0.50.0" dependencies = [ "anyhow", "atty", @@ -308,11 +306,11 @@ dependencies = [ "clap", "core-foundation", "crates-io", - "crossbeam-utils 0.7.2", + "crossbeam-utils 0.8.0", "crypto-hash", "curl", "curl-sys", - "env_logger 0.7.1", + "env_logger 0.8.1", "filetime", "flate2", "fwdansi", @@ -688,6 +686,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -821,6 +825,18 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "const_fn", + "lazy_static", +] + [[package]] name = "crypto-hash" version = "0.3.4" @@ -1029,6 +1045,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54532e3223c5af90a6a757c90b5c5521564b07e5e7a958681bcd2afad421cdcd" +dependencies = [ + "atty", + "humantime 2.0.1", + "log", + "regex", + "termcolor", +] + [[package]] name = "error_index_generator" version = "0.0.0" @@ -2677,13 +2706,13 @@ dependencies = [ "lazy_static", "log", "rls-span", - "rustc-ap-rustc_ast 679.0.0", - "rustc-ap-rustc_ast_pretty 679.0.0", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_errors 679.0.0", - "rustc-ap-rustc_parse 679.0.0", - "rustc-ap-rustc_session 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast_pretty", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_parse", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", ] [[package]] @@ -2991,43 +3020,16 @@ dependencies = [ "mdbook", ] -[[package]] -name = "rustc-ap-rustc_arena" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2958af0d6e0458434a25cd3a96f6e19f24f71bf50b900add520dec52e212866b" -dependencies = [ - "rustc-ap-rustc_data_structures 677.0.0", - "smallvec 1.4.2", -] - [[package]] name = "rustc-ap-rustc_arena" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8e941a8fc3878a111d2bbfe78e39522d884136f0b412b12592195f26f653476" dependencies = [ - "rustc-ap-rustc_data_structures 679.0.0", + "rustc-ap-rustc_data_structures", "smallvec 1.4.2", ] -[[package]] -name = "rustc-ap-rustc_ast" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c82c2510460f2133548e62399e5acd30c25ae6ece30245baab3d1e00c2fefac" -dependencies = [ - "bitflags", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_index 677.0.0", - "rustc-ap-rustc_lexer 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "smallvec 1.4.2", - "tracing", -] - [[package]] name = "rustc-ap-rustc_ast" version = "679.0.0" @@ -3035,44 +3037,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b58b6b035710df7f339a2bf86f6dafa876efd95439540970e24609e33598ca6" dependencies = [ "bitflags", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_index 679.0.0", - "rustc-ap-rustc_lexer 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_index", + "rustc-ap-rustc_lexer", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_span", "smallvec 1.4.2", "tracing", ] [[package]] name = "rustc-ap-rustc_ast_passes" -version = "677.0.0" +version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83977da57f81c6edd89bad47e49136680eaa33288de4abb702e95358c2a0fc6c" +checksum = "3d379a900d6a1f098490d92ab83e87487dcee2e4ec3f04c3ac4512b5117b64e2" dependencies = [ - "itertools 0.8.2", - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_ast_pretty 677.0.0", + "itertools 0.9.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast_pretty", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", - "rustc-ap-rustc_feature 677.0.0", - "rustc-ap-rustc_parse 677.0.0", - "rustc-ap-rustc_session 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "tracing", -] - -[[package]] -name = "rustc-ap-rustc_ast_pretty" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "becf4ca1638b214694c71a8752192683048ab8bd47947cc481f57bd48157eeb9" -dependencies = [ - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "rustc-ap-rustc_target 677.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_feature", + "rustc-ap-rustc_parse", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", "tracing", ] @@ -3082,60 +3072,29 @@ version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658d925c0da9e3c5cddc5e54f4fa8c03b41aff1fc6dc5e41837c1118ad010ac0" dependencies = [ - "rustc-ap-rustc_ast 679.0.0", - "rustc-ap-rustc_span 679.0.0", - "rustc-ap-rustc_target 679.0.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_span", + "rustc-ap-rustc_target", "tracing", ] [[package]] name = "rustc-ap-rustc_attr" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f21ca5dadce8a40d75a2756b77eab75b4c2d827f645c622dd93ee2285599640" -dependencies = [ - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_ast_pretty 677.0.0", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", - "rustc-ap-rustc_feature 677.0.0", - "rustc-ap-rustc_lexer 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_session 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "version_check", -] - -[[package]] -name = "rustc-ap-rustc_data_structures" -version = "677.0.0" +version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4cd204764727fde9abf75333eb661f058bfc7242062d91019440fe1b240688b" -dependencies = [ - "bitflags", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "ena", - "indexmap", - "jobserver", - "lazy_static", - "libc", - "measureme 0.7.1", - "parking_lot 0.10.2", - "rustc-ap-rustc_graphviz 677.0.0", - "rustc-ap-rustc_index 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-hash", - "rustc-rayon", - "rustc-rayon-core", - "smallvec 1.4.2", - "stable_deref_trait", - "stacker", - "tempfile", - "tracing", - "winapi 0.3.9", +checksum = "3f387037534f34c148aed753622677500e42d190a095670e7ac3fffc09811a59" +dependencies = [ + "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast_pretty", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_feature", + "rustc-ap-rustc_lexer", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", + "version_check", ] [[package]] @@ -3153,10 +3112,10 @@ dependencies = [ "libc", "measureme 0.7.1", "parking_lot 0.11.0", - "rustc-ap-rustc_graphviz 679.0.0", - "rustc-ap-rustc_index 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_graphviz", + "rustc-ap-rustc_index", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", "rustc-hash", "rustc-rayon", "rustc-rayon-core", @@ -3168,25 +3127,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "rustc-ap-rustc_errors" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58116f119e37f14c029f99077b347069621118e048a69df74695b98204e7c136" -dependencies = [ - "annotate-snippets 0.8.0", - "atty", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "termcolor", - "termize", - "tracing", - "unicode-width", - "winapi 0.3.9", -] - [[package]] name = "rustc-ap-rustc_errors" version = "679.0.0" @@ -3195,10 +3135,10 @@ checksum = "2b3263ddcfa9eb911e54a4e8088878dd9fd10e00d8b99b01033ba4a2733fe91d" dependencies = [ "annotate-snippets 0.8.0", "atty", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_span", "termcolor", "termize", "tracing", @@ -3208,83 +3148,49 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_expand" -version = "677.0.0" +version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e3c4bda9b64b92805bebe7431fdb8e24fd112b35a8c6d2174827441f10a6b2" +checksum = "e1ab7e68cede8a2273fd8b8623002ce9dc832e061dfc3330e9bcc1fc2a722d73" dependencies = [ - "rustc-ap-rustc_ast 677.0.0", + "rustc-ap-rustc_ast", "rustc-ap-rustc_ast_passes", - "rustc-ap-rustc_ast_pretty 677.0.0", + "rustc-ap-rustc_ast_pretty", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", - "rustc-ap-rustc_feature 677.0.0", - "rustc-ap-rustc_lexer 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_parse 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_session 677.0.0", - "rustc-ap-rustc_span 677.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_feature", + "rustc-ap-rustc_lexer", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_parse", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", "smallvec 1.4.2", "tracing", ] -[[package]] -name = "rustc-ap-rustc_feature" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b612bb67d3fc49f395b03fc4ea4384a0145b05afbadab725803074ec827632b" -dependencies = [ - "lazy_static", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_span 677.0.0", -] - [[package]] name = "rustc-ap-rustc_feature" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea2dc95421bc19bbd4d939399833a882c46b684283b4267ad1fcf982fc043d9" dependencies = [ - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_span", ] -[[package]] -name = "rustc-ap-rustc_fs_util" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7630ad1a73a8434ee920676148cb5440ac57509bd20e94ec41087fb0b1d11c28" - [[package]] name = "rustc-ap-rustc_fs_util" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e44c1804f09635f83f6cf1e04c2e92f8aeb7b4e850ac6c53d373dab02c13053" -[[package]] -name = "rustc-ap-rustc_graphviz" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a603fca4817062eb4fb23ff129d475bd66a69fb32f34ed4362ae950cf814b49d" - [[package]] name = "rustc-ap-rustc_graphviz" version = "679.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc491f2b9be6e928f6df6b287549b8d50c48e8eff8638345155f40fa2cfb785d" -[[package]] -name = "rustc-ap-rustc_index" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9850c4a5d7c341513e10802bca9588bf8f452ceea2d5cfa87b934246a52622bc" -dependencies = [ - "arrayvec", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", -] - [[package]] name = "rustc-ap-rustc_index" version = "679.0.0" @@ -3292,17 +3198,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa73f3fed413cdb6290738a10267da17b9ae8e02087334778b9a8c9491c5efc0" dependencies = [ "arrayvec", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", -] - -[[package]] -name = "rustc-ap-rustc_lexer" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d86722e5a1a615b198327d0d794cd9cbc8b9db4542276fc51fe078924de68ea" -dependencies = [ - "unicode-xid", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", ] [[package]] @@ -3314,18 +3211,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "rustc-ap-rustc_macros" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3fc8482e44cabdda7ac9a8e224aef62ebdf95274d629dac8db3b42321025fea" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "rustc-ap-rustc_macros" version = "679.0.0" @@ -3338,26 +3223,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "rustc-ap-rustc_parse" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3716cdcd978a91dbd4a2788400e90e809527f841426fbeb92f882f9b8582f3ab" -dependencies = [ - "bitflags", - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_ast_pretty 677.0.0", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", - "rustc-ap-rustc_feature 677.0.0", - "rustc-ap-rustc_lexer 677.0.0", - "rustc-ap-rustc_session 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "smallvec 1.4.2", - "tracing", - "unicode-normalization", -] - [[package]] name = "rustc-ap-rustc_parse" version = "679.0.0" @@ -3365,29 +3230,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0342675835251571471d3dca9ea1576a853a8dfa1f4b0084db283c861223cb60" dependencies = [ "bitflags", - "rustc-ap-rustc_ast 679.0.0", - "rustc-ap-rustc_ast_pretty 679.0.0", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_errors 679.0.0", - "rustc-ap-rustc_feature 679.0.0", - "rustc-ap-rustc_lexer 679.0.0", - "rustc-ap-rustc_session 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast_pretty", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_feature", + "rustc-ap-rustc_lexer", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", "smallvec 1.4.2", "tracing", "unicode-normalization", ] -[[package]] -name = "rustc-ap-rustc_serialize" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68046d07988b349b2e1c8bc1c9664a1d06519354aa677b9df358c5c5c058da0" -dependencies = [ - "indexmap", - "smallvec 1.4.2", -] - [[package]] name = "rustc-ap-rustc_serialize" version = "679.0.0" @@ -3398,27 +3253,6 @@ dependencies = [ "smallvec 1.4.2", ] -[[package]] -name = "rustc-ap-rustc_session" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85735553501a4de0c8904e37b7ccef79cc1c585a7d7f2cfa02cc38e0d149f982" -dependencies = [ - "bitflags", - "getopts", - "num_cpus", - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", - "rustc-ap-rustc_feature 677.0.0", - "rustc-ap-rustc_fs_util 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "rustc-ap-rustc_target 677.0.0", - "tracing", -] - [[package]] name = "rustc-ap-rustc_session" version = "679.0.0" @@ -3428,37 +3262,18 @@ dependencies = [ "bitflags", "getopts", "num_cpus", - "rustc-ap-rustc_ast 679.0.0", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_errors 679.0.0", - "rustc-ap-rustc_feature 679.0.0", - "rustc-ap-rustc_fs_util 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", - "rustc-ap-rustc_span 679.0.0", - "rustc-ap-rustc_target 679.0.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", + "rustc-ap-rustc_feature", + "rustc-ap-rustc_fs_util", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_span", + "rustc-ap-rustc_target", "tracing", ] -[[package]] -name = "rustc-ap-rustc_span" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c49ae8a0d3b9e27c6ffe8febeaa30f899294fff012de70625f9ee81c54fda85" -dependencies = [ - "cfg-if 0.1.10", - "md-5", - "rustc-ap-rustc_arena 677.0.0", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_index 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "scoped-tls", - "sha-1", - "tracing", - "unicode-width", -] - [[package]] name = "rustc-ap-rustc_span" version = "679.0.0" @@ -3467,32 +3282,17 @@ checksum = "1c267f15c3cfc82a8a441d2bf86bcccf299d1eb625822468e3d8ee6f7c5a1c89" dependencies = [ "cfg-if 0.1.10", "md-5", - "rustc-ap-rustc_arena 679.0.0", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_index 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", + "rustc-ap-rustc_arena", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_index", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", "scoped-tls", "sha-1", "tracing", "unicode-width", ] -[[package]] -name = "rustc-ap-rustc_target" -version = "677.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1765f447594740c501c7b666b87639aa7c1dae2bf8c3166d5d2dca16646fd034" -dependencies = [ - "bitflags", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_index 677.0.0", - "rustc-ap-rustc_macros 677.0.0", - "rustc-ap-rustc_serialize 677.0.0", - "rustc-ap-rustc_span 677.0.0", - "tracing", -] - [[package]] name = "rustc-ap-rustc_target" version = "679.0.0" @@ -3500,11 +3300,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1b4b266c4d44aac0f7f83b6741d8f0545b03d1ce32f3b5254f2014225cb96c" dependencies = [ "bitflags", - "rustc-ap-rustc_data_structures 679.0.0", - "rustc-ap-rustc_index 679.0.0", - "rustc-ap-rustc_macros 679.0.0", - "rustc-ap-rustc_serialize 679.0.0", - "rustc-ap-rustc_span 679.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_index", + "rustc-ap-rustc_macros", + "rustc-ap-rustc_serialize", + "rustc-ap-rustc_span", "tracing", ] @@ -4513,7 +4313,7 @@ dependencies = [ [[package]] name = "rustfmt-nightly" -version = "1.4.21" +version = "1.4.22" dependencies = [ "annotate-snippets 0.6.1", "anyhow", @@ -4529,15 +4329,15 @@ dependencies = [ "lazy_static", "log", "regex", - "rustc-ap-rustc_ast 677.0.0", - "rustc-ap-rustc_ast_pretty 677.0.0", + "rustc-ap-rustc_ast", + "rustc-ap-rustc_ast_pretty", "rustc-ap-rustc_attr", - "rustc-ap-rustc_data_structures 677.0.0", - "rustc-ap-rustc_errors 677.0.0", + "rustc-ap-rustc_data_structures", + "rustc-ap-rustc_errors", "rustc-ap-rustc_expand", - "rustc-ap-rustc_parse 677.0.0", - "rustc-ap-rustc_session 677.0.0", - "rustc-ap-rustc_span 677.0.0", + "rustc-ap-rustc_parse", + "rustc-ap-rustc_session", + "rustc-ap-rustc_span", "rustc-workspace-hack", "rustfmt-config_proc_macro", "serde", @@ -5772,7 +5572,3 @@ checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" dependencies = [ "linked-hash-map", ] - -[[patch.unused]] -name = "backtrace" -version = "0.3.50" diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index 36d6a831287d6..b96612fa16671 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -226,6 +226,8 @@ pub struct ObligationForest { done: bool, /// Reusable vector for storing unblocked nodes whose watch should be removed. temp_unblocked_nodes: Vec, + + reused_node_vec: Vec, } /// Helper struct for use with `BinaryHeap` to process nodes in the order that they were added to @@ -390,6 +392,10 @@ enum NodeState { #[derive(Debug)] pub struct Outcome { + /// Obligations that were completely evaluated, including all + /// (transitive) subobligations. Only computed if requested. + pub completed: Option>, + /// Backtrace of obligations that were found to be in error. pub errors: Vec>, } @@ -426,6 +432,7 @@ impl ObligationForest { temp_unblocked_nodes: Default::default(), watcher_offset: None, done: false, + reused_node_vec: Vec::new(), } } @@ -545,7 +552,8 @@ impl ObligationForest { .map(|&index| Error { error: error.clone(), backtrace: self.error_at(index) }) .collect(); - self.compress(|_| assert!(false)); + let successful_obligations = self.compress(DoCompleted::Yes); + assert!(successful_obligations.unwrap().is_empty()); errors } @@ -576,7 +584,11 @@ impl ObligationForest { /// be called in a loop until `outcome.stalled` is false. /// /// This _cannot_ be unrolled (presently, at least). - pub fn process_obligations

(&mut self, processor: &mut P) -> Outcome + pub fn process_obligations

( + &mut self, + processor: &mut P, + do_completed: DoCompleted, + ) -> Outcome where P: ObligationProcessor, { @@ -965,7 +977,7 @@ impl ObligationForest { where P: ObligationProcessor, { - let mut stack = vec![]; + let mut stack = mem::take(&mut self.reused_node_vec); let success_or_waiting_nodes = mem::take(&mut self.success_or_waiting_nodes); for &index in &success_or_waiting_nodes { diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index ae5539c5e143c..271f335f7f23c 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -1,7 +1,7 @@ use crate::infer::{InferCtxt, TyOrConstInferVar, WatcherOffset}; use rustc_data_structures::captures::Captures; use rustc_data_structures::obligation_forest::ProcessResult; -use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome}; +use rustc_data_structures::obligation_forest::{DoCompleted, Error, ForestObligation}; use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor}; use rustc_errors::ErrorReported; use rustc_infer::traits::{TraitEngine, TraitEngineExt as _, TraitObligation}; From d3d3fc47d7fc445ee20d75ad8a49cbb767a89061 Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Tue, 27 Oct 2020 00:56:48 +0100 Subject: [PATCH 24/25] Always use the simple obligation forest --- compiler/rustc_data_structures/src/obligation_forest/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index b96612fa16671..69ca9f7974f19 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -594,7 +594,7 @@ impl ObligationForest { { if self.watcher_offset.is_none() { assert!(!self.done); - if self.nodes.len() > 100 { + if false && self.nodes.len() > 100 { self.watcher_offset = Some(processor.register_variable_watcher()); } if let Some(outcome) = self.process_obligations_simple(processor, do_completed) { From b27911ad6e08740ae5b1626b51d2aa159b6b51cf Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Tue, 27 Oct 2020 22:20:22 +0100 Subject: [PATCH 25/25] Hack out unify log for perf --- .../src/logged_unification_table.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/compiler/rustc_data_structures/src/logged_unification_table.rs b/compiler/rustc_data_structures/src/logged_unification_table.rs index 717763a69c149..902389736cd4c 100644 --- a/compiler/rustc_data_structures/src/logged_unification_table.rs +++ b/compiler/rustc_data_structures/src/logged_unification_table.rs @@ -106,9 +106,6 @@ where where K::Value: ut::UnifyValue, { - if self.storage.unify_log.needs_log(vid) { - self.storage.modified_set.set(&mut self.undo_log, vid); - } let vid = vid.into(); let mut relations = self.relations(); debug_assert!(relations.find(vid) == vid); @@ -131,10 +128,7 @@ where vid: I, value: K::Value, ) -> Result<(), ::Error> { - let vid = self.find(vid).into(); - if self.storage.unify_log.needs_log(vid) { - self.storage.modified_set.set(&mut self.undo_log, vid); - } + let vid = self.find(vid); self.relations().unify_var_value(vid, value) } @@ -148,11 +142,6 @@ where relations.unify_var_var(a, b)?; - if a == relations.find(a) { - self.storage.unify_log.unify(&mut self.undo_log, a.into(), b.into()); - } else { - self.storage.unify_log.unify(&mut self.undo_log, b.into(), a.into()); - } Ok(()) }