[compiler-v2] Ast generator from stackless bytecode, Source gen from Ast

Converter from stackless bytecode into Model AST ('astifier'). (We already have one from binary format to stackless.) Also adds a converter from AST to source ('sourcifier'). This adds the major pieces for a working decompiler. However, the pieces are not fully connected to a tool yet, as more sidework needs to be done first. Specifically, this PR adds breaks/continues to labeled outer loops, which are not yet supported by the compiler. There are some tests in a new test crate `ast-generator-tests`. These call the v2 compiler to compile up to file format, then decompile from there into stackless bytecode, into AST, and via sourcifier back to source. However, desired execution roundtrip comparison tests can only be implemented once the full toolchain is ready. Another set of tests for the 'sourcifier' is via compiler-v2 baseline files. Where AST dump is requested in those tests, we dump now also the sourcified AST. This can live side-by-side for a while until we may decide to remove one of the outputs.
aptos-labs · Oct 3, 2024 · d74e23c · d74e23c
1 parent 6938caa
commit d74e23c
Show file tree

Hide file tree

Showing 333 changed files with 17,744 additions and 42 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -220,6 +220,7 @@ members = [
     "third_party/move/move-model",
     "third_party/move/move-model/bytecode",
     "third_party/move/move-model/bytecode-test-utils",
+    "third_party/move/move-model/bytecode/ast-generator-tests",
     "third_party/move/move-prover",
     "third_party/move/move-prover/boogie-backend",
     "third_party/move/move-prover/bytecode-pipeline",
@@ -790,6 +791,7 @@ tonic = { version = "0.11.0", features = [
     "zstd",
 ] }
 tonic-reflection = "0.11.0"
+topological-sort = "0.2.2"
 triomphe = "0.1.9"
 tui = "0.19.0"
 typed-arena = "2.0.2"

diff --git a/third_party/move/move-compiler-v2/src/bytecode_generator.rs b/third_party/move/move-compiler-v2/src/bytecode_generator.rs
@@ -447,7 +447,7 @@ impl<'env> Generator<'env> {
                 self.emit_with(*id, |attr| Bytecode::Jump(attr, continue_label));
                 self.emit_with(*id, |attr| Bytecode::Label(attr, break_label));
             },
-            ExpData::LoopCont(id, do_continue) => {
+            ExpData::LoopCont(id, 0, do_continue) => {
                 if let Some(LoopContext {
                     continue_label,
                     break_label,
@@ -463,6 +463,9 @@ impl<'env> Generator<'env> {
                     self.error(*id, "missing enclosing loop statement")
                 }
             },
+            ExpData::LoopCont(_, _, _) => {
+                unimplemented!("continue/break with nesting")
+            },
             ExpData::SpecBlock(id, spec) => {
                 // Map locals in spec to assigned temporaries.
                 let mut replacer = |id, target| {

diff --git a/third_party/move/move-compiler-v2/src/env_pipeline/inliner.rs b/third_party/move/move-compiler-v2/src/env_pipeline/inliner.rs
@@ -800,7 +800,7 @@ impl<'env, 'rewriter> InlinedRewriter<'env, 'rewriter> {
                          (lambda expressions)",
                     )
                 },
-                ExpData::LoopCont(node_id, is_continue) if !post && in_loop == 0 => {
+                ExpData::LoopCont(node_id, _, is_continue) if !post && in_loop == 0 => {
                     let node_loc = env.get_node_loc(*node_id);
                     env.error(
                         &node_loc,
@@ -1046,7 +1046,7 @@ impl<'env, 'rewriter> ExpRewriterFunctions for InlinedRewriter<'env, 'rewriter>
                 self.in_loop += 1;
                 true
             },
-            ExpData::LoopCont(node_id, is_continue) if self.in_loop == 0 => {
+            ExpData::LoopCont(node_id, _, is_continue) if self.in_loop == 0 => {
                 let node_loc = self.env.get_node_loc(*node_id);
                 self.env.error(
                     &node_loc,

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/assign.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/assign.exp
@@ -25,6 +25,30 @@ module 0x42::assign {
     }
 } // end 0x42::assign
 
+// -- Sourcified model before bytecode pipeline
+module 0x42::assign {
+    struct T has drop {
+        h: u64,
+    }
+    struct S has drop {
+        f: u64,
+        g: T,
+    }
+    fun assign_field(s: &mut S, f: u64) {
+        s.f = f;
+    }
+    fun assign_int(x: &mut u64) {
+        *x = 42;
+    }
+    fun assign_pattern(s: S, f: u64, h: u64): u64 {
+        S{f: f,g: T{h: h}} = s;
+        f + h
+    }
+    fun assign_struct(s: &mut S) {
+        *s = S{f: 42,g: T{h: 42}};
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/assign_inline.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/assign_inline.exp
@@ -8,6 +8,16 @@ module 0x42::assign {
     }
 } // end 0x42::assign
 
+// -- Sourcified model before bytecode pipeline
+module 0x42::assign {
+    public inline fun expose(x: u64): (u64, u64) {
+        (1, x)
+    }
+    public fun main(): (u64, u64) {
+        (1, 3)
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/borrow.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/borrow.exp
@@ -61,6 +61,41 @@ module 0x42::borrow {
     }
 } // end 0x42::borrow
 
+// -- Sourcified model before bytecode pipeline
+module 0x42::borrow {
+    struct S {
+        f: u64,
+    }
+    fun field(s: &S): u64 {
+        let r = &s.f;
+        *r
+    }
+    fun local(param: u64): u64 {
+        let r = &33;
+        *r
+    }
+    fun param(param: u64): u64 {
+        let r = &param;
+        *r
+    }
+    fun mut_field(s: &mut S): u64 {
+        let r = &mut s.f;
+        *r = 22;
+        *r
+    }
+    fun mut_local(param: u64): u64 {
+        let local = 33;
+        let r = &mut local;
+        *r = 22;
+        *r
+    }
+    fun mut_param(param: u64): u64 {
+        let r = &mut param;
+        *r = 22;
+        *r
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/borrow_deref_optimize.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/borrow_deref_optimize.exp
@@ -36,6 +36,32 @@ module 0x42::test {
     }
 } // end 0x42::test
 
+// -- Sourcified model before bytecode pipeline
+module 0x42::test {
+    struct X has copy, drop, key {
+        value: bool,
+    }
+    fun no_optimize_resource(): bool
+        acquires X
+    {
+        let x = &mut *borrow_global<X>(0x1);
+        x.value
+    }
+    fun no_optimize_vector() {
+        let _ = vector::borrow_mut<u64>(&mut *vector::borrow<vector<u64>>(&vector[vector[1, 2]], 0), 1);
+    }
+    fun optimize_resource(): bool
+        acquires X
+    {
+        let x = &*borrow_global<X>(0x1);
+        x.value
+    }
+    fun optimize_vector() {
+        let x = vector[vector[1, 2]];
+        let _ = vector::borrow_mut<u64>(&mut *vector::borrow_mut<vector<u64>>(&mut x, 0), 1);
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/..._party/move/move-compiler-v2/tests/bytecode-generator/bug_14300_update_variant_select.exp b/..._party/move/move-compiler-v2/tests/bytecode-generator/bug_14300_update_variant_select.exp
@@ -37,6 +37,39 @@ module 0x815::m {
     }
 } // end 0x815::m
 
+// -- Sourcified model before bytecode pipeline
+module 0x815::m {
+    enum CommonFields has drop {
+        Foo {
+            x: u64,
+            y: u8,
+        }
+        Bar {
+            x: u64,
+            y: u8,
+            z: u32,
+        }
+        Baz {
+            y: u8,
+        }
+    }
+    fun update_common_field(): u64 {
+        let common = CommonFields::Bar{x: 30,y: 40u8,z: 50u32};
+        common.Foo.x = 15;
+        common.Foo.x
+    }
+    fun update_common_field_different_offset(): u8 {
+        let common = CommonFields::Bar{x: 30,y: 40u8,z: 50u32};
+        common.Foo.y = 15u8;
+        common.Foo.y
+    }
+    fun update_non_common_field(): u32 {
+        let common = CommonFields::Bar{x: 30,y: 40u8,z: 50u32};
+        common.Bar.z = 15u32;
+        common.Bar.z
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/...party/move/move-compiler-v2/tests/bytecode-generator/bug_14300_variant_select_autoref.exp b/...party/move/move-compiler-v2/tests/bytecode-generator/bug_14300_variant_select_autoref.exp
@@ -17,6 +17,23 @@ module 0x815::m {
     }
 } // end 0x815::m
 
+// -- Sourcified model before bytecode pipeline
+module 0x815::m {
+    enum Positional has drop {
+        A {
+            0: u8,
+        }
+        B {
+            0: u8,
+        }
+    }
+    fun test_common_access(): u8 {
+        let x = Positional::A{0: 42u8};
+        x.A.0 = 19u8;
+        20u8
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/bug_14471_receiver_inference.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/bug_14471_receiver_inference.exp
@@ -34,6 +34,34 @@ module 0x815::m {
     }
 } // end 0x815::m
 
+// -- Sourcified model before bytecode pipeline
+module 0x815::m {
+    struct MyMap has key {
+        table: Table<address, ValueWrap>,
+    }
+    struct Table<T1, T2> has store {
+        x: T1,
+        y: T2,
+    }
+    struct ValueWrap has drop, store {
+        val: u64,
+    }
+    fun contains<T1: drop, T2: drop>(self: &Table<T1, T2>, _key: T1): bool {
+        true
+    }
+    fun add<T1: drop, T2: drop>(self: &mut Table<T1, T2>, _key: T1, _val: T2) {
+    }
+    public fun add_when_missing(key: address, val: u64)
+        acquires MyMap
+    {
+        let my_map = borrow_global_mut<MyMap>(0x815);
+        if (!contains<address,ValueWrap>(&my_map.table, key)) {
+            let wrap = ValueWrap{val: val};
+            add<address,ValueWrap>(&mut my_map.table, key, wrap);
+        }
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]

diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/conditional_borrow.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/conditional_borrow.exp
@@ -96,6 +96,62 @@ module 0x8675::M {
     }
 } // end 0x8675::M
 
+// -- Sourcified model before bytecode pipeline
+module 0x8675::M {
+    struct S has copy, drop {
+        f: u64,
+    }
+    public fun test(): u64 {
+        test1(7) + test1(2)
+    }
+    fun test1(r: u64): u64 {
+        let tref = &mut (if (r < 4) r else 3);
+        *tref = 10;
+        let y = r;
+        let tref2 = &mut y;
+        *tref2 = *tref2 + 1;
+        let z = y;
+        let tref3 = &mut (z + 0);
+        *tref3 = *tref3 + 2;
+        let a = z;
+        let tref4 = &mut a;
+        *tref4 = *tref4 + 4;
+        let tref5 = &mut a;
+        *tref5 = *tref5 + 8;
+        let tref6 = &mut {
+            3;
+            a
+        };
+        *tref6 = *tref6 + 16;
+        a
+    }
+    fun test1b(r: S): u64 {
+        let x = S{f: 3};
+        let tref = &mut (if (r.f < 4) r else x);
+        (*tref).f = 10;
+        let y = r;
+        let tref2 = &mut y;
+        (*tref2).f = (*tref2).f + 1;
+        let z = y;
+        let tref3 = &mut z.f;
+        *tref3 = *tref3 + 1;
+        let a = z;
+        let tref4 = &mut a.f;
+        *tref4 = *tref4 + 1;
+        let tref5 = &mut a.f;
+        *tref5 = *tref5 + 8;
+        let tref6 = &mut {
+            3;
+            a.f
+        };
+        *tref6 = *tref6 + 16;
+        a.f
+    }
+    public fun testb(): u64 {
+        test1b(S{f: 7}) + test1b(S{f: 2})
+    }
+}
+
 ============ initial bytecode ================
 
 [variant baseline]