[gas-profiler] support storage refunds and generate report in html

aptos-move/aptos-gas-profiling/Cargo.toml

@@ -14,8 +14,10 @@ rust-version = { workspace = true }
 [dependencies]
 anyhow = { workspace = true }
 bcs = { workspace = true }
+handlebars = { workspace = true }
 inferno = { workspace = true }
 regex = { workspace = true }
+serde_json = { workspace = true }
 smallvec = { workspace = true }
 
 aptos-framework = { workspace = true }

aptos-move/aptos-gas-profiling/src/erased.rs

@@ -9,28 +9,37 @@ use crate::{
     render::Render,
     FrameName, TransactionGasLog,
 };
-use aptos_gas_algebra::{Fee, GasQuantity, GasScalingFactor, InternalGas, InternalGasUnit, Octa};
+use aptos_gas_algebra::{Fee, GasScalingFactor, InternalGas};
+use std::ops::{Add, AddAssign};
 
-/// Represents a node in a general tree structure with some text & cost attached to each node.
-pub struct Node<U> {
+/// Represents a node in a general tree structure where each node is tagged with
+/// some text & a numerical value.
+#[derive(Clone)]
+pub struct Node<N> {
     pub text: String,
-    pub cost: GasQuantity<U>,
-    pub children: Vec<Node<U>>,
+    pub val: N,
+    pub children: Vec<Node<N>>,
 }
 
 /// A type-erased gas log for execution and IO costs.
 #[derive(Clone)]
 pub struct TypeErasedExecutionAndIoCosts {
     pub gas_scaling_factor: GasScalingFactor,
     pub total: InternalGas,
-    pub tree: Node<InternalGasUnit>,
+    pub tree: Node<InternalGas>,
+}
+
+#[derive(Clone, Copy)]
+pub struct StoragePair {
+    pub cost: Fee,
+    pub refund: Fee,
 }
 
 /// A type-erased gas log for storage fees.
 #[derive(Clone)]
 pub struct TypeErasedStorageFees {
     pub total: Fee,
-    pub tree: Node<Octa>,
+    pub tree: Node<StoragePair>,
 }
 
 /// A gas log with some of the type information erased.
@@ -44,49 +53,83 @@ pub struct TypeErasedGasLog {
     pub storage: TypeErasedStorageFees,
 }
 
-impl<U> Clone for Node<U> {
-    fn clone(&self) -> Self {
+impl StoragePair {
+    pub fn zero() -> Self {
         Self {
-            text: self.text.clone(),
-            cost: self.cost,
-            children: self.children.clone(),
+            cost: 0.into(),
+            refund: 0.into(),
         }
     }
+
+    pub fn new(cost: Fee, refund: Fee) -> Self {
+        Self { cost, refund }
+    }
 }
 
-impl<U> Node<U> {
-    pub fn new(name: impl Into<String>, cost: impl Into<GasQuantity<U>>) -> Self {
+impl Add<Self> for StoragePair {
+    type Output = Self;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        Self {
+            cost: self.cost + rhs.cost,
+            refund: self.refund + rhs.refund,
+        }
+    }
+}
+
+impl AddAssign<Self> for StoragePair {
+    fn add_assign(&mut self, rhs: Self) {
+        self.cost += rhs.cost;
+        self.refund += rhs.refund;
+    }
+}
+
+impl<A, B> From<(A, B)> for StoragePair
+where
+    A: Into<Fee>,
+    B: Into<Fee>,
+{
+    fn from((cost, refund): (A, B)) -> Self {
         Self {
-            text: name.into(),
             cost: cost.into(),
+            refund: refund.into(),
+        }
+    }
+}
+
+impl<N> Node<N> {
+    pub fn new(name: impl Into<String>, data: impl Into<N>) -> Self {
+        Self {
+            text: name.into(),
+            val: data.into(),
             children: vec![],
         }
     }
 
     pub fn new_with_children(
         name: impl Into<String>,
-        cost: impl Into<GasQuantity<U>>,
+        data: impl Into<N>,
         children: impl IntoIterator<Item = Self>,
     ) -> Self {
         Self {
             text: name.into(),
-            cost: cost.into(),
+            val: data.into(),
             children: children.into_iter().collect(),
         }
     }
 
-    pub fn preorder_traversel(&self, mut f: impl FnMut(usize, &str, GasQuantity<U>)) {
+    pub fn preorder_traversel(&self, mut f: impl FnMut(usize, &str, &N)) {
         let mut stack = vec![(self, 0)];
 
         while let Some((node, depth)) = stack.pop() {
-            f(depth, &node.text, node.cost);
+            f(depth, &node.text, &node.val);
             stack.extend(node.children.iter().map(|child| (child, depth + 1)).rev());
         }
     }
 }
 
 impl ExecutionGasEvent {
-    fn to_erased(&self) -> Node<InternalGasUnit> {
+    fn to_erased(&self) -> Node<InternalGas> {
         use ExecutionGasEvent::*;
 
         match self {
@@ -113,7 +156,7 @@ impl ExecutionGasEvent {
 }
 
 impl CallFrame {
-    fn to_erased(&self) -> Node<InternalGasUnit> {
+    fn to_erased(&self) -> Node<InternalGas> {
         let name = match &self.name {
             FrameName::Script => "script".to_string(),
             FrameName::Function {
@@ -139,7 +182,7 @@ impl CallFrame {
 }
 
 impl WriteTransient {
-    fn to_erased(&self) -> Node<InternalGasUnit> {
+    fn to_erased(&self) -> Node<InternalGas> {
         Node::new(
             format!("{}<{}>", Render(&self.op_type), Render(&self.key)),
             self.cost,
@@ -173,17 +216,17 @@ impl ExecutionAndIOCosts {
 }
 
 impl WriteStorage {
-    fn to_erased(&self) -> Node<Octa> {
+    fn to_erased(&self) -> Node<StoragePair> {
         Node::new(
             format!("{}<{}>", Render(&self.op_type), Render(&self.key)),
-            self.cost,
+            (self.cost, self.refund),
         )
     }
 }
 
 impl EventStorage {
-    fn to_erased(&self) -> Node<Octa> {
-        Node::new(format!("{}", self.ty), self.cost)
+    fn to_erased(&self) -> Node<StoragePair> {
+        Node::new(format!("{}", self.ty), (self.cost, Fee::zero()))
     }
 }
 
@@ -193,21 +236,21 @@ impl StorageFees {
     pub fn to_erased(&self) -> TypeErasedStorageFees {
         let mut nodes = vec![];
 
-        nodes.push(Node::new("transaction", self.txn_storage));
+        nodes.push(Node::new("transaction", (self.txn_storage, Fee::zero())));
         nodes.push(Node::new_with_children(
             "writes",
-            0,
+            (Fee::zero(), Fee::zero()),
             self.write_set_storage.iter().map(|write| write.to_erased()),
         ));
         nodes.push(Node::new_with_children(
             "events",
-            0,
+            (Fee::zero(), Fee::zero()),
             self.events.iter().map(|event| event.to_erased()),
         ));
 
         TypeErasedStorageFees {
             total: self.total,
-            tree: Node::new_with_children("storage fees (APT)", 0, nodes),
+            tree: Node::new_with_children("storage fees (APT)", (Fee::zero(), Fee::zero()), nodes),
         }
     }
 }
@@ -222,11 +265,14 @@ impl TransactionGasLog {
     }
 }
 
-impl<U> Node<U> {
+impl<N> Node<N>
+where
+    N: AddAssign<N> + Copy,
+{
     pub fn include_child_costs(&mut self) {
         for child in &mut self.children {
             child.include_child_costs();
-            self.cost += child.cost;
+            self.val += child.val;
         }
     }
 }

aptos-move/aptos-gas-profiling/src/lib.rs

@@ -8,7 +8,7 @@ mod log;
 mod misc;
 mod profiler;
 mod render;
-mod textualize;
+mod report;
 
 pub use log::{FrameName, TransactionGasLog};
 pub use profiler::GasProfiler;

aptos-move/aptos-gas-profiling/src/log.rs

@@ -81,6 +81,7 @@ pub struct WriteStorage {
     pub key: StateKey,
     pub op_type: WriteOpType,
     pub cost: Fee,
+    pub refund: Fee,
 }
 
 #[derive(Debug)]
@@ -104,6 +105,7 @@ pub struct ExecutionAndIOCosts {
 // Struct containing all types of storage fees.
 pub struct StorageFees {
     pub total: Fee,
+    pub total_refund: Fee,
 
     pub write_set_storage: Vec<WriteStorage>,
     pub events: Vec<EventStorage>,
@@ -168,13 +170,72 @@ impl CallFrame {
     }
 }
 
+impl StorageFees {
+    pub(crate) fn assert_consistency(&self) {
+        let mut total = Fee::zero();
+        let mut total_refund = Fee::zero();
+
+        for write in &self.write_set_storage {
+            total += write.cost;
+            total_refund += write.refund;
+        }
+
+        for event in &self.events {
+            total += event.cost;
+        }
+
+        total += self.txn_storage;
+
+        if total != self.total {
+            panic!(
+                "Storage fees do not add up. Check if the gas meter & the gas profiler have been implemented correctly. From gas meter: {}. Calculated: {}.",
+                self.total, total
+            );
+        }
+        if total_refund != self.total_refund {
+            panic!(
+                "Storage refunds do not add up. Check if the gas meter & the gas profiler have been implemented correctly. From gas meter: {}. Calculated: {}.",
+                self.total_refund, total
+            );
+        }
+    }
+}
+
 impl ExecutionAndIOCosts {
     #[allow(clippy::needless_lifetimes)]
     pub fn gas_events<'a>(&'a self) -> GasEventIter<'a> {
         GasEventIter {
             stack: smallvec![(&self.call_graph, 0)],
         }
     }
+
+    pub(crate) fn assert_consistency(&self) {
+        use ExecutionGasEvent::{Bytecode, Call, CallNative, LoadResource, Loc};
+
+        let mut total = InternalGas::zero();
+
+        total += self.intrinsic_cost;
+
+        for op in self.gas_events() {
+            match op {
+                Loc(..) | Call(..) => (),
+                Bytecode { cost, .. } | CallNative { cost, .. } | LoadResource { cost, .. } => {
+                    total += *cost
+                },
+            }
+        }
+
+        for write in &self.write_set_transient {
+            total += write.cost;
+        }
+
+        if total != self.total {
+            panic!(
+                "Execution & IO costs do not add up. Check if the gas meter & the gas profiler have been implemented correctly. From gas meter: {}. Calculated: {}.",
+                self.total, total
+            )
+        }
+    }
 }
 
 impl TransactionGasLog {