Implement REPLACE_DEFERRED protocol feature

libraries/chain/apply_context.cpp

@@ -363,35 +363,45 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
    if ( auto ptr = db.find<generated_transaction_object,by_sender_id>(boost::make_tuple(receiver, sender_id)) ) {
       EOS_ASSERT( replace_existing, deferred_tx_duplicate, "deferred transaction with the same sender_id and payer already exists" );
 
-      // TODO: Remove the following subjective check when the deferred trx replacement RAM bug has been fixed with a hard fork.
-      EOS_ASSERT( !control.is_producing_block(), subjective_block_production_exception,
+      bool replace_deferred_activated = control.is_builtin_activated(builtin_protocol_feature_t::replace_deferred);
+
+      EOS_ASSERT( replace_deferred_activated || !control.is_producing_block()
+                     || control.all_subjective_mitigations_disabled(),
+                  subjective_block_production_exception,
                   "Replacing a deferred transaction is temporarily disabled." );
 
-      // TODO: The logic of the next line needs to be incorporated into the next hard fork.
-      // add_ram_usage( ptr->payer, -(config::billable_size_v<generated_transaction_object> + ptr->packed_trx.size()) );
+      uint64_t orig_trx_ram_bytes = config::billable_size_v<generated_transaction_object> + ptr->packed_trx.size();
+      if( replace_deferred_activated ) {
+         add_ram_usage( ptr->payer, -static_cast<int64_t>( orig_trx_ram_bytes ) );
+      } else {
+         control.add_to_ram_correction( ptr->payer, orig_trx_ram_bytes );
+      }
 
       db.modify<generated_transaction_object>( *ptr, [&]( auto& gtx ) {
-            gtx.sender      = receiver;
-            gtx.sender_id   = sender_id;
-            gtx.payer       = payer;
-            gtx.published   = control.pending_block_time();
-            gtx.delay_until = gtx.published + delay;
-            gtx.expiration  = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window);
-
-            trx_size = gtx.set( trx );
-         });
+         if( replace_deferred_activated ) {
+            gtx.trx_id = trx.id();
+         }
+         gtx.sender      = receiver;
+         gtx.sender_id   = sender_id;
+         gtx.payer       = payer;
+         gtx.published   = control.pending_block_time();
+         gtx.delay_until = gtx.published + delay;
+         gtx.expiration  = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window);
+
+         trx_size = gtx.set( trx );
+      } );
    } else {
       db.create<generated_transaction_object>( [&]( auto& gtx ) {
-            gtx.trx_id      = trx.id();
-            gtx.sender      = receiver;
-            gtx.sender_id   = sender_id;
-            gtx.payer       = payer;
-            gtx.published   = control.pending_block_time();
-            gtx.delay_until = gtx.published + delay;
-            gtx.expiration  = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window);
-
-            trx_size = gtx.set( trx );
-         });
+         gtx.trx_id      = trx.id();
+         gtx.sender      = receiver;
+         gtx.sender_id   = sender_id;
+         gtx.payer       = payer;
+         gtx.published   = control.pending_block_time();
+         gtx.delay_until = gtx.published + delay;
+         gtx.expiration  = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window);
+
+         trx_size = gtx.set( trx );
+      } );
    }
 
    EOS_ASSERT( control.is_ram_billing_in_notify_allowed() || (receiver == act.account) || (receiver == payer) || privileged,

libraries/chain/controller.cpp

@@ -35,6 +35,7 @@ using resource_limits::resource_limits_manager;
 using controller_index_set = index_set<
    account_index,
    account_sequence_index,
+   account_ram_correction_index,
    global_property_multi_index,
    protocol_state_multi_index,
    dynamic_global_property_multi_index,
@@ -306,6 +307,7 @@ struct controller_impl {
       );
 
       set_activation_handler<builtin_protocol_feature_t::preactivate_feature>();
+      set_activation_handler<builtin_protocol_feature_t::replace_deferred>();
 
 
 #define SET_APP_HANDLER( receiver, contract, action) \
@@ -885,6 +887,7 @@ struct controller_impl {
       });
 
       db.create<protocol_state_object>([&](auto& pso ){
+         pso.num_supported_key_types = 2;
          for( const auto& i : genesis_intrinsics ) {
             add_intrinsic_to_whitelist( pso.whitelisted_intrinsics, i );
          }
@@ -2946,6 +2949,24 @@ const flat_set<account_name> &controller::get_resource_greylist() const {
    return  my->conf.resource_greylist;
 }
 
+
+void controller::add_to_ram_correction( account_name account, uint64_t ram_bytes ) {
+   if( auto ptr = my->db.find<account_ram_correction_object, by_name>( account ) ) {
+      my->db.modify<account_ram_correction_object>( *ptr, [&]( auto& rco ) {
+         rco.ram_correction += ram_bytes;
+      } );
+   } else {
+      my->db.create<account_ram_correction_object>( [&]( auto& rco ) {
+         rco.name = account;
+         rco.ram_correction = ram_bytes;
+      } );
+   }
+}
+
+bool controller::all_subjective_mitigations_disabled()const {
+   return my->conf.disable_all_subjective_mitigations;
+}
+
 /// Protocol feature activation handlers:
 
 template<>
@@ -2956,6 +2977,23 @@ void controller_impl::on_activation<builtin_protocol_feature_t::preactivate_feat
    } );
 }
 
+template<>
+void controller_impl::on_activation<builtin_protocol_feature_t::replace_deferred>() {
+   const auto& indx = db.get_index<account_ram_correction_index, by_id>();
+   for( auto itr = indx.begin(); itr != indx.end(); itr = indx.begin() ) {
+      int64_t current_ram_usage = resource_limits.get_account_ram_usage( itr->name );
+      int64_t ram_delta = -static_cast<int64_t>(itr->ram_correction);
+      if( itr->ram_correction > static_cast<uint64_t>(current_ram_usage) ) {
+         ram_delta = -current_ram_usage;
+         elog( "account ${name} was to be reduced by ${adjust} bytes of RAM despite only using ${current} bytes of RAM",
+               ("name", itr->name)("adjust", itr->ram_correction)("current", current_ram_usage) );
+      }
+
+      resource_limits.add_pending_ram_usage( itr->name, ram_delta );
+      db.remove( *itr );
+   }
+}
+
 /// End of protocol feature activation handlers
 
 } } /// eosio::chain

libraries/chain/include/eosio/chain/account_object.hpp

@@ -75,11 +75,31 @@ namespace eosio { namespace chain {
       >
    >;
 
+   class account_ram_correction_object : public chainbase::object<account_ram_correction_object_type, account_ram_correction_object>
+   {
+      OBJECT_CTOR(account_ram_correction_object);
+
+      id_type      id;
+      account_name name;
+      uint64_t     ram_correction = 0;
+   };
+
+   struct by_name;
+   using account_ram_correction_index = chainbase::shared_multi_index_container<
+      account_ram_correction_object,
+      indexed_by<
+         ordered_unique<tag<by_id>, member<account_ram_correction_object, account_ram_correction_object::id_type, &account_ram_correction_object::id>>,
+         ordered_unique<tag<by_name>, member<account_ram_correction_object, account_name, &account_ram_correction_object::name>>
+      >
+   >;
+
 } } // eosio::chain
 
 CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_object, eosio::chain::account_index)
 CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_sequence_object, eosio::chain::account_sequence_index)
+CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_ram_correction_object, eosio::chain::account_ram_correction_index)
 
 
 FC_REFLECT(eosio::chain::account_object, (name)(vm_type)(vm_version)(privileged)(last_code_update)(code_version)(creation_date)(code)(abi))
-FC_REFLECT(eosio::chain::account_sequence_object, (name)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence))
+FC_REFLECT(eosio::chain::account_sequence_object, (name)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence))
+FC_REFLECT(eosio::chain::account_ram_correction_object, (name)(ram_correction))

libraries/chain/include/eosio/chain/chain_snapshot.hpp

@@ -12,10 +12,13 @@ struct chain_snapshot_header {
    /**
     * Version history
     *   1: initial version
+    *   2: Updated chain snapshot for v1.8.0 initial protocol features release:
+    *         - Incompatible with version 1.
+    *         - Adds new indices for: protocol_state_object and account_ram_correction_object
     */
 
-   static constexpr uint32_t minimum_compatible_version = 1;
-   static constexpr uint32_t current_version = 1;
+   static constexpr uint32_t minimum_compatible_version = 2;
+   static constexpr uint32_t current_version = 2;
 
    uint32_t version = current_version;
 
@@ -31,4 +34,4 @@ struct chain_snapshot_header {
 
 } }
 
-FC_REFLECT(eosio::chain::chain_snapshot_header,(version))
+FC_REFLECT(eosio::chain::chain_snapshot_header,(version))

libraries/chain/include/eosio/chain/controller.hpp

@@ -76,6 +76,7 @@ namespace eosio { namespace chain {
             bool                     disable_replay_opts    =  false;
             bool                     contracts_console      =  false;
             bool                     allow_ram_billing_in_notify = false;
+            bool                     disable_all_subjective_mitigations = false; //< for testing purposes only
 
             genesis_state            genesis;
             wasm_interface::vm_type  wasm_runtime = chain::config::default_wasm_runtime;
@@ -271,6 +272,9 @@ namespace eosio { namespace chain {
 
          void set_subjective_cpu_leeway(fc::microseconds leeway);
 
+         void add_to_ram_correction( account_name account, uint64_t ram_bytes );
+         bool all_subjective_mitigations_disabled()const;
+
          signal<void(const signed_block_ptr&)>         pre_accepted_block;
          signal<void(const block_state_ptr&)>          accepted_block_header;
          signal<void(const block_state_ptr&)>          accepted_block;

libraries/chain/include/eosio/chain/protocol_feature_manager.hpp

@@ -15,7 +15,8 @@ enum class protocol_feature_t : uint32_t {
 
 enum class builtin_protocol_feature_t : uint32_t {
    preactivate_feature,
-   only_link_to_existing_permission
+   only_link_to_existing_permission,
+   replace_deferred
 };
 
 struct protocol_feature_subjective_restrictions {

libraries/chain/include/eosio/chain/protocol_state_object.hpp

@@ -40,6 +40,7 @@ namespace eosio { namespace chain {
       shared_vector<activated_protocol_feature>  activated_protocol_features;
       shared_vector<digest_type>                 preactivated_protocol_features;
       whitelisted_intrinsics_type                whitelisted_intrinsics;
+      uint32_t                                   num_supported_key_types = 0;
    };
 
    using protocol_state_multi_index = chainbase::shared_multi_index_container<
@@ -55,6 +56,7 @@ namespace eosio { namespace chain {
       vector<protocol_state_object::activated_protocol_feature> activated_protocol_features;
       vector<digest_type>                                       preactivated_protocol_features;
       std::set<std::string>                                     whitelisted_intrinsics;
+      uint32_t                                                  num_supported_key_types = 0;
    };
 
    namespace detail {
@@ -81,9 +83,9 @@ FC_REFLECT(eosio::chain::protocol_state_object::activated_protocol_feature,
           )
 
 FC_REFLECT(eosio::chain::protocol_state_object,
-            (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)
+            (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)(num_supported_key_types)
           )
 
 FC_REFLECT(eosio::chain::snapshot_protocol_state_object,
-            (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)
+            (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)(num_supported_key_types)
           )

libraries/chain/include/eosio/chain/types.hpp

@@ -190,6 +190,7 @@ namespace eosio { namespace chain {
       action_history_object_type,               ///< Defined by history_plugin
       reversible_block_object_type,
       protocol_state_object_type,
+      account_ram_correction_object_type,
       OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types
    };
 

libraries/chain/protocol_feature_manager.cpp

@@ -38,6 +38,18 @@ Pre-activated protocol features must be activated in the next block.
 Builtin protocol feature: ONLY_LINK_TO_EXISTING_PERMISSION
 
 Disallows linking an action to a non-existing permission.
+*/
+            {}
+         } )
+         (  builtin_protocol_feature_t::replace_deferred, builtin_protocol_feature_spec{
+            "REPLACE_DEFERRED",
+            fc::variant("9908b3f8413c8474ab2a6be149d3f4f6d0421d37886033f27d4759c47a26d944").as<digest_type>(),
+            // SHA256 hash of the raw message below within the comment delimiters (do not modify message below).
+/*
+Builtin protocol feature: REPLACE_DEFERRED
+
+Fix the problems associated with replacing an existing deferred transaction.
+Also corrects the RAM usage of accounts affected by the replace deferred transaction bug.
 */
             {}
          } )

libraries/chain/protocol_state_object.cpp

@@ -26,6 +26,8 @@ namespace eosio { namespace chain {
 
          res.whitelisted_intrinsics = convert_intrinsic_whitelist_to_set( value.whitelisted_intrinsics );
 
+         res.num_supported_key_types = value.num_supported_key_types;
+
          return res;
       }
 
@@ -47,6 +49,8 @@ namespace eosio { namespace chain {
          }
 
          reset_intrinsic_whitelist( value.whitelisted_intrinsics, row.whitelisted_intrinsics );
+
+         value.num_supported_key_types = row.num_supported_key_types;
       }
 
    }

unittests/api_tests.cpp

@@ -1164,8 +1164,6 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try {
 
    produce_blocks(10);
 
-#warning re-enable deferred transaction replacement test after bug has been fixed
-   #if 0
    //schedule twice with replace_existing flag (second deferred transaction should replace first one)
    {
       transaction_trace_ptr trace;
@@ -1186,7 +1184,6 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try {
       BOOST_CHECK_EQUAL( 1, trace->action_traces.size() );
       c.disconnect();
    }
-   #endif
 
    produce_blocks(10);