Merge branch 'main' into hieu/comet_service

.github/.codespellignore

@@ -5,4 +5,5 @@ pastTime
 hasTables
 Nam
 EyT
-upTo
+upTo
+pullRequests

CHANGELOG.md

@@ -142,7 +142,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 * (genutil) [#21701](https://github.com/cosmos/cosmos-sdk/pull/21701) Improved error messages for genesis validation.
 * (runtime) [#21704](https://github.com/cosmos/cosmos-sdk/pull/21704) Move `upgradetypes.StoreLoader` to runtime and alias it in upgrade for backward compatibility.
 * (sims)[#21613](https://github.com/cosmos/cosmos-sdk/pull/21613) Add sims2 framework and factory methods for simpler message factories in modules
-* (modules) [#21963](https://github.com/cosmos/cosmos-sdk/pull/21963) Duplicatable metrics are no more collected in modules. They were unecessary overhead.
+* (modules) [#21963](https://github.com/cosmos/cosmos-sdk/pull/21963) Duplicatable metrics are no more collected in modules. They were unnecessary overhead.
 * (crypto/ledger) [#22116](https://github.com/cosmos/cosmos-sdk/pull/22116) Improve error message when deriving paths using index >100
 
 ### Bug Fixes

baseapp/abci.go

@@ -999,6 +999,10 @@ func (app *BaseApp) Commit() (*abci.CommitResponse, error) {
 				app.logger.Error("Commit listening hook failed", "height", blockHeight, "err", err)
 				if app.streamingManager.StopNodeOnErr {
 					err = fmt.Errorf("Commit listening hook failed: %w", err)
+					if blockHeight == 1 {
+						// can't rollback to height 0, so just return the error
+						return nil, fmt.Errorf("failed to commit block 1, can't automatically rollback: %w", err)
+					}
 					rollbackErr := app.cms.RollbackToVersion(blockHeight - 1)
 					if rollbackErr != nil {
 						return nil, errors.Join(err, rollbackErr)

baseapp/streaming.go

@@ -2,6 +2,7 @@ package baseapp
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strconv"
@@ -20,6 +21,7 @@ import (
 
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
 const (
@@ -48,7 +50,7 @@ func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*stor
 	app.cms.AddListeners(exposedKeys)
 
 	app.streamingManager = storetypes.StreamingManager{
-		ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener.Listener}},
+		ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener.Listener, app.txDecoder}},
 		StopNodeOnErr: true,
 	}
 
@@ -144,9 +146,10 @@ func exposeStoreKeysSorted(keysStr []string, keys map[string]*storetypes.KVStore
 	return exposeStoreKeys
 }
 
-func eventToAppDataEvent(event abci.Event) (appdata.Event, error) {
+func eventToAppDataEvent(event abci.Event, height int64) (appdata.Event, error) {
 	appdataEvent := appdata.Event{
-		Type: event.Type,
+		BlockNumber: uint64(height),
+		Type:        event.Type,
 		Attributes: func() ([]appdata.EventAttribute, error) {
 			attrs := make([]appdata.EventAttribute, len(event.Attributes))
 			for j, attr := range event.Attributes {
@@ -197,7 +200,8 @@ func eventToAppDataEvent(event abci.Event) (appdata.Event, error) {
 }
 
 type listenerWrapper struct {
-	listener appdata.Listener
+	listener  appdata.Listener
+	txDecoder sdk.TxDecoder
 }
 
 // NewListenerWrapper creates a new listenerWrapper.
@@ -208,20 +212,35 @@ func NewListenerWrapper(listener appdata.Listener) listenerWrapper {
 
 func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.FinalizeBlockRequest, res abci.FinalizeBlockResponse) error {
 	if p.listener.StartBlock != nil {
+		// clean up redundant data
+		reqWithoutTxs := req
+		reqWithoutTxs.Txs = nil
+
 		if err := p.listener.StartBlock(appdata.StartBlockData{
 			Height:      uint64(req.Height),
 			HeaderBytes: nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009
-			HeaderJSON:  nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009
+			HeaderJSON: func() (json.RawMessage, error) {
+				return json.Marshal(reqWithoutTxs)
+			},
 		}); err != nil {
 			return err
 		}
 	}
 	if p.listener.OnTx != nil {
 		for i, tx := range req.Txs {
 			if err := p.listener.OnTx(appdata.TxData{
-				TxIndex: int32(i),
-				Bytes:   func() ([]byte, error) { return tx, nil },
-				JSON:    nil, // TODO: https://github.com/cosmos/cosmos-sdk/issues/22009
+				BlockNumber: uint64(req.Height),
+				TxIndex:     int32(i),
+				Bytes:       func() ([]byte, error) { return tx, nil },
+				JSON: func() (json.RawMessage, error) {
+					sdkTx, err := p.txDecoder(tx)
+					if err != nil {
+						// if the transaction cannot be decoded, return the error as JSON
+						// as there are some txs that might not be decodeable by the txDecoder
+						return json.Marshal(err)
+					}
+					return json.Marshal(sdkTx)
+				},
 			}); err != nil {
 				return err
 			}
@@ -231,14 +250,14 @@ func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.Finaliz
 		events := make([]appdata.Event, len(res.Events))
 		var err error
 		for i, event := range res.Events {
-			events[i], err = eventToAppDataEvent(event)
+			events[i], err = eventToAppDataEvent(event, req.Height)
 			if err != nil {
 				return err
 			}
 		}
 		for _, txResult := range res.TxResults {
 			for _, event := range txResult.Events {
-				appdataEvent, err := eventToAppDataEvent(event)
+				appdataEvent, err := eventToAppDataEvent(event, req.Height)
 				if err != nil {
 					return err
 				}

codec/collections.go

@@ -398,6 +398,7 @@ func protoCol(f protoreflect.FieldDescriptor) schema.Field {
 			col.Kind = schema.StringKind
 		case protoreflect.BytesKind:
 			col.Kind = schema.BytesKind
+			col.Nullable = true
 		case protoreflect.EnumKind:
 			// TODO: support enums
 			col.Kind = schema.EnumKind

docs/architecture/adr-065-store-v2.md

@@ -79,13 +79,11 @@ We propose to build upon some of the great ideas introduced in [ADR-040](./adr-0
 while being a bit more flexible with the underlying implementations and overall
 less intrusive. Specifically, we propose to:
 
-* Separate the concerns of state commitment (**SC**), needed for consensus, and
-  state storage (**SS**), needed for state machine and clients.
 * Reduce layers of abstractions necessary between the RMS and underlying stores.
 * Remove unnecessary store types and implementations such as `CacheKVStore`.
-* Simplify the branching logic.
+* Remove the branching logic from the store package.
 * Ensure the `RootStore` interface remains as lightweight as possible.
-* Allow application developers to easily swap out SS and SC backends.
+* Allow application developers to easily swap out SC backends.
 
 Furthermore, we will keep IAVL as the default [SC](https://cryptography.fandom.com/wiki/Commitment_scheme)
 backend for the time being. While we might not fully settle on the use of IAVL in
@@ -95,18 +93,12 @@ to change the backing commitment store in the future should evidence arise to
 warrant a better alternative. However there is promising work being done to IAVL
 that should result in significant performance improvement <sup>[1,2]</sup>.
 
-Note, we will provide applications with the ability to use IAVL v1 and IAVL v2 as
+Note, we will provide applications with the ability to use IAVL v1,  IAVL v2 and MemIAVL as
 either SC backend, with the latter showing extremely promising performance improvements
 over IAVL v0 and v1, at the cost of a state migration.
 
-### Separating SS and SC
 
-By separating SS and SC, it will allow for us to optimize against primary use cases
-and access patterns to state. Specifically, The SS layer will be responsible for
-direct access to data in the form of (key, value) pairs, whereas the SC layer (e.g. IAVL)
-will be responsible for committing to data and providing Merkle proofs.
-
-#### State Commitment (SC)
+### State Commitment (SC)
 
 A foremost design goal is that SC backends should be easily swappable, i.e. not
 necessarily IAVL.  To this end, the scope of SC has been reduced, it must only:
@@ -121,45 +113,6 @@ due to the time and space constraints, but since store v2 defines an API for his
 proofs there should be at least one configuration of a given SC backend which
 supports this.
 
-#### State Storage (SS)
-
-The goal of SS is to provide a modular storage backend, i.e. multiple implementations,
-to facilitate storing versioned raw key/value pairs in a fast embedded database.
-The responsibility and functions of SS include the following:
-
-* Provided fast and efficient queries for versioned raw key/value pairs
-* Provide versioned CRUD operations
-* Provide versioned batching functionality
-* Provide versioned iteration (forward and reverse) functionality
-* Provide pruning functionality
-
-All of the functionality provided by an SS backend should work under a versioned
-scheme, i.e. a user should be able to get, store, and iterate over keys for the latest
-and historical versions efficiently and a store key, which is used for name-spacing
-purposes.
-
-We propose to have three defaulting SS backends for applications to choose from:
-
-* RocksDB
-    * CGO based
-    * Usage of User-Defined Timestamps as a built-in versioning mechanism
-* PebbleDB
-    * Native
-    * Manual implementation of MVCC keys for versioning
-* SQLite
-    * CGO based
-    * Single table for all state
-
-Since operators might want pruning strategies to differ in SS compared to SC,
-e.g. having a very tight pruning strategy in SC while having a looser pruning
-strategy for SS, we propose to introduce an additional pruning configuration,
-with parameters that are identical to what exists in the SDK today, and allow
-operators to control the pruning strategy of the SS layer independently of the
-SC layer.
-
-Note, the SC pruning strategy must be congruent with the operator's state sync
-configuration. This is so as to allow state sync snapshots to execute successfully,
-otherwise, a snapshot could be triggered on a height that is not available in SC.
 
 #### State Sync
 
@@ -179,7 +132,7 @@ the primary interface for the application to interact with. The `RootStore` will
 be responsible for housing SS and SC backends. Specifically, a `RootStore` will
 provide the following functionality:
 
-* Manage commitment of state (both SS and SC)
+* Manage commitment of state 
 * Provide modules access to state
 * Query delegation (i.e. get a value for a <key, height> tuple)
 * Providing commitment proofs
@@ -197,12 +150,7 @@ solely provide key prefixing/namespacing functionality for modules.
 
 #### Proofs
 
-Since the SS layer is naturally a storage layer only, without any commitments
-to (key, value) pairs, it cannot provide Merkle proofs to clients during queries.
-
-So providing inclusion and exclusion proofs, via a `CommitmentOp` type, will be
-the responsibility of the SC backend. Retrieving proofs will be done through the
-a `RootStore`, which will internally route the request to the SC backend.
+Providing a `CommitmentOp` type, will be the responsibility of the SC backend. Retrieving proofs will be done through the a `RootStore`, which will internally route the request to the SC backend.
 
 #### Commitment
 
@@ -231,9 +179,6 @@ and storage backends for further performance, in addition to a reduced amount of
 abstraction around KVStores making operations such as caching and state branching
 more intuitive.
 
-However, due to the proposed design, there are drawbacks around providing state
-proofs for historical queries.
-
 ### Backwards Compatibility
 
 This ADR proposes changes to the storage implementation in the Cosmos SDK through
@@ -243,17 +188,14 @@ be broken or modified.
 
 ### Positive
 
-* Improved performance of independent SS and SC layers
+* Improved performance of SC layers
 * Reduced layers of abstraction making storage primitives easier to understand
-* Atomic commitments for SC
 * Redesign of storage types and interfaces will allow for greater experimentation
   such as different physical storage backends and different commitment schemes
   for different application modules
 
 ### Negative
 
-* Providing proofs for historical state is challenging
-
 ### Neutral
 
 * Removal of OCAP-based store keys in favor of simple strings for state retrieval

docs/architecture/adr-75-v2.md

@@ -56,7 +56,7 @@ The alternative to doing a rewrite is to spend more time cleaning up baseapp. Th
 
 ## Decision
 
-The Descision is to rewrite the core componenets (baseapp, server, store) of the SDK into smaller modules. 
+The Decision is to rewrite the core components (baseapp, server, store) of the SDK into smaller modules. 
 
 These components will be broken into separate go.mods. The modules consist of the following:
 
@@ -135,7 +135,7 @@ The state transition function interface is simple and meant to be as light weigh
 
 ```mermaid
 graph TD    
-    subgraph STF[State Transition Funciton]
+    subgraph STF[State Transition Function]
         BR --> DB
         subgraph DB[DeliverBlock]
             PB[PreBlock]
@@ -205,9 +205,9 @@ The design of the node comes with a number of tradeoffs.
 
 ### Backwards Compatibility
 
-The state machine was made to not affect modules that are not using the state transition function. If a user would like to migrate to v2 they will need to migrate to `appmodule.Environment` from `sdk.Context`.  `sdk.Context` is a struct which is a global in the state machine, this desing limits the concurrency. 
+The state machine was made to not affect modules that are not using the state transition function. If a user would like to migrate to v2 they will need to migrate to `appmodule.Environment` from `sdk.Context`.  `sdk.Context` is a struct which is a global in the state machine, this design limits the concurrency. 
 
-V2 will have a breaking changes in regards to how CometBFT handles certain fields in ABCI. Previously, the Cosmos SDK paniced and recovered in the case of out of gas, providing an error to CometBFT which we do not return in the new design. 
+V2 will have a breaking changes in regards to how CometBFT handles certain fields in ABCI. Previously, the Cosmos SDK panicked and recovered in the case of out of gas, providing an error to CometBFT which we do not return in the new design. 
 
 V2 only works with `Store/v2`, `IAVL V1` can be used with `Store/v2`. This allows chains to continue with existing databases. There will be a migration happening to convert the database to the separation of Storage and Commitment. Once the migration is completed the state machine will query information from the rawDB unless otherwise specified. 
 

docs/build/building-apps/00-runtime.md

@@ -5,5 +5,23 @@ sidebar_position: 1
 # What is `runtime`?
 
 The `runtime` package is the Cosmos SDK package that combines the building blocks of your blockchain together. It wires together the modules, the applications, the codecs, and the stores.
+It is a layer of abstraction between `baseapp` and the application modules that simplifies the process of building a Cosmos SDK application.
 
+## Modules wiring
+
+Runtime is responsible for wiring the modules together. It uses `depinject` to inject the dependencies of the modules.
+
+## App wiring
+
+Runtime is the base boilerplate of a Cosmos SDK application.
 A user only needs to import `runtime` in their `app.go` and instantiate a `runtime.App`.
+
+## Services
+
+Modules have access to a multitude of services that are provided by the runtime.
+These services include the `store`, the `event manager`, the `context`, and the `logger`.
+As runtime is doing the wiring of modules, it can ensure that the services are scoped to their respective modules.
+
+```go reference
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.2/runtime/module.go#L250-L279
+```

docs/build/building-modules/00-intro.md

@@ -58,6 +58,13 @@ While there are no definitive guidelines for writing modules, here are some impo
 * **Specialization**: A direct consequence of the **composability** feature is that modules should be **specialized**. Developers should carefully establish the scope of their module and not batch multiple functionalities into the same module. This separation of concerns enables modules to be re-used in other projects and improves the upgradability of the application. **Specialization** also plays an important role in the [object-capabilities model](https://docs.cosmos.network/main/learn/advanced/ocap#ocaps-in-practice) of the Cosmos SDK.
 * **Capabilities**: Most modules need to read and/or write to the store(s) of other modules. However, in an open-source environment, it is possible for some modules to be malicious. That is why module developers need to carefully think not only about how their module interacts with other modules, but also about how to give access to the module's store(s). The Cosmos SDK takes a capabilities-oriented approach to inter-module security. This means that each store defined by a module is accessed by a `key`, which is held by the module's [`keeper`](./06-keeper.md). This `keeper` defines how to access the store(s) and under what conditions. Access to the module's store(s) is done by passing a reference to the module's `keeper`.
 
+## Core APIs for Modules
+
+The SDK provides a set of APIs that a module can implement, and a set of services that a module can use.
+Those APIs are defined in the `cosmossdk.io/core/appmodule` package, and are used to defined the module capabilities, which is used by `runtime` during the wiring of the application. 
+
+Learn more about the core APIs for modules [here](../../learn/advanced/02-core.md).
+
 ## Main Components of Cosmos SDK Modules
 
 Modules are by convention defined in the `./x/` subfolder (e.g. the `bank` module will be defined in the `./x/bank` folder). They generally share the same core components: