perf: parse chain-id from big genesis file could be slow (backport #1…

…8204) (#18267)

Co-authored-by: yihuang <[email protected]>
Co-authored-by: marbar3778 <[email protected]>

CHANGELOG.md

@@ -50,6 +50,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (x/staking) [#18035](https://github.com/cosmos/cosmos-sdk/pull/18035) Hoisted out of the redelegation loop, the non-changing validator and delegator addresses parsing.
 * (keyring) [#17913](https://github.com/cosmos/cosmos-sdk/pull/17913) Add `NewAutoCLIKeyring` for creating an AutoCLI keyring from a SDK keyring.
 * (x/consensus) [#18041](https://github.com/cosmos/cosmos-sdk/pull/18041) Let `ToProtoConsensusParams()` return an error.
+* [#18204](https://github.com/cosmos/cosmos-sdk/pull/18204) Use streaming json parser to parse chain-id from genesis file.
 
 ### Bug Fixes
 

server/util.go

@@ -475,12 +475,16 @@ func DefaultBaseappOptions(appOpts types.AppOptions) []func(*baseapp.BaseApp) {
 	chainID := cast.ToString(appOpts.Get(flags.FlagChainID))
 	if chainID == "" {
 		// fallback to genesis chain-id
-		appGenesis, err := genutiltypes.AppGenesisFromFile(filepath.Join(homeDir, "config", "genesis.json"))
+		reader, err := os.Open(filepath.Join(homeDir, "config", "genesis.json"))
 		if err != nil {
 			panic(err)
 		}
+		defer reader.Close()
 
-		chainID = appGenesis.ChainID
+		chainID, err = genutiltypes.ParseChainIDFromGenesis(reader)
+		if err != nil {
+			panic(fmt.Errorf("failed to parse chain-id from genesis file: %w", err))
+		}
 	}
 
 	snapshotStore, err := GetSnapshotStore(appOpts)

x/genutil/types/chain_id.go

@@ -0,0 +1,69 @@
+package types
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cometbft/cometbft/types"
+)
+
+const ChainIDFieldName = "chain_id"
+
+// ParseChainIDFromGenesis parses the `chain_id` from a genesis JSON file, aborting early after finding the `chain_id`.
+// For efficiency, it's recommended to place the `chain_id` field before any large entries in the JSON file.
+// Returns an error if the `chain_id` field is not found.
+func ParseChainIDFromGenesis(r io.Reader) (string, error) {
+	dec := json.NewDecoder(r)
+
+	t, err := dec.Token()
+	if err != nil {
+		return "", err
+	}
+	if t != json.Delim('{') {
+		return "", fmt.Errorf("expected {, got %s", t)
+	}
+
+	for dec.More() {
+		t, err = dec.Token()
+		if err != nil {
+			return "", err
+		}
+		key, ok := t.(string)
+		if !ok {
+			return "", fmt.Errorf("expected string for the key type, got %s", t)
+		}
+
+		if key == ChainIDFieldName {
+			var chainID string
+			if err := dec.Decode(&chainID); err != nil {
+				return "", err
+			}
+			if err := validateChainID(chainID); err != nil {
+				return "", err
+			}
+			return chainID, nil
+		}
+
+		// skip the value
+		var value json.RawMessage
+		if err := dec.Decode(&value); err != nil {
+			return "", err
+		}
+	}
+
+	return "", errors.New("missing chain-id in genesis file")
+}
+
+func validateChainID(chainID string) error {
+	if strings.TrimSpace(chainID) == "" {
+		return errors.New("genesis doc must include non-empty chain_id")
+	}
+	if len(chainID) > types.MaxChainIDLen {
+		return fmt.Errorf("chain_id in genesis doc is too long (max: %d)", types.MaxChainIDLen)
+	}
+
+	return nil
+}

x/genutil/types/chain_id_test.go

@@ -0,0 +1,130 @@
+package types_test
+
+import (
+	_ "embed"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cosmos/cosmos-sdk/x/genutil/types"
+)
+
+//go:embed testdata/parse_chain_id.json
+var BenchmarkGenesis string
+
+func TestParseChainIDFromGenesis(t *testing.T) {
+	testCases := []struct {
+		name       string
+		json       string
+		expChainID string
+		expError   string
+	}{
+		{
+			"success",
+			`{
+				"state": {
+				  "accounts": {
+					"a": {}
+				  }
+				},
+				"chain_id": "test-chain-id"
+			}`,
+			"test-chain-id",
+			"",
+		},
+		{
+			"nested",
+			`{
+				"state": {
+				  "accounts": {
+					"a": {}
+				  },
+				  "chain_id": "test-chain-id"
+				}
+			}`,
+			"",
+			"missing chain-id in genesis file",
+		},
+		{
+			"not exist",
+			`{
+				"state": {
+				  "accounts": {
+					"a": {}
+				  }
+				},
+				"chain-id": "test-chain-id"
+			}`,
+			"",
+			"missing chain-id in genesis file",
+		},
+		{
+			"invalid type",
+			`{
+				"chain-id": 1,
+			}`,
+			"",
+			"invalid character '}' looking for beginning of object key string",
+		},
+		{
+			"invalid json",
+			`[ " ': }`,
+			"",
+			"expected {, got [",
+		},
+		{
+			"empty chain_id",
+			`{"chain_id": ""}`,
+			"",
+			"genesis doc must include non-empty chain_id",
+		},
+		{
+			"whitespace chain_id",
+			`{"chain_id": "   "}`,
+			"",
+			"genesis doc must include non-empty chain_id",
+		},
+		{
+			"chain_id too long",
+			`{"chain_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}`,
+			"",
+			"chain_id in genesis doc is too long",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			chainID, err := types.ParseChainIDFromGenesis(strings.NewReader(tc.json))
+			if tc.expChainID == "" {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expError)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expChainID, chainID)
+			}
+		})
+	}
+}
+
+func BenchmarkParseChainID(b *testing.B) {
+	expChainID := "cronosmainnet_25-1"
+	b.ReportAllocs()
+	b.Run("new", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			chainID, err := types.ParseChainIDFromGenesis(strings.NewReader(BenchmarkGenesis))
+			require.NoError(b, err)
+			require.Equal(b, expChainID, chainID)
+		}
+	})
+
+	b.Run("old", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			doc, err := types.AppGenesisFromReader(strings.NewReader(BenchmarkGenesis))
+			require.NoError(b, err)
+			require.Equal(b, expChainID, doc.ChainID)
+		}
+	})
+}

x/genutil/types/testdata/parse_chain_id.json

@@ -0,0 +1 @@
+{"genesis_time":"2021-11-08T01:00:00Z","chain_id":"cronosmainnet_25-1","initial_height":"1","consensus_params":{"block":{"max_bytes":"1048576","max_gas":"10000000","time_iota_ms":"1000"},"evidence":{"max_age_num_blocks":"403200","max_age_duration":"2419200000000000","max_bytes":"150000"},"validator":{"pub_key_types":["ed25519"]},"version":{}},"app_hash":"","app_state":{"auth":{"params":{"max_memo_characters":"256","tx_sig_limit":"7","tx_size_cost_per_byte":"10","sig_verify_cost_ed25519":"590","sig_verify_cost_secp256k1":"1000"},"accounts":[{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1q7q2mmmcx2nlw6ptw4a9a3danlnu8z6tq32gv9","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1s8372smy0erx5k4usf84s39tpgy3kmrvjwdejw","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1efw0q0ggzxtmuf80wpcgr77h70c3avpdp9nq5k","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1f7r687vm68jc6qw7rsut07puh9n7s9kzdj0zph","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1aaxs058pksrq8cx3k0nrxv60p2a9c7nq527949","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc17s50usvlr5934tr2fxsesr89k4twtm25vjl6zs","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc1tasmcx3rqpglmsafdt7vylnsdkc3yjlzfv3u63","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},{"@type":"/ethermint.types.v1.EthAccount","base_account":{"address":"crc17m20ajc6d7mu9j34q956q5x5sw7c0wyrem3s7n","pub_key":null,"account_number":"0","sequence":"0"},"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"}]},"authz":{"authorization":[]},"bank":{"params":{"send_enabled":[{"denom":"stake","enabled":true},{"denom":"basecro","enabled":false}],"default_send_enabled":true},"balances":[{"address":"crc1q7q2mmmcx2nlw6ptw4a9a3danlnu8z6tq32gv9","coins":[{"denom":"stake","amount":"10000000000"}]},{"address":"crc1f7r687vm68jc6qw7rsut07puh9n7s9kzdj0zph","coins":[{"denom":"stake","amount":"10000000000"}]},{"address":"crc1tasmcx3rqpglmsafdt7vylnsdkc3yjlzfv3u63","coins":[{"denom":"stake","amount":"600000000000"}]},{"address":"crc1s8372smy0erx5k4usf84s39tpgy3kmrvjwdejw","coins":[{"denom":"stake","amount":"10000000000"}]},{"address":"crc1efw0q0ggzxtmuf80wpcgr77h70c3avpdp9nq5k","coins":[{"denom":"stake","amount":"10000000000"}]},{"address":"crc1aaxs058pksrq8cx3k0nrxv60p2a9c7nq527949","coins":[{"denom":"stake","amount":"10000"}]},{"address":"crc17s50usvlr5934tr2fxsesr89k4twtm25vjl6zs","coins":[{"denom":"stake","amount":"1000000000000000"}]},{"address":"crc17m20ajc6d7mu9j34q956q5x5sw7c0wyrem3s7n","coins":[{"denom":"stake","amount":"50000000"}]}],"supply":[{"denom":"stake","amount":"1000640050010000"}],"denom_metadata":[]},"capability":{"index":"1","owners":[]},"crisis":{"constant_fee":{"denom":"stake","amount":"1000"}},"cronos":{"params":{"ibc_cro_denom":"ibc/6411AE2ADA1E73DB59DB151A8988F9B7D5E7E233D8414DB6817F8F1A01611F86","ibc_timeout":"86400000000000","cronos_admin":"crc1tasmcx3rqpglmsafdt7vylnsdkc3yjlzfv3u63","enable_auto_deployment":false},"external_contracts":[],"auto_contracts":[]},"distribution":{"params":{"community_tax":"0","base_proposer_reward":"0","bonus_proposer_reward":"0","withdraw_addr_enabled":true},"fee_pool":{"community_pool":[]},"delegator_withdraw_infos":[],"previous_proposer":"","outstanding_rewards":[],"validator_accumulated_commissions":[],"validator_historical_rewards":[],"validator_current_rewards":[],"delegator_starting_infos":[],"validator_slash_events":[]},"evidence":{"evidence":[]},"evm":{"accounts":[],"params":{"evm_denom":"basecro","enable_create":true,"enable_call":true,"extra_eips":["2929","2200","1884","1344"],"chain_config":{"homestead_block":"0","dao_fork_block":"0","dao_fork_support":true,"eip150_block":"0","eip150_hash":"0x0000000000000000000000000000000000000000000000000000000000000000","eip155_block":"0","eip158_block":"0","byzantium_block":"0","constantinople_block":"0","petersburg_block":"0","istanbul_block":"0","muir_glacier_block":"0","berlin_block":"0","catalyst_block":null,"london_block":"9223372036854775808"}}},"feegrant":{"allowances":[]},"genutil":{"gen_txs":[{"body":{"messages":[{"@type":"/cosmos.staking.v1beta1.MsgCreateValidator","description":{"moniker":"Pioneer 11","identity":"BB602FB166F21C6E","website":"https://cronos.crypto.org","security_contact":"[email protected]","details":""},"commission":{"rate":"1.000000000000000000","max_rate":"1.000000000000000000","max_change_rate":"1.000000000000000000"},"min_self_delegation":"1","delegator_address":"crc1q7q2mmmcx2nlw6ptw4a9a3danlnu8z6tq32gv9","validator_address":"crcvaloper1q7q2mmmcx2nlw6ptw4a9a3danlnu8z6t2wmyjv","pubkey":{"@type":"/cosmos.crypto.ed25519.PubKey","key":"H8pPcYZvZYd/cI2pKuwcmyst7ZTWm5+QkXXuCoGg1P0="},"value":{"denom":"stake","amount":"10000000000"}}],"memo":"","timeout_height":0,"extension_options":[],"non_critical_extension_options":[]},"auth_info":{"signer_infos":[{"public_key":{"@type":"/ethermint.crypto.v1.ethsecp256k1.PubKey","key":"A/S9y8Yhb2FVVInVKmdPOtgIiKUlm7CE8ixT14GmNYm1"},"mode_info":{"single":{"mode":"SIGN_MODE_DIRECT"}},"sequence":0}],"fee":{"amount":[],"gas_limit":200000,"payer":"","granter":""}},"signatures":["35KHI9qc+w+2cOKwlmgXzx1QaPfsK9lxs1Z7INaiNHgHXqRHBUTgl8eKzIHTCnFJU89gpLPEqjUsLDasiHavggA="]},{"body":{"messages":[{"@type":"/cosmos.staking.v1beta1.MsgCreateValidator","description":{"moniker":"Voyager 1","identity":"BB602FB166F21C6E","website":"https://cronos.crypto.org","security_contact":"[email protected]","details":""},"commission":{"rate":"1.000000000000000000","max_rate":"1.000000000000000000","max_change_rate":"1.000000000000000000"},"min_self_delegation":"1","delegator_address":"crc1s8372smy0erx5k4usf84s39tpgy3kmrvjwdejw","validator_address":"crcvaloper1s8372smy0erx5k4usf84s39tpgy3kmrvc3u4v8","pubkey":{"@type":"/cosmos.crypto.ed25519.PubKey","key":"7WE80ID9TzUjIb5Lu9A+ncidsND5+sJAUL6l/NNn4KE="},"value":{"denom":"stake","amount":"10000000000"}}],"memo":"","timeout_height":0,"extension_options":[],"non_critical_extension_options":[]},"auth_info":{"signer_infos":[{"public_key":{"@type":"/ethermint.crypto.v1.ethsecp256k1.PubKey","key":"AxI/HqRT4KKIQhonXtJBe9H/wRq4BkvAZO+4TxSQSjZt"},"mode_info":{"single":{"mode":"SIGN_MODE_DIRECT"}},"sequence":0}],"fee":{"amount":[],"gas_limit":200000,"payer":"","granter":""}},"signatures":["TlS6sZBiFNI64lr13x+sr3rrwV+U7icm/V9ksIMEtlhSZlXNJhS+2hQywyGutb6JhA+Ov+Wjln4puHm/MSA3iAE="]},{"body":{"messages":[{"@type":"/cosmos.staking.v1beta1.MsgCreateValidator","description":{"moniker":"Voyager 2","identity":"BB602FB166F21C6E","website":"https://cronos.crypto.org","security_contact":"[email protected]","details":""},"commission":{"rate":"1.000000000000000000","max_rate":"1.000000000000000000","max_change_rate":"1.000000000000000000"},"min_self_delegation":"1","delegator_address":"crc1efw0q0ggzxtmuf80wpcgr77h70c3avpdp9nq5k","validator_address":"crcvaloper1efw0q0ggzxtmuf80wpcgr77h70c3avpdt6zv2l","pubkey":{"@type":"/cosmos.crypto.ed25519.PubKey","key":"TXH4O/GhYibZffcBo5UAERwi9T4VgPAiVCDyl4rdFDE="},"value":{"denom":"stake","amount":"10000000000"}}],"memo":"","timeout_height":0,"extension_options":[],"non_critical_extension_options":[]},"auth_info":{"signer_infos":[{"public_key":{"@type":"/ethermint.crypto.v1.ethsecp256k1.PubKey","key":"AzR06WzkRRXNNE+A4VFG18w170ZicRa/mVsLvBGlBAxz"},"mode_info":{"single":{"mode":"SIGN_MODE_DIRECT"}},"sequence":0}],"fee":{"amount":[],"gas_limit":200000,"payer":"","granter":""}},"signatures":["mx//YsSSwUP4G020FLRS0532nP2AylVPqRUXb9EdauM4OWdXF0+Ftx8jbcFinzBw8PHGof3XHcg2yw2P2qNBMAE="]},{"body":{"messages":[{"@type":"/cosmos.staking.v1beta1.MsgCreateValidator","description":{"moniker":"Huygens","identity":"BB602FB166F21C6E","website":"https://cronos.crypto.org","security_contact":"[email protected]","details":""},"commission":{"rate":"1.000000000000000000","max_rate":"1.000000000000000000","max_change_rate":"1.000000000000000000"},"min_self_delegation":"1","delegator_address":"crc1f7r687vm68jc6qw7rsut07puh9n7s9kzdj0zph","validator_address":"crcvaloper1f7r687vm68jc6qw7rsut07puh9n7s9kz8d7wl7","pubkey":{"@type":"/cosmos.crypto.ed25519.PubKey","key":"f826QMXnfR9pEAJDEQR4t228BwQVJ7xe2EwGZN8doQY="},"value":{"denom":"stake","amount":"10000000000"}}],"memo":"","timeout_height":0,"extension_options":[],"non_critical_extension_options":[]},"auth_info":{"signer_infos":[{"public_key":{"@type":"/ethermint.crypto.v1.ethsecp256k1.PubKey","key":"AkDcLOeNUKVUS52hUPx+cSo+ohG6vW3P2RYkbW6dwdMG"},"mode_info":{"single":{"mode":"SIGN_MODE_DIRECT"}},"sequence":0}],"fee":{"amount":[],"gas_limit":200000,"payer":"","granter":""}},"signatures":["Rz89OXlfJQgasEeKxsywF/XyiYg7Xdgg/qgwtJcOMuIgJ3hfsE0KrreXlfHOGPIe9nVduffC9W3gKvw2sHe9ugE="]}]},"gov":{"starting_proposal_id":"1","deposits":[],"votes":[],"proposals":[],"deposit_params":{"min_deposit":[{"denom":"basecro","amount":"20000000000000000000000"}],"max_deposit_period":"21600000000000ns"},"voting_params":{"voting_period":"259200000000000ns"},"tally_params":{"quorum":"0.334","threshold":"0.5","veto_threshold":"0.334"}},"ibc":{"client_genesis":{"clients":[],"clients_consensus":[],"clients_metadata":[],"params":{"allowed_clients":["06-solomachine","07-tendermint"]},"create_localhost":false,"next_client_sequence":"0"},"connection_genesis":{"connections":[],"client_connection_paths":[],"next_connection_sequence":"0","params":{"max_expected_time_per_block":"30000000000"}},"channel_genesis":{"channels":[],"acknowledgements":[],"commitments":[],"receipts":[],"send_sequences":[],"recv_sequences":[],"ack_sequences":[],"next_channel_sequence":"0"}},"mint":{"minter":{"inflation":"0.000000000000000000","annual_provisions":"0.000000000000000000"},"params":{"mint_denom":"stake","inflation_rate_change":"0","inflation_max":"0","inflation_min":"0","goal_bonded":"1","blocks_per_year":"6311520"}},"params":null,"slashing":{"params":{"signed_blocks_window":"10000","min_signed_per_window":"0.5","downtime_jail_duration":"28800s","slash_fraction_double_sign":"0","slash_fraction_downtime":"0"},"signing_infos":[],"missed_blocks":[]},"staking":{"params":{"unbonding_time":"2419200000000000ns","max_validators":"50","max_entries":"7","historical_entries":"10000","bond_denom":"stake"},"last_total_power":"0","last_validator_powers":[],"validators":[],"delegations":[],"unbonding_delegations":[],"redelegations":[],"exported":false},"transfer":{"port_id":"transfer","denom_traces":[],"params":{"send_enabled":true,"receive_enabled":true}},"upgrade":{},"vesting":{}}}