use dedicated query node for tx and query RPC (#367)

* add quiery node to integration tests Closes: #263 * use query node for tx and query RPC * ignore unused function declaration * connect relayer query node instead of validator * refactor integration test scripts after review Co-authored-by: Jehan <[email protected]> Co-authored-by: Daniel T <[email protected]>
cosmos · Oct 5, 2022 · 149b42e · 149b42e
1 parent 1540617
commit 149b42e
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 43 deletions.
diff --git a/tests/integration/actions.go b/tests/integration/actions.go
@@ -377,7 +377,7 @@ func (tr TestRun) startConsumerChain(
 		"query", "provider", "consumer-genesis",
 		string(tr.chainConfigs[action.consumerChain].chainId),
 
-		`--node`, tr.getValidatorNode(action.providerChain, tr.getDefaultValidator(action.providerChain)),
+		`--node`, tr.getQueryNode(action.providerChain),
 		`-o`, `json`,
 	)
 
@@ -438,12 +438,12 @@ func (tr TestRun) addChainToRelayer(
 	action addChainToRelayerAction,
 	verbose bool,
 ) {
-	valIp := tr.getValidatorIP(action.chain, action.validator)
+	queryNodeIP := tr.getQueryNodeIP(action.chain)
 	chainId := tr.chainConfigs[action.chain].chainId
-	keyName := "validator" + fmt.Sprint(action.validator)
-	rpcAddr := "http://" + valIp + ":26658"
-	grpcAddr := "tcp://" + valIp + ":9091"
-	wsAddr := "ws://" + valIp + ":26657/websocket"
+	keyName := "query"
+	rpcAddr := "http://" + queryNodeIP + ":26658"
+	grpcAddr := "tcp://" + queryNodeIP + ":9091"
+	wsAddr := "ws://" + queryNodeIP + ":26657/websocket"
 
 	chainConfig := fmt.Sprintf(hermesChainConfigTemplate,
 		grpcAddr,

diff --git a/tests/integration/config.go b/tests/integration/config.go
@@ -158,7 +158,11 @@ func (s *TestRun) ValidateStringLiterals() {
 			panic(fmt.Sprintf("ip suffix must be an int: %v\n", err))
 		}
 
-		if ipSuffix < 1 || ipSuffix > 253 {
+		if ipSuffix == 253 {
+			panic("ip suffix 253 is reserved for query node")
+		}
+
+		if ipSuffix < 1 || ipSuffix > 252 {
 			panic("ip suffix out of range, need to change config")
 		}
 	}

diff --git a/tests/integration/state.go b/tests/integration/state.go
@@ -6,7 +6,6 @@ import (
 	"os/exec"
 	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	clienttypes "github.com/cosmos/ibc-go/v3/modules/core/02-client/types"
@@ -127,7 +126,7 @@ func (tr TestRun) getBlockHeight(chain chainID) uint {
 
 		"query", "tendermint-validator-set",
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 	).CombinedOutput()
 
 	if err != nil {
@@ -229,7 +228,7 @@ func (tr TestRun) getReward(chain chainID, validator validatorID, blockHeight ui
 		tr.validatorConfigs[validator].delAddress,
 
 		`--height`, fmt.Sprint(blockHeight),
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	).CombinedOutput()
 
@@ -252,7 +251,7 @@ func (tr TestRun) getBalance(chain chainID, validator validatorID) uint {
 		"query", "bank", "balances",
 		tr.validatorConfigs[validator].delAddress,
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	).CombinedOutput()
 
@@ -275,7 +274,7 @@ func (tr TestRun) getProposal(chain chainID, proposal uint) Proposal {
 		"query", "gov", "proposal",
 		fmt.Sprint(proposal),
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	).CombinedOutput()
 
@@ -360,7 +359,7 @@ func (tr TestRun) getValPower(chain chainID, validator validatorID) uint {
 
 		"query", "tendermint-validator-set",
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 	).CombinedOutput()
 
 	if err != nil {
@@ -406,7 +405,7 @@ func (tr TestRun) getRepresentativePower(chain chainID, validator validatorID) u
 		"query", "staking", "validator",
 		tr.validatorConfigs[validator].valoperAddress,
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	).CombinedOutput()
 
@@ -427,7 +426,7 @@ func (tr TestRun) getParam(chain chainID, param Param) string {
 		param.Subspace,
 		param.Key,
 
-		`--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)),
+		`--node`, tr.getQueryNode(chain),
 		`-o`, `json`,
 	).CombinedOutput()
 
@@ -440,31 +439,6 @@ func (tr TestRun) getParam(chain chainID, param Param) string {
 	return value.String()
 }
 
-// Gets a default validator for txs and queries using the first subdirectory
-// of the directory of the input chain, which will be the home directory
-// of one of the validators.
-// TODO: Best solution for default validator fulfilling queries etc. is a dedicated, non validating, full node.
-// See https://github.com/cosmos/interchain-security/issues/263
-func (s TestRun) getDefaultValidator(chain chainID) validatorID {
-	//#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments.
-	bz, err := exec.Command("docker", "exec", s.containerConfig.instanceName, "bash", "-c",
-		`cd /`+string(s.chainConfigs[chain].chainId)+
-			`; ls -d */ | awk '{print $1}' | head -n 1`).CombinedOutput()
-
-	if err != nil {
-		log.Fatal(err, "\n", string(bz))
-	}
-
-	// Returned string will be of format: "validator<valID literal>/"
-	bzPrefixTrimmed := strings.TrimPrefix(string(bz), "validator")
-	bzFullyTrimmed := bzPrefixTrimmed[:len(bzPrefixTrimmed)-2]
-	if bzPrefixTrimmed == string(bz) || bzFullyTrimmed == string(bz) {
-		log.Fatal("unexpected validator subdirectory name: ", bz)
-	}
-
-	return validatorID(bzFullyTrimmed)
-}
-
 func (tr TestRun) getValidatorNode(chain chainID, validator validatorID) string {
 	return "tcp://" + tr.getValidatorIP(chain, validator) + ":26658"
 }
@@ -476,3 +450,14 @@ func (tr TestRun) getValidatorIP(chain chainID, validator validatorID) string {
 func (tr TestRun) getValidatorHome(chain chainID, validator validatorID) string {
 	return `/` + string(tr.chainConfigs[chain].chainId) + `/validator` + fmt.Sprint(validator)
 }
+
+// getQueryNode returns query node tcp address on chain.
+func (tr TestRun) getQueryNode(chain chainID) string {
+	return fmt.Sprintf("tcp://%s:26658", tr.getQueryNodeIP(chain))
+}
+
+// getQueryNodeIP returns query node IP for chain,
+// ipSuffix is hardcoded to be 253 on all query nodes.
+func (tr TestRun) getQueryNodeIP(chain chainID) string {
+	return fmt.Sprintf("%s.253", tr.chainConfigs[chain].ipPrefix)
+}
diff --git a/tests/integration/testnet-scripts/start-chain.sh b/tests/integration/testnet-scripts/start-chain.sh
@@ -230,12 +230,65 @@ do
     ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/validator$VAL_ID/logs &
 done
 
+## SETUP QUERY NODE NETWORK NAMESPACE
+QUERY_NODE_ID="query"
+QUERY_IP_SUFFIX="253"
+QUERY_NET_NAMESPACE_NAME="$CHAIN_ID-$QUERY_NODE_ID"
+QUERY_IP_ADDR="$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX/24"
+
+ip netns add $QUERY_NET_NAMESPACE_NAME
+ip link add $QUERY_NET_NAMESPACE_NAME-in type veth peer name $QUERY_NET_NAMESPACE_NAME-out
+ip link set $QUERY_NET_NAMESPACE_NAME-in netns $QUERY_NET_NAMESPACE_NAME
+ip netns exec $QUERY_NET_NAMESPACE_NAME ip addr add $QUERY_IP_ADDR dev $QUERY_NET_NAMESPACE_NAME-in
+ip link set $QUERY_NET_NAMESPACE_NAME-out master virtual-bridge
+## DONE ADD SETUP QUERY NODE NETWORK NAMESPACE
+
+## QUERY NODE ENABLE DEVICE
+ip link set $QUERY_NET_NAMESPACE_NAME-out up
+ip netns exec $QUERY_NET_NAMESPACE_NAME ip link set dev $QUERY_NET_NAMESPACE_NAME-in up
+ip netns exec $QUERY_NET_NAMESPACE_NAME ip link set dev lo up
+## DONE QUERY NODE ENABLE DEVICE
+
+## INIT QUERY NODE
+$BIN init --home /$CHAIN_ID/$QUERY_NODE_ID --chain-id=$CHAIN_ID $QUERY_NODE_ID > /dev/null
+cp /$CHAIN_ID/genesis.json /$CHAIN_ID/$QUERY_NODE_ID/config/genesis.json
+## DONE INIT QUERY NODE
+
+
+## START QUERY NODE
+QUERY_GAIA_HOME="--home /$CHAIN_ID/$QUERY_NODE_ID"
+QUERY_RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658"
+QUERY_GRPC_ADDRESS="--grpc.address $CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:9091"
+QUERY_LISTEN_ADDRESS="--address tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26655"
+QUERY_P2P_ADDRESS="--p2p.laddr tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26656"
+QUERY_LOG_LEVEL="--log_level trace" # Keep as "trace" to see panic messages
+QUERY_ENABLE_WEBGRPC="--grpc-web.enable=false"
+
+QUERY_PERSISTENT_PEERS=""
+
+## add validators to persistend peers of QUERY node
+for j in $(seq 0 $(($NODES - 1)));
+do
+    PEER_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$j].val_id")
+    PEER_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$j].ip_suffix")
+    NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_ID/validator$PEER_VAL_ID)
+    ADDRESS="$NODE_ID@$CHAIN_IP_PREFIX.$PEER_VAL_IP_SUFFIX:26656"
+    QUERY_PERSISTENT_PEERS="$QUERY_PERSISTENT_PEERS,$ADDRESS"
+done
+
+# Remove leading comma and concat to flag
+QUERY_PERSISTENT_PEERS="--p2p.persistent_peers ${QUERY_PERSISTENT_PEERS:1}"
+
+## START NODE
+ARGS="$QUERY_GAIA_HOME $QUERY_LISTEN_ADDRESS $QUERY_RPC_ADDRESS $QUERY_GRPC_ADDRESS $QUERY_LOG_LEVEL $QUERY_P2P_ADDRESS $QUERY_ENABLE_WEBGRPC $QUERY_PERSISTENT_PEERS"
+ip netns exec $QUERY_NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/$QUERY_NODE_ID/logs &
+## DONE START NODE
 
 
 
 # poll for chain start
 set +e
-until $BIN query block --node "tcp://$CHAIN_IP_PREFIX.$FIRST_VAL_IP_SUFFIX:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done
+until $BIN query block --node "tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done
 set -e
 
 echo "done!!!!!!!!"

diff --git a/tests/integration/testnet-scripts/start-docker.sh b/tests/integration/testnet-scripts/start-docker.sh
@@ -36,5 +36,4 @@ rm -rf ./cosmos-sdk/
 # Run new test container instance with extended privileges.
 # Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) 
 # See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
-docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh
-`
+docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh &