test(sim): connext instability tests

test/simulation/actions.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/big"
 	"time"
 
 	"github.com/roasbeef/btcutil"
@@ -58,6 +59,37 @@ func (a *actions) init(node *xudtest.HarnessNode) {
 	}
 }
 
+func (a *actions) initConnext(net *xudtest.NetworkHarness, node *xudtest.HarnessNode, fund bool) {
+	// Wait for node's connext connection to catch-up.
+	err := waitConnextReady(node)
+	a.assert.NoError(err)
+
+	// Fund node's wallet.
+	if fund {
+		resInfo, err := node.Client.GetInfo(context.Background(), &xudrpc.GetInfoRequest{})
+		a.assert.NoError(err)
+		amount := big.NewInt(2000000000000000000)
+		err = net.ConnextNetwork.Wallet.SendEth(resInfo.Connext.Address, amount)
+		a.assert.NoError(err)
+
+		time.Sleep(15 * time.Second)
+	}
+
+	// Init node.
+	ETHTokenAddress := "0x0000000000000000000000000000000000000000"
+	a.addCurrency(node, "ETH", 2, ETHTokenAddress, 18)
+	a.addPair(node, "BTC", "ETH")
+	err = net.RestartNode(node)
+	a.assert.NoError(err)
+
+	// Verify node's ETH balance.
+	if fund {
+		resBal, err := node.Client.GetBalance(a.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"})
+		a.assert.NoError(err)
+		a.assert.Equal(uint64(200000000), resBal.Balances["ETH"].WalletBalance)
+	}
+}
+
 func (a *actions) addCurrency(node *xudtest.HarnessNode, currency string, swapClient xudrpc.Currency_SwapClient, tokenAddress string, decimalPlaces uint32) {
 	if len(tokenAddress) > 0 {
 		req := &xudrpc.Currency{Currency: currency, SwapClient: swapClient, TokenAddress: tokenAddress, DecimalPlaces: decimalPlaces}

test/simulation/connexttest/harness.go

@@ -1,6 +1,7 @@
 package connexttest
 
 import (
+	"fmt"
 	"sync"
 )
 
@@ -10,7 +11,7 @@ type ConnextError struct {
 }
 
 var (
-	clientPath = "/connext-vol/rest-api-client"
+	ClientPath = "/connext-vol/rest-api-client"
 
 	// 172.17.0.1 is the static IP equivalent of `host.docker.internal` for accessing
 	// the `localhost` of the host machine.
@@ -28,7 +29,7 @@ type NetworkHarness struct {
 
 	Wallet *EthWallet
 
-	errorChan chan *ConnextError
+	ErrorChan chan *ConnextError
 
 	quit chan struct{}
 
@@ -39,14 +40,14 @@ type NetworkHarness struct {
 func NewNetworkHarness() *NetworkHarness {
 	n := NetworkHarness{
 		ActiveClients: make(map[int]*HarnessClient),
-		errorChan:     make(chan *ConnextError),
+		ErrorChan:     make(chan *ConnextError),
 		quit:          make(chan struct{}),
 	}
 	return &n
 }
 
-func (n *NetworkHarness) newClient(name string, path string) (*HarnessClient, error) {
-	client, err := newClient(name, path, EthProviderURL, NodeURL)
+func (n *NetworkHarness) NewClient(name string) (*HarnessClient, error) {
+	client, err := newClient(name, ClientPath, EthProviderURL, NodeURL)
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +69,7 @@ func (n *NetworkHarness) Start() error {
 		client := _client
 		go func() {
 			defer wg.Done()
-			if err := client.Start(n.errorChan); err != nil {
+			if err := client.Start(n.ErrorChan); err != nil {
 				if err != nil {
 					errChan <- err
 					return
@@ -97,21 +98,21 @@ func (n *NetworkHarness) Start() error {
 func (n *NetworkHarness) SetUp() error {
 	var err error
 
-	n.Alice, err = n.newClient("Alice", clientPath)
+	n.Alice, err = n.NewClient("Alice")
 	if err != nil {
 		return err
 	}
-	n.Bob, err = n.newClient("Bob", clientPath)
+	n.Bob, err = n.NewClient("Bob")
 	if err != nil {
 		return err
 	}
 
-	n.Carol, err = n.newClient("Carol", clientPath)
+	n.Carol, err = n.NewClient("Carol")
 	if err != nil {
 		return err
 	}
 
-	n.Dave, err = n.newClient("Dave", clientPath)
+	n.Dave, err = n.NewClient("Dave")
 	if err != nil {
 		return err
 	}
@@ -120,10 +121,10 @@ func (n *NetworkHarness) SetUp() error {
 }
 
 func (n *NetworkHarness) ProcessErrors() <-chan *ConnextError {
-	return n.errorChan
+	return n.ErrorChan
 }
 
-// TearDownAll tears down all active nodes.
+// TearDownAll tears down all active clients.
 func (n *NetworkHarness) TearDownAll() error {
 	for _, client := range n.ActiveClients {
 		if err := client.shutdown(true, true); err != nil {
@@ -133,8 +134,23 @@ func (n *NetworkHarness) TearDownAll() error {
 		delete(n.ActiveClients, client.ID)
 	}
 
-	close(n.errorChan)
+	close(n.ErrorChan)
 	close(n.quit)
 
 	return nil
 }
+
+// TearDown tears down a specific client.
+func (n *NetworkHarness) TearDown(id int) error {
+	client, ok := n.ActiveClients[id]
+	if !ok {
+		return fmt.Errorf("client (%d) not found", id)
+	}
+	if err := client.shutdown(true, true); err != nil {
+		return err
+	}
+
+	delete(n.ActiveClients, id)
+
+	return nil
+}

test/simulation/custom-xud.patch

@@ -5,7 +5,7 @@ index 08402caa8..c9972d258 100644
 @@ -87,6 +87,11 @@ class Xud extends EventEmitter {
        this.logger.info('config file loaded');
      }
- 
+
 +    this.logger.info('CUSTOM-XUD');
 +    if (process.env.CUSTOM_SCENARIO) {
 +      this.logger.info(`CUSTOM_SCENARIO=${process.env.CUSTOM_SCENARIO}`);
@@ -19,22 +19,22 @@ index 090618c4b..cd33a5d58 100644
 --- a/lib/swaps/SwapRecovery.ts
 +++ b/lib/swaps/SwapRecovery.ts
 @@ -28,7 +28,15 @@ class SwapRecovery extends EventEmitter {
- 
+
    public beginTimer = () => {
      if (!this.pendingSwapsTimer) {
 -      this.pendingSwapsTimer = setInterval(this.checkPendingSwaps, SwapRecovery.PENDING_SWAP_RECHECK_INTERVAL);
 +      let interval = SwapRecovery.PENDING_SWAP_RECHECK_INTERVAL;
-+      if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') {
-+        interval = 5000;
++      if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') {
++        interval = 2000;
 +      }
 +      if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CRASH_AFTER_SEND') {
-+        interval = 5000;
++        interval = 2000;
 +      }
 +
 +      this.pendingSwapsTimer = setInterval(this.checkPendingSwaps, interval);
      }
    }
- 
+
 diff --git a/lib/swaps/Swaps.ts b/lib/swaps/Swaps.ts
 index 7fd02cbb8..2bc60fe51 100644
 --- a/lib/swaps/Swaps.ts
@@ -53,10 +53,10 @@ index 7fd02cbb8..2bc60fe51 100644
 +          await setTimeoutPromise(5000);
 +        }
 +
-+        if (deal && deal.role === SwapRole.Maker && process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') {
++        if (deal && deal.role === SwapRole.Maker && process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') {
 +          this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`);
-+          this.logger.info(`LNDLTC_PID: ${process.env.LNDLTC_PID}`);
-+          process.kill(parseInt(process.env.LNDLTC_PID!, 10));
++          this.logger.info(`CLIENT_TYPE ${process.env.CLIENT_TYPE}, CLIENT_PID: ${process.env.CLIENT_PID}`);
++          process.kill(parseInt(process.env.CLIENT_PID!, 10));
 +          await setTimeoutPromise(1000);
 +        }
 +
@@ -82,7 +82,7 @@ index 7fd02cbb8..2bc60fe51 100644
      const { quantity, rHash, makerCltvDelta } = responsePacket.body;
      const deal = this.getDeal(rHash);
 @@ -821,6 +849,11 @@ class Swaps extends EventEmitter {
- 
+
      try {
        await makerSwapClient.sendPayment(deal);
 +
@@ -94,9 +94,9 @@ index 7fd02cbb8..2bc60fe51 100644
        // first we must handle the edge case where the maker has paid us but failed to claim our payment
        // in this case, we've already marked the swap as having been paid and completed
 @@ -1002,6 +1035,18 @@ class Swaps extends EventEmitter {
- 
+
        this.logger.debug('Executing maker code to resolve hash');
- 
+
 +      if (process.env.CUSTOM_SCENARIO === 'SECURITY::MAKER_1ST_HTLC_STALL') {
 +        this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`);
 +        const makerSwapClient = this.swapClientManager.get(deal.makerCurrency)!;
@@ -110,11 +110,11 @@ index 7fd02cbb8..2bc60fe51 100644
 +      }
 +
        const swapClient = this.swapClientManager.get(deal.takerCurrency)!;
- 
+
        // we update the phase persist the deal to the database before we attempt to send payment
 @@ -1012,6 +1057,13 @@ class Swaps extends EventEmitter {
        assert(deal.state !== SwapState.Error, `cannot send payment for failed swap ${deal.rHash}`);
- 
+
        try {
 +        if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CRASH_AFTER_SEND') {
 +          setTimeout(() => {
@@ -129,7 +129,7 @@ index 7fd02cbb8..2bc60fe51 100644
 @@ -1073,6 +1125,16 @@ class Swaps extends EventEmitter {
        assert(htlcCurrency === undefined || htlcCurrency === deal.takerCurrency, 'incoming htlc does not match expected deal currency');
        this.logger.debug('Executing taker code to resolve hash');
- 
+
 +      if (process.env.CUSTOM_SCENARIO === 'SECURITY::TAKER_2ND_HTLC_STALL') {
 +        this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`);
 +        return '';
@@ -155,15 +155,15 @@ index 7fd02cbb8..2bc60fe51 100644
 +        swapClient.removeInvoice(deal.rHash).catch(this.logger.error); // we don't need to await the remove invoice call
 +      }
      }
- 
+
      this.logger.trace(`emitting swap.failed event for ${deal.rHash}`);
 @@ -1313,9 +1378,14 @@ class Swaps extends EventEmitter {
- 
+
          if (deal.role === SwapRole.Maker) {
            // the maker begins execution of the swap upon accepting the deal
 +
 +          let interval = Swaps.SWAP_COMPLETE_TIMEOUT + Swaps.SWAP_COMPLETE_MAKER_BUFFER;
-+          if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') {
++          if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') {
 +            interval = 5000;
 +          }
            this.timeouts.set(rHash, setTimeout(

test/simulation/docker-build.sh

@@ -6,7 +6,7 @@ then
     pushd temp
     git clone https://github.com/ConnextProject/indra.git
     cd indra
-    git checkout indra-7.0.0-alpha.6
+    git checkout indra-7.0.0-alpha.14
     make
     popd
 fi

test/simulation/docker-connext/Dockerfile

@@ -6,5 +6,5 @@ WORKDIR /app
 
 RUN git clone https://github.com/connext/rest-api-client.git
 RUN cd rest-api-client && git pull
-RUN cd rest-api-client && git checkout 64dd1eceba441297f4c31f3389f3910a453d550c
+RUN cd rest-api-client && git checkout fe1876a0411e0b9928825ae51104e16a9fd7b787
 RUN cd rest-api-client && npm install && npm run build

test/simulation/docker-run.sh

@@ -16,9 +16,4 @@ pushd temp/indra
 make reset
 popd
 
-if [[ retVal != 0 ]]
-then
-    ./logs.sh
-fi
-
 exit $retVal

test/simulation/logs.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
 find $PWD/temp/logs -type f -name xud*.log -printf "\n%f\n\n" -exec cat {} \;
+find $PWD/temp/logs -type f -name connext*.log -printf "\n%f\n\n" -exec cat {} \;