diff --git a/test/simulation/actions.go b/test/simulation/actions.go index 139f24879..d3937098d 100644 --- a/test/simulation/actions.go +++ b/test/simulation/actions.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/big" "time" "github.com/roasbeef/btcutil" @@ -58,6 +59,37 @@ func (a *actions) init(node *xudtest.HarnessNode) { } } +func (a *actions) initConnext(net *xudtest.NetworkHarness, node *xudtest.HarnessNode, fund bool) { + // Wait for node's connext connection to catch-up. + err := waitConnextReady(node) + a.assert.NoError(err) + + // Fund node's wallet. + if fund { + resInfo, err := node.Client.GetInfo(context.Background(), &xudrpc.GetInfoRequest{}) + a.assert.NoError(err) + amount := big.NewInt(2000000000000000000) + err = net.ConnextNetwork.Wallet.SendEth(resInfo.Connext.Address, amount) + a.assert.NoError(err) + + time.Sleep(15 * time.Second) + } + + // Init node. + ETHTokenAddress := "0x0000000000000000000000000000000000000000" + a.addCurrency(node, "ETH", 2, ETHTokenAddress, 18) + a.addPair(node, "BTC", "ETH") + err = net.RestartNode(node) + a.assert.NoError(err) + + // Verify node's ETH balance. + if fund { + resBal, err := node.Client.GetBalance(a.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + a.assert.NoError(err) + a.assert.Equal(uint64(200000000), resBal.Balances["ETH"].WalletBalance) + } +} + func (a *actions) addCurrency(node *xudtest.HarnessNode, currency string, swapClient xudrpc.Currency_SwapClient, tokenAddress string, decimalPlaces uint32) { if len(tokenAddress) > 0 { req := &xudrpc.Currency{Currency: currency, SwapClient: swapClient, TokenAddress: tokenAddress, DecimalPlaces: decimalPlaces} diff --git a/test/simulation/connexttest/harness.go b/test/simulation/connexttest/harness.go index bd12fffa1..82726156b 100644 --- a/test/simulation/connexttest/harness.go +++ b/test/simulation/connexttest/harness.go @@ -1,6 +1,7 @@ package connexttest import ( + "fmt" "sync" ) @@ -10,7 +11,7 @@ type ConnextError struct { } var ( - clientPath = "/connext-vol/rest-api-client" + ClientPath = "/connext-vol/rest-api-client" // 172.17.0.1 is the static IP equivalent of `host.docker.internal` for accessing // the `localhost` of the host machine. @@ -28,7 +29,7 @@ type NetworkHarness struct { Wallet *EthWallet - errorChan chan *ConnextError + ErrorChan chan *ConnextError quit chan struct{} @@ -39,14 +40,14 @@ type NetworkHarness struct { func NewNetworkHarness() *NetworkHarness { n := NetworkHarness{ ActiveClients: make(map[int]*HarnessClient), - errorChan: make(chan *ConnextError), + ErrorChan: make(chan *ConnextError), quit: make(chan struct{}), } return &n } -func (n *NetworkHarness) newClient(name string, path string) (*HarnessClient, error) { - client, err := newClient(name, path, EthProviderURL, NodeURL) +func (n *NetworkHarness) NewClient(name string) (*HarnessClient, error) { + client, err := newClient(name, ClientPath, EthProviderURL, NodeURL) if err != nil { return nil, err } @@ -68,7 +69,7 @@ func (n *NetworkHarness) Start() error { client := _client go func() { defer wg.Done() - if err := client.Start(n.errorChan); err != nil { + if err := client.Start(n.ErrorChan); err != nil { if err != nil { errChan <- err return @@ -97,21 +98,21 @@ func (n *NetworkHarness) Start() error { func (n *NetworkHarness) SetUp() error { var err error - n.Alice, err = n.newClient("Alice", clientPath) + n.Alice, err = n.NewClient("Alice") if err != nil { return err } - n.Bob, err = n.newClient("Bob", clientPath) + n.Bob, err = n.NewClient("Bob") if err != nil { return err } - n.Carol, err = n.newClient("Carol", clientPath) + n.Carol, err = n.NewClient("Carol") if err != nil { return err } - n.Dave, err = n.newClient("Dave", clientPath) + n.Dave, err = n.NewClient("Dave") if err != nil { return err } @@ -120,10 +121,10 @@ func (n *NetworkHarness) SetUp() error { } func (n *NetworkHarness) ProcessErrors() <-chan *ConnextError { - return n.errorChan + return n.ErrorChan } -// TearDownAll tears down all active nodes. +// TearDownAll tears down all active clients. func (n *NetworkHarness) TearDownAll() error { for _, client := range n.ActiveClients { if err := client.shutdown(true, true); err != nil { @@ -133,8 +134,23 @@ func (n *NetworkHarness) TearDownAll() error { delete(n.ActiveClients, client.ID) } - close(n.errorChan) + close(n.ErrorChan) close(n.quit) return nil } + +// TearDown tears down a specific client. +func (n *NetworkHarness) TearDown(id int) error { + client, ok := n.ActiveClients[id] + if !ok { + return fmt.Errorf("client (%d) not found", id) + } + if err := client.shutdown(true, true); err != nil { + return err + } + + delete(n.ActiveClients, id) + + return nil +} diff --git a/test/simulation/custom-xud.patch b/test/simulation/custom-xud.patch index 04ac96d39..91573f9e2 100644 --- a/test/simulation/custom-xud.patch +++ b/test/simulation/custom-xud.patch @@ -5,7 +5,7 @@ index 08402caa8..c9972d258 100644 @@ -87,6 +87,11 @@ class Xud extends EventEmitter { this.logger.info('config file loaded'); } - + + this.logger.info('CUSTOM-XUD'); + if (process.env.CUSTOM_SCENARIO) { + this.logger.info(`CUSTOM_SCENARIO=${process.env.CUSTOM_SCENARIO}`); @@ -19,22 +19,22 @@ index 090618c4b..cd33a5d58 100644 --- a/lib/swaps/SwapRecovery.ts +++ b/lib/swaps/SwapRecovery.ts @@ -28,7 +28,15 @@ class SwapRecovery extends EventEmitter { - + public beginTimer = () => { if (!this.pendingSwapsTimer) { - this.pendingSwapsTimer = setInterval(this.checkPendingSwaps, SwapRecovery.PENDING_SWAP_RECHECK_INTERVAL); + let interval = SwapRecovery.PENDING_SWAP_RECHECK_INTERVAL; -+ if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') { -+ interval = 5000; ++ if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') { ++ interval = 2000; + } + if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CRASH_AFTER_SEND') { -+ interval = 5000; ++ interval = 2000; + } + + this.pendingSwapsTimer = setInterval(this.checkPendingSwaps, interval); } } - + diff --git a/lib/swaps/Swaps.ts b/lib/swaps/Swaps.ts index 7fd02cbb8..2bc60fe51 100644 --- a/lib/swaps/Swaps.ts @@ -53,10 +53,10 @@ index 7fd02cbb8..2bc60fe51 100644 + await setTimeoutPromise(5000); + } + -+ if (deal && deal.role === SwapRole.Maker && process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') { ++ if (deal && deal.role === SwapRole.Maker && process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') { + this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`); -+ this.logger.info(`LNDLTC_PID: ${process.env.LNDLTC_PID}`); -+ process.kill(parseInt(process.env.LNDLTC_PID!, 10)); ++ this.logger.info(`CLIENT_TYPE ${process.env.CLIENT_TYPE}, CLIENT_PID: ${process.env.CLIENT_PID}`); ++ process.kill(parseInt(process.env.CLIENT_PID!, 10)); + await setTimeoutPromise(1000); + } + @@ -82,7 +82,7 @@ index 7fd02cbb8..2bc60fe51 100644 const { quantity, rHash, makerCltvDelta } = responsePacket.body; const deal = this.getDeal(rHash); @@ -821,6 +849,11 @@ class Swaps extends EventEmitter { - + try { await makerSwapClient.sendPayment(deal); + @@ -94,9 +94,9 @@ index 7fd02cbb8..2bc60fe51 100644 // first we must handle the edge case where the maker has paid us but failed to claim our payment // in this case, we've already marked the swap as having been paid and completed @@ -1002,6 +1035,18 @@ class Swaps extends EventEmitter { - + this.logger.debug('Executing maker code to resolve hash'); - + + if (process.env.CUSTOM_SCENARIO === 'SECURITY::MAKER_1ST_HTLC_STALL') { + this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`); + const makerSwapClient = this.swapClientManager.get(deal.makerCurrency)!; @@ -110,11 +110,11 @@ index 7fd02cbb8..2bc60fe51 100644 + } + const swapClient = this.swapClientManager.get(deal.takerCurrency)!; - + // we update the phase persist the deal to the database before we attempt to send payment @@ -1012,6 +1057,13 @@ class Swaps extends EventEmitter { assert(deal.state !== SwapState.Error, `cannot send payment for failed swap ${deal.rHash}`); - + try { + if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CRASH_AFTER_SEND') { + setTimeout(() => { @@ -129,7 +129,7 @@ index 7fd02cbb8..2bc60fe51 100644 @@ -1073,6 +1125,16 @@ class Swaps extends EventEmitter { assert(htlcCurrency === undefined || htlcCurrency === deal.takerCurrency, 'incoming htlc does not match expected deal currency'); this.logger.debug('Executing taker code to resolve hash'); - + + if (process.env.CUSTOM_SCENARIO === 'SECURITY::TAKER_2ND_HTLC_STALL') { + this.logger.info(`CUSTOM_SCENARIO: ${process.env.CUSTOM_SCENARIO}`); + return ''; @@ -155,15 +155,15 @@ index 7fd02cbb8..2bc60fe51 100644 + swapClient.removeInvoice(deal.rHash).catch(this.logger.error); // we don't need to await the remove invoice call + } } - + this.logger.trace(`emitting swap.failed event for ${deal.rHash}`); @@ -1313,9 +1378,14 @@ class Swaps extends EventEmitter { - + if (deal.role === SwapRole.Maker) { // the maker begins execution of the swap upon accepting the deal + + let interval = Swaps.SWAP_COMPLETE_TIMEOUT + Swaps.SWAP_COMPLETE_MAKER_BUFFER; -+ if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE') { ++ if (process.env.CUSTOM_SCENARIO === 'INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE') { + interval = 5000; + } this.timeouts.set(rHash, setTimeout( diff --git a/test/simulation/docker-build.sh b/test/simulation/docker-build.sh index d168cae51..bdf314f9e 100755 --- a/test/simulation/docker-build.sh +++ b/test/simulation/docker-build.sh @@ -6,7 +6,7 @@ then pushd temp git clone https://github.com/ConnextProject/indra.git cd indra - git checkout indra-7.0.0-alpha.6 + git checkout indra-7.0.0-alpha.14 make popd fi diff --git a/test/simulation/docker-connext/Dockerfile b/test/simulation/docker-connext/Dockerfile index 0834be52b..8b34f440b 100644 --- a/test/simulation/docker-connext/Dockerfile +++ b/test/simulation/docker-connext/Dockerfile @@ -6,5 +6,5 @@ WORKDIR /app RUN git clone https://github.com/connext/rest-api-client.git RUN cd rest-api-client && git pull -RUN cd rest-api-client && git checkout 64dd1eceba441297f4c31f3389f3910a453d550c +RUN cd rest-api-client && git checkout fe1876a0411e0b9928825ae51104e16a9fd7b787 RUN cd rest-api-client && npm install && npm run build diff --git a/test/simulation/docker-run.sh b/test/simulation/docker-run.sh index 701b91dff..de01a13fe 100755 --- a/test/simulation/docker-run.sh +++ b/test/simulation/docker-run.sh @@ -16,9 +16,4 @@ pushd temp/indra make reset popd -if [[ retVal != 0 ]] -then - ./logs.sh -fi - exit $retVal diff --git a/test/simulation/logs.sh b/test/simulation/logs.sh index bc6e63ed6..e70ac855c 100755 --- a/test/simulation/logs.sh +++ b/test/simulation/logs.sh @@ -1,3 +1,4 @@ #!/bin/bash find $PWD/temp/logs -type f -name xud*.log -printf "\n%f\n\n" -exec cat {} \; +find $PWD/temp/logs -type f -name connext*.log -printf "\n%f\n\n" -exec cat {} \; diff --git a/test/simulation/tests-instability.go b/test/simulation/tests-instability.go index 31a792364..6ee2cfbf1 100644 --- a/test/simulation/tests-instability.go +++ b/test/simulation/tests-instability.go @@ -27,9 +27,21 @@ var instabilityTestCases = []*testCase{ test: testMakerLndCrashedBeforeSettlement, }, { - name: "maker crashed after send payment with delayed settlement", // replacing Alice + Bob + name: "maker connext client crashed before order settlement", // replacing Alice + test: testMakerConnextClientCrashedBeforeSettlement, + }, + { + name: "maker crashed after send payment with delayed settlement; incoming: lnd, outgoing: lnd", // replacing Alice + Bob test: testMakerCrashedAfterSendDelayedSettlement, }, + { + name: "maker crashed after send payment with delayed settlement; incoming: connext, outgoing: lnd", // replacing Alice + Bob + test: testMakerCrashedAfterSendDelayedSettlementConnextIn, + }, + { + name: "maker crashed after send payment with delayed settlement; incoming: lnd, outgoing: connext", // replacing Alice + Bob + test: testMakerCrashedAfterSendDelayedSettlementConnextOut, + }, } // testMakerLndCrashedBeforeSettlement @@ -85,6 +97,150 @@ func testMakerCrashedAfterSend(net *xudtest.NetworkHarness, ht *harnessTest) { ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not receive LTC") } +func testMakerLndCrashedBeforeSettlement(net *xudtest.NetworkHarness, ht *harnessTest) { + var err error + net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{ + "CUSTOM_SCENARIO=INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE", + "CLIENT_TYPE=LndLtc", + fmt.Sprintf("CLIENT_PID=%d", net.Alice.LndLtcNode.Cmd.Process.Pid), + }) + ht.assert.NoError(err) + ht.act.init(net.Alice) + + // Connect Alice to Bob. + ht.act.connect(net.Alice, net.Bob) + ht.act.verifyConnectivity(net.Alice, net.Bob) + + // Save the initial balance. + alicePrevBalance, err := getBalance(ht.ctx, net.Alice) + ht.assert.NoError(err) + alicePrevLtcBalance := alicePrevBalance.ltc.channel.GetBalance() + + // Place an order on Alice. + aliceOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "maker_order_id", + Price: 0.02, + Quantity: uint64(ltcQuantity), + PairId: "LTC/BTC", + Side: xudrpc.OrderSide_BUY, + } + ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq) + + // Place a matching order on Bob. + bobOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "taker_order_id", + Price: aliceOrderReq.Price, + Quantity: aliceOrderReq.Quantity, + PairId: aliceOrderReq.PairId, + Side: xudrpc.OrderSide_SELL, + } + go net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq) + + // Alice's lnd-ltc is expected to be killed by Alice's custom xud. + <-net.Alice.LndLtcNode.ProcessExit + + // Wait a bit so that Alice's call to lnd-ltc for settlement would fail. + time.Sleep(5 * time.Second) + + // Restart Alice's lnd-ltc. + err = net.Alice.LndLtcNode.Start(nil) + ht.assert.NoError(err) + + // Brief delay to allow for swap to be recovered consistently. + // The pending swap recheck interval is usually 5m, but was adjusted in + // Alice's custom xud to 5s (as well as the swap completion timeout interval). + time.Sleep(10 * time.Second) + + // Verify that alice received her LTC. + aliceBalance, err := getBalance(ht.ctx, net.Alice) + ht.assert.NoError(err) + aliceLtcBalance := aliceBalance.ltc.channel.GetBalance() + ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not recover LTC funds") +} + +func testMakerConnextClientCrashedBeforeSettlement(net *xudtest.NetworkHarness, ht *harnessTest) { + var err error + net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{ + "CUSTOM_SCENARIO=INSTABILITY::MAKER_CLIENT_CRASHED_BEFORE_SETTLE", + "CLIENT_TYPE=ConnextClient", + // connext-client should be replaced, so we're not specifying its current PID, + // as in other client types. + }) + + ht.assert.NoError(err) + ht.act.init(net.Alice) + + ht.act.initConnext(net, net.Alice, false) + ht.act.initConnext(net, net.Bob, true) + + // Connect Alice to Bob. + ht.act.connect(net.Alice, net.Bob) + ht.act.verifyConnectivity(net.Alice, net.Bob) + + err = openETHChannel(ht.ctx, net.Bob, 400, 0) + ht.assert.NoError(err) + + // Save the initial balances. + alicePrevBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + alicePrevEthBalance := alicePrevBalance.Balances["ETH"] + + bobPrevBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + bobPrevBtcBalance := bobPrevBalance.Balances["BTC"] + + // Place an order on Alice. + aliceOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "maker_order_id", + Price: 40, + Quantity: 1, + PairId: "BTC/ETH", + Side: xudrpc.OrderSide_SELL, + } + ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq) + + // Place a matching order on Bob. + bobOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "taker_order_id", + Price: aliceOrderReq.Price, + Quantity: aliceOrderReq.Quantity, + PairId: aliceOrderReq.PairId, + Side: xudrpc.OrderSide_BUY, + } + go net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq) + + // Alice's connext-client is expected to be killed by Alice's custom xud. + <-net.Alice.ConnextClient.ProcessExit + + // Wait a bit so that Alice's call to connext-client for settlement would fail. + time.Sleep(5 * time.Second) + + // Restart Alice's connext-client. + err = net.Alice.ConnextClient.Start(nil) + ht.assert.NoError(err) + + err = waitConnextReady(net.Alice) + ht.assert.NoError(err) + + // Brief delay to allow for swap to be recovered consistently. + // The pending swap recheck interval is usually 5m, but was adjusted in + // Alice's custom xud to 5s (as well as the swap completion timeout interval). + time.Sleep(10 * time.Second) + + // Verify that both parties received their payment. + bobBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + bobBtcBalance := bobBalance.Balances["BTC"] + diff := bobOrderReq.Quantity + ht.assert.Equal(bobPrevBtcBalance.ChannelBalance+diff, bobBtcBalance.ChannelBalance) + + aliceBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + aliceEthBalance := aliceBalance.Balances["ETH"] + diff = uint64(float64(aliceOrderReq.Quantity) * aliceOrderReq.Price) + ht.assert.Equal(alicePrevEthBalance.ChannelBalance+diff, aliceEthBalance.ChannelBalance, "alice did not recover ETH funds") +} + func testMakerCrashedAfterSendDelayedSettlement(net *xudtest.NetworkHarness, ht *harnessTest) { var err error net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{"CUSTOM_SCENARIO=INSTABILITY::MAKER_CRASH_AFTER_SEND"}) @@ -147,30 +303,42 @@ func testMakerCrashedAfterSendDelayedSettlement(net *xudtest.NetworkHarness, ht ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not recover LTC funds") } -func testMakerLndCrashedBeforeSettlement(net *xudtest.NetworkHarness, ht *harnessTest) { +func testMakerCrashedAfterSendDelayedSettlementConnextOut(net *xudtest.NetworkHarness, ht *harnessTest) { var err error - net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{ - "CUSTOM_SCENARIO=INSTABILITY::MAKER_LND_CRASHED_BEFORE_SETTLE", - fmt.Sprintf("LNDLTC_PID=%d", net.Alice.LndLtcNode.Cmd.Process.Pid), - }) + net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{"CUSTOM_SCENARIO=INSTABILITY::MAKER_CRASH_AFTER_SEND"}) + ht.assert.NoError(err) + + net.Bob, err = net.SetCustomXud(ht.ctx, ht, net.Bob, []string{"CUSTOM_SCENARIO=INSTABILITY::TAKER_DELAY_BEFORE_SETTLE"}) ht.assert.NoError(err) + ht.act.init(net.Alice) + ht.act.initConnext(net, net.Alice, true) + + ht.act.init(net.Bob) + ht.act.initConnext(net, net.Bob, false) // Connect Alice to Bob. ht.act.connect(net.Alice, net.Bob) ht.act.verifyConnectivity(net.Alice, net.Bob) - // Save the initial balance. - alicePrevBalance, err := getBalance(ht.ctx, net.Alice) + err = openETHChannel(ht.ctx, net.Alice, 400, 0) ht.assert.NoError(err) - alicePrevLtcBalance := alicePrevBalance.ltc.channel.GetBalance() + + // Save the initial balances. + alicePrevBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + alicePrevBtcBalance := alicePrevBalance.Balances["BTC"] + + bobPrevBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + bobPrevEthBalance := bobPrevBalance.Balances["ETH"] // Place an order on Alice. aliceOrderReq := &xudrpc.PlaceOrderRequest{ OrderId: "maker_order_id", - Price: 0.02, - Quantity: uint64(ltcQuantity), - PairId: "LTC/BTC", + Price: 40, + Quantity: 1, + PairId: "BTC/ETH", Side: xudrpc.OrderSide_BUY, } ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq) @@ -185,24 +353,128 @@ func testMakerLndCrashedBeforeSettlement(net *xudtest.NetworkHarness, ht *harnes } go net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq) - // Alice's lnd-ltc is expected to be killed by Alice's custom xud. - <-net.Alice.LndLtcNode.ProcessExit + <-net.Alice.ProcessExit - // Wait a bit so that Alice's call to lnd-ltc for settlement would fail. - time.Sleep(5 * time.Second) + err = net.Alice.Start(nil) + ht.assert.NoError(err) - // Restart Alice's lnd-ltc. - err = net.Alice.LndLtcNode.Start(nil) + err = waitConnextReady(net.Alice) ht.assert.NoError(err) - // Brief delay to allow for swap to be recovered consistently. - // The pending swap recheck interval is usually 5m, but was adjusted in - // Alice's custom xud to 5s (as well as the swap completion timeout interval). + // Verify that alice hasn't claimed her BTC yet. The incoming BTC payment + // cannot be settled until the outgoing ETH payment is settled by bob, + // which is being intentionally delayed. + aliceIntermediateBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + aliceIntermediateBtcBalance := aliceIntermediateBalance.Balances["BTC"] + ht.assert.Equal(alicePrevBtcBalance.ChannelBalance, aliceIntermediateBtcBalance.ChannelBalance) + + bobIntermediateBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + bobIntermediateEthBalance := bobIntermediateBalance.Balances["ETH"] + ht.assert.Equal(bobPrevEthBalance.ChannelBalance, bobIntermediateEthBalance.ChannelBalance) + + // Wait to allow the ETH payment to be claimed by bob and then recovered by alice. time.Sleep(10 * time.Second) - // Verify that alice received her LTC. - aliceBalance, err := getBalance(ht.ctx, net.Alice) + // Verify that both parties received their payment. + bobBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) ht.assert.NoError(err) - aliceLtcBalance := aliceBalance.ltc.channel.GetBalance() - ht.assert.Equal(alicePrevLtcBalance+ltcQuantity, aliceLtcBalance, "alice did not recover LTC funds") + bobEthBalance := bobBalance.Balances["ETH"] + diff := uint64(float64(bobOrderReq.Quantity) * bobOrderReq.Price) + ht.assert.Equal(bobPrevEthBalance.ChannelBalance+diff, bobEthBalance.ChannelBalance) + + aliceBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + aliceBtcBalance := aliceBalance.Balances["BTC"] + diff = aliceOrderReq.Quantity + ht.assert.Equal(alicePrevBtcBalance.ChannelBalance+diff, aliceBtcBalance.ChannelBalance, "alice did not recover BTC funds") +} + +func testMakerCrashedAfterSendDelayedSettlementConnextIn(net *xudtest.NetworkHarness, ht *harnessTest) { + var err error + net.Alice, err = net.SetCustomXud(ht.ctx, ht, net.Alice, []string{"CUSTOM_SCENARIO=INSTABILITY::MAKER_CRASH_AFTER_SEND"}) + ht.assert.NoError(err) + + net.Bob, err = net.SetCustomXud(ht.ctx, ht, net.Bob, []string{"CUSTOM_SCENARIO=INSTABILITY::TAKER_DELAY_BEFORE_SETTLE"}) + ht.assert.NoError(err) + + ht.act.init(net.Alice) + ht.act.initConnext(net, net.Alice, false) + + ht.act.init(net.Bob) + ht.act.initConnext(net, net.Bob, true) + + // Connect Alice to Bob. + ht.act.connect(net.Alice, net.Bob) + ht.act.verifyConnectivity(net.Alice, net.Bob) + + err = openETHChannel(ht.ctx, net.Bob, 400, 0) + ht.assert.NoError(err) + + // Save the initial balances. + alicePrevBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + alicePrevEthBalance := alicePrevBalance.Balances["ETH"] + + bobPrevBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + bobPrevBtcBalance := bobPrevBalance.Balances["BTC"] + + // Place an order on Alice. + aliceOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "maker_order_id", + Price: 40, + Quantity: 1, + PairId: "BTC/ETH", + Side: xudrpc.OrderSide_SELL, + } + ht.act.placeOrderAndBroadcast(net.Alice, net.Bob, aliceOrderReq) + + // Place a matching order on Bob. + bobOrderReq := &xudrpc.PlaceOrderRequest{ + OrderId: "taker_order_id", + Price: aliceOrderReq.Price, + Quantity: aliceOrderReq.Quantity, + PairId: aliceOrderReq.PairId, + Side: xudrpc.OrderSide_BUY, + } + go net.Bob.Client.PlaceOrderSync(ht.ctx, bobOrderReq) + + <-net.Alice.ProcessExit + + err = net.Alice.Start(nil) + ht.assert.NoError(err) + + err = waitConnextReady(net.Alice) + ht.assert.NoError(err) + + // Verify that alice hasn't claimed her ETH yet. The incoming ETH payment + // cannot be settled until the outgoing BTC payment is settled by bob, + // which is being intentionally delayed. + aliceIntermediateBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + aliceIntermediateEthBalance := aliceIntermediateBalance.Balances["ETH"] + ht.assert.Equal(alicePrevEthBalance.ChannelBalance, aliceIntermediateEthBalance.ChannelBalance) + + bobIntermediateBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + bobIntermediateBtcBalance := bobIntermediateBalance.Balances["BTC"] + ht.assert.Equal(bobPrevBtcBalance.ChannelBalance, bobIntermediateBtcBalance.ChannelBalance) + + // Wait to allow the ETH payment to be claimed by bob and then recovered by alice. + time.Sleep(10 * time.Second) + + // Verify that both parties received their payment. + bobBalance, err := net.Bob.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "BTC"}) + ht.assert.NoError(err) + bobBtcBalance := bobBalance.Balances["BTC"] + diff := bobOrderReq.Quantity + ht.assert.Equal(bobPrevBtcBalance.ChannelBalance+diff, bobBtcBalance.ChannelBalance) + + aliceBalance, err := net.Alice.Client.GetBalance(ht.ctx, &xudrpc.GetBalanceRequest{Currency: "ETH"}) + ht.assert.NoError(err) + aliceEthBalance := aliceBalance.Balances["ETH"] + diff = uint64(float64(aliceOrderReq.Quantity) * aliceOrderReq.Price) + ht.assert.Equal(alicePrevEthBalance.ChannelBalance+diff, aliceEthBalance.ChannelBalance, "alice did not recover ETH funds") } diff --git a/test/simulation/xud_test.go b/test/simulation/xud_test.go index 4a58c5cf0..f224f5b45 100644 --- a/test/simulation/xud_test.go +++ b/test/simulation/xud_test.go @@ -163,7 +163,7 @@ func TestInstability(t *testing.T) { for _, testCase := range instabilityTestCases { success := t.Run(testCase.name, func(t1 *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) ht := newHarnessTest(ctx, cancel, t1) defer ht.teardown() diff --git a/test/simulation/xudtest/harness.go b/test/simulation/xudtest/harness.go index 97935da50..01f169c73 100644 --- a/test/simulation/xudtest/harness.go +++ b/test/simulation/xudtest/harness.go @@ -2,6 +2,7 @@ package xudtest import ( "context" + "fmt" "github.com/ExchangeUnion/xud-simulation/connexttest" "sync" "time" @@ -49,37 +50,54 @@ type CtxSetter interface { SetCtx(ctx context.Context, cancel context.CancelFunc) } -func (n *NetworkHarness) SetCustomXud(ctx context.Context, ctxSetter CtxSetter, node *HarnessNode, envVars []string) (*HarnessNode, error) { - err := node.shutdown(true, true) +func (n *NetworkHarness) newConnextClient(ctx context.Context, node *HarnessNode, envVars *[]string) (*connexttest.HarnessClient, error) { + if err := n.ConnextNetwork.TearDown(node.ConnextClient.ID); err != nil { + return nil, err + } + + client, err := n.ConnextNetwork.NewClient(node.Name) if err != nil { return nil, err } - delete(n.ActiveNodes, node.ID) + if err := client.Start(n.ConnextNetwork.ErrorChan); err != nil { + return nil, err + } + for _, kv := range *envVars { + if kv == "CLIENT_TYPE=ConnextClient" { + *envVars = append(*envVars, fmt.Sprintf("CLIENT_PID=%d", client.Cmd.Process.Pid)) + break + } + } + + return client, nil +} + +func (n *NetworkHarness) SetCustomXud(ctx context.Context, ctxSetter CtxSetter, node *HarnessNode, envVars []string) (*HarnessNode, error) { t := time.Now() - customNode, err := n.newNode(node.Name, "/custom-xud-vol", true) + connextClient, err := n.newConnextClient(ctx, node, &envVars) if err != nil { return nil, err } + if err := node.shutdown(true, true); err != nil { + return nil, err + } + delete(n.ActiveNodes, node.ID) + + customNode, err := n.newNode(node.Name, "/custom-xud-vol", true) + if err != nil { + return nil, err + } customNode.SetEnvVars(envVars) customNode.SetLnd(node.LndBtcNode, "BTC") customNode.SetLnd(node.LndLtcNode, "LTC") + customNode.SetConnextClient(connextClient) - var wg sync.WaitGroup - wg.Add(1) - errChan := make(chan error, 1) - go func() { - defer wg.Done() - if err := customNode.Start(n.errorChan); err != nil { - if err != nil { - errChan <- err - return - } - } - }() - wg.Wait() + if err := customNode.Start(n.errorChan); err != nil { + return nil, err + } // Adjust the ctx deadline so that time spent here // won't consume the timeout duration.