From 3ae992592f0794291e5c4493d33c241e32250151 Mon Sep 17 00:00:00 2001 From: Debnil Sur Date: Wed, 15 May 2019 12:03:42 -0700 Subject: [PATCH 1/7] services/friendbot: adds Stellar channels to friendbot (#1187) --- services/friendbot/friendbot.cfg | 1 + services/friendbot/init_friendbot.go | 106 ++++++++++-- services/friendbot/internal/account.go | 42 +++++ services/friendbot/internal/friendbot.go | 159 ++---------------- .../friendbot/internal/friendbot_handler.go | 1 - services/friendbot/internal/friendbot_test.go | 61 ++++--- services/friendbot/internal/minion.go | 109 ++++++++++++ services/friendbot/loadtest/loadtest.go | 80 +++++++++ services/friendbot/main.go | 8 +- 9 files changed, 385 insertions(+), 182 deletions(-) create mode 100644 services/friendbot/internal/account.go create mode 100644 services/friendbot/internal/minion.go create mode 100644 services/friendbot/loadtest/loadtest.go diff --git a/services/friendbot/friendbot.cfg b/services/friendbot/friendbot.cfg index fe36943ca1..d1fb913c4b 100644 --- a/services/friendbot/friendbot.cfg +++ b/services/friendbot/friendbot.cfg @@ -3,3 +3,4 @@ friendbot_secret = "SA6WAWCCMRZSEFD5PEN5GK5BHK347YEQ552XZM72NV553TAGWFVHW2G5" network_passphrase = "Test SDF Network ; September 2015" horizon_url = "https://horizon-testnet.stellar.org" starting_balance = "10000.00" +num_minions = 100 diff --git a/services/friendbot/init_friendbot.go b/services/friendbot/init_friendbot.go index 3987309311..569f55dc94 100644 --- a/services/friendbot/init_friendbot.go +++ b/services/friendbot/init_friendbot.go @@ -1,11 +1,15 @@ package main import ( + "log" "net/http" - "github.com/stellar/go/clients/horizon" + "github.com/stellar/go/clients/horizonclient" + "github.com/stellar/go/keypair" "github.com/stellar/go/services/friendbot/internal" "github.com/stellar/go/strkey" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/txnbuild" ) func initFriendbot( @@ -13,24 +17,94 @@ func initFriendbot( networkPassphrase string, horizonURL string, startingBalance string, -) *internal.Bot { - - if friendbotSecret == "" || networkPassphrase == "" || horizonURL == "" || startingBalance == "" { - return nil + numMinions int, +) (*internal.Bot, error) { + if friendbotSecret == "" || networkPassphrase == "" || horizonURL == "" || startingBalance == "" || numMinions < 0 { + return nil, errors.New("invalid input param(s)") } - // ensure its a seed if its not blank + // Guarantee that friendbotSecret is a seed, if not blank. strkey.MustDecode(strkey.VersionByteSeed, friendbotSecret) - return &internal.Bot{ - Secret: friendbotSecret, - Horizon: &horizon.Client{ - URL: horizonURL, - HTTP: http.DefaultClient, - AppName: "friendbot", - }, - Network: networkPassphrase, - StartingBalance: startingBalance, - SubmitTransaction: internal.AsyncSubmitTransaction, + hclient := &horizonclient.Client{ + HorizonURL: horizonURL, + HTTP: http.DefaultClient, + AppName: "friendbot", + } + + botKP, err := keypair.Parse(friendbotSecret) + if err != nil { + return nil, errors.Wrap(err, "parsing bot keypair") + } + + // Casting from the interface type will work, since we + // already confirmed that friendbotSecret is a seed. + botKeypair := botKP.(*keypair.Full) + botAccount := internal.Account{AccountID: botKeypair.Address()} + minionBalance := "101.00" + minions, err := createMinionAccounts(botAccount, botKeypair, networkPassphrase, startingBalance, minionBalance, numMinions, hclient) + if err != nil { + return nil, errors.Wrap(err, "creating minion accounts") + } + + return &internal.Bot{Minions: minions}, nil +} + +func createMinionAccounts(botAccount internal.Account, botKeypair *keypair.Full, networkPassphrase, newAccountBalance, minionBalance string, numMinions int, hclient *horizonclient.Client) ([]internal.Minion, error) { + var minions []internal.Minion + numRemainingMinions := numMinions + minionBatchSize := 100 + for numRemainingMinions > 0 { + var ops []txnbuild.Operation + // Refresh the sequence number before submitting a new transaction. + err := botAccount.RefreshSequenceNumber(hclient) + if err != nil { + return nil, errors.Wrap(err, "refreshing bot seqnum") + } + // The tx will create min(numRemainingMinions, 100) Minion accounts. + numCreateMinions := minionBatchSize + if numRemainingMinions < minionBatchSize { + numCreateMinions = numRemainingMinions + } + for i := 0; i < numCreateMinions; i++ { + minionKeypair, err := keypair.Random() + if err != nil { + return []internal.Minion{}, errors.Wrap(err, "making keypair") + } + minions = append(minions, internal.Minion{ + Account: internal.Account{AccountID: minionKeypair.Address()}, + Keypair: minionKeypair, + BotAccount: botAccount, + BotKeypair: botKeypair, + Horizon: hclient, + Network: networkPassphrase, + StartingBalance: newAccountBalance, + SubmitTransaction: internal.SubmitTransaction, + }) + + ops = append(ops, &txnbuild.CreateAccount{ + Destination: minionKeypair.Address(), + Amount: minionBalance, + }) + } + numRemainingMinions -= numCreateMinions + + // Build and submit batched account creation tx. + txn := txnbuild.Transaction{ + SourceAccount: botAccount, + Operations: ops, + Timebounds: txnbuild.NewTimeout(300), + Network: networkPassphrase, + } + txe, err := txn.BuildSignEncode(botKeypair) + if err != nil { + return []internal.Minion{}, errors.Wrap(err, "making create accounts tx") + } + resp, err := hclient.SubmitTransactionXDR(txe) + if err != nil { + log.Print(resp) + return []internal.Minion{}, errors.Wrap(err, "submitting create accounts tx") + } } + return minions, nil } diff --git a/services/friendbot/internal/account.go b/services/friendbot/internal/account.go new file mode 100644 index 0000000000..f4f42a7fca --- /dev/null +++ b/services/friendbot/internal/account.go @@ -0,0 +1,42 @@ +package internal + +import ( + "strconv" + + "github.com/stellar/go/clients/horizonclient" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" +) + +// Account implements the `txnbuild.Account` interface. +type Account struct { + AccountID string + Sequence xdr.SequenceNumber +} + +// GetAccountID returns the Account ID. +func (a Account) GetAccountID() string { + return a.AccountID +} + +// IncrementSequenceNumber increments the internal record of the +// account's sequence number by 1. +func (a Account) IncrementSequenceNumber() (xdr.SequenceNumber, error) { + a.Sequence++ + return a.Sequence, nil +} + +// RefreshSequenceNumber gets an Account's correct in-memory sequence number from Horizon. +func (a *Account) RefreshSequenceNumber(hclient *horizonclient.Client) error { + accountRequest := horizonclient.AccountRequest{AccountID: a.GetAccountID()} + accountDetail, err := hclient.AccountDetail(accountRequest) + if err != nil { + return errors.Wrap(err, "getting account detail") + } + seq, err := strconv.ParseInt(accountDetail.Sequence, 10, 64) + if err != nil { + return errors.Wrap(err, "parsing account seqnum") + } + a.Sequence = xdr.SequenceNumber(seq) + return nil +} diff --git a/services/friendbot/internal/friendbot.go b/services/friendbot/internal/friendbot.go index a7dced4490..10e9fdd542 100644 --- a/services/friendbot/internal/friendbot.go +++ b/services/friendbot/internal/friendbot.go @@ -1,152 +1,29 @@ package internal import ( - "strconv" - "sync" - - b "github.com/stellar/go/build" - "github.com/stellar/go/clients/horizon" - "github.com/stellar/go/keypair" - "github.com/stellar/go/support/errors" + hProtocol "github.com/stellar/go/protocols/horizon" ) -// TxResult is the result from the asynchronous submit transaction method over a channel -type TxResult struct { - maybeTransactionSuccess *horizon.TransactionSuccess - maybeErr error -} - -// Bot represents the friendbot subsystem. +// Bot represents the friendbot subsystem and primarily delegates work +// to its Minions. type Bot struct { - Horizon *horizon.Client - Secret string - Network string - StartingBalance string - SubmitTransaction func(bot *Bot, channel chan TxResult, signed string) - - // uninitialized - sequence uint64 - forceRefreshSequence bool - lock sync.Mutex -} - -// Pay funds the account at `destAddress` -func (bot *Bot) Pay(destAddress string) (*horizon.TransactionSuccess, error) { - channel := make(chan TxResult) - err := bot.lockedPay(channel, destAddress) - if err != nil { - return nil, err - } - - v := <-channel - return v.maybeTransactionSuccess, v.maybeErr + Minions []Minion + nextMinionIndex int } -func (bot *Bot) lockedPay(channel chan TxResult, destAddress string) error { - bot.lock.Lock() - defer bot.lock.Unlock() - - err := bot.checkSequenceRefresh() - if err != nil { - return err - } - - signed, err := bot.makeTx(destAddress) - if err != nil { - return err - } - - go bot.SubmitTransaction(bot, channel, signed) - return nil -} - -// AsyncSubmitTransaction should be passed into the bot -func AsyncSubmitTransaction(bot *Bot, channel chan TxResult, signed string) { - result, err := bot.Horizon.SubmitTransaction(signed) - if err != nil { - switch e := err.(type) { - case *horizon.Error: - bot.checkHandleBadSequence(e) - } - - channel <- TxResult{ - maybeTransactionSuccess: nil, - maybeErr: err, - } - } else { - channel <- TxResult{ - maybeTransactionSuccess: &result, - maybeErr: nil, - } - } -} - -func (bot *Bot) checkHandleBadSequence(err *horizon.Error) { - resCode, e := err.ResultCodes() - isTxBadSeqCode := e == nil && resCode.TransactionCode == "tx_bad_seq" - if !isTxBadSeqCode { - return - } - bot.forceRefreshSequence = true -} - -// establish initial sequence if needed -func (bot *Bot) checkSequenceRefresh() error { - if bot.sequence != 0 && !bot.forceRefreshSequence { - return nil - } - return bot.refreshSequence() -} - -func (bot *Bot) makeTx(destAddress string) (string, error) { - txn, err := b.Transaction( - b.SourceAccount{AddressOrSeed: bot.Secret}, - b.Sequence{Sequence: bot.sequence + 1}, - b.Network{Passphrase: bot.Network}, - b.CreateAccount( - b.Destination{AddressOrSeed: destAddress}, - b.NativeAmount{Amount: bot.StartingBalance}, - ), - ) - - if err != nil { - return "", errors.Wrap(err, "Error building a transaction") - } - - txs, err := txn.Sign(bot.Secret) - if err != nil { - return "", errors.Wrap(err, "Error signing a transaction") - } - - base64, err := txs.Base64() - - // only increment the in-memory sequence number if we are going to submit the transaction, while we hold the lock - if err == nil { - bot.sequence++ - } - return base64, err -} - -// refreshes the sequence from the bot account -func (bot *Bot) refreshSequence() error { - botAccount, err := bot.Horizon.LoadAccount(bot.address()) - if err != nil { - bot.sequence = 0 - return err - } - - seq, err := strconv.ParseInt(botAccount.Sequence, 10, 64) - if err != nil { - bot.sequence = 0 - return err - } - - bot.sequence = uint64(seq) - bot.forceRefreshSequence = false - return nil +// SubmitResult is the result from the asynchronous tx submission. +type SubmitResult struct { + maybeTransactionSuccess *hProtocol.TransactionSuccess + maybeErr error } -func (bot *Bot) address() string { - kp := keypair.MustParse(bot.Secret) - return kp.Address() +// Pay funds the account at `destAddress`. +func (bot *Bot) Pay(destAddress string) (*hProtocol.TransactionSuccess, error) { + minion := bot.Minions[bot.nextMinionIndex] + resultChan := make(chan SubmitResult) + go minion.Run(destAddress, resultChan) + bot.nextMinionIndex = (bot.nextMinionIndex + 1) % len(bot.Minions) + maybeSubmitResult := <-resultChan + close(resultChan) + return maybeSubmitResult.maybeTransactionSuccess, maybeSubmitResult.maybeErr } diff --git a/services/friendbot/internal/friendbot_handler.go b/services/friendbot/internal/friendbot_handler.go index 0db9191a9f..a21b598571 100644 --- a/services/friendbot/internal/friendbot_handler.go +++ b/services/friendbot/internal/friendbot_handler.go @@ -42,7 +42,6 @@ func (handler *FriendbotHandler) doHandle(r *http.Request) (*horizon.Transaction if err != nil { return nil, problem.MakeInvalidFieldProblem("addr", err) } - return handler.loadResult(address) } diff --git a/services/friendbot/internal/friendbot_test.go b/services/friendbot/internal/friendbot_test.go index 675622a6c8..8f70a3f798 100644 --- a/services/friendbot/internal/friendbot_test.go +++ b/services/friendbot/internal/friendbot_test.go @@ -1,52 +1,69 @@ package internal import ( + "sync" "testing" - "github.com/stellar/go/clients/horizon" + "github.com/stellar/go/clients/horizonclient" + "github.com/stellar/go/keypair" + hProtocol "github.com/stellar/go/protocols/horizon" "github.com/stretchr/testify/assert" - - "sync" ) func TestFriendbot_Pay(t *testing.T) { - mockSubmitTransaction := func(bot *Bot, channel chan TxResult, signed string) { - txSuccess := horizon.TransactionSuccess{Env: signed} - // we don't want to actually submit the tx here but emulate a success instead - channel <- TxResult{ - maybeTransactionSuccess: &txSuccess, - maybeErr: nil, - } + mockSubmitTransaction := func(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) { + // Instead of submitting the tx, we emulate a success. + txSuccess := hProtocol.TransactionSuccess{Env: tx} + return &txSuccess, nil + } + + // Public key: GD25B4QI6KWVDWXDW25CIM7EKR6A6PBSWE2RCNSAC4NJQDQJXZJYMMKR + botSeed := "SCWNLYELENPBXN46FHYXETT5LJCYBZD5VUQQVW4KZPHFO2YTQJUWT4D5" + botKeypair, err := keypair.Parse(botSeed) + if !assert.NoError(t, err) { + return + } + botAccount := Account{AccountID: botKeypair.Address()} + + // Public key: GD4AGPPDFFHKK3Z2X4XZDRXX6GZQKP4FMLVQ5T55NDEYGG3GIP7BQUHM + minionSeed := "SDTNSEERJPJFUE2LSDNYBFHYGVTPIWY7TU2IOJZQQGLWO2THTGB7NU5A" + minionKeypair, err := keypair.Parse(minionSeed) + if !assert.NoError(t, err) { + return } - fb := &Bot{ - Secret: "SAQWC7EPIYF3XGILYVJM4LVAVSLZKT27CTEI3AFBHU2VRCMQ3P3INPG5", + minion := Minion{ + Account: Account{ + AccountID: minionKeypair.Address(), + Sequence: 1, + }, + Keypair: minionKeypair.(*keypair.Full), + BotAccount: botAccount, + BotKeypair: botKeypair.(*keypair.Full), Network: "Test SDF Network ; September 2015", - StartingBalance: "100.00", + StartingBalance: "10000.00", SubmitTransaction: mockSubmitTransaction, - sequence: 2, } + fb := &Bot{Minions: []Minion{minion}} - txSuccess, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") + recipientAddress := "GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z" + txSuccess, err := fb.Pay(recipientAddress) if !assert.NoError(t, err) { return } - expectedTxn := "AAAAAPuYf7x7KGvFX9fjCR9WIaoTX3yHJYwX6ZSx6w76HPjEAAAAZAAAAAAAAAADAAAAAAAAAAAAAAAB" + - "AAAAAAAAAAAAAAAA0ob63nrm9S1s7+lvdyfgUTpejMQOhgMlxcvOvzUFhhQAAAAAO5rKAAAAAAAAAAAB+hz4xAAAAEC" + - "zNV2yXevMYKzm7OhXX2gYwmLZ5V37yeRHUX3Vhb6eT8wkUtpj2vJsUwzLWjdKMyGonFCPkaG4twRFUVqBRLEH" + expectedTxn := "AAAAAPgDPeMpTqVvOr8vkcb38bMFP4Vi6w7PvWjJgxtmQ/4YAAAAZAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAAA9dDyCPKtUdrjtrokM+RUfA88MrE1ETZAFxqYDgm+U4YAAAAAAAAAANKG+t565vUtbO/pb3cn4FE6XozEDoYDJcXLzr81BYYUAAAAF0h26AAAAAAAAAAAAmZD/hgAAABANEsSWMNVgAudOT2YNx5AR3k+uNDITctQCOy0jJNYfm39M/3T0XrpOAR8EUozFIoXp+Rrtm49xKzjSLHgCiYSCgm+U4YAAABA9Iazzw7Be5vPtRPqcWG+EXjsRB9o6yaIiw6SODNSuYGjKklBOYwxuB6LHSR1t8epLvn6J58ml1cs0UOt4afGAQ==" assert.Equal(t, expectedTxn, txSuccess.Env) + // Don't assert on tx values below, since the completion order is unknown. var wg sync.WaitGroup wg.Add(2) go func() { - _, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") - // don't assert on the txn value here because the ordering is not guaranteed between these 2 goroutines + _, err := fb.Pay(recipientAddress) assert.NoError(t, err) wg.Done() }() go func() { - _, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") - // don't assert on the txn value here because the ordering is not guaranteed between these 2 goroutines + _, err := fb.Pay(recipientAddress) assert.NoError(t, err) wg.Done() }() diff --git a/services/friendbot/internal/minion.go b/services/friendbot/internal/minion.go new file mode 100644 index 0000000000..0f72de374e --- /dev/null +++ b/services/friendbot/internal/minion.go @@ -0,0 +1,109 @@ +package internal + +import ( + "github.com/stellar/go/clients/horizonclient" + "github.com/stellar/go/keypair" + hProtocol "github.com/stellar/go/protocols/horizon" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/txnbuild" +) + +const createAccountInitialAmount = "1.0" + +// Minion contains a Stellar channel account and Go channels to communicate with friendbot. +type Minion struct { + Account Account + Keypair *keypair.Full + BotAccount txnbuild.Account + BotKeypair *keypair.Full + Horizon *horizonclient.Client + Network string + StartingBalance string + SubmitTransaction func(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) + + // Uninitialized. + forceRefreshSequence bool +} + +// Run reads a payment destination address and an output channel. It attempts +// to pay that address and submits the result to the channel. +func (minion *Minion) Run(destAddress string, resultChan chan SubmitResult) { + err := minion.checkSequenceRefresh(minion.Horizon) + if err != nil { + resultChan <- SubmitResult{ + maybeTransactionSuccess: nil, + maybeErr: errors.Wrap(err, "checking minion seq"), + } + } + txStr, err := minion.makeTx(destAddress) + if err != nil { + resultChan <- SubmitResult{ + maybeTransactionSuccess: nil, + maybeErr: errors.Wrap(err, "making payment tx"), + } + } + succ, err := minion.SubmitTransaction(minion, minion.Horizon, txStr) + resultChan <- SubmitResult{ + maybeTransactionSuccess: succ, + maybeErr: errors.Wrap(err, "submitting tx to minion"), + } +} + +// SubmitTransaction should be passed to the Minion. +func SubmitTransaction(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) { + result, err := hclient.SubmitTransactionXDR(tx) + if err != nil { + switch e := err.(type) { + case *horizonclient.Error: + minion.checkHandleBadSequence(e) + } + return nil, errors.Wrap(err, "submitting tx to horizon") + } + return &result, nil +} + +// Establishes the minion's initial sequence number, if needed. +func (minion *Minion) checkSequenceRefresh(hclient *horizonclient.Client) error { + if minion.Account.Sequence != 0 && !minion.forceRefreshSequence { + return nil + } + err := minion.Account.RefreshSequenceNumber(hclient) + if err != nil { + return errors.Wrap(err, "refreshing minion seqnum") + } + minion.forceRefreshSequence = false + return nil +} + +func (minion *Minion) checkHandleBadSequence(err *horizonclient.Error) { + resCode, e := err.ResultCodes() + isTxBadSeqCode := e == nil && resCode.TransactionCode == "tx_bad_seq" + if !isTxBadSeqCode { + return + } + minion.forceRefreshSequence = true +} + +func (minion *Minion) makeTx(destAddress string) (string, error) { + createAccountOp := txnbuild.CreateAccount{ + Destination: destAddress, + SourceAccount: minion.BotAccount, + Amount: minion.StartingBalance, + } + txn := txnbuild.Transaction{ + SourceAccount: minion.Account, + Operations: []txnbuild.Operation{&createAccountOp}, + Network: minion.Network, + Timebounds: txnbuild.NewInfiniteTimeout(), + } + txe, err := txn.BuildSignEncode(minion.Keypair, minion.BotKeypair) + if err != nil { + return "", errors.Wrap(err, "making account payment tx") + } + // Increment the in-memory sequence number, since the tx will be submitted. + _, err = minion.Account.IncrementSequenceNumber() + if err != nil { + return "", errors.Wrap(err, "incrementing minion seq") + } + return txe, err +} diff --git a/services/friendbot/loadtest/loadtest.go b/services/friendbot/loadtest/loadtest.go new file mode 100644 index 0000000000..e40409ee27 --- /dev/null +++ b/services/friendbot/loadtest/loadtest.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "flag" + "log" + "net/http" + "net/url" + "time" + + "github.com/stellar/go/keypair" + "github.com/stellar/go/support/errors" +) + +type maybeDuration struct { + maybeDuration time.Duration + maybeError error +} + +func main() { + // Friendbot must be running as a local server. Get Friendbot URL from CL. + fbURL := flag.String("url", "http://0.0.0.0:8000/", "URL of friendbot") + numRequests := flag.Int("requests", 500, "number of requests") + flag.Parse() + durationChannel := make(chan maybeDuration, *numRequests) + for i := 0; i < *numRequests; i++ { + kp, err := keypair.Random() + if err != nil { + panic(err) + } + address := kp.Address() + go makeFriendbotRequest(address, *fbURL, durationChannel) + + time.Sleep(time.Duration(500) * time.Millisecond) + } + durations := []maybeDuration{} + for i := 0; i < *numRequests; i++ { + durations = append(durations, <-durationChannel) + } + close(durationChannel) + log.Printf("Got %d times with average %s", *numRequests, mean(durations)) +} + +func makeFriendbotRequest(address, fbURL string, durationChannel chan maybeDuration) { + start := time.Now() + formData := url.Values{ + "addr": {address}, + } + resp, err := http.PostForm(fbURL, formData) + if err != nil { + log.Printf("Got post error: %s", err) + durationChannel <- maybeDuration{maybeError: errors.Wrap(err, "posting form")} + } + var result map[string]interface{} + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + log.Printf("Got decode error: %s", err) + durationChannel <- maybeDuration{maybeError: errors.Wrap(err, "decoding json")} + } + timeTrack(start, "makeFriendbotRequest", durationChannel) +} + +func timeTrack(start time.Time, name string, durationChannel chan maybeDuration) { + elapsed := time.Since(start) + log.Printf("%s took %s", name, elapsed) + durationChannel <- maybeDuration{maybeDuration: elapsed} +} + +func mean(durations []maybeDuration) time.Duration { + var total time.Duration + count := 0 + for _, d := range durations { + if d.maybeError != nil { + continue + } + total += d.maybeDuration + count++ + } + return total / time.Duration(count) +} diff --git a/services/friendbot/main.go b/services/friendbot/main.go index 9ff4d62c30..d6e2f15cc4 100644 --- a/services/friendbot/main.go +++ b/services/friendbot/main.go @@ -25,6 +25,7 @@ type Config struct { HorizonURL string `toml:"horizon_url" valid:"required"` StartingBalance string `toml:"starting_balance" valid:"required"` TLS *config.TLS `valid:"optional"` + NumMinions int `toml:"num_minions" valid:"optional"` } func main() { @@ -57,8 +58,11 @@ func run(cmd *cobra.Command, args []string) { } os.Exit(1) } - - fb := initFriendbot(cfg.FriendbotSecret, cfg.NetworkPassphrase, cfg.HorizonURL, cfg.StartingBalance) + fb, err := initFriendbot(cfg.FriendbotSecret, cfg.NetworkPassphrase, cfg.HorizonURL, cfg.StartingBalance, cfg.NumMinions) + if err != nil { + log.Error(err) + os.Exit(1) + } router := initRouter(fb) registerProblems() From c2688a84e42abf4e7fdf88e502aa6385be2099af Mon Sep 17 00:00:00 2001 From: Johnny Goodnow Date: Wed, 15 May 2019 13:57:01 -0700 Subject: [PATCH 2/7] docs: Update order of SDK's to reflect docs sidebar. (#1280) * Update order of SDK's to reflect docs sidebar. * Fix Go link. --- services/horizon/internal/docs/readme.md | 9 +++++---- services/horizon/internal/docs/reference/readme.md | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/services/horizon/internal/docs/readme.md b/services/horizon/internal/docs/readme.md index 395eadef00..2a1015332a 100644 --- a/services/horizon/internal/docs/readme.md +++ b/services/horizon/internal/docs/readme.md @@ -12,12 +12,13 @@ SDF runs a instance of Horizon that is connected to the test net [https://horizo SDF maintained libraries:
- [JavaScript](https://github.com/stellar/js-stellar-sdk) +- [Go](https://github.com/stellar/go/tree/master/clients/horizonclient) - [Java](https://github.com/stellar/java-stellar-sdk) -- [Go](https://github.com/stellar/go) Community maintained libraries (in various states of completeness) for interacting with Horizon in other languages:
-- [Ruby](https://github.com/stellar/ruby-stellar-sdk) - [Python](https://github.com/StellarCN/py-stellar-base) -- [C# .NET 2.0](https://github.com/QuantozTechnology/csharp-stellar-base) - [C# .NET Core 2.x](https://github.com/elucidsoft/dotnetcore-stellar-sdk) -- [C++](https://bitbucket.org/bnogal/stellarqore/wiki/Home) +- [Ruby](https://github.com/bloom-solutions/ruby-stellar-sdk) +- [iOS and macOS](https://github.com/Soneso/stellar-ios-mac-sdk) +- [Scala SDK](https://github.com/synesso/scala-stellar-sdk) +- [C++ SDK](https://github.com/bnogalm/StellarQtSDK) diff --git a/services/horizon/internal/docs/reference/readme.md b/services/horizon/internal/docs/reference/readme.md index 1373e265d8..44e55ea867 100644 --- a/services/horizon/internal/docs/reference/readme.md +++ b/services/horizon/internal/docs/reference/readme.md @@ -16,10 +16,13 @@ SDF runs a instance of Horizon that is connected to the test net: [https://horiz SDF maintained libraries:
- [JavaScript](https://github.com/stellar/js-stellar-sdk) +- [Go](https://github.com/stellar/go/tree/master/clients/horizonclient) - [Java](https://github.com/stellar/java-stellar-sdk) -- [Go](https://github.com/stellar/go) Community maintained libraries (in various states of completeness) for interacting with Horizon in other languages:
-- [Ruby](https://github.com/stellar/ruby-stellar-sdk) - [Python](https://github.com/StellarCN/py-stellar-base) -- [C#](https://github.com/elucidsoft/dotnet-stellar-sdk) +- [C# .NET Core 2.x](https://github.com/elucidsoft/dotnetcore-stellar-sdk) +- [Ruby](https://github.com/bloom-solutions/ruby-stellar-sdk) +- [iOS and macOS](https://github.com/Soneso/stellar-ios-mac-sdk) +- [Scala SDK](https://github.com/synesso/scala-stellar-sdk) +- [C++ SDK](https://github.com/bnogalm/StellarQtSDK) From 457ffe01d3d5af4a7d37d4360c7a878f225cf7dc Mon Sep 17 00:00:00 2001 From: Debnil Sur Date: Wed, 15 May 2019 14:44:56 -0700 Subject: [PATCH 3/7] Revert "services/friendbot: adds Stellar channels to friendbot (#1187)" (#1282) This reverts commit 3ae992592f0794291e5c4493d33c241e32250151. --- services/friendbot/friendbot.cfg | 1 - services/friendbot/init_friendbot.go | 106 ++---------- services/friendbot/internal/account.go | 42 ----- services/friendbot/internal/friendbot.go | 159 ++++++++++++++++-- .../friendbot/internal/friendbot_handler.go | 1 + services/friendbot/internal/friendbot_test.go | 61 +++---- services/friendbot/internal/minion.go | 109 ------------ services/friendbot/loadtest/loadtest.go | 80 --------- services/friendbot/main.go | 8 +- 9 files changed, 182 insertions(+), 385 deletions(-) delete mode 100644 services/friendbot/internal/account.go delete mode 100644 services/friendbot/internal/minion.go delete mode 100644 services/friendbot/loadtest/loadtest.go diff --git a/services/friendbot/friendbot.cfg b/services/friendbot/friendbot.cfg index d1fb913c4b..fe36943ca1 100644 --- a/services/friendbot/friendbot.cfg +++ b/services/friendbot/friendbot.cfg @@ -3,4 +3,3 @@ friendbot_secret = "SA6WAWCCMRZSEFD5PEN5GK5BHK347YEQ552XZM72NV553TAGWFVHW2G5" network_passphrase = "Test SDF Network ; September 2015" horizon_url = "https://horizon-testnet.stellar.org" starting_balance = "10000.00" -num_minions = 100 diff --git a/services/friendbot/init_friendbot.go b/services/friendbot/init_friendbot.go index 569f55dc94..3987309311 100644 --- a/services/friendbot/init_friendbot.go +++ b/services/friendbot/init_friendbot.go @@ -1,15 +1,11 @@ package main import ( - "log" "net/http" - "github.com/stellar/go/clients/horizonclient" - "github.com/stellar/go/keypair" + "github.com/stellar/go/clients/horizon" "github.com/stellar/go/services/friendbot/internal" "github.com/stellar/go/strkey" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/txnbuild" ) func initFriendbot( @@ -17,94 +13,24 @@ func initFriendbot( networkPassphrase string, horizonURL string, startingBalance string, - numMinions int, -) (*internal.Bot, error) { - if friendbotSecret == "" || networkPassphrase == "" || horizonURL == "" || startingBalance == "" || numMinions < 0 { - return nil, errors.New("invalid input param(s)") - } - - // Guarantee that friendbotSecret is a seed, if not blank. - strkey.MustDecode(strkey.VersionByteSeed, friendbotSecret) +) *internal.Bot { - hclient := &horizonclient.Client{ - HorizonURL: horizonURL, - HTTP: http.DefaultClient, - AppName: "friendbot", + if friendbotSecret == "" || networkPassphrase == "" || horizonURL == "" || startingBalance == "" { + return nil } - botKP, err := keypair.Parse(friendbotSecret) - if err != nil { - return nil, errors.Wrap(err, "parsing bot keypair") - } - - // Casting from the interface type will work, since we - // already confirmed that friendbotSecret is a seed. - botKeypair := botKP.(*keypair.Full) - botAccount := internal.Account{AccountID: botKeypair.Address()} - minionBalance := "101.00" - minions, err := createMinionAccounts(botAccount, botKeypair, networkPassphrase, startingBalance, minionBalance, numMinions, hclient) - if err != nil { - return nil, errors.Wrap(err, "creating minion accounts") - } - - return &internal.Bot{Minions: minions}, nil -} - -func createMinionAccounts(botAccount internal.Account, botKeypair *keypair.Full, networkPassphrase, newAccountBalance, minionBalance string, numMinions int, hclient *horizonclient.Client) ([]internal.Minion, error) { - var minions []internal.Minion - numRemainingMinions := numMinions - minionBatchSize := 100 - for numRemainingMinions > 0 { - var ops []txnbuild.Operation - // Refresh the sequence number before submitting a new transaction. - err := botAccount.RefreshSequenceNumber(hclient) - if err != nil { - return nil, errors.Wrap(err, "refreshing bot seqnum") - } - // The tx will create min(numRemainingMinions, 100) Minion accounts. - numCreateMinions := minionBatchSize - if numRemainingMinions < minionBatchSize { - numCreateMinions = numRemainingMinions - } - for i := 0; i < numCreateMinions; i++ { - minionKeypair, err := keypair.Random() - if err != nil { - return []internal.Minion{}, errors.Wrap(err, "making keypair") - } - minions = append(minions, internal.Minion{ - Account: internal.Account{AccountID: minionKeypair.Address()}, - Keypair: minionKeypair, - BotAccount: botAccount, - BotKeypair: botKeypair, - Horizon: hclient, - Network: networkPassphrase, - StartingBalance: newAccountBalance, - SubmitTransaction: internal.SubmitTransaction, - }) - - ops = append(ops, &txnbuild.CreateAccount{ - Destination: minionKeypair.Address(), - Amount: minionBalance, - }) - } - numRemainingMinions -= numCreateMinions + // ensure its a seed if its not blank + strkey.MustDecode(strkey.VersionByteSeed, friendbotSecret) - // Build and submit batched account creation tx. - txn := txnbuild.Transaction{ - SourceAccount: botAccount, - Operations: ops, - Timebounds: txnbuild.NewTimeout(300), - Network: networkPassphrase, - } - txe, err := txn.BuildSignEncode(botKeypair) - if err != nil { - return []internal.Minion{}, errors.Wrap(err, "making create accounts tx") - } - resp, err := hclient.SubmitTransactionXDR(txe) - if err != nil { - log.Print(resp) - return []internal.Minion{}, errors.Wrap(err, "submitting create accounts tx") - } + return &internal.Bot{ + Secret: friendbotSecret, + Horizon: &horizon.Client{ + URL: horizonURL, + HTTP: http.DefaultClient, + AppName: "friendbot", + }, + Network: networkPassphrase, + StartingBalance: startingBalance, + SubmitTransaction: internal.AsyncSubmitTransaction, } - return minions, nil } diff --git a/services/friendbot/internal/account.go b/services/friendbot/internal/account.go deleted file mode 100644 index f4f42a7fca..0000000000 --- a/services/friendbot/internal/account.go +++ /dev/null @@ -1,42 +0,0 @@ -package internal - -import ( - "strconv" - - "github.com/stellar/go/clients/horizonclient" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" -) - -// Account implements the `txnbuild.Account` interface. -type Account struct { - AccountID string - Sequence xdr.SequenceNumber -} - -// GetAccountID returns the Account ID. -func (a Account) GetAccountID() string { - return a.AccountID -} - -// IncrementSequenceNumber increments the internal record of the -// account's sequence number by 1. -func (a Account) IncrementSequenceNumber() (xdr.SequenceNumber, error) { - a.Sequence++ - return a.Sequence, nil -} - -// RefreshSequenceNumber gets an Account's correct in-memory sequence number from Horizon. -func (a *Account) RefreshSequenceNumber(hclient *horizonclient.Client) error { - accountRequest := horizonclient.AccountRequest{AccountID: a.GetAccountID()} - accountDetail, err := hclient.AccountDetail(accountRequest) - if err != nil { - return errors.Wrap(err, "getting account detail") - } - seq, err := strconv.ParseInt(accountDetail.Sequence, 10, 64) - if err != nil { - return errors.Wrap(err, "parsing account seqnum") - } - a.Sequence = xdr.SequenceNumber(seq) - return nil -} diff --git a/services/friendbot/internal/friendbot.go b/services/friendbot/internal/friendbot.go index 10e9fdd542..a7dced4490 100644 --- a/services/friendbot/internal/friendbot.go +++ b/services/friendbot/internal/friendbot.go @@ -1,29 +1,152 @@ package internal import ( - hProtocol "github.com/stellar/go/protocols/horizon" + "strconv" + "sync" + + b "github.com/stellar/go/build" + "github.com/stellar/go/clients/horizon" + "github.com/stellar/go/keypair" + "github.com/stellar/go/support/errors" ) -// Bot represents the friendbot subsystem and primarily delegates work -// to its Minions. +// TxResult is the result from the asynchronous submit transaction method over a channel +type TxResult struct { + maybeTransactionSuccess *horizon.TransactionSuccess + maybeErr error +} + +// Bot represents the friendbot subsystem. type Bot struct { - Minions []Minion - nextMinionIndex int + Horizon *horizon.Client + Secret string + Network string + StartingBalance string + SubmitTransaction func(bot *Bot, channel chan TxResult, signed string) + + // uninitialized + sequence uint64 + forceRefreshSequence bool + lock sync.Mutex } -// SubmitResult is the result from the asynchronous tx submission. -type SubmitResult struct { - maybeTransactionSuccess *hProtocol.TransactionSuccess - maybeErr error +// Pay funds the account at `destAddress` +func (bot *Bot) Pay(destAddress string) (*horizon.TransactionSuccess, error) { + channel := make(chan TxResult) + err := bot.lockedPay(channel, destAddress) + if err != nil { + return nil, err + } + + v := <-channel + return v.maybeTransactionSuccess, v.maybeErr +} + +func (bot *Bot) lockedPay(channel chan TxResult, destAddress string) error { + bot.lock.Lock() + defer bot.lock.Unlock() + + err := bot.checkSequenceRefresh() + if err != nil { + return err + } + + signed, err := bot.makeTx(destAddress) + if err != nil { + return err + } + + go bot.SubmitTransaction(bot, channel, signed) + return nil +} + +// AsyncSubmitTransaction should be passed into the bot +func AsyncSubmitTransaction(bot *Bot, channel chan TxResult, signed string) { + result, err := bot.Horizon.SubmitTransaction(signed) + if err != nil { + switch e := err.(type) { + case *horizon.Error: + bot.checkHandleBadSequence(e) + } + + channel <- TxResult{ + maybeTransactionSuccess: nil, + maybeErr: err, + } + } else { + channel <- TxResult{ + maybeTransactionSuccess: &result, + maybeErr: nil, + } + } +} + +func (bot *Bot) checkHandleBadSequence(err *horizon.Error) { + resCode, e := err.ResultCodes() + isTxBadSeqCode := e == nil && resCode.TransactionCode == "tx_bad_seq" + if !isTxBadSeqCode { + return + } + bot.forceRefreshSequence = true +} + +// establish initial sequence if needed +func (bot *Bot) checkSequenceRefresh() error { + if bot.sequence != 0 && !bot.forceRefreshSequence { + return nil + } + return bot.refreshSequence() +} + +func (bot *Bot) makeTx(destAddress string) (string, error) { + txn, err := b.Transaction( + b.SourceAccount{AddressOrSeed: bot.Secret}, + b.Sequence{Sequence: bot.sequence + 1}, + b.Network{Passphrase: bot.Network}, + b.CreateAccount( + b.Destination{AddressOrSeed: destAddress}, + b.NativeAmount{Amount: bot.StartingBalance}, + ), + ) + + if err != nil { + return "", errors.Wrap(err, "Error building a transaction") + } + + txs, err := txn.Sign(bot.Secret) + if err != nil { + return "", errors.Wrap(err, "Error signing a transaction") + } + + base64, err := txs.Base64() + + // only increment the in-memory sequence number if we are going to submit the transaction, while we hold the lock + if err == nil { + bot.sequence++ + } + return base64, err +} + +// refreshes the sequence from the bot account +func (bot *Bot) refreshSequence() error { + botAccount, err := bot.Horizon.LoadAccount(bot.address()) + if err != nil { + bot.sequence = 0 + return err + } + + seq, err := strconv.ParseInt(botAccount.Sequence, 10, 64) + if err != nil { + bot.sequence = 0 + return err + } + + bot.sequence = uint64(seq) + bot.forceRefreshSequence = false + return nil } -// Pay funds the account at `destAddress`. -func (bot *Bot) Pay(destAddress string) (*hProtocol.TransactionSuccess, error) { - minion := bot.Minions[bot.nextMinionIndex] - resultChan := make(chan SubmitResult) - go minion.Run(destAddress, resultChan) - bot.nextMinionIndex = (bot.nextMinionIndex + 1) % len(bot.Minions) - maybeSubmitResult := <-resultChan - close(resultChan) - return maybeSubmitResult.maybeTransactionSuccess, maybeSubmitResult.maybeErr +func (bot *Bot) address() string { + kp := keypair.MustParse(bot.Secret) + return kp.Address() } diff --git a/services/friendbot/internal/friendbot_handler.go b/services/friendbot/internal/friendbot_handler.go index a21b598571..0db9191a9f 100644 --- a/services/friendbot/internal/friendbot_handler.go +++ b/services/friendbot/internal/friendbot_handler.go @@ -42,6 +42,7 @@ func (handler *FriendbotHandler) doHandle(r *http.Request) (*horizon.Transaction if err != nil { return nil, problem.MakeInvalidFieldProblem("addr", err) } + return handler.loadResult(address) } diff --git a/services/friendbot/internal/friendbot_test.go b/services/friendbot/internal/friendbot_test.go index 8f70a3f798..675622a6c8 100644 --- a/services/friendbot/internal/friendbot_test.go +++ b/services/friendbot/internal/friendbot_test.go @@ -1,69 +1,52 @@ package internal import ( - "sync" "testing" - "github.com/stellar/go/clients/horizonclient" - "github.com/stellar/go/keypair" - hProtocol "github.com/stellar/go/protocols/horizon" + "github.com/stellar/go/clients/horizon" "github.com/stretchr/testify/assert" + + "sync" ) func TestFriendbot_Pay(t *testing.T) { - mockSubmitTransaction := func(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) { - // Instead of submitting the tx, we emulate a success. - txSuccess := hProtocol.TransactionSuccess{Env: tx} - return &txSuccess, nil - } - - // Public key: GD25B4QI6KWVDWXDW25CIM7EKR6A6PBSWE2RCNSAC4NJQDQJXZJYMMKR - botSeed := "SCWNLYELENPBXN46FHYXETT5LJCYBZD5VUQQVW4KZPHFO2YTQJUWT4D5" - botKeypair, err := keypair.Parse(botSeed) - if !assert.NoError(t, err) { - return - } - botAccount := Account{AccountID: botKeypair.Address()} - - // Public key: GD4AGPPDFFHKK3Z2X4XZDRXX6GZQKP4FMLVQ5T55NDEYGG3GIP7BQUHM - minionSeed := "SDTNSEERJPJFUE2LSDNYBFHYGVTPIWY7TU2IOJZQQGLWO2THTGB7NU5A" - minionKeypair, err := keypair.Parse(minionSeed) - if !assert.NoError(t, err) { - return + mockSubmitTransaction := func(bot *Bot, channel chan TxResult, signed string) { + txSuccess := horizon.TransactionSuccess{Env: signed} + // we don't want to actually submit the tx here but emulate a success instead + channel <- TxResult{ + maybeTransactionSuccess: &txSuccess, + maybeErr: nil, + } } - minion := Minion{ - Account: Account{ - AccountID: minionKeypair.Address(), - Sequence: 1, - }, - Keypair: minionKeypair.(*keypair.Full), - BotAccount: botAccount, - BotKeypair: botKeypair.(*keypair.Full), + fb := &Bot{ + Secret: "SAQWC7EPIYF3XGILYVJM4LVAVSLZKT27CTEI3AFBHU2VRCMQ3P3INPG5", Network: "Test SDF Network ; September 2015", - StartingBalance: "10000.00", + StartingBalance: "100.00", SubmitTransaction: mockSubmitTransaction, + sequence: 2, } - fb := &Bot{Minions: []Minion{minion}} - recipientAddress := "GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z" - txSuccess, err := fb.Pay(recipientAddress) + txSuccess, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") if !assert.NoError(t, err) { return } - expectedTxn := "AAAAAPgDPeMpTqVvOr8vkcb38bMFP4Vi6w7PvWjJgxtmQ/4YAAAAZAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAAA9dDyCPKtUdrjtrokM+RUfA88MrE1ETZAFxqYDgm+U4YAAAAAAAAAANKG+t565vUtbO/pb3cn4FE6XozEDoYDJcXLzr81BYYUAAAAF0h26AAAAAAAAAAAAmZD/hgAAABANEsSWMNVgAudOT2YNx5AR3k+uNDITctQCOy0jJNYfm39M/3T0XrpOAR8EUozFIoXp+Rrtm49xKzjSLHgCiYSCgm+U4YAAABA9Iazzw7Be5vPtRPqcWG+EXjsRB9o6yaIiw6SODNSuYGjKklBOYwxuB6LHSR1t8epLvn6J58ml1cs0UOt4afGAQ==" + expectedTxn := "AAAAAPuYf7x7KGvFX9fjCR9WIaoTX3yHJYwX6ZSx6w76HPjEAAAAZAAAAAAAAAADAAAAAAAAAAAAAAAB" + + "AAAAAAAAAAAAAAAA0ob63nrm9S1s7+lvdyfgUTpejMQOhgMlxcvOvzUFhhQAAAAAO5rKAAAAAAAAAAAB+hz4xAAAAEC" + + "zNV2yXevMYKzm7OhXX2gYwmLZ5V37yeRHUX3Vhb6eT8wkUtpj2vJsUwzLWjdKMyGonFCPkaG4twRFUVqBRLEH" assert.Equal(t, expectedTxn, txSuccess.Env) - // Don't assert on tx values below, since the completion order is unknown. var wg sync.WaitGroup wg.Add(2) go func() { - _, err := fb.Pay(recipientAddress) + _, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") + // don't assert on the txn value here because the ordering is not guaranteed between these 2 goroutines assert.NoError(t, err) wg.Done() }() go func() { - _, err := fb.Pay(recipientAddress) + _, err := fb.Pay("GDJIN6W6PLTPKLLM57UW65ZH4BITUXUMYQHIMAZFYXF45PZVAWDBI77Z") + // don't assert on the txn value here because the ordering is not guaranteed between these 2 goroutines assert.NoError(t, err) wg.Done() }() diff --git a/services/friendbot/internal/minion.go b/services/friendbot/internal/minion.go deleted file mode 100644 index 0f72de374e..0000000000 --- a/services/friendbot/internal/minion.go +++ /dev/null @@ -1,109 +0,0 @@ -package internal - -import ( - "github.com/stellar/go/clients/horizonclient" - "github.com/stellar/go/keypair" - hProtocol "github.com/stellar/go/protocols/horizon" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/txnbuild" -) - -const createAccountInitialAmount = "1.0" - -// Minion contains a Stellar channel account and Go channels to communicate with friendbot. -type Minion struct { - Account Account - Keypair *keypair.Full - BotAccount txnbuild.Account - BotKeypair *keypair.Full - Horizon *horizonclient.Client - Network string - StartingBalance string - SubmitTransaction func(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) - - // Uninitialized. - forceRefreshSequence bool -} - -// Run reads a payment destination address and an output channel. It attempts -// to pay that address and submits the result to the channel. -func (minion *Minion) Run(destAddress string, resultChan chan SubmitResult) { - err := minion.checkSequenceRefresh(minion.Horizon) - if err != nil { - resultChan <- SubmitResult{ - maybeTransactionSuccess: nil, - maybeErr: errors.Wrap(err, "checking minion seq"), - } - } - txStr, err := minion.makeTx(destAddress) - if err != nil { - resultChan <- SubmitResult{ - maybeTransactionSuccess: nil, - maybeErr: errors.Wrap(err, "making payment tx"), - } - } - succ, err := minion.SubmitTransaction(minion, minion.Horizon, txStr) - resultChan <- SubmitResult{ - maybeTransactionSuccess: succ, - maybeErr: errors.Wrap(err, "submitting tx to minion"), - } -} - -// SubmitTransaction should be passed to the Minion. -func SubmitTransaction(minion *Minion, hclient *horizonclient.Client, tx string) (*hProtocol.TransactionSuccess, error) { - result, err := hclient.SubmitTransactionXDR(tx) - if err != nil { - switch e := err.(type) { - case *horizonclient.Error: - minion.checkHandleBadSequence(e) - } - return nil, errors.Wrap(err, "submitting tx to horizon") - } - return &result, nil -} - -// Establishes the minion's initial sequence number, if needed. -func (minion *Minion) checkSequenceRefresh(hclient *horizonclient.Client) error { - if minion.Account.Sequence != 0 && !minion.forceRefreshSequence { - return nil - } - err := minion.Account.RefreshSequenceNumber(hclient) - if err != nil { - return errors.Wrap(err, "refreshing minion seqnum") - } - minion.forceRefreshSequence = false - return nil -} - -func (minion *Minion) checkHandleBadSequence(err *horizonclient.Error) { - resCode, e := err.ResultCodes() - isTxBadSeqCode := e == nil && resCode.TransactionCode == "tx_bad_seq" - if !isTxBadSeqCode { - return - } - minion.forceRefreshSequence = true -} - -func (minion *Minion) makeTx(destAddress string) (string, error) { - createAccountOp := txnbuild.CreateAccount{ - Destination: destAddress, - SourceAccount: minion.BotAccount, - Amount: minion.StartingBalance, - } - txn := txnbuild.Transaction{ - SourceAccount: minion.Account, - Operations: []txnbuild.Operation{&createAccountOp}, - Network: minion.Network, - Timebounds: txnbuild.NewInfiniteTimeout(), - } - txe, err := txn.BuildSignEncode(minion.Keypair, minion.BotKeypair) - if err != nil { - return "", errors.Wrap(err, "making account payment tx") - } - // Increment the in-memory sequence number, since the tx will be submitted. - _, err = minion.Account.IncrementSequenceNumber() - if err != nil { - return "", errors.Wrap(err, "incrementing minion seq") - } - return txe, err -} diff --git a/services/friendbot/loadtest/loadtest.go b/services/friendbot/loadtest/loadtest.go deleted file mode 100644 index e40409ee27..0000000000 --- a/services/friendbot/loadtest/loadtest.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "log" - "net/http" - "net/url" - "time" - - "github.com/stellar/go/keypair" - "github.com/stellar/go/support/errors" -) - -type maybeDuration struct { - maybeDuration time.Duration - maybeError error -} - -func main() { - // Friendbot must be running as a local server. Get Friendbot URL from CL. - fbURL := flag.String("url", "http://0.0.0.0:8000/", "URL of friendbot") - numRequests := flag.Int("requests", 500, "number of requests") - flag.Parse() - durationChannel := make(chan maybeDuration, *numRequests) - for i := 0; i < *numRequests; i++ { - kp, err := keypair.Random() - if err != nil { - panic(err) - } - address := kp.Address() - go makeFriendbotRequest(address, *fbURL, durationChannel) - - time.Sleep(time.Duration(500) * time.Millisecond) - } - durations := []maybeDuration{} - for i := 0; i < *numRequests; i++ { - durations = append(durations, <-durationChannel) - } - close(durationChannel) - log.Printf("Got %d times with average %s", *numRequests, mean(durations)) -} - -func makeFriendbotRequest(address, fbURL string, durationChannel chan maybeDuration) { - start := time.Now() - formData := url.Values{ - "addr": {address}, - } - resp, err := http.PostForm(fbURL, formData) - if err != nil { - log.Printf("Got post error: %s", err) - durationChannel <- maybeDuration{maybeError: errors.Wrap(err, "posting form")} - } - var result map[string]interface{} - err = json.NewDecoder(resp.Body).Decode(&result) - if err != nil { - log.Printf("Got decode error: %s", err) - durationChannel <- maybeDuration{maybeError: errors.Wrap(err, "decoding json")} - } - timeTrack(start, "makeFriendbotRequest", durationChannel) -} - -func timeTrack(start time.Time, name string, durationChannel chan maybeDuration) { - elapsed := time.Since(start) - log.Printf("%s took %s", name, elapsed) - durationChannel <- maybeDuration{maybeDuration: elapsed} -} - -func mean(durations []maybeDuration) time.Duration { - var total time.Duration - count := 0 - for _, d := range durations { - if d.maybeError != nil { - continue - } - total += d.maybeDuration - count++ - } - return total / time.Duration(count) -} diff --git a/services/friendbot/main.go b/services/friendbot/main.go index d6e2f15cc4..9ff4d62c30 100644 --- a/services/friendbot/main.go +++ b/services/friendbot/main.go @@ -25,7 +25,6 @@ type Config struct { HorizonURL string `toml:"horizon_url" valid:"required"` StartingBalance string `toml:"starting_balance" valid:"required"` TLS *config.TLS `valid:"optional"` - NumMinions int `toml:"num_minions" valid:"optional"` } func main() { @@ -58,11 +57,8 @@ func run(cmd *cobra.Command, args []string) { } os.Exit(1) } - fb, err := initFriendbot(cfg.FriendbotSecret, cfg.NetworkPassphrase, cfg.HorizonURL, cfg.StartingBalance, cfg.NumMinions) - if err != nil { - log.Error(err) - os.Exit(1) - } + + fb := initFriendbot(cfg.FriendbotSecret, cfg.NetworkPassphrase, cfg.HorizonURL, cfg.StartingBalance) router := initRouter(fb) registerProblems() From 57f23ffc6ffb68ed83cb6c1b9d9b52a4fe505f29 Mon Sep 17 00:00:00 2001 From: Alex Cordeiro Date: Thu, 16 May 2019 10:43:17 -0300 Subject: [PATCH 4/7] exp/ticker: ensure trade pair names use their anchor assets' codes whenever possible (#1283) * exp/ticker: ensure aggregated markets use anchor asset code whenever available * exp/ticker: empty commit to trigger CI tests on previously draft PR --- .../internal/tickerdb/queries_market.go | 18 ++- .../internal/tickerdb/queries_market_test.go | 109 ++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/exp/ticker/internal/tickerdb/queries_market.go b/exp/ticker/internal/tickerdb/queries_market.go index ca0c5812e9..c197a573f3 100644 --- a/exp/ticker/internal/tickerdb/queries_market.go +++ b/exp/ticker/internal/tickerdb/queries_market.go @@ -140,7 +140,11 @@ SELECT FROM ( SELECT -- All valid trades for 24h period - concat(bAsset.code, '_', cAsset.code) as trade_pair_name, + concat( + COALESCE(NULLIF(bAsset.anchor_asset_code, ''), bAsset.code), + '_', + COALESCE(NULLIF(cAsset.anchor_asset_code, ''), cAsset.code) + ) as trade_pair_name, sum(t.base_amount) AS base_volume_24h, sum(t.counter_amount) AS counter_volume_24h, count(t.base_amount) AS trade_count_24h, @@ -160,7 +164,11 @@ FROM ( ) t1 RIGHT JOIN ( SELECT -- All valid trades for 7d period - concat(bAsset.code, '_', cAsset.code) as trade_pair_name, + concat( + COALESCE(NULLIF(bAsset.anchor_asset_code, ''), bAsset.code), + '_', + COALESCE(NULLIF(cAsset.anchor_asset_code, ''), cAsset.code) + ) as trade_pair_name, sum(t.base_amount) AS base_volume_7d, sum(t.counter_amount) AS counter_volume_7d, count(t.base_amount) AS trade_count_7d, @@ -243,7 +251,11 @@ SELECT COALESCE(aob.lowest_ask, 0.0) AS lowest_ask FROM ( SELECT - concat(bAsset.code, '_', cAsset.code) as trade_pair_name, + concat( + COALESCE(NULLIF(bAsset.anchor_asset_code, ''), bAsset.code), + '_', + COALESCE(NULLIF(cAsset.anchor_asset_code, ''), cAsset.code) + ) as trade_pair_name, sum(t.base_amount) AS base_volume, sum(t.counter_amount) AS counter_volume, count(t.base_amount) AS trade_count, diff --git a/exp/ticker/internal/tickerdb/queries_market_test.go b/exp/ticker/internal/tickerdb/queries_market_test.go index 625d189bb3..f6f112c5f4 100644 --- a/exp/ticker/internal/tickerdb/queries_market_test.go +++ b/exp/ticker/internal/tickerdb/queries_market_test.go @@ -751,3 +751,112 @@ func Test24hStatsFallback(t *testing.T) { assert.Equal(t, 0.5, mkt.OpenPrice24h) assert.Equal(t, 0.5, mkt.HighestPrice24h) } + +func TestPreferAnchorAssetCode(t *testing.T) { + db := dbtest.Postgres(t) + defer db.Close() + + var session TickerSession + session.DB = db.Open() + defer session.DB.Close() + + // Run migrations to make sure the tests are run + // on the most updated schema version + migrations := &migrate.FileMigrationSource{ + Dir: "./migrations", + } + _, err := migrate.Exec(session.DB.DB, "postgres", migrations, migrate.Up) + require.NoError(t, err) + + // Adding a seed issuer to be used later: + tbl := session.GetTable("issuers") + _, err = tbl.Insert(Issuer{ + PublicKey: "GCF3TQXKZJNFJK7HCMNE2O2CUNKCJH2Y2ROISTBPLC7C5EIA5NNG2XZB", + Name: "FOO BAR", + }).IgnoreCols("id").Exec() + require.NoError(t, err) + var issuer Issuer + err = session.GetRaw(&issuer, ` + SELECT * + FROM issuers + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // Adding a seed asset to be used later: + err = session.InsertOrUpdateAsset(&Asset{ + Code: "XLM", + IssuerID: issuer.ID, + IsValid: true, + }, []string{"code", "issuer_id"}) + require.NoError(t, err) + var xlmAsset Asset + err = session.GetRaw(&xlmAsset, ` + SELECT * + FROM assets + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // Adding another asset to be used later: + err = session.InsertOrUpdateAsset(&Asset{ + Code: "EURT", + IssuerID: issuer.ID, + IsValid: true, + AnchorAssetCode: "EUR", + }, []string{"code", "issuer_id"}) + require.NoError(t, err) + var btcAsset Asset + err = session.GetRaw(&btcAsset, ` + SELECT * + FROM assets + ORDER BY id DESC + LIMIT 1`, + ) + require.NoError(t, err) + + // A few times to be used: + now := time.Now() + twoDaysAgo := now.AddDate(0, 0, -3) + threeDaysAgo := now.AddDate(0, 0, -3) + + // Now let's create the trades: + trades := []Trade{ + Trade{ + HorizonID: "hrzid1", + BaseAssetID: xlmAsset.ID, + BaseAmount: 1.0, + CounterAssetID: btcAsset.ID, + CounterAmount: 1.0, + Price: 0.5, // close price & lowest price + LedgerCloseTime: twoDaysAgo, + }, + Trade{ // BTC_ETH trade (ETH is from issuer 2) + HorizonID: "hrzid2", + BaseAssetID: xlmAsset.ID, + BaseAmount: 1.0, + CounterAssetID: btcAsset.ID, + CounterAmount: 1.0, + Price: 1.0, // open price & highest price + LedgerCloseTime: threeDaysAgo, + }, + } + err = session.BulkInsertTrades(trades) + require.NoError(t, err) + + markets, err := session.RetrieveMarketData() + require.NoError(t, err) + require.Equal(t, 1, len(markets)) + for _, mkt := range markets { + require.Equal(t, "XLM_EUR", mkt.TradePair) + } + + partialAggMkts, err := session.RetrievePartialAggMarkets(nil, 168) + require.NoError(t, err) + assert.Equal(t, 1, len(partialAggMkts)) + for _, aggMkt := range partialAggMkts { + require.Equal(t, "XLM_EUR", aggMkt.TradePairName) + } +} From 7864802dac5bda97c2381a2e0c2acde025b6ec48 Mon Sep 17 00:00:00 2001 From: Alex Cordeiro Date: Thu, 16 May 2019 13:54:46 -0300 Subject: [PATCH 5/7] docs: update README.md to use Circle CI status badge (#1289) * docs: update README.md CI status badge * docs: empty commit to trigger CI tests on previously draft PR --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d25b07ae0..fb247c129c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Stellar Go -[![Build Status](https://travis-ci.org/stellar/go.svg?branch=master)](https://travis-ci.org/stellar/go) +[![Build Status](https://circleci.com/gh/stellar/go.svg?style=shield)](https://circleci.com/gh/stellar/go) [![GoDoc](https://godoc.org/github.com/stellar/go?status.svg)](https://godoc.org/github.com/stellar/go) [![Go Report Card](https://goreportcard.com/badge/github.com/stellar/go)](https://goreportcard.com/report/github.com/stellar/go) From 47397fabc07707065dd791f8b9ca575c89644b24 Mon Sep 17 00:00:00 2001 From: Debnil Sur Date: Thu, 16 May 2019 12:02:42 -0700 Subject: [PATCH 6/7] clients/horizonclient: fix server time storage (#1284) * Initial commit. * Add check for server date. --- clients/horizonclient/internal.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/clients/horizonclient/internal.go b/clients/horizonclient/internal.go index 6b35f73404..deb1b8500a 100644 --- a/clients/horizonclient/internal.go +++ b/clients/horizonclient/internal.go @@ -111,6 +111,9 @@ func addQueryParams(params ...interface{}) string { // setCurrentServerTime saves the current time returned by a horizon server func setCurrentServerTime(host string, serverDate []string) { + if len(serverDate) == 0 { + return + } st, err := time.Parse(time.RFC1123, serverDate[0]) if err != nil { return @@ -122,7 +125,9 @@ func setCurrentServerTime(host string, serverDate []string) { // currentServerTime returns the current server time for a given horizon server func currentServerTime(host string) int64 { + serverTimeMapMutex.Lock() st := ServerTimeMap[host] + serverTimeMapMutex.Unlock() if &st == nil { return 0 } From 1fe3f2ae1d3e36d934d54482e04c5da77b4e6a96 Mon Sep 17 00:00:00 2001 From: Tom Quisel Date: Thu, 16 May 2019 12:04:11 -0700 Subject: [PATCH 7/7] Fix build image (#1301) --- services/horizon/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/README.md b/services/horizon/README.md index 277f208ba6..de4ea9a6fe 100644 --- a/services/horizon/README.md +++ b/services/horizon/README.md @@ -1,5 +1,5 @@ # Horizon -[![Build Status](https://travis-ci.org/stellar/go.svg?branch=master)](https://travis-ci.org/stellar/go) +[![Build Status](https://circleci.com/gh/stellar/go.svg?style=shield)](https://circleci.com/gh/stellar/go) Horizon is the client facing API server for the [Stellar ecosystem](https://www.stellar.org/developers/guides/get-started/). It acts as the interface between [Stellar Core](https://www.stellar.org/developers/stellar-core/software/admin.html) and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.