From 34c154127b6e354a906e5d253e023ae864772482 Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Wed, 17 Jun 2020 16:32:50 +0200 Subject: [PATCH 1/4] Remove ingest failed transactions flag --- services/horizon/CHANGELOG.md | 1 + services/horizon/cmd/db.go | 9 ++- services/horizon/cmd/ingest.go | 18 +++--- services/horizon/cmd/root.go | 7 --- services/horizon/internal/action_test.go | 2 +- .../horizon/internal/actions/operation.go | 9 +-- .../internal/actions/operation_test.go | 43 ++++--------- .../internal/actions_operation_fee_stats.go | 16 ----- services/horizon/internal/app.go | 2 +- services/horizon/internal/config.go | 2 - .../internal/expingest/db_integration_test.go | 1 - services/horizon/internal/expingest/main.go | 2 - .../horizon/internal/expingest/main_test.go | 1 - .../internal/expingest/processor_runner.go | 19 +----- .../expingest/processor_runner_test.go | 44 +------------- services/horizon/internal/handler.go | 9 +-- services/horizon/internal/helpers_test.go | 7 +-- services/horizon/internal/init.go | 6 +- .../horizon/internal/txsub/results/db/main.go | 60 ------------------- .../internal/txsub/results/db/main_test.go | 30 +--------- services/horizon/internal/web.go | 28 +++------ 21 files changed, 46 insertions(+), 270 deletions(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 2022f052e9..ef4561fd07 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -7,6 +7,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).x * Add transaction set operation count to `history_ledger`([#2690](https://github.com/stellar/go/pull/2690)). Extend ingestion to store the total number of operations in the transaction set and expose it in the ledger resource via `tx_set_operation_count`. This feature allow you to assess the used capacity of a transaction set. +* Remove `--ingest-failed-transactions` flag. From now on Horizon will always ingest failed transactions ([#2702](https://github.com/stellar/go/pull/2702)). ## v1.4.0 diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 55dd716db1..119ff21a13 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -173,11 +173,10 @@ var dbReingestRangeCmd = &cobra.Command{ } ingestConfig := expingest.Config{ - CoreSession: coreSession, - NetworkPassphrase: config.NetworkPassphrase, - HistorySession: horizonSession, - HistoryArchiveURL: config.HistoryArchiveURLs[0], - IngestFailedTransactions: config.IngestFailedTransactions, + CoreSession: coreSession, + NetworkPassphrase: config.NetworkPassphrase, + HistorySession: horizonSession, + HistoryArchiveURL: config.HistoryArchiveURLs[0], } if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index b04e789809..efcbc0eabd 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -102,11 +102,10 @@ var ingestVerifyRangeCmd = &cobra.Command{ } ingestConfig := expingest.Config{ - CoreSession: coreSession, - NetworkPassphrase: config.NetworkPassphrase, - HistorySession: horizonSession, - HistoryArchiveURL: config.HistoryArchiveURLs[0], - IngestFailedTransactions: config.IngestFailedTransactions, + CoreSession: coreSession, + NetworkPassphrase: config.NetworkPassphrase, + HistorySession: horizonSession, + HistoryArchiveURL: config.HistoryArchiveURLs[0], } if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath @@ -182,11 +181,10 @@ var ingestStressTestCmd = &cobra.Command{ } ingestConfig := expingest.Config{ - CoreSession: coreSession, - NetworkPassphrase: config.NetworkPassphrase, - HistorySession: horizonSession, - HistoryArchiveURL: config.HistoryArchiveURLs[0], - IngestFailedTransactions: config.IngestFailedTransactions, + CoreSession: coreSession, + NetworkPassphrase: config.NetworkPassphrase, + HistorySession: horizonSession, + HistoryArchiveURL: config.HistoryArchiveURLs[0], } if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index 12ac89236f..a64db31bc8 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -329,13 +329,6 @@ var configOpts = support.ConfigOptions{ FlagDefault: false, Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", }, - &support.ConfigOption{ - Name: "ingest-failed-transactions", - ConfigKey: &config.IngestFailedTransactions, - OptType: types.Bool, - FlagDefault: false, - Usage: "causes this horizon process to ingest failed transactions data", - }, &support.ConfigOption{ Name: "cursor-name", EnvVar: "CURSOR_NAME", diff --git a/services/horizon/internal/action_test.go b/services/horizon/internal/action_test.go index b59b068d27..02a646214a 100644 --- a/services/horizon/internal/action_test.go +++ b/services/horizon/internal/action_test.go @@ -22,7 +22,7 @@ func TestGetTransactionPage(t *testing.T) { defer tt.Finish() ctx := context.Background() - w := mustInitWeb(ctx, &history.Q{tt.HorizonSession()}, time.Duration(5), 0, true) + w := mustInitWeb(ctx, &history.Q{tt.HorizonSession()}, time.Duration(5), 0) // filter by account params := &indexActionQueryParams{ diff --git a/services/horizon/internal/actions/operation.go b/services/horizon/internal/actions/operation.go index a5119f83f9..66b4383c01 100644 --- a/services/horizon/internal/actions/operation.go +++ b/services/horizon/internal/actions/operation.go @@ -50,8 +50,7 @@ func (qp OperationsQuery) Validate() error { // GetOperationsHandler is the action handler for all end-points returning a list of operations. type GetOperationsHandler struct { - OnlyPayments bool - IngestingFailedTransactions bool + OnlyPayments bool } // GetResourcePage returns a page of operations. @@ -74,12 +73,6 @@ func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Requ return nil, err } - if qp.IncludeFailedTransactions && !handler.IngestingFailedTransactions { - err = errors.New("`include_failed` parameter is unavailable when Horizon is not ingesting failed " + - "transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.") - return nil, problem.MakeInvalidFieldProblem("include_failed", err) - } - historyQ, err := HistoryQFromRequest(r) if err != nil { return nil, err diff --git a/services/horizon/internal/actions/operation_test.go b/services/horizon/internal/actions/operation_test.go index 0a0b369f06..3e2a72dc4b 100644 --- a/services/horizon/internal/actions/operation_test.go +++ b/services/horizon/internal/actions/operation_test.go @@ -19,7 +19,7 @@ func TestGetOperationsWithoutFilter(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} records, err := handler.GetResourcePage( httptest.NewRecorder(), @@ -37,7 +37,7 @@ func TestGetOperationsExclusiveFilters(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} testCases := []struct { desc string @@ -92,7 +92,7 @@ func TestGetOperationsFilterByAccountID(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} testCases := []struct { accountID string @@ -133,7 +133,7 @@ func TestGetOperationsFilterByTxID(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} testCases := []struct { desc string @@ -207,7 +207,7 @@ func TestGetOperationsIncludeFailed(t *testing.T) { tt.Scenario("failed_transactions") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} records, err := handler.GetResourcePage( httptest.NewRecorder(), @@ -329,28 +329,6 @@ func TestGetOperationsIncludeFailed(t *testing.T) { "Filter should be true or false", p.Extras["reason"], ) - - handler = GetOperationsHandler{ - IngestingFailedTransactions: false, - } - - _, err = handler.GetResourcePage( - httptest.NewRecorder(), - makeRequest( - t, map[string]string{ - "include_failed": "true", - }, map[string]string{}, q.Session, - ), - ) - tt.Assert.Error(err) - tt.Assert.IsType(&problem.P{}, err) - p = err.(*problem.P) - tt.Assert.Equal("bad_request", p.Type) - tt.Assert.Equal("include_failed", p.Extras["invalid_field"]) - tt.Assert.Equal( - "`include_failed` parameter is unavailable when Horizon is not ingesting failed transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.", - p.Extras["reason"], - ) } func TestGetOperationsFilterByLedgerID(t *testing.T) { @@ -359,7 +337,7 @@ func TestGetOperationsFilterByLedgerID(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} testCases := []struct { ledgerID string @@ -430,8 +408,7 @@ func TestGetOperationsOnlyPayments(t *testing.T) { q := &history.Q{tt.HorizonSession()} handler := GetOperationsHandler{ - IngestingFailedTransactions: true, - OnlyPayments: true, + OnlyPayments: true, } records, err := handler.GetResourcePage( @@ -510,7 +487,7 @@ func TestOperation_CreatedAt(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} records, err := handler.GetResourcePage( httptest.NewRecorder(), @@ -535,7 +512,7 @@ func TestGetOperationsPagination(t *testing.T) { tt.Scenario("base") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} records, err := handler.GetResourcePage( httptest.NewRecorder(), @@ -592,7 +569,7 @@ func TestGetOperations_IncludeTransactions(t *testing.T) { tt.Scenario("failed_transactions") q := &history.Q{tt.HorizonSession()} - handler := GetOperationsHandler{IngestingFailedTransactions: true} + handler := GetOperationsHandler{} _, err := handler.GetResourcePage( httptest.NewRecorder(), diff --git a/services/horizon/internal/actions_operation_fee_stats.go b/services/horizon/internal/actions_operation_fee_stats.go index 43e0b959e5..0f6142f6c8 100644 --- a/services/horizon/internal/actions_operation_fee_stats.go +++ b/services/horizon/internal/actions_operation_fee_stats.go @@ -1,14 +1,12 @@ package horizon import ( - "net/http" "strconv" hProtocol "github.com/stellar/go/protocols/horizon" "github.com/stellar/go/services/horizon/internal/actions" "github.com/stellar/go/services/horizon/internal/operationfeestats" "github.com/stellar/go/support/render/httpjson" - "github.com/stellar/go/support/render/problem" ) // This file contains the actions: @@ -26,20 +24,6 @@ type FeeStatsAction struct { // JSON is a method for actions.JSON func (action *FeeStatsAction) JSON() error { - if !action.App.config.IngestFailedTransactions { - // If Horizon is not ingesting failed transaction it does not make sense to display - // operation fee stats because they will be incorrect. - p := problem.P{ - Type: "endpoint_not_available", - Title: "Endpoint Not Available", - Status: http.StatusNotImplemented, - Detail: "/fee_stats is unavailable when Horizon is not ingesting failed " + - "transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.", - } - problem.Render(action.R.Context(), action.W, p) - return nil - } - action.Do( action.loadRecords, func() { diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index f472697a46..7f98cb8dce 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -466,7 +466,7 @@ func (a *App) init() { a.reaper = reap.New(a.config.HistoryRetentionCount, a.HorizonSession(context.Background())) // web.init - a.web = mustInitWeb(a.ctx, a.historyQ, a.config.SSEUpdateFrequency, a.config.StaleThreshold, a.config.IngestFailedTransactions) + a.web = mustInitWeb(a.ctx, a.historyQ, a.config.SSEUpdateFrequency, a.config.StaleThreshold) // web.rate-limiter a.web.rateLimiter = maybeInitWebRateLimiter(a.config.RateQuota) diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index e559872312..22e1c0a5c6 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -45,8 +45,6 @@ type Config struct { TLSKey string // Ingest toggles whether this horizon instance should run the data ingestion subsystem. Ingest bool - // IngestFailedTransactions toggles whether to ingest failed transactions - IngestFailedTransactions bool // CursorName is the cursor used for ingesting from stellar-core. // Setting multiple cursors in different Horizon instances allows multiple // Horizons to ingest from the same stellar-core instance without cursor diff --git a/services/horizon/internal/expingest/db_integration_test.go b/services/horizon/internal/expingest/db_integration_test.go index 700f7d99c3..4b1fc406a6 100644 --- a/services/horizon/internal/expingest/db_integration_test.go +++ b/services/horizon/internal/expingest/db_integration_test.go @@ -83,7 +83,6 @@ func (s *DBTestSuite) SetupTest() { HistoryArchiveURL: "http://ignore.test", MaxStreamRetries: 3, DisableStateVerification: false, - IngestFailedTransactions: true, }) s.Assert().NoError(err) diff --git a/services/horizon/internal/expingest/main.go b/services/horizon/internal/expingest/main.go index cca155edbf..64bf9e62e0 100644 --- a/services/horizon/internal/expingest/main.go +++ b/services/horizon/internal/expingest/main.go @@ -66,8 +66,6 @@ type Config struct { // errors while streaming xdr bucket entries from the history archive. // Set MaxStreamRetries to 0 if there should be no retry attempts MaxStreamRetries int - - IngestFailedTransactions bool } const ( diff --git a/services/horizon/internal/expingest/main_test.go b/services/horizon/internal/expingest/main_test.go index 5b9226b85a..c1463bebfe 100644 --- a/services/horizon/internal/expingest/main_test.go +++ b/services/horizon/internal/expingest/main_test.go @@ -87,7 +87,6 @@ func TestNewSystem(t *testing.T) { }, DisableStateVerification: true, HistoryArchiveURL: "https://history.stellar.org/prd/core-live/core_live_001", - IngestFailedTransactions: true, } system, err := NewSystem(config) diff --git a/services/horizon/internal/expingest/processor_runner.go b/services/horizon/internal/expingest/processor_runner.go index 214f2683b7..934df6ba49 100644 --- a/services/horizon/internal/expingest/processor_runner.go +++ b/services/horizon/internal/expingest/processor_runner.go @@ -108,17 +108,6 @@ func (s *ProcessorRunner) buildChangeProcessor( } } -type skipFailedTransactions struct { - horizonTransactionProcessor -} - -func (p skipFailedTransactions) ProcessTransaction(tx io.LedgerTransaction) error { - if !tx.Result.Successful() { - return nil - } - return p.horizonTransactionProcessor.ProcessTransaction(tx) -} - func (s *ProcessorRunner) buildTransactionProcessor( ledgerTransactionStats *io.StatsLedgerTransactionProcessor, ledger xdr.LedgerHeaderHistoryEntry, @@ -128,7 +117,7 @@ func (s *ProcessorRunner) buildTransactionProcessor( } sequence := uint32(ledger.Header.LedgerSeq) - group := groupTransactionProcessors{ + return groupTransactionProcessors{ statsLedgerTransactionProcessor, processors.NewEffectProcessor(s.historyQ, sequence), processors.NewLedgerProcessor(s.historyQ, ledger, CurrentVersion), @@ -137,12 +126,6 @@ func (s *ProcessorRunner) buildTransactionProcessor( processors.NewParticipantsProcessor(s.historyQ, sequence), processors.NewTransactionProcessor(s.historyQ, sequence), } - - if s.config.IngestFailedTransactions { - return group - } - - return skipFailedTransactions{group} } // validateBucketList validates if the bucket list hash in history archive diff --git a/services/horizon/internal/expingest/processor_runner_test.go b/services/horizon/internal/expingest/processor_runner_test.go index aa068d2847..5bae0e7177 100644 --- a/services/horizon/internal/expingest/processor_runner_test.go +++ b/services/horizon/internal/expingest/processor_runner_test.go @@ -239,9 +239,7 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) { Return(&history.MockTransactionsBatchInsertBuilder{}).Twice() runner := ProcessorRunner{ - config: Config{ - IngestFailedTransactions: true, - }, + config: Config{}, historyQ: q, } @@ -257,16 +255,6 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) { assert.IsType(t, &processors.TradeProcessor{}, processor.(groupTransactionProcessors)[4]) assert.IsType(t, &processors.ParticipantsProcessor{}, processor.(groupTransactionProcessors)[5]) assert.IsType(t, &processors.TransactionProcessor{}, processor.(groupTransactionProcessors)[6]) - - runner = ProcessorRunner{ - config: Config{ - IngestFailedTransactions: false, - }, - historyQ: q, - } - - processor = runner.buildTransactionProcessor(stats, ledger) - assert.IsType(t, skipFailedTransactions{}, processor) } func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { @@ -340,33 +328,3 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { _, _, err := runner.RunAllProcessorsOnLedger(63) assert.NoError(t, err) } - -func TestSkipFailedTransactions(t *testing.T) { - mockProcessor := &mockHorizonTransactionProcessor{} - successfulTx := io.LedgerTransaction{ - Result: xdr.TransactionResultPair{ - Result: xdr.TransactionResult{ - Result: xdr.TransactionResultResult{ - Code: xdr.TransactionResultCodeTxSuccess, - }, - }, - }, - } - mockProcessor.On("ProcessTransaction", successfulTx).Return(nil) - defer mock.AssertExpectationsForObjects(t, mockProcessor) - - processor := skipFailedTransactions{mockProcessor} - err := processor.ProcessTransaction(io.LedgerTransaction{ - Result: xdr.TransactionResultPair{ - Result: xdr.TransactionResult{ - Result: xdr.TransactionResultResult{ - Code: xdr.TransactionResultCodeTxBadAuth, - }, - }, - }, - }) - assert.NoError(t, err) - - err = processor.ProcessTransaction(successfulTx) - assert.NoError(t, err) -} diff --git a/services/horizon/internal/handler.go b/services/horizon/internal/handler.go index 0ef8091656..f041d823ef 100644 --- a/services/horizon/internal/handler.go +++ b/services/horizon/internal/handler.go @@ -200,7 +200,7 @@ func (we *web) streamIndexActionHandler(jfn interface{}, sfn streamFunc) http.Ha return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - params, err := getIndexActionQueryParams(r, we.ingestFailedTx) + params, err := getIndexActionQueryParams(r) if err != nil { problem.Render(ctx, w, err) return @@ -282,7 +282,7 @@ func getShowActionQueryParams(r *http.Request, requireAccountID bool) (*showActi } // getIndexActionQueryParams gets the available query params for all indexable endpoints. -func getIndexActionQueryParams(r *http.Request, ingestFailedTransactions bool) (*indexActionQueryParams, error) { +func getIndexActionQueryParams(r *http.Request) (*indexActionQueryParams, error) { addr, err := getAccountID(r, "account_id", false) if err != nil { return nil, errors.Wrap(err, "getting account id") @@ -307,11 +307,6 @@ func getIndexActionQueryParams(r *http.Request, ingestFailedTransactions bool) ( if err != nil { return nil, errors.Wrap(err, "getting include_failed param") } - if includeFailedTx && !ingestFailedTransactions { - return nil, problem.MakeInvalidFieldProblem("include_failed", - errors.New("`include_failed` parameter is unavailable when Horizon is not ingesting failed "+ - "transactions. Set `INGEST_FAILED_TRANSACTIONS=true` to start ingesting them.")) - } return &indexActionQueryParams{ AccountID: addr, diff --git a/services/horizon/internal/helpers_test.go b/services/horizon/internal/helpers_test.go index a5352a64e2..3bf6111754 100644 --- a/services/horizon/internal/helpers_test.go +++ b/services/horizon/internal/helpers_test.go @@ -28,10 +28,9 @@ func NewTestConfig() Config { MaxRate: throttled.PerHour(1000), MaxBurst: 100, }, - ConnectionTimeout: 55 * time.Second, // Default - LogLevel: supportLog.InfoLevel, - NetworkPassphrase: network.TestNetworkPassphrase, - IngestFailedTransactions: true, + ConnectionTimeout: 55 * time.Second, // Default + LogLevel: supportLog.InfoLevel, + NetworkPassphrase: network.TestNetworkPassphrase, } } diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index 066e867fda..e335742fb2 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -90,7 +90,6 @@ func initExpIngester(app *App) { StellarCoreCursor: app.config.CursorName, MaxStreamRetries: 3, DisableStateVerification: app.config.IngestDisableStateVerification, - IngestFailedTransactions: app.config.IngestFailedTransactions, }) if err != nil { log.Fatal(err) @@ -225,9 +224,8 @@ func initSubmissionSystem(app *App) { Submitter: txsub.NewDefaultSubmitter(http.DefaultClient, app.config.StellarCoreURL), SubmissionQueue: sequence.NewManager(), Results: &results.DB{ - Core: &core.Q{Session: app.CoreSession(context.Background())}, - History: &history.Q{Session: app.HorizonSession(context.Background())}, - SkipCoreChecks: app.config.IngestFailedTransactions, + Core: &core.Q{Session: app.CoreSession(context.Background())}, + History: &history.Q{Session: app.HorizonSession(context.Background())}, }, Sequences: &history.Q{Session: app.HorizonSession(context.Background())}, } diff --git a/services/horizon/internal/txsub/results/db/main.go b/services/horizon/internal/txsub/results/db/main.go index 5581af72fa..30bdc70b08 100644 --- a/services/horizon/internal/txsub/results/db/main.go +++ b/services/horizon/internal/txsub/results/db/main.go @@ -3,13 +3,9 @@ package results import ( - "bytes" "context" - "encoding/base64" - "github.com/stellar/go/services/horizon/internal/db2/core" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/ledger" "github.com/stellar/go/services/horizon/internal/txsub" "github.com/stellar/go/xdr" ) @@ -19,16 +15,12 @@ import ( type DB struct { Core *core.Q History *history.Q - // SkipCoreChecks makes DB skip checking transaction result in Core DB if `true`. - SkipCoreChecks bool } var _ txsub.ResultProvider = &DB{} // ResultByHash implements txsub.ResultProvider func (rp *DB) ResultByHash(ctx context.Context, hash string) txsub.Result { - historyLatest := ledger.CurrentState().HistoryLatest - // query history database var hr history.Transaction err := rp.History.TransactionByHash(&hr, hash) @@ -40,35 +32,6 @@ func (rp *DB) ResultByHash(ctx context.Context, hash string) txsub.Result { return txsub.Result{Err: err} } - if !rp.SkipCoreChecks { - // query core database - var cr core.Transaction - // In the past we were searching for the transaction in core DB *after* the - // latest ingested ledger. This was incorrect because history DB contains - // successful transactions only. So it was possible that the transaction was - // never found and clients were receiving Timeout errors. - // However we can't change it to simply find a transaction by hash because - // `txhistory` table does not have an index on `txid` field. Because of this - // we query the last 120 ledgers (~10 minutes) to not kill the DB by searching - // for a value on a table with millions of rows but also to support returning - // the failed tx result (when resubmitting) for 10 minutes (or before core - // clears `txhistory` table, whatever is first). - // If you are modifying the code here, please do not make this error again. - err = rp.Core.TransactionByHashAfterLedger(&cr, hash, historyLatest-120) - if err == nil { - if err = txSucceeded(cr); err == nil { - // The transaction succeeded and was found in Stellar Core. The transaction should also - // be present in Horizon's DB but it could be that horizon ingestion is behind Stellar Core. - // We return ErrNoResults so the txsub will retry the horizon db lookup - return txsub.Result{Err: txsub.ErrNoResults} - } else { - return txsub.Result{Err: err} - } - } else if !rp.Core.NoRows(err) { - return txsub.Result{Err: err} - } - } - // if no result was found in either db, return ErrNoResults return txsub.Result{Err: txsub.ErrNoResults} } @@ -86,26 +49,3 @@ func txResultFromHistory(tx history.Transaction) txsub.Result { return txsub.Result{Err: err, Transaction: tx} } - -// txSucceeded returns nil if the the transaction succeeded -// otherwise, it returns an error -func txSucceeded(tx core.Transaction) error { - // re-encode result to base64 - var raw bytes.Buffer - _, err := xdr.Marshal(&raw, tx.Result.Result) - if err != nil { - return err - } - - trx := base64.StdEncoding.EncodeToString(raw.Bytes()) - - // if result is success, send a normal resposne - if tx.Result.Successful() { - return nil - } - - // if failed, produce a FailedTransactionError - return &txsub.FailedTransactionError{ - ResultXDR: trx, - } -} diff --git a/services/horizon/internal/txsub/results/db/main_test.go b/services/horizon/internal/txsub/results/db/main_test.go index 9d5d19d7c1..e831ea9602 100644 --- a/services/horizon/internal/txsub/results/db/main_test.go +++ b/services/horizon/internal/txsub/results/db/main_test.go @@ -32,9 +32,8 @@ func TestResultProviderHorizonOnly(t *testing.T) { defer tt.Finish() rp := &DB{ - Core: &core.Q{Session: tt.CoreSession()}, - History: &history.Q{Session: tt.HorizonSession()}, - SkipCoreChecks: true, + Core: &core.Q{Session: tt.CoreSession()}, + History: &history.Q{Session: tt.HorizonSession()}, } hash := "adf1efb9fd253f53cbbe6230c131d2af19830328e52b610464652d67d2fb7195" @@ -69,28 +68,3 @@ func TestResultFailed(t *testing.T) { tt.Require.Error(ret.Err) tt.Assert.Equal("AAAAAAAAAGT/////AAAAAQAAAAAAAAAB/////gAAAAA=", ret.Err.(*txsub.FailedTransactionError).ResultXDR) } - -func TestResultFailedNotInHorizonDB(t *testing.T) { - tt := test.Start(t).Scenario("failed_transactions") - defer tt.Finish() - - rp := &DB{ - Core: &core.Q{Session: tt.CoreSession()}, - History: &history.Q{Session: tt.HorizonSession()}, - SkipCoreChecks: false, - } - - hash := "aa168f12124b7c196c0adaee7c73a64d37f99428cacb59a91ff389626845e7cf" - - // remove tx from horizon db - _, err := tt.HorizonSession().ExecRaw( - `DELETE FROM history_transactions WHERE transaction_hash = ?`, - hash, - ) - tt.Require.NoError(err) - - ret := rp.ResultByHash(tt.Ctx, hash) - - tt.Require.Error(ret.Err) - tt.Assert.Equal("AAAAAAAAAGT/////AAAAAQAAAAAAAAAB/////gAAAAA=", ret.Err.(*txsub.FailedTransactionError).ResultXDR) -} diff --git a/services/horizon/internal/web.go b/services/horizon/internal/web.go index 148cf0e6e5..82799c081e 100644 --- a/services/horizon/internal/web.go +++ b/services/horizon/internal/web.go @@ -43,7 +43,6 @@ type web struct { rateLimiter *throttled.HTTPRateLimiter sseUpdateFrequency time.Duration staleThreshold uint - ingestFailedTx bool historyQ *history.Q @@ -66,7 +65,7 @@ func init() { } // mustInitWeb installed a new Web instance onto the provided app object. -func mustInitWeb(ctx context.Context, hq *history.Q, updateFreq time.Duration, threshold uint, ingestFailedTx bool) *web { +func mustInitWeb(ctx context.Context, hq *history.Q, updateFreq time.Duration, threshold uint) *web { if hq == nil { log.Fatal("missing history DB for installing the web instance") } @@ -78,7 +77,6 @@ func mustInitWeb(ctx context.Context, hq *history.Q, updateFreq time.Duration, t historyQ: hq, sseUpdateFrequency: updateFreq, staleThreshold: threshold, - ingestFailedTx: ingestFailedTx, requestTimer: metrics.NewTimer(), failureMeter: metrics.NewMeter(), successMeter: metrics.NewMeter(), @@ -215,12 +213,10 @@ func (w *web) mustInstallActions(config Config, pathFinder paths.Finder, session r.Group(func(r chi.Router) { r.Use(historyMiddleware) r.Method(http.MethodGet, "/accounts/{account_id:\\w+}/operations", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: false, + OnlyPayments: false, }, streamHandler)) r.Method(http.MethodGet, "/accounts/{account_id:\\w+}/payments", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: true, + OnlyPayments: true, }, streamHandler)) }) }) @@ -234,12 +230,10 @@ func (w *web) mustInstallActions(config Config, pathFinder paths.Finder, session r.Group(func(r chi.Router) { r.Use(historyMiddleware) r.Method(http.MethodGet, "/operations", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: false, + OnlyPayments: false, }, streamHandler)) r.Method(http.MethodGet, "/payments", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: true, + OnlyPayments: true, }, streamHandler)) }) }) @@ -254,12 +248,10 @@ func (w *web) mustInstallActions(config Config, pathFinder paths.Finder, session r.Group(func(r chi.Router) { r.Use(historyMiddleware) r.Method(http.MethodGet, "/operations", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: false, + OnlyPayments: false, }, streamHandler)) r.Method(http.MethodGet, "/payments", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: true, + OnlyPayments: true, }, streamHandler)) }) }) @@ -268,8 +260,7 @@ func (w *web) mustInstallActions(config Config, pathFinder paths.Finder, session // operation actions r.Route("/operations", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: false, + OnlyPayments: false, }, streamHandler)) r.Get("/{id}", OperationShowAction{}.Handle) r.Get("/{op_id}/effects", EffectIndexAction{}.Handle) @@ -278,8 +269,7 @@ func (w *web) mustInstallActions(config Config, pathFinder paths.Finder, session r.Group(func(r chi.Router) { // payment actions r.With(historyMiddleware).Method(http.MethodGet, "/payments", streamableHistoryPageHandler(actions.GetOperationsHandler{ - IngestingFailedTransactions: w.ingestFailedTx, - OnlyPayments: true, + OnlyPayments: true, }, streamHandler)) // effect actions From 30a6e1084f8087f9d2b83f412305efed700e6dad Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Wed, 17 Jun 2020 17:28:48 +0200 Subject: [PATCH 2/4] Remove core.Q from txsub --- services/horizon/internal/txsub/results/db/main.go | 2 -- services/horizon/internal/txsub/results/db/main_test.go | 4 ---- 2 files changed, 6 deletions(-) diff --git a/services/horizon/internal/txsub/results/db/main.go b/services/horizon/internal/txsub/results/db/main.go index 30bdc70b08..ea34811628 100644 --- a/services/horizon/internal/txsub/results/db/main.go +++ b/services/horizon/internal/txsub/results/db/main.go @@ -4,7 +4,6 @@ package results import ( "context" - "github.com/stellar/go/services/horizon/internal/db2/core" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/txsub" "github.com/stellar/go/xdr" @@ -13,7 +12,6 @@ import ( // DB provides transaction submission results by querying the // connected horizon and, if set, stellar core databases. type DB struct { - Core *core.Q History *history.Q } diff --git a/services/horizon/internal/txsub/results/db/main_test.go b/services/horizon/internal/txsub/results/db/main_test.go index e831ea9602..91d5eaf459 100644 --- a/services/horizon/internal/txsub/results/db/main_test.go +++ b/services/horizon/internal/txsub/results/db/main_test.go @@ -3,7 +3,6 @@ package results import ( "testing" - "github.com/stellar/go/services/horizon/internal/db2/core" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/services/horizon/internal/txsub" @@ -14,7 +13,6 @@ func TestResultProvider(t *testing.T) { defer tt.Finish() rp := &DB{ - Core: &core.Q{Session: tt.CoreSession()}, History: &history.Q{Session: tt.HorizonSession()}, } @@ -32,7 +30,6 @@ func TestResultProviderHorizonOnly(t *testing.T) { defer tt.Finish() rp := &DB{ - Core: &core.Q{Session: tt.CoreSession()}, History: &history.Q{Session: tt.HorizonSession()}, } @@ -50,7 +47,6 @@ func TestResultFailed(t *testing.T) { defer tt.Finish() rp := &DB{ - Core: &core.Q{Session: tt.CoreSession()}, History: &history.Q{Session: tt.HorizonSession()}, } From 0aa3e6a579874f110a9108e87109909a6ae7cb51 Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Wed, 17 Jun 2020 17:37:39 +0200 Subject: [PATCH 3/4] Remove dead code from db2/core package --- services/horizon/internal/db2/core/account.go | 68 ------ .../horizon/internal/db2/core/account_data.go | 81 ------- .../internal/db2/core/ledger_header.go | 26 --- services/horizon/internal/db2/core/main.go | 175 --------------- .../horizon/internal/db2/core/main_test.go | 181 --------------- services/horizon/internal/db2/core/offer.go | 207 ------------------ .../horizon/internal/db2/core/offer_test.go | 71 ------ services/horizon/internal/db2/core/signer.go | 58 ----- .../horizon/internal/db2/core/transaction.go | 156 ------------- .../internal/db2/core/transaction_fee.go | 26 --- .../internal/db2/core/transaction_fee_test.go | 20 -- .../internal/db2/core/transaction_test.go | 101 --------- .../horizon/internal/db2/core/trustline.go | 127 ----------- .../internal/db2/core/trustline_test.go | 58 ----- services/horizon/internal/init.go | 1 - 15 files changed, 1356 deletions(-) delete mode 100644 services/horizon/internal/db2/core/account.go delete mode 100644 services/horizon/internal/db2/core/account_data.go delete mode 100644 services/horizon/internal/db2/core/ledger_header.go delete mode 100644 services/horizon/internal/db2/core/offer.go delete mode 100644 services/horizon/internal/db2/core/offer_test.go delete mode 100644 services/horizon/internal/db2/core/signer.go delete mode 100644 services/horizon/internal/db2/core/transaction.go delete mode 100644 services/horizon/internal/db2/core/transaction_fee.go delete mode 100644 services/horizon/internal/db2/core/transaction_fee_test.go delete mode 100644 services/horizon/internal/db2/core/transaction_test.go delete mode 100644 services/horizon/internal/db2/core/trustline.go delete mode 100644 services/horizon/internal/db2/core/trustline_test.go diff --git a/services/horizon/internal/db2/core/account.go b/services/horizon/internal/db2/core/account.go deleted file mode 100644 index aad6b15930..0000000000 --- a/services/horizon/internal/db2/core/account.go +++ /dev/null @@ -1,68 +0,0 @@ -package core - -import ( - "encoding/base64" - - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" -) - -// IsAuthRequired returns true if the account has the "AUTH_REQUIRED" option -// turned on. -func (ac Account) IsAuthRequired() bool { - return (ac.Flags & xdr.AccountFlagsAuthRequiredFlag) != 0 -} - -// IsAuthRevocable returns true if the account has the "AUTH_REVOCABLE" option -// turned on. -func (ac Account) IsAuthRevocable() bool { - return (ac.Flags & xdr.AccountFlagsAuthRevocableFlag) != 0 -} - -// IsAuthImmutable returns true if the account has the "AUTH_IMMUTABLE" option -// turned on. -func (ac Account) IsAuthImmutable() bool { - return (ac.Flags & xdr.AccountFlagsAuthImmutableFlag) != 0 -} - -// AccountByAddress loads a row from `accounts`, by address -func (q *Q) AccountByAddress(dest *Account, addy string) error { - sql := selectAccount.Limit(1).Where("accountid = ?", addy) - err := q.Get(dest, sql) - if err != nil { - return err - } - - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - if schemaVersion >= 9 { - // Since schema version 9, home_domain is base64 encoded. - decoded, err := base64.StdEncoding.DecodeString(dest.HomeDomain.String) - if err != nil { - return errors.Wrap(err, "Unable to base64 decode HomeDomain") - } - dest.HomeDomain.String = string(decoded) - } - - return nil -} - -var selectAccount = sq.Select( - "a.accountid", - "a.balance", - "a.seqnum", - "a.numsubentries", - "a.inflationdest", - "a.homedomain", - "a.thresholds", - "a.flags", - "a.lastmodified", - // Liabilities can be NULL so can error without `coalesce`: - // `Invalid value for xdr.Int64` - "coalesce(a.buyingliabilities, 0) as buyingliabilities", - "coalesce(a.sellingliabilities, 0) as sellingliabilities", -).From("accounts a") diff --git a/services/horizon/internal/db2/core/account_data.go b/services/horizon/internal/db2/core/account_data.go deleted file mode 100644 index 5f3f80d301..0000000000 --- a/services/horizon/internal/db2/core/account_data.go +++ /dev/null @@ -1,81 +0,0 @@ -package core - -import ( - "encoding/base64" - "fmt" - - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/errors" -) - -// Raw returns the decoded, raw value of the account data -func (ad AccountData) Raw() ([]byte, error) { - return base64.StdEncoding.DecodeString(ad.Value) -} - -// AccountDataByKey loads a row from `accountdata`, by key -func (q *Q) AccountDataByKey(dest *AccountData, addy string, key string) error { - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - queryKey := key - if schemaVersion >= 9 { - // Since schema version 9, keys are base64 encoded. - queryKey = base64.StdEncoding.EncodeToString([]byte(key)) - } - - sql := selectAccountData.Limit(1). - Where("accountid = ?", addy). - Where("dataname = ?", queryKey) - - err = q.Get(dest, sql) - if err != nil { - return err - } - - if schemaVersion >= 9 { - dest.Key = key - } - - return nil -} - -// AllDataByAddress loads all data for `addy` -func (q *Q) AllDataByAddress(dest interface{}, addy string) error { - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - sql := selectAccountData.Where("accountid = ?", addy) - err = q.Select(dest, sql) - if err != nil { - return err - } - - if schemaVersion >= 9 { - // Since schema version 9, keys are base64 encoded. - d, ok := dest.(*[]AccountData) - if !ok { - return errors.New("Cannot ensure []AccountData type") - } - - for i, val := range *d { - decoded, err := base64.StdEncoding.DecodeString(val.Key) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Error decoding data entry: %s", val.Key)) - } - (*d)[i].Key = string(decoded) - } - } - - return nil -} - -var selectAccountData = sq.Select( - "ad.accountid", - "ad.dataname", - "ad.datavalue", -).From("accountdata ad") diff --git a/services/horizon/internal/db2/core/ledger_header.go b/services/horizon/internal/db2/core/ledger_header.go deleted file mode 100644 index 2ba7fd68ce..0000000000 --- a/services/horizon/internal/db2/core/ledger_header.go +++ /dev/null @@ -1,26 +0,0 @@ -package core - -import ( - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/xdr" -) - -// DataXDR returns the base64 encoded ledger header -func (lh *LedgerHeader) DataXDR() string { - out, err := xdr.MarshalBase64(lh.Data) - if err != nil { - panic(err) - } - return out -} - -// LedgerHeaderBySequence is a query that loads a single row from the -// `ledgerheaders` table. -func (q *Q) LedgerHeaderBySequence(dest interface{}, seq int32) error { - sql := sq.Select("clh.*"). - From("ledgerheaders clh"). - Limit(1). - Where("clh.ledgerseq = ?", seq) - - return q.Get(dest, sql) -} diff --git a/services/horizon/internal/db2/core/main.go b/services/horizon/internal/db2/core/main.go index 13359ff3c0..5c3a35b76c 100644 --- a/services/horizon/internal/db2/core/main.go +++ b/services/horizon/internal/db2/core/main.go @@ -3,177 +3,15 @@ package core import ( - "strconv" - - "github.com/guregu/null" - "github.com/stellar/go/strkey" "github.com/stellar/go/support/db" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" ) -// Account is a row of data from the `accounts` table -type Account struct { - Accountid string - Balance xdr.Int64 - Seqnum string - Numsubentries int32 - Inflationdest null.String - HomeDomain null.String - Thresholds xdr.Thresholds - Flags xdr.AccountFlags - LastModified uint32 - BuyingLiabilities xdr.Int64 `db:"buyingliabilities"` - SellingLiabilities xdr.Int64 `db:"sellingliabilities"` -} - -// AccountData is a row of data from the `accountdata` table -type AccountData struct { - Accountid string - Key string `db:"dataname"` - Value string `db:"datavalue"` -} - -// LedgerHeader is row of data from the `ledgerheaders` table -type LedgerHeader struct { - LedgerHash string `db:"ledgerhash"` - PrevHash string `db:"prevhash"` - BucketListHash string `db:"bucketlisthash"` - CloseTime int64 `db:"closetime"` - Sequence uint32 `db:"ledgerseq"` - Data xdr.LedgerHeader `db:"data"` -} - -// Offer is row of data from the `offers` table from stellar-core -type Offer struct { - SellerID string `db:"sellerid"` - OfferID int64 `db:"offerid"` - - SellingAsset xdr.Asset `db:"sellingasset"` - BuyingAsset xdr.Asset `db:"buyingasset"` - - Amount xdr.Int64 `db:"amount"` - Pricen int32 `db:"pricen"` - Priced int32 `db:"priced"` - Price float64 `db:"price"` - Flags int32 `db:"flags"` - Lastmodified int32 `db:"lastmodified"` -} - -// get returns Offer. Useful in `internalOffer` context, when Offer is embedded. -func (o Offer) get() Offer { - return o -} - -// internalOffer is row of data from the `offers` table from stellar-core used -// internally only to support schema <=8. -type internalOffer struct { - Offer - - // Schema v8 fields, for compatibility only. - SellingAssetType xdr.AssetType `db:"sellingassettype"` - SellingAssetCode null.String `db:"sellingassetcode"` - SellingIssuer null.String `db:"sellingissuer"` - - BuyingAssetType xdr.AssetType `db:"buyingassettype"` - BuyingAssetCode null.String `db:"buyingassetcode"` - BuyingIssuer null.String `db:"buyingissuer"` -} - // Q is a helper struct on which to hang common queries against a stellar // core database. type Q struct { *db.Session } -// Signer is a row of data from the `signers` table from stellar-core -type Signer struct { - Accountid string - Publickey string - Weight int32 -} - -// Transaction is row of data from the `txhistory` table from stellar-core -type Transaction struct { - TransactionHash string `db:"txid"` - LedgerSequence int32 `db:"ledgerseq"` - Index int32 `db:"txindex"` - Envelope xdr.TransactionEnvelope `db:"txbody"` - Result xdr.TransactionResultPair `db:"txresult"` - ResultMeta xdr.TransactionMeta `db:"txmeta"` -} - -// TransactionFee is row of data from the `txfeehistory` table from stellar-core -type TransactionFee struct { - TransactionHash string `db:"txid"` - LedgerSequence int32 `db:"ledgerseq"` - Index int32 `db:"txindex"` - Changes xdr.LedgerEntryChanges `db:"txchanges"` -} - -// Trustline is a row of data from the `trustlines` table from stellar-core -type Trustline struct { - Accountid string - Assettype xdr.AssetType - Issuer string - Assetcode string - Tlimit xdr.Int64 - Balance xdr.Int64 - Flags int32 - LastModified uint32 - BuyingLiabilities xdr.Int64 `db:"buyingliabilities"` - SellingLiabilities xdr.Int64 `db:"sellingliabilities"` -} - -// AssetFromDB produces an xdr.Asset by combining the constituent type, code and -// issuer, as often retrieved from the DB in 3 separate columns. -func AssetFromDB(typ xdr.AssetType, code string, issuer string) (result xdr.Asset, err error) { - switch typ { - case xdr.AssetTypeAssetTypeNative: - result, err = xdr.NewAsset(xdr.AssetTypeAssetTypeNative, nil) - case xdr.AssetTypeAssetTypeCreditAlphanum4: - var ( - an xdr.AssetAlphaNum4 - decoded []byte - pkey xdr.Uint256 - ) - - copy(an.AssetCode[:], []byte(code)) - decoded, err = strkey.Decode(strkey.VersionByteAccountID, issuer) - if err != nil { - return - } - - copy(pkey[:], decoded) - an.Issuer, err = xdr.NewAccountId(xdr.PublicKeyTypePublicKeyTypeEd25519, pkey) - if err != nil { - return - } - result, err = xdr.NewAsset(xdr.AssetTypeAssetTypeCreditAlphanum4, an) - case xdr.AssetTypeAssetTypeCreditAlphanum12: - var ( - an xdr.AssetAlphaNum12 - decoded []byte - pkey xdr.Uint256 - ) - - copy(an.AssetCode[:], []byte(code)) - decoded, err = strkey.Decode(strkey.VersionByteAccountID, issuer) - if err != nil { - return - } - - copy(pkey[:], decoded) - an.Issuer, err = xdr.NewAccountId(xdr.PublicKeyTypePublicKeyTypeEd25519, pkey) - if err != nil { - return - } - result, err = xdr.NewAsset(xdr.AssetTypeAssetTypeCreditAlphanum12, an) - } - - return -} - // ElderLedger represents the oldest "ingestable" ledger known to the // stellar-core database this ingestion system is communicating with. Horizon, // which wants to operate on a contiguous range of ledger data (i.e. free from @@ -183,14 +21,12 @@ func AssetFromDB(typ xdr.AssetType, code string, issuer string) (result xdr.Asse func (q *Q) ElderLedger(dest *int32) error { err := q.GetRaw(dest, ` SELECT COALESCE(ledgerseq, 0) - FROM ( SELECT ledgerseq, LAG(ledgerseq, 1) OVER ( ORDER BY ledgerseq) as prev FROM ledgerheaders ) seqs - WHERE COALESCE(prev, -1) < ledgerseq - 1 ORDER BY ledgerseq DESC LIMIT 1; @@ -203,14 +39,3 @@ func (q *Q) ElderLedger(dest *int32) error { func (q *Q) LatestLedger(dest interface{}) error { return q.GetRaw(dest, `SELECT COALESCE(MAX(ledgerseq), 0) FROM ledgerheaders`) } - -// SchemaVersion returns Core DB schema version -func (q *Q) SchemaVersion() (int, error) { - var version string - err := q.GetRaw(&version, `SELECT state FROM storestate WHERE statename = 'databaseschema'`) - if err != nil { - return 0, errors.Wrap(err, "Error getting 'databaseschema'") - } - - return strconv.Atoi(version) -} diff --git a/services/horizon/internal/db2/core/main_test.go b/services/horizon/internal/db2/core/main_test.go index 343f3b7bc8..787492adc7 100644 --- a/services/horizon/internal/db2/core/main_test.go +++ b/services/horizon/internal/db2/core/main_test.go @@ -3,9 +3,7 @@ package core import ( "testing" - "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" - "github.com/stellar/go/xdr" ) func TestLatestLedger(t *testing.T) { @@ -63,182 +61,3 @@ func TestElderLedger(t *testing.T) { tt.Assert.Equal(elder, int32(20)) } } - -func TestSchemaVersion8(t *testing.T) { - tt := test.Start(t).ScenarioWithoutHorizon("core_database_schema_version_8") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - var account Account - err := q.AccountByAddress(&account, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") - if tt.Assert.NoError(err) { - tt.Assert.True(account.HomeDomain.Valid) - tt.Assert.Equal("stellar.org", account.HomeDomain.String) - } - - var data []AccountData - err = q.AllDataByAddress(&data, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(data)) - tt.Assert.Equal("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", data[0].Accountid) - tt.Assert.Equal("aaa", data[0].Key) - tt.Assert.Equal("bWFu", data[0].Value) - } - - var singleData AccountData - err = q.AccountDataByKey(&singleData, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", "aaa") - if tt.Assert.NoError(err) { - tt.Assert.Equal("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", singleData.Accountid) - tt.Assert.Equal("aaa", singleData.Key) - tt.Assert.Equal("bWFu", singleData.Value) - } - - var signers []Signer - err = q.SignersByAddress(&signers, "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(signers)) - tt.Assert.Equal("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", signers[0].Accountid) - tt.Assert.Equal("GAFEES4MDE5Z7Q6JBB2BYMLS7YWEHTPNR7ICANZA7TAOLMSRELE4H4S2", signers[0].Publickey) - tt.Assert.Equal(int32(2), signers[0].Weight) - } - - pq, err := db2.NewPageQuery("", true, "asc", db2.DefaultPageSize) - if !tt.Assert.NoError(err) { - return - } - - var offers []Offer - err = q.OffersByAddress(&offers, "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU", pq) - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(offers)) - tt.Assert.True(offers[0].SellingAsset.Equals(xdr.MustNewNativeAsset())) - tt.Assert.True(offers[0].BuyingAsset.Equals(xdr.MustNewCreditAsset("USD", "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU"))) - } - - offers = []Offer{} - err = q.OffersByAddress(&offers, "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD", pq) - if tt.Assert.NoError(err) { - tt.Assert.Equal(4, len(offers)) - - tt.Assert.True(offers[0].SellingAsset.Equals(xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD"))) - tt.Assert.True(offers[0].BuyingAsset.Equals(xdr.MustNewNativeAsset())) - } - - var assets []xdr.Asset - err = q.ConnectedAssets(&assets, xdr.MustNewNativeAsset()) - if tt.Assert.NoError(err) { - tt.Assert.Equal(2, len(assets)) - connectedAsset := xdr.MustNewCreditAsset("USD", "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU") - tt.Assert.True(assets[0].Equals(connectedAsset)) - } - - assets = []xdr.Asset{} - err = q.ConnectedAssets(&assets, xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD")) - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(assets)) - connectedAsset := xdr.MustNewNativeAsset() - tt.Assert.True(assets[0].Equals(connectedAsset)) - } -} - -func TestSchemaVersion9(t *testing.T) { - tt := test.Start(t).ScenarioWithoutHorizon("core_database_schema_version_9") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - var account Account - err := q.AccountByAddress(&account, "GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU") - if tt.Assert.NoError(err) { - tt.Assert.True(account.HomeDomain.Valid) - tt.Assert.Equal("lobstr.co", account.HomeDomain.String) - } - - var data []AccountData - err = q.AllDataByAddress(&data, "GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU") - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(data)) - tt.Assert.Equal("GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU", data[0].Accountid) - tt.Assert.Equal("jam", data[0].Key) - tt.Assert.Equal("bWFu", data[0].Value) - } - - var singleData AccountData - err = q.AccountDataByKey(&singleData, "GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU", "jam") - if tt.Assert.NoError(err) { - tt.Assert.Equal("GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU", singleData.Accountid) - tt.Assert.Equal("jam", singleData.Key) - tt.Assert.Equal("bWFu", singleData.Value) - } - - var signers []Signer - err = q.SignersByAddress(&signers, "GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU") - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(signers)) - tt.Assert.Equal("GDZOBPTVEECUYFCHSQ5NCEUVAV4JKRZI6KO5HFOM7HGQT22E3XIGRHNU", signers[0].Accountid) - tt.Assert.Equal("GC7BWB2ME4LII3TVWTHUIT7KGJXU4D5M6JUNLQ57WA7JERDNSAEXLOAN", signers[0].Publickey) - tt.Assert.Equal(int32(10), signers[0].Weight) - } - - var signers2 []Signer - err = q.SignersByAddress(&signers2, "GD7HOGYRECGFKFR2GGOWEF2FT3DVR3GU4K7BVRGGPWVSXAVKGSYKTXOH") - if tt.Assert.NoError(err) { - tt.Assert.Equal(0, len(signers2)) - } - - pq, err := db2.NewPageQuery("", true, "asc", db2.DefaultPageSize) - if !tt.Assert.NoError(err) { - return - } - - var offers []Offer - err = q.OffersByAddress(&offers, "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU", pq) - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(offers)) - tt.Assert.True(offers[0].SellingAsset.Equals(xdr.MustNewNativeAsset())) - tt.Assert.True(offers[0].BuyingAsset.Equals(xdr.MustNewCreditAsset("USD", "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU"))) - } - - offers = []Offer{} - err = q.OffersByAddress(&offers, "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD", pq) - if tt.Assert.NoError(err) { - tt.Assert.Equal(4, len(offers)) - - tt.Assert.Equal(4, len(offers)) - - tt.Assert.True(offers[0].SellingAsset.Equals(xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD"))) - tt.Assert.True(offers[0].BuyingAsset.Equals(xdr.MustNewNativeAsset())) - - tt.Assert.True(offers[1].SellingAsset.Equals(xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD"))) - tt.Assert.True(offers[1].BuyingAsset.Equals(xdr.MustNewNativeAsset())) - - tt.Assert.True(offers[2].SellingAsset.Equals(xdr.MustNewNativeAsset())) - tt.Assert.True(offers[2].BuyingAsset.Equals(xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD"))) - - tt.Assert.True(offers[3].SellingAsset.Equals(xdr.MustNewNativeAsset())) - tt.Assert.True(offers[3].BuyingAsset.Equals(xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD"))) - } - - var assets []xdr.Asset - err = q.ConnectedAssets(&assets, xdr.MustNewNativeAsset()) - if tt.Assert.NoError(err) { - tt.Assert.Equal(2, len(assets)) - connectedAssetA := xdr.MustNewCreditAsset("USD", "GAXMF43TGZHW3QN3REOUA2U5PW5BTARXGGYJ3JIFHW3YT6QRKRL3CPPU") - connectedAssetB := xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD") - // It looks like there are some ordering changes between Postgres versions. - // We should really stick to a single version. - if assets[0].Equals(connectedAssetA) { - tt.Assert.True(assets[1].Equals(connectedAssetB), "%s %s", assets[0], assets[1]) - } - if assets[0].Equals(connectedAssetB) { - tt.Assert.True(assets[1].Equals(connectedAssetA), "%s %s", assets[0], assets[1]) - } - } - - assets = []xdr.Asset{} - err = q.ConnectedAssets(&assets, xdr.MustNewCreditAsset("USD", "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD")) - if tt.Assert.NoError(err) { - tt.Assert.Equal(1, len(assets)) - connectedAsset := xdr.MustNewNativeAsset() - tt.Assert.True(assets[0].Equals(connectedAsset)) - } -} diff --git a/services/horizon/internal/db2/core/offer.go b/services/horizon/internal/db2/core/offer.go deleted file mode 100644 index eb9d18da10..0000000000 --- a/services/horizon/internal/db2/core/offer.go +++ /dev/null @@ -1,207 +0,0 @@ -package core - -import ( - "fmt" - "math/big" - - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/services/horizon/internal/db2" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" -) - -// PagingToken returns a suitable paging token for the Offer -func (r Offer) PagingToken() string { - return fmt.Sprintf("%d", r.OfferID) -} - -// PriceAsString return the price fraction as a floating point approximate. -func (r Offer) PriceAsString() string { - return big.NewRat(int64(r.Pricen), int64(r.Priced)).FloatString(7) -} - -// ConnectedAssets loads xdr.Asset records for the purposes of path -// finding. Given the input asset type, a list of xdr.Assets is returned that -// each have some available trades for the input asset. -func (q *Q) ConnectedAssets(dest interface{}, selling xdr.Asset) error { - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - if schemaVersion < 9 { - return q.connectedAssetsSchema8(dest, selling) - } else { - return q.connectedAssetsSchema9(dest, selling) - } -} - -func (q *Q) connectedAssetsSchema9(dest interface{}, selling xdr.Asset) error { - assets, ok := dest.(*[]xdr.Asset) - if !ok { - return errors.New("dest is not *[]xdr.Asset") - } - - sellingAssetXDRString, err := xdr.MarshalBase64(selling) - if err != nil { - return errors.Wrap(err, "Error marshaling selling") - } - - sql := sq.Select("buyingasset"). - From("offers"). - Where(sq.Eq{"sellingasset": sellingAssetXDRString}). - GroupBy("buyingasset") - - var rows []struct { - Asset xdr.Asset `db:"buyingasset"` - } - - err = q.Select(&rows, sql) - - if err != nil { - return err - } - - results := make([]xdr.Asset, len(rows)) - *assets = results - - for i, r := range rows { - results[i] = r.Asset - } - - return nil -} - -// ConnectedAssets loads xdr.Asset records for the purposes of path -// finding. Given the input asset type, a list of xdr.Assets is returned that -// each have some available trades for the input asset. -func (q *Q) connectedAssetsSchema8(dest interface{}, selling xdr.Asset) error { - assets, ok := dest.(*[]xdr.Asset) - if !ok { - return errors.New("dest is not *[]xdr.Asset") - } - - var ( - t xdr.AssetType - c string - i string - ) - - err := selling.Extract(&t, &c, &i) - if err != nil { - return err - } - - sql := sq.Select( - "buyingassettype AS type", - "coalesce(buyingassetcode, '') AS code", - "coalesce(buyingissuer, '') AS issuer"). - From("offers"). - Where(sq.Eq{"sellingassettype": t}). - GroupBy("buyingassettype", "buyingassetcode", "buyingissuer") - - if t != xdr.AssetTypeAssetTypeNative { - sql = sql.Where(sq.Eq{"sellingassetcode": c, "sellingissuer": i}) - } - - var rows []struct { - Type xdr.AssetType - Code string - Issuer string - } - - err = q.Select(&rows, sql) - - if err != nil { - return err - } - - results := make([]xdr.Asset, len(rows)) - *assets = results - - for i, r := range rows { - results[i], err = AssetFromDB(r.Type, r.Code, r.Issuer) - if err != nil { - return err - } - } - - return nil -} - -// OffersByAddress loads a page of active offers for the given -// address. -func (q *Q) OffersByAddress(dest interface{}, addy string, pq db2.PageQuery) error { - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - offers := []internalOffer{} - - sql := sq.Select("co.*"). - From("offers co"). - Where("co.sellerid = ?", addy). - Limit(uint64(pq.Limit)) - - cursor, err := pq.CursorInt64() - if err != nil { - return err - } - - switch pq.Order { - case "asc": - sql = sql.Where("co.offerid > ?", cursor).OrderBy("co.offerid asc") - case "desc": - sql = sql.Where("co.offerid < ?", cursor).OrderBy("co.offerid desc") - } - - err = q.Select(&offers, sql) - if err != nil { - return err - } - - newOffers := make([]Offer, len(offers)) - - for i, offer := range offers { - newOffers[i] = offer.get() - } - - if schemaVersion >= 9 { - *dest.(*[]Offer) = newOffers - return nil - } - - // Convert schema 8 results to xdr.Assets - for i, offer := range offers { - var sellingAsset, buyingAsset xdr.Asset - - if offer.SellingAssetType == xdr.AssetTypeAssetTypeNative { - sellingAsset.SetNative() - } else { - var account xdr.AccountId - err := account.SetAddress(offer.SellingIssuer.String) - if err != nil { - return errors.Wrap(err, "Error setting offer.SellingIssuer") - } - sellingAsset.SetCredit(offer.SellingAssetCode.String, account) - } - - if offer.BuyingAssetType == xdr.AssetTypeAssetTypeNative { - buyingAsset.SetNative() - } else { - var account xdr.AccountId - err := account.SetAddress(offer.BuyingIssuer.String) - if err != nil { - return errors.Wrap(err, "Error setting offer.BuyingIssuer") - } - buyingAsset.SetCredit(offer.BuyingAssetCode.String, account) - } - - newOffers[i].SellingAsset = sellingAsset - newOffers[i].BuyingAsset = buyingAsset - } - - *dest.(*[]Offer) = newOffers - return nil -} diff --git a/services/horizon/internal/db2/core/offer_test.go b/services/horizon/internal/db2/core/offer_test.go deleted file mode 100644 index 3461272ddd..0000000000 --- a/services/horizon/internal/db2/core/offer_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package core - -import ( - "testing" - - "github.com/stellar/go/services/horizon/internal/db2" - "github.com/stellar/go/services/horizon/internal/test" -) - -func TestOffersByAddress(t *testing.T) { - tt := test.Start(t).Scenario("trades") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - var offers []Offer - - load := func(addy, cursor, order string, limit uint64) bool { - offers = []Offer{} - pq, err := db2.NewPageQuery(cursor, true, order, limit) - if !tt.Assert.NoError(err) { - return false - } - - err = q.OffersByAddress(&offers, addy, pq) - if !tt.Assert.NoError(err) { - return false - } - return true - } - - // Works for native offers - if load("GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU", "", "asc", db2.DefaultPageSize) { - tt.Assert.Len(offers, 1) - tt.Assert.Equal(int64(4), offers[0].OfferID) - } - - // Filters properly - if load("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", "", "asc", db2.DefaultPageSize) { - tt.Assert.Len(offers, 0) - } - - if load("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", "", "asc", db2.DefaultPageSize) { - tt.Assert.Len(offers, 3) - } - - // limits properly - if load("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", "", "asc", 2) { - tt.Assert.Len(offers, 2) - } - - // ordering works - if load("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", "", "desc", db2.DefaultPageSize) { - for i := range offers { - // if there is no next element, break - if i+1 == len(offers) { - break - } - tt.Assert.True(offers[i].OfferID > offers[i+1].OfferID, "Results are not in order") - } - } - - // cursor works - if load("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", "1", "asc", db2.DefaultPageSize) { - tt.Assert.Len(offers, 2) - tt.Assert.Equal(int64(2), offers[0].OfferID) - } - if load("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", "3", "desc", db2.DefaultPageSize) { - tt.Assert.Len(offers, 2) - tt.Assert.Equal(int64(2), offers[0].OfferID) - } -} diff --git a/services/horizon/internal/db2/core/signer.go b/services/horizon/internal/db2/core/signer.go deleted file mode 100644 index aaba416f1a..0000000000 --- a/services/horizon/internal/db2/core/signer.go +++ /dev/null @@ -1,58 +0,0 @@ -package core - -import ( - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" -) - -// SignersByAddress loads all signer rows for `addy` -func (q *Q) SignersByAddress(dest interface{}, addy string) error { - schemaVersion, err := q.SchemaVersion() - if err != nil { - return err - } - - if schemaVersion < 9 { - sql := selectSigner.Where("accountid = ?", addy) - return q.Select(dest, sql) - } - - var signersXDRString *string - sql := selectSignerVersion9.Where("accountid = ?", addy) - err = q.Get(&signersXDRString, sql) - if err != nil { - return err - } - - if signersXDRString == nil { - *dest.(*[]Signer) = []Signer{} - return nil - } - - var signersXDR []xdr.Signer - err = xdr.SafeUnmarshalBase64(*signersXDRString, &signersXDR) - if err != nil { - return errors.Wrap(err, "Error decoding []xdr.Signer") - } - - signers := make([]Signer, 0, len(signersXDR)) - for _, signer := range signersXDR { - signers = append(signers, Signer{ - Accountid: addy, - Publickey: signer.Key.Address(), - Weight: int32(signer.Weight), - }) - } - - *dest.(*[]Signer) = signers - return nil -} - -var selectSigner = sq.Select( - "si.accountid", - "si.publickey", - "si.weight", -).From("signers si") - -var selectSignerVersion9 = sq.Select("a.signers").From("accounts a") diff --git a/services/horizon/internal/db2/core/transaction.go b/services/horizon/internal/db2/core/transaction.go deleted file mode 100644 index 9a7de776ae..0000000000 --- a/services/horizon/internal/db2/core/transaction.go +++ /dev/null @@ -1,156 +0,0 @@ -package core - -import ( - "encoding/base64" - "fmt" - - "strings" - - sq "github.com/Masterminds/squirrel" - "github.com/guregu/null" - "github.com/stellar/go/services/horizon/internal/utf8" - "github.com/stellar/go/xdr" -) - -// Base64Signatures returns a slice of strings where each element is a base64 -// encoded representation of a signature attached to this transaction. -func (tx *Transaction) Base64Signatures() []string { - raw := tx.Envelope.Signatures() - results := make([]string, len(raw)) - - for i := range raw { - results[i] = base64.StdEncoding.EncodeToString(raw[i].Signature) - } - return results -} - -// EnvelopeXDR returns the XDR encoded envelope for this transaction -func (tx *Transaction) EnvelopeXDR() string { - out, err := xdr.MarshalBase64(tx.Envelope) - if err != nil { - panic(err) - } - return out -} - -// Fee returns the max fee that was set for `tx` -func (tx *Transaction) MaxFee() int32 { - return int32(tx.Envelope.Fee()) -} - -// FeeCharged returns the fee that was actually charged for `tx` -func (tx *Transaction) FeeCharged() int32 { - return int32(tx.Result.Result.FeeCharged) -} - -// IsSuccessful returns true when the transaction was successful. -func (tx *Transaction) IsSuccessful() bool { - return tx.Result.Successful() -} - -// Memo returns the memo for this transaction, if there is one. -func (tx *Transaction) Memo() null.String { - var ( - value string - valid bool - ) - memo := tx.Envelope.Memo() - switch memo.Type { - case xdr.MemoTypeMemoNone: - value, valid = "", false - case xdr.MemoTypeMemoText: - scrubbed := utf8.Scrub(memo.MustText()) - notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") - value, valid = notnull, true - case xdr.MemoTypeMemoId: - value, valid = fmt.Sprintf("%d", memo.MustId()), true - case xdr.MemoTypeMemoHash: - hash := memo.MustHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - case xdr.MemoTypeMemoReturn: - hash := memo.MustRetHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - default: - panic(fmt.Errorf("invalid memo type: %v", memo.Type)) - } - - return null.NewString(value, valid) -} - -// MemoType returns the memo type for this transaction -func (tx *Transaction) MemoType() string { - switch tx.Envelope.Memo().Type { - case xdr.MemoTypeMemoNone: - return "none" - case xdr.MemoTypeMemoText: - return "text" - case xdr.MemoTypeMemoId: - return "id" - case xdr.MemoTypeMemoHash: - return "hash" - case xdr.MemoTypeMemoReturn: - return "return" - default: - panic(fmt.Errorf("invalid memo type: %v", tx.Envelope.Memo().Type)) - } -} - -// ResultXDR returns the XDR encoded result for this transaction -func (tx *Transaction) ResultXDR() string { - out, err := xdr.MarshalBase64(tx.Result.Result) - if err != nil { - panic(err) - } - return out -} - -// ResultMetaXDR returns the XDR encoded result meta for this transaction -func (tx *Transaction) ResultMetaXDR() string { - out, err := xdr.MarshalBase64(tx.ResultMeta) - if err != nil { - panic(err) - } - return out -} - -// Sequence returns the sequence number for `tx` -func (tx *Transaction) Sequence() int64 { - return int64(tx.Envelope.SeqNum()) -} - -// SourceAddress returns the strkey-encoded account id that paid the fee for -// `tx`. -func (tx *Transaction) SourceAddress() string { - sa := tx.Envelope.SourceAccount().ToAccountId() - return sa.Address() -} - -// TransactionByHashAfterLedger is a query that loads a single row from the `txhistory`. -func (q *Q) TransactionByHashAfterLedger( - dest interface{}, - hash string, - ledger int32, -) error { - sql := sq.Select("ctxh.*"). - From("txhistory ctxh"). - Limit(1). - Where("ctxh.txid = ?", hash). - Where("ctxh.ledgerseq > ?", ledger) - - return q.Get(dest, sql) -} - -// TransactionsByLedger is a query that loads all rows from `txhistory` where -// ledgerseq matches `Sequence.` -func (q *Q) TransactionsByLedger(dest interface{}, seq int32) error { - sql := sq.Select("ctxh.*"). - From("txhistory ctxh"). - OrderBy("ctxh.txindex ASC"). - Where("ctxh.ledgerseq = ?", seq) - - return q.Select(dest, sql) -} diff --git a/services/horizon/internal/db2/core/transaction_fee.go b/services/horizon/internal/db2/core/transaction_fee.go deleted file mode 100644 index d7e7467bb8..0000000000 --- a/services/horizon/internal/db2/core/transaction_fee.go +++ /dev/null @@ -1,26 +0,0 @@ -package core - -import ( - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/xdr" -) - -// ChangesXDR returns the XDR encoded changes for this transaction fee -func (fee *TransactionFee) ChangesXDR() string { - out, err := xdr.MarshalBase64(fee.Changes) - if err != nil { - panic(err) - } - return out -} - -// TransactionFeesByLedger is a query that loads all rows from `txfeehistory` -// where ledgerseq matches `Sequence.` -func (q *Q) TransactionFeesByLedger(dest interface{}, seq int32) error { - sql := sq.Select("ctxfh.*"). - From("txfeehistory ctxfh"). - OrderBy("ctxfh.txindex ASC"). - Where("ctxfh.ledgerseq = ?", seq) - - return q.Select(dest, sql) -} diff --git a/services/horizon/internal/db2/core/transaction_fee_test.go b/services/horizon/internal/db2/core/transaction_fee_test.go deleted file mode 100644 index fd32e08172..0000000000 --- a/services/horizon/internal/db2/core/transaction_fee_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package core - -import ( - "testing" - - "github.com/stellar/go/services/horizon/internal/test" -) - -func TestTransactionFeesByLedger(t *testing.T) { - tt := test.Start(t).Scenario("base") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - var fees []TransactionFee - err := q.TransactionFeesByLedger(&fees, 2) - - if tt.Assert.NoError(err) { - tt.Assert.Len(fees, 3) - } -} diff --git a/services/horizon/internal/db2/core/transaction_test.go b/services/horizon/internal/db2/core/transaction_test.go deleted file mode 100644 index cb7dfd1052..0000000000 --- a/services/horizon/internal/db2/core/transaction_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package core - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/stellar/go/services/horizon/internal/test" - "github.com/stellar/go/xdr" -) - -func TestTransactionsQueries(t *testing.T) { - tt := test.Start(t).Scenario("base") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - // Test TransactionsByLedger - var txs []Transaction - err := q.TransactionsByLedger(&txs, 2) - - if tt.Assert.NoError(err) { - tt.Assert.Len(txs, 3) - } - - // Test TransactionByHashAfterLedger - var tx Transaction - err = q.TransactionByHashAfterLedger(&tx, "cebb875a00ff6e1383aef0fd251a76f22c1f9ab2a2dffcb077855736ade2659a", 2) - - if tt.Assert.NoError(err) { - tt.Assert.Equal(int32(3), tx.LedgerSequence) - } - - err = q.TransactionByHashAfterLedger(&tx, "cebb875a00ff6e1383aef0fd251a76f22c1f9ab2a2dffcb077855736ade2659a", 3) - - if tt.Assert.Error(err) { - tt.Assert.True(q.NoRows(err)) - } -} - -func TestMemo(t *testing.T) { - tt := test.Start(t).Scenario("base") - defer tt.Finish() - - var tx Transaction - - xdr.SafeUnmarshalBase64("AAAAAMvoFDdcyQrJAcBmRdyEnW6047pvlk4MS/4r0n/1WH8VAAAAZAACnMAAAAACAAAAAAAAAAEAAAARADEuMC4xb3dlcnJpZGUgbWUAAAAAAAABAAAAAQAAAACJzogbLxrrmN7N5JVQceSxl8jkED26RGzbyyRIpwTh6wAAAAoAAAAWaSBzaG91bGQgYmUgb3dlcnJpZGRlbgAAAAAAAQAAABVpIHNob3VsZCBiZSBvd2VycmlkZW4AAAAAAAAAAAAAAacE4esAAABA0GuCIEmKyQ2DRqt5+BOIqjVlHisjY6rK1IcOtzjIKCDgSAoiv5yhYe09PohBH91TXvAQ/LZJj5hVMihfMjtgCw==", &tx.Envelope) - - tt.Assert.Equal("1.0.1owerride me", tx.Memo().String) -} - -func TestSignatures(t *testing.T) { - tt := test.Start(t).Scenario("base") - defer tt.Finish() - - var tx Transaction - - // https://github.com/stellar/stellar-core/issues/1225 - xdr.SafeUnmarshalBase64("AAAAAMIK9djC7k75ziKOLJcvMAIBG7tnBuoeI34x+Pi6zqcZAAAAZAAZphYAAAABAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAynnCTTyw53VVRLOWX6XKTva63IM1LslPNW01YB0hz/8AAAAAAAAAAlQL5AAAAAAAAAAAAh0hz/8AAABA8qkkeKaKfsbgInyIkzXJhqJE5/Ufxri2LdxmyKkgkT6I3sPmvrs5cPWQSzEQyhV750IW2ds97xTHqTpOfuZCAnhSuFUAAAAA", &tx.Envelope) - - signatures := tx.Base64Signatures() - - tt.Assert.Equal(2, len(signatures)) - tt.Assert.Equal("8qkkeKaKfsbgInyIkzXJhqJE5/Ufxri2LdxmyKkgkT6I3sPmvrs5cPWQSzEQyhV750IW2ds97xTHqTpOfuZCAg==", signatures[0]) - tt.Assert.Equal("", signatures[1]) -} - -func TestTransaction_SourceAddress_MuxedAccount(t *testing.T) { - aid := xdr.MustAddress("GA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVSGZ") - - muxed := xdr.MuxedAccount{ - Type: xdr.CryptoKeyTypeKeyTypeMuxedEd25519, - Med25519: &xdr.MuxedAccountMed25519{ - Id: 0xcafebabe, - Ed25519: *aid.Ed25519, - }, - } - var tx Transaction - tx.Envelope = xdr.TransactionEnvelope{ - Type: xdr.EnvelopeTypeEnvelopeTypeTx, - V1: &xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - SourceAccount: muxed, - Operations: []xdr.Operation{ - { - SourceAccount: &muxed, - Body: xdr.OperationBody{ - Type: xdr.OperationTypePayment, - PaymentOp: &xdr.PaymentOp{ - Destination: muxed, - Asset: xdr.Asset{Type: xdr.AssetTypeAssetTypeNative}, - Amount: 100, - }, - }, - }, - }, - }, - }, - } - - assert.Equal(t, "GA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVSGZ", tx.SourceAddress()) -} diff --git a/services/horizon/internal/db2/core/trustline.go b/services/horizon/internal/db2/core/trustline.go deleted file mode 100644 index d0cac3012e..0000000000 --- a/services/horizon/internal/db2/core/trustline.go +++ /dev/null @@ -1,127 +0,0 @@ -package core - -import ( - "errors" - - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/xdr" -) - -func (tl Trustline) IsAuthorized() bool { - return (tl.Flags & int32(xdr.TrustLineFlagsAuthorizedFlag)) != 0 -} - -func (tl Trustline) IsAuthorizedToMaintainLiabilities() bool { - return (tl.Flags & int32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag)) != 0 -} - -// AssetsForAddress returns a list of assets and balances for those assets held by -// a given address. -func (q *Q) AssetsForAddress(addy string) ([]xdr.Asset, []xdr.Int64, error) { - var tls []Trustline - var account Account - - if err := q.AccountByAddress(&account, addy); q.NoRows(err) { - // if there is no account for the given address then - // we return an empty list of assets and balances - return []xdr.Asset{}, []xdr.Int64{}, nil - } else if err != nil { - return nil, nil, err - } - - if err := q.TrustlinesByAddress(&tls, addy); err != nil { - return nil, nil, err - } - - assets := make([]xdr.Asset, len(tls)+1) - balances := make([]xdr.Int64, len(tls)+1) - - var err error - for i, tl := range tls { - assets[i], err = AssetFromDB(tl.Assettype, tl.Assetcode, tl.Issuer) - if err != nil { - return nil, nil, err - } - balances[i] = tl.Balance - } - - assets[len(assets)-1], err = xdr.NewAsset(xdr.AssetTypeAssetTypeNative, nil) - balances[len(assets)-1] = account.Balance - - return assets, balances, err -} - -// AllAssets loads all (unique) assets from core DB -func (q *Q) AllAssets(dest interface{}) error { - var tls []Trustline - - sql := sq.Select( - "tl.assettype", - "tl.issuer", - "tl.assetcode", - ).From("trustlines tl").GroupBy("(tl.assettype, tl.issuer, tl.assetcode)") - err := q.Select(&tls, sql) - if err != nil { - return err - } - - dtl, ok := dest.(*[]xdr.Asset) - if !ok { - return errors.New("Invalid destination") - } - - result := make([]xdr.Asset, len(tls)) - *dtl = result - - for i, tl := range tls { - result[i], err = AssetFromDB(tl.Assettype, tl.Assetcode, tl.Issuer) - if err != nil { - return err - } - } - - return nil -} - -// TrustlinesByAddress loads all trustlines for `addy` -func (q *Q) TrustlinesByAddress(dest interface{}, addy string) error { - sql := selectTrustline.Where("accountid = ?", addy) - return q.Select(dest, sql) -} - -// BalancesForAsset returns all the balances by asset type, code, issuer -func (q *Q) BalancesForAsset( - assetType int32, - assetCode string, - assetIssuer string, -) (int32, string, error) { - sql := selectBalances.Where(sq.Eq{ - "assettype": assetType, - "assetcode": assetCode, - "issuer": assetIssuer, - "flags": 1, - }) - result := struct { - Count int32 `db:"count"` - Sum string `db:"sum"` - }{} - err := q.Get(&result, sql) - return result.Count, result.Sum, err -} - -var selectTrustline = sq.Select( - "tl.accountid", - "tl.assettype", - "tl.issuer", - "tl.assetcode", - "tl.tlimit", - "tl.balance", - "tl.flags", - "tl.lastmodified", - // Liabilities can be NULL so can error without `coalesce`: - // `Invalid value for xdr.Int64` - "coalesce(tl.buyingliabilities, 0) as buyingliabilities", - "coalesce(tl.sellingliabilities, 0) as sellingliabilities", -).From("trustlines tl") - -var selectBalances = sq.Select("COUNT(*)", "COALESCE(SUM(balance), 0) as sum").From("trustlines") diff --git a/services/horizon/internal/db2/core/trustline_test.go b/services/horizon/internal/db2/core/trustline_test.go deleted file mode 100644 index c88e8bbeda..0000000000 --- a/services/horizon/internal/db2/core/trustline_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package core - -import ( - "testing" - - "github.com/stellar/go/services/horizon/internal/test" - "github.com/stellar/go/xdr" -) - -func assetsToSymbols(assets []xdr.Asset) []string { - symbols := make([]string, len(assets)) - for i, asset := range assets { - symbols[i] = asset.String() - } - return symbols -} - -func TestAssetsForAddress(t *testing.T) { - tt := test.Start(t).Scenario("order_books") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - assets, balances, err := q.AssetsForAddress( - "GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON", - ) - tt.Assert.NoError(err) - assetSymbols := assetsToSymbols(assets) - - assetsToBalance := map[string]xdr.Int64{} - for i, symbol := range assetSymbols { - assetsToBalance[symbol] = balances[i] - } - - expected := map[string]xdr.Int64{ - "credit_alphanum4/BTC/GC23QF2HUE52AMXUFUH3AYJAXXGXXV2VHXYYR6EYXETPKDXZSAW67XO4": 60000000000, - "credit_alphanum4/USD/GC23QF2HUE52AMXUFUH3AYJAXXGXXV2VHXYYR6EYXETPKDXZSAW67XO4": 50000000000, - "native": 99999999200, - } - - tt.Assert.Equal(expected, assetsToBalance) -} - -func TestAssetsForAddressWithoutAccount(t *testing.T) { - tt := test.Start(t).Scenario("order_books") - defer tt.Finish() - q := &Q{tt.CoreSession()} - - var account Account - err := q.AccountByAddress(&account, "GD5PM5X7Q5MM54ERO2P5PXW3HD6HVZI5IRZGEDWS4OPFBGHNTF6XOWQO") - tt.Assert.True(q.NoRows(err)) - - assets, balances, err := q.AssetsForAddress( - "GD5PM5X7Q5MM54ERO2P5PXW3HD6HVZI5IRZGEDWS4OPFBGHNTF6XOWQO", - ) - tt.Assert.NoError(err) - tt.Assert.Empty(assets) - tt.Assert.Empty(balances) -} diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index e335742fb2..e8dc49fd55 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -224,7 +224,6 @@ func initSubmissionSystem(app *App) { Submitter: txsub.NewDefaultSubmitter(http.DefaultClient, app.config.StellarCoreURL), SubmissionQueue: sequence.NewManager(), Results: &results.DB{ - Core: &core.Q{Session: app.CoreSession(context.Background())}, History: &history.Q{Session: app.HorizonSession(context.Background())}, }, Sequences: &history.Q{Session: app.HorizonSession(context.Background())}, From 55a3da9a39d2b7afa01c515d60a1d98fa7247556 Mon Sep 17 00:00:00 2001 From: tamirms Date: Wed, 17 Jun 2020 18:56:42 +0200 Subject: [PATCH 4/4] Update services/horizon/CHANGELOG.md Co-authored-by: Bartek Nowotarski --- services/horizon/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index ef4561fd07..68712e113f 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -7,7 +7,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).x * Add transaction set operation count to `history_ledger`([#2690](https://github.com/stellar/go/pull/2690)). Extend ingestion to store the total number of operations in the transaction set and expose it in the ledger resource via `tx_set_operation_count`. This feature allow you to assess the used capacity of a transaction set. -* Remove `--ingest-failed-transactions` flag. From now on Horizon will always ingest failed transactions ([#2702](https://github.com/stellar/go/pull/2702)). +* Remove `--ingest-failed-transactions` flag. From now on Horizon will always ingest failed transactions. WARNING: if your application is using Horizon DB directly (not recommended!) remember that now it will also contain failed txs. ([#2702](https://github.com/stellar/go/pull/2702)). ## v1.4.0