From d730264ac550eb98c3d7d648429158bc031c1ab3 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Mon, 18 Jul 2022 16:25:00 +0200 Subject: [PATCH] historyarchive: Add User-Agent header to history archive HTTP requests --- exp/services/captivecore/main.go | 1 + exp/tools/dump-ledger-state/main.go | 8 +++- exp/tools/dump-orderbook/main.go | 8 +++- historyarchive/archive.go | 2 + historyarchive/http_archive.go | 45 +++++++++++--------- ingest/ledgerbackend/captive_core_backend.go | 4 +- services/horizon/internal/ingest/main.go | 4 ++ tools/archive-reader/archive_reader.go | 1 + 8 files changed, 47 insertions(+), 26 deletions(-) diff --git a/exp/services/captivecore/main.go b/exp/services/captivecore/main.go index 7fde277d69..69d7da0b5f 100644 --- a/exp/services/captivecore/main.go +++ b/exp/services/captivecore/main.go @@ -139,6 +139,7 @@ func main() { CheckpointFrequency: checkpointFrequency, Log: logger.WithField("subservice", "stellar-core"), Toml: captiveCoreToml, + UserAgent: "captivecore", } var dbConn *db.Session diff --git a/exp/tools/dump-ledger-state/main.go b/exp/tools/dump-ledger-state/main.go index ce3ef58f5e..1523cbbacb 100644 --- a/exp/tools/dump-ledger-state/main.go +++ b/exp/tools/dump-ledger-state/main.go @@ -307,13 +307,17 @@ func archive(testnet bool) (*historyarchive.Archive, error) { if testnet { return historyarchive.Connect( "https://history.stellar.org/prd/core-testnet/core_testnet_001", - historyarchive.ConnectOptions{}, + historyarchive.ConnectOptions{ + UserAgent: "dump-ledger-state", + }, ) } return historyarchive.Connect( fmt.Sprintf("https://history.stellar.org/prd/core-live/core_live_001/"), - historyarchive.ConnectOptions{}, + historyarchive.ConnectOptions{ + UserAgent: "dump-ledger-state", + }, ) } diff --git a/exp/tools/dump-orderbook/main.go b/exp/tools/dump-orderbook/main.go index eee12037b5..ce54252c46 100644 --- a/exp/tools/dump-orderbook/main.go +++ b/exp/tools/dump-orderbook/main.go @@ -109,12 +109,16 @@ func archive(testnet bool) (*historyarchive.Archive, error) { if testnet { return historyarchive.Connect( "https://history.stellar.org/prd/core-testnet/core_testnet_001", - historyarchive.ConnectOptions{}, + historyarchive.ConnectOptions{ + UserAgent: "dump-orderbook", + }, ) } return historyarchive.Connect( fmt.Sprintf("https://history.stellar.org/prd/core-live/core_live_001/"), - historyarchive.ConnectOptions{}, + historyarchive.ConnectOptions{ + UserAgent: "dump-orderbook", + }, ) } diff --git a/historyarchive/archive.go b/historyarchive/archive.go index 099f16c540..2d470a8026 100644 --- a/historyarchive/archive.go +++ b/historyarchive/archive.go @@ -49,6 +49,8 @@ type ConnectOptions struct { // CheckpointFrequency is the number of ledgers between checkpoints // if unset, DefaultCheckpointFrequency will be used CheckpointFrequency uint32 + // UserAgent is the value of `User-Agent` header. Applicable only for HTTP client. + UserAgent string } type Ledger struct { diff --git a/historyarchive/http_archive.go b/historyarchive/http_archive.go index 2e00675f0f..aa941853a9 100644 --- a/historyarchive/http_archive.go +++ b/historyarchive/http_archive.go @@ -16,9 +16,10 @@ import ( ) type HttpArchiveBackend struct { - ctx context.Context - client http.Client - base url.URL + ctx context.Context + client http.Client + base url.URL + userAgent string } func checkResp(r *http.Response) error { @@ -33,14 +34,7 @@ func checkResp(r *http.Response) error { func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error) { derived := b.base derived.Path = path.Join(derived.Path, pth) - req, err := http.NewRequest("GET", derived.String(), nil) - if err != nil { - return nil, err - } - req = req.WithContext(b.ctx) - logReq(req) - resp, err := b.client.Do(req) - logResp(resp) + resp, err := b.makeSendRequest("GET", derived.String()) if err != nil { if resp != nil && resp.Body != nil { resp.Body.Close() @@ -60,14 +54,7 @@ func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error) { func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error) { derived := b.base derived.Path = path.Join(derived.Path, pth) - req, err := http.NewRequest("HEAD", derived.String(), nil) - if err != nil { - return nil, err - } - req = req.WithContext(b.ctx) - logReq(req) - resp, err := b.client.Do(req) - logResp(resp) + resp, err := b.makeSendRequest("HEAD", derived.String()) if err != nil { return nil, err } @@ -79,6 +66,21 @@ func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error) { return resp, nil } +func (b *HttpArchiveBackend) makeSendRequest(method, url string) (*http.Response, error) { + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + req = req.WithContext(b.ctx) + logReq(req) + if b.userAgent != "" { + req.Header.Set("User-Agent", b.userAgent) + } + resp, err := b.client.Do(req) + logResp(resp) + return resp, err +} + func (b *HttpArchiveBackend) Exists(pth string) (bool, error) { resp, err := b.Head(pth) if err != nil { @@ -127,7 +129,8 @@ func (b *HttpArchiveBackend) CanListFiles() bool { func makeHttpBackend(base *url.URL, opts ConnectOptions) ArchiveBackend { return &HttpArchiveBackend{ - ctx: opts.Context, - base: *base, + ctx: opts.Context, + userAgent: opts.UserAgent, + base: *base, } } diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index 1154096a8f..7543abb660 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -103,7 +103,9 @@ type CaptiveCoreConfig struct { NetworkPassphrase string // HistoryArchiveURLs are a list of history archive urls HistoryArchiveURLs []string - Toml *CaptiveCoreToml + // UserAgent is the value of `User-Agent` header that will be send along http archive requests. + UserAgent string + Toml *CaptiveCoreToml // Optional fields diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index f49af5e3b1..4e20967e76 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net/http" + "runtime" "sync" "time" @@ -18,6 +19,7 @@ import ( "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest/filters" + apkg "github.com/stellar/go/support/app" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" logpkg "github.com/stellar/go/support/log" @@ -210,6 +212,7 @@ func NewSystem(config Config) (System, error) { Context: ctx, NetworkPassphrase: config.NetworkPassphrase, CheckpointFrequency: config.CheckpointFrequency, + UserAgent: fmt.Sprintf("horizon/%s golang/%s", apkg.Version(), runtime.Version()), }, ) if err != nil { @@ -239,6 +242,7 @@ func NewSystem(config Config) (System, error) { LedgerHashStore: ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession), Log: logger, Context: ctx, + UserAgent: fmt.Sprintf("captivecore horizon/%s golang/%s", apkg.Version(), runtime.Version()), }, ) if err != nil { diff --git a/tools/archive-reader/archive_reader.go b/tools/archive-reader/archive_reader.go index 0639268696..c5b03694d1 100644 --- a/tools/archive-reader/archive_reader.go +++ b/tools/archive-reader/archive_reader.go @@ -67,6 +67,7 @@ func archive() (*historyarchive.Archive, error) { historyarchive.ConnectOptions{ S3Region: "eu-west-1", UnsignedRequests: true, + UserAgent: "archive-reader", }, ) }