historyarchive: Add User-Agent header to history archive HTTP requests

exp/services/captivecore/main.go

@@ -139,6 +139,7 @@ func main() {
 				CheckpointFrequency: checkpointFrequency,
 				Log:                 logger.WithField("subservice", "stellar-core"),
 				Toml:                captiveCoreToml,
+				UserAgent:           "captivecore",
 			}
 
 			var dbConn *db.Session

exp/tools/dump-ledger-state/main.go

@@ -307,13 +307,17 @@ func archive(testnet bool) (*historyarchive.Archive, error) {
 	if testnet {
 		return historyarchive.Connect(
 			"https://history.stellar.org/prd/core-testnet/core_testnet_001",
-			historyarchive.ConnectOptions{},
+			historyarchive.ConnectOptions{
+				UserAgent: "dump-ledger-state",
+			},
 		)
 	}
 
 	return historyarchive.Connect(
 		fmt.Sprintf("https://history.stellar.org/prd/core-live/core_live_001/"),
-		historyarchive.ConnectOptions{},
+		historyarchive.ConnectOptions{
+			UserAgent: "dump-ledger-state",
+		},
 	)
 }
 

exp/tools/dump-orderbook/main.go

@@ -109,12 +109,16 @@ func archive(testnet bool) (*historyarchive.Archive, error) {
 	if testnet {
 		return historyarchive.Connect(
 			"https://history.stellar.org/prd/core-testnet/core_testnet_001",
-			historyarchive.ConnectOptions{},
+			historyarchive.ConnectOptions{
+				UserAgent: "dump-orderbook",
+			},
 		)
 	}
 
 	return historyarchive.Connect(
 		fmt.Sprintf("https://history.stellar.org/prd/core-live/core_live_001/"),
-		historyarchive.ConnectOptions{},
+		historyarchive.ConnectOptions{
+			UserAgent: "dump-orderbook",
+		},
 	)
 }

historyarchive/archive.go

@@ -49,6 +49,8 @@ type ConnectOptions struct {
 	// CheckpointFrequency is the number of ledgers between checkpoints
 	// if unset, DefaultCheckpointFrequency will be used
 	CheckpointFrequency uint32
+	// UserAgent is the value of `User-Agent` header. Applicable only for HTTP client.
+	UserAgent string
 }
 
 type Ledger struct {

historyarchive/http_archive.go

@@ -16,9 +16,10 @@ import (
 )
 
 type HttpArchiveBackend struct {
-	ctx    context.Context
-	client http.Client
-	base   url.URL
+	ctx       context.Context
+	client    http.Client
+	base      url.URL
+	userAgent string
 }
 
 func checkResp(r *http.Response) error {
@@ -33,14 +34,7 @@ func checkResp(r *http.Response) error {
 func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error) {
 	derived := b.base
 	derived.Path = path.Join(derived.Path, pth)
-	req, err := http.NewRequest("GET", derived.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(b.ctx)
-	logReq(req)
-	resp, err := b.client.Do(req)
-	logResp(resp)
+	resp, err := b.makeSendRequest("GET", derived.String())
 	if err != nil {
 		if resp != nil && resp.Body != nil {
 			resp.Body.Close()
@@ -60,14 +54,7 @@ func (b *HttpArchiveBackend) GetFile(pth string) (io.ReadCloser, error) {
 func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error) {
 	derived := b.base
 	derived.Path = path.Join(derived.Path, pth)
-	req, err := http.NewRequest("HEAD", derived.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(b.ctx)
-	logReq(req)
-	resp, err := b.client.Do(req)
-	logResp(resp)
+	resp, err := b.makeSendRequest("HEAD", derived.String())
 	if err != nil {
 		return nil, err
 	}
@@ -79,6 +66,21 @@ func (b *HttpArchiveBackend) Head(pth string) (*http.Response, error) {
 	return resp, nil
 }
 
+func (b *HttpArchiveBackend) makeSendRequest(method, url string) (*http.Response, error) {
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(b.ctx)
+	logReq(req)
+	if b.userAgent != "" {
+		req.Header.Set("User-Agent", b.userAgent)
+	}
+	resp, err := b.client.Do(req)
+	logResp(resp)
+	return resp, err
+}
+
 func (b *HttpArchiveBackend) Exists(pth string) (bool, error) {
 	resp, err := b.Head(pth)
 	if err != nil {
@@ -127,7 +129,8 @@ func (b *HttpArchiveBackend) CanListFiles() bool {
 
 func makeHttpBackend(base *url.URL, opts ConnectOptions) ArchiveBackend {
 	return &HttpArchiveBackend{
-		ctx:  opts.Context,
-		base: *base,
+		ctx:       opts.Context,
+		userAgent: opts.UserAgent,
+		base:      *base,
 	}
 }

ingest/ledgerbackend/captive_core_backend.go

@@ -103,7 +103,9 @@ type CaptiveCoreConfig struct {
 	NetworkPassphrase string
 	// HistoryArchiveURLs are a list of history archive urls
 	HistoryArchiveURLs []string
-	Toml               *CaptiveCoreToml
+	// UserAgent is the value of `User-Agent` header that will be send along http archive requests.
+	UserAgent string
+	Toml      *CaptiveCoreToml
 
 	// Optional fields
 

services/horizon/internal/ingest/main.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"runtime"
 	"sync"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/ingest/filters"
+	apkg "github.com/stellar/go/support/app"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	logpkg "github.com/stellar/go/support/log"
@@ -210,6 +212,7 @@ func NewSystem(config Config) (System, error) {
 			Context:             ctx,
 			NetworkPassphrase:   config.NetworkPassphrase,
 			CheckpointFrequency: config.CheckpointFrequency,
+			UserAgent:           fmt.Sprintf("horizon/%s golang/%s", apkg.Version(), runtime.Version()),
 		},
 	)
 	if err != nil {
@@ -239,6 +242,7 @@ func NewSystem(config Config) (System, error) {
 					LedgerHashStore:     ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession),
 					Log:                 logger,
 					Context:             ctx,
+					UserAgent:           fmt.Sprintf("captivecore horizon/%s golang/%s", apkg.Version(), runtime.Version()),
 				},
 			)
 			if err != nil {

tools/archive-reader/archive_reader.go

@@ -67,6 +67,7 @@ func archive() (*historyarchive.Archive, error) {
 		historyarchive.ConnectOptions{
 			S3Region:         "eu-west-1",
 			UnsignedRequests: true,
+			UserAgent:        "archive-reader",
 		},
 	)
 }