Merge branch 'protocol-22' into master

.github/workflows/horizon.yml

@@ -13,7 +13,7 @@ jobs:
         os: [ubuntu-20.04, ubuntu-22.04]
         go: ["1.21", "1.22"]
         pg: [12, 16]
-        protocol-version: [21]
+        protocol-version: [21, 22]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -36,6 +36,9 @@ jobs:
       PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal
       PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.3.1-2007.4ede19620.focal
       PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.5.1
+      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.0.0-2095.rc2.1bccbc921.focal
+      PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.0.0-2095.rc2.1bccbc921.focal
+      PROTOCOL_22_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:22.0.0-rc2-99
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres

.gitignore

@@ -5,7 +5,7 @@
 /services/horizon/captive-core
 /services/horizon/horizon
 /services/horizon/stellar-horizon
-/bucket-cache
+**bucket-cache/
 .vscode
 .idea
 debug

Makefile

@@ -18,7 +18,7 @@ XDRS = $(DOWNLOADABLE_XDRS) xdr/Stellar-exporter.x
 
 
 XDRGEN_COMMIT=e2cac557162d99b12ae73b846cf3d5bfe16636de
-XDR_COMMIT=70180d5e8d9caee9e8645ed8a38c36a8cf403cd9
+XDR_COMMIT=529d5176f24c73eeccfa5eba481d4e89c19b1181
 
 .PHONY: xdr xdr-clean xdr-update
 

clients/horizonclient/internal.go

@@ -1,13 +1,16 @@
 package horizonclient
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/stellar/go/protocols/horizon"
 	"github.com/stellar/go/support/clock"
 	"github.com/stellar/go/support/errors"
 )
@@ -27,10 +30,11 @@ func decodeResponse(resp *http.Response, object interface{}, horizonUrl string,
 	}
 	setCurrentServerTime(u.Hostname(), resp.Header["Date"], clock)
 
-	// While this part of code assumes that any error < 200 or error >= 300 is a Horizon problem, it is not
-	// true for the response from /transactions_async endpoint which does give these codes for certain responses
-	// from core.
-	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) && (resp.Request == nil || resp.Request.URL == nil || resp.Request.URL.Path != "/transactions_async") {
+	if isStatusCodeAnError(resp.StatusCode) {
+		if isAsyncTxSubRequest(resp) {
+			return decodeAsyncTxSubResponse(resp, object)
+		}
+
 		horizonError := &Error{
 			Response: resp,
 		}
@@ -47,6 +51,42 @@ func decodeResponse(resp *http.Response, object interface{}, horizonUrl string,
 	return
 }
 
+func isStatusCodeAnError(statusCode int) bool {
+	return !(statusCode >= 200 && statusCode < 300)
+}
+
+func isAsyncTxSubRequest(resp *http.Response) bool {
+	return resp.Request != nil && resp.Request.URL != nil && resp.Request.URL.Path == "/transactions_async"
+}
+
+func decodeAsyncTxSubResponse(resp *http.Response, object interface{}) error {
+	// We need to read the entire body in order to create 2 decoders later.
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "error reading response body")
+	}
+
+	// The first decoder converts the response to AsyncTransactionSubmissionResponse and checks
+	// the hash of the transaction. If the response was not a valid AsyncTransactionSubmissionResponse object,
+	// the hash of the converted object will be empty.
+	asyncRespDecoder := json.NewDecoder(bytes.NewReader(bodyBytes))
+	err = asyncRespDecoder.Decode(&object)
+	if asyncResp, ok := object.(*horizon.AsyncTransactionSubmissionResponse); err == nil && ok && asyncResp.Hash != "" {
+		return nil
+	}
+
+	// Create a new reader for the second decoding. The second decoder decodes to Horizon.Problem object.
+	problemDecoder := json.NewDecoder(bytes.NewReader(bodyBytes))
+	horizonError := Error{
+		Response: resp,
+	}
+	err = problemDecoder.Decode(&horizonError.Problem)
+	if err != nil {
+		return errors.Wrap(err, "error decoding horizon error")
+	}
+	return horizonError
+}
+
 // countParams counts the number of parameters provided
 func countParams(params ...interface{}) int {
 	counter := 0

clients/horizonclient/main_test.go

@@ -480,8 +480,6 @@ func TestAssetsRequest(t *testing.T) {
 		assert.Equal(t, record.Asset.Code, "ABC")
 		assert.Equal(t, record.Asset.Issuer, "GCLWGQPMKXQSPF776IU33AH4PZNOOWNAWGGKVTBQMIC5IMKUNP3E6NVU")
 		assert.Equal(t, record.PT, "1")
-		assert.Equal(t, record.NumAccounts, int32(3))
-		assert.Equal(t, record.Amount, "105.0000000")
 		assert.Equal(t, record.Flags.AuthRevocable, false)
 		assert.Equal(t, record.Flags.AuthRequired, true)
 		assert.Equal(t, record.Flags.AuthImmutable, false)

clients/stellarcore/client.go

@@ -1,11 +1,11 @@
 package stellarcore
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -21,7 +21,7 @@ import (
 // Client represents a client that is capable of communicating with a
 // stellar-core server using HTTP
 type Client struct {
-	// HTTP is the client to use when communicating with stellar-core.  If nil,
+	// HTTP is the client to use when communicating with stellar-core. If nil,
 	// http.DefaultClient will be used.
 	HTTP HTTP
 
@@ -36,7 +36,7 @@ type Client struct {
 // in case an error was encountered during either the draining or closing of the
 // stream, that error would be returned.
 func drainReponse(hresp *http.Response, close bool, err *error) (outerror error) {
-	_, err2 := io.Copy(ioutil.Discard, hresp.Body)
+	_, err2 := io.Copy(io.Discard, hresp.Body)
 	if err2 != nil {
 		if err != nil && *err == nil {
 			*err = errors.Wrap(err2, "unable to read excess data from response")
@@ -75,54 +75,12 @@ func (c *Client) Upgrade(ctx context.Context, version int) (err error) {
 	}
 	defer drainReponse(hresp, true, &err) //nolint:errcheck
 
-	if !(hresp.StatusCode >= 200 && hresp.StatusCode < 300) {
-		err = errors.New("http request failed with non-200 status code")
-		return
+	if hresp.StatusCode < 200 || hresp.StatusCode >= 300 {
+		return errors.New("http request failed with non-200 status code")
 	}
 	return nil
 }
 
-// GetLedgerEntry submits a request to the stellar core instance to get the latest
-// state of a given ledger entry.
-func (c *Client) GetLedgerEntry(ctx context.Context, ledgerKey xdr.LedgerKey) (proto.GetLedgerEntryResponse, error) {
-	b64, err := xdr.MarshalBase64(ledgerKey)
-	if err != nil {
-		return proto.GetLedgerEntryResponse{}, errors.Wrap(err, "failed to marshal ledger key")
-	}
-	q := url.Values{}
-	q.Set("key", b64)
-
-	req, err := c.simpleGet(ctx, "getledgerentry", q)
-	if err != nil {
-		return proto.GetLedgerEntryResponse{}, errors.Wrap(err, "failed to create request")
-	}
-
-	hresp, err := c.http().Do(req)
-	if err != nil {
-		return proto.GetLedgerEntryResponse{}, errors.Wrap(err, "http request errored")
-	}
-	defer hresp.Body.Close()
-
-	if !(hresp.StatusCode >= 200 && hresp.StatusCode < 300) {
-		if drainReponse(hresp, false, &err) != nil {
-			return proto.GetLedgerEntryResponse{}, err
-		}
-		return proto.GetLedgerEntryResponse{}, errors.New("http request failed with non-200 status code")
-	}
-
-	responseBytes, err := io.ReadAll(hresp.Body)
-	if err != nil {
-		return proto.GetLedgerEntryResponse{}, errors.Wrap(err, "could not read response")
-	}
-
-	var response proto.GetLedgerEntryResponse
-	if err = json.Unmarshal(responseBytes, &response); err != nil {
-		return proto.GetLedgerEntryResponse{}, errors.Wrap(err, "json decode failed: "+string(responseBytes))
-	}
-
-	return response, nil
-}
-
 // Info calls the `info` command on the connected stellar core and returns the
 // provided response
 func (c *Client) Info(ctx context.Context) (resp *proto.InfoResponse, err error) {
@@ -180,7 +138,7 @@ func (c *Client) SetCursor(ctx context.Context, id string, cursor int32) (err er
 	}
 
 	var raw []byte
-	raw, err = ioutil.ReadAll(hresp.Body)
+	raw, err = io.ReadAll(hresp.Body)
 	if err != nil {
 		return err
 	}
@@ -193,9 +151,13 @@ func (c *Client) SetCursor(ctx context.Context, id string, cursor int32) (err er
 	return nil
 }
 
+func (c *Client) GetLedgerEntryRaw(ctx context.Context, ledgerSeq uint32, keys ...xdr.LedgerKey) (proto.GetLedgerEntryRawResponse, error) {
+	var resp proto.GetLedgerEntryRawResponse
+	return resp, c.makeLedgerKeyRequest(ctx, &resp, "getledgerentryraw", ledgerSeq, keys...)
+}
+
 // SubmitTransaction calls the `tx` command on the connected stellar core with the provided envelope
 func (c *Client) SubmitTransaction(ctx context.Context, envelope string) (resp *proto.TXResponse, err error) {
-
 	q := url.Values{}
 	q.Set("blob", envelope)
 
@@ -214,13 +176,12 @@ func (c *Client) SubmitTransaction(ctx context.Context, envelope string) (resp *
 	}
 	defer drainReponse(hresp, true, &err) //nolint:errcheck
 
-	if !(hresp.StatusCode >= 200 && hresp.StatusCode < 300) {
+	if hresp.StatusCode < 200 || hresp.StatusCode >= 300 {
 		err = errors.New("http request failed with non-200 status code")
 		return
 	}
 
 	err = json.NewDecoder(hresp.Body).Decode(&resp)
-
 	if err != nil {
 		err = errors.Wrap(err, "json decode failed")
 		return
@@ -232,7 +193,6 @@ func (c *Client) SubmitTransaction(ctx context.Context, envelope string) (resp *
 // WaitForNetworkSync continually polls the connected stellar-core until it
 // receives a response that indicated the node has synced with the network
 func (c *Client) WaitForNetworkSync(ctx context.Context) error {
-
 	for {
 		info, err := c.Info(ctx)
 
@@ -256,7 +216,6 @@ func (c *Client) WaitForNetworkSync(ctx context.Context) error {
 
 // ManualClose closes a ledger when Core is running in `MANUAL_CLOSE` mode
 func (c *Client) ManualClose(ctx context.Context) (err error) {
-
 	q := url.Values{}
 
 	var req *http.Request
@@ -274,7 +233,7 @@ func (c *Client) ManualClose(ctx context.Context) (err error) {
 	}
 	defer drainReponse(hresp, true, &err) //nolint:errcheck
 
-	if !(hresp.StatusCode >= 200 && hresp.StatusCode < 300) {
+	if hresp.StatusCode < 200 || hresp.StatusCode >= 300 {
 		err = errors.New("http request failed with non-200 status code")
 		return
 	}
@@ -312,7 +271,6 @@ func (c *Client) simpleGet(
 	newPath string,
 	query url.Values,
 ) (*http.Request, error) {
-
 	u, err := url.Parse(c.URL)
 	if err != nil {
 		return nil, errors.Wrap(err, "unparseable url")
@@ -332,3 +290,93 @@ func (c *Client) simpleGet(
 
 	return req, nil
 }
+
+// rawPost returns a new POST request to the connected stellar-core using the
+// provided path and the params values encoded as the request body to construct
+// the result.
+func (c *Client) rawPost(
+	ctx context.Context,
+	newPath string,
+	body string,
+) (*http.Request, error) {
+	u, err := url.Parse(c.URL)
+	if err != nil {
+		return nil, errors.Wrap(err, "unparseable url")
+	}
+
+	u.Path = path.Join(u.Path, newPath)
+	newURL := u.String()
+
+	var req *http.Request
+	req, err = http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		newURL,
+		bytes.NewBuffer([]byte(body)))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create request")
+	}
+
+	return req, nil
+}
+
+// makeLedgerKeyRequest is a generic method to perform a request in the form
+// `key=...&key=...&ledgerSeq=...` which is useful because several Stellar Core
+// endpoints all use this request format. Be sure to pass `target` by reference.
+func (c *Client) makeLedgerKeyRequest(
+	ctx context.Context,
+	target interface{},
+	endpoint string,
+	ledgerSeq uint32,
+	keys ...xdr.LedgerKey,
+) error {
+	if len(keys) == 0 {
+		return errors.New("no keys specified in request")
+	}
+
+	q, err := buildMultiKeyRequest(keys...)
+	if err != nil {
+		return err
+	}
+	if ledgerSeq >= 2 { // optional param
+		q += fmt.Sprintf("&ledgerSeq=%d", ledgerSeq)
+	}
+
+	var req *http.Request
+	req, err = c.rawPost(ctx, endpoint, q)
+	if err != nil {
+		return err
+	}
+
+	var hresp *http.Response
+	hresp, err = c.http().Do(req)
+	if err != nil {
+		return errors.Wrap(err, "http request errored")
+	}
+	defer drainReponse(hresp, true, &err) //nolint:errcheck
+
+	if hresp.StatusCode < 200 || hresp.StatusCode >= 300 {
+		return fmt.Errorf("http request failed with non-200 status code (%d)", hresp.StatusCode)
+	}
+
+	// wrap returns nil if the inner error is nil
+	return errors.Wrap(json.NewDecoder(hresp.Body).Decode(&target), "json decode failed")
+}
+
+// buildMultiKeyRequest is a workaround helper because, unfortunately,
+// url.Values does not support multiple keys via Set(), so we have to build our
+// URL parameters manually.
+func buildMultiKeyRequest(keys ...xdr.LedgerKey) (string, error) {
+	stringKeys := make([]string, 0, len(keys))
+
+	for _, key := range keys {
+		keyB64, err := key.MarshalBinaryBase64()
+		if err != nil {
+			return "", errors.Wrap(err, "failed to encode LedgerKey")
+		}
+
+		stringKeys = append(stringKeys, "key="+url.QueryEscape(keyB64))
+	}
+
+	return strings.Join(stringKeys, "&"), nil
+}