protocols/horizon: Add memo_bytes field to transaction response (#2485)

Add memo_bytes field to the Horizon transaction response. memo_bytes contains the base64 encoding of the original byte sequence set in the transaction envelope's memo field.

protocols/horizon/main.go

@@ -459,6 +459,7 @@ type Transaction struct {
 	ResultMetaXdr      string              `json:"result_meta_xdr"`
 	FeeMetaXdr         string              `json:"fee_meta_xdr"`
 	MemoType           string              `json:"memo_type"`
+	MemoBytes          string              `json:"memo_bytes,omitempty"`
 	Memo               string              `json:"memo,omitempty"`
 	Signatures         []string            `json:"signatures"`
 	ValidAfter         string              `json:"valid_after,omitempty"`

services/horizon/CHANGELOG.md

@@ -3,6 +3,17 @@
 All notable changes to this project will be documented in this
 file. This project adheres to [Semantic Versioning](http://semver.org/).
 
+## v1.2.0
+
+### Changes
+* The XDR definition of a transaction memo is a string.
+However, XDR strings are actually binary blobs with no enforced encoding. 
+It is possible to set the memo in a transaction envelope to a binary sequence which is not valid ASCII or unicode. 
+Previously, if you wanted to recover the original binary sequence for a transaction memo, you would have to decode the transaction's envelope.
+In this release, we have added a `memo_bytes` field to the Horizon transaction response.
+`memo_bytes` stores the base 64 encoding of the memo bytes set in the transaction envelope.
+
+
 ## v1.1.0
 
 ### **IMPORTANT**: Database migration

services/horizon/internal/actions/transaction.go

@@ -31,7 +31,10 @@ func TransactionPage(ctx context.Context, hq *history.Q, accountID string, ledge
 	for _, record := range records {
 		// TODO: make PopulateTransaction return horizon.Transaction directly.
 		var res horizon.Transaction
-		resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record)
+		err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record)
+		if err != nil {
+			return hal.Page{}, errors.Wrap(err, "could not populate transaction")
+		}
 		page.Add(res)
 	}
 
@@ -100,7 +103,10 @@ func StreamTransactions(ctx context.Context, s *sse.Stream, hq *history.Q, accou
 	records := allRecords[s.SentCount():]
 	for _, record := range records {
 		var res horizon.Transaction
-		resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record)
+		err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record)
+		if err != nil {
+			return errors.Wrap(err, "could not populate transaction")
+		}
 		s.Send(sse.Event{ID: res.PagingToken(), Data: res})
 	}
 
@@ -118,6 +124,8 @@ func TransactionResource(ctx context.Context, hq *history.Q, txHash string) (hor
 		return resource, errors.Wrap(err, "loading transaction record")
 	}
 
-	resourceadapter.PopulateTransaction(ctx, txHash, &resource, record)
+	if err = resourceadapter.PopulateTransaction(ctx, txHash, &resource, record); err != nil {
+		return resource, errors.Wrap(err, "could not populate transaction")
+	}
 	return resource, nil
 }

services/horizon/internal/actions_transaction.go

@@ -102,7 +102,7 @@ func (action *TransactionCreateAction) loadResult() {
 
 func (action *TransactionCreateAction) loadResource() {
 	if action.Result.Err == nil {
-		resourceadapter.PopulateTransaction(
+		action.Err = resourceadapter.PopulateTransaction(
 			action.R.Context(),
 			action.TX.hash,
 			&action.Resource,

services/horizon/internal/docs/reference/resources/transaction.md

@@ -28,8 +28,9 @@ To learn more about the concept of transactions in the Stellar network, take a l
 | result_xdr              | string                   | A base64 encoded string of the raw `TransactionResult` xdr struct for this transaction                                         |
 | result_meta_xdr         | string                   | A base64 encoded string of the raw `TransactionMeta` xdr struct for this transaction                                           |
 | fee_meta_xdr            | string                   | A base64 encoded string of the raw `LedgerEntryChanges` xdr struct produced by taking fees for this transaction.               |
-| memo_type               | string                   |                                                                                                                                |
-| memo                    | string                   |                                                                                                                                |
+| memo_type               | string                   | The type of memo set in the transaction. Possible values are `none`, `text`, `id`, `hash`, and `return`.                       |
+| memo                    | string                   | The string representation of the memo set in the transaction. When `memo_type` is `id`, the `memo` is a decimal string representation of an unsigned 64 bit integer. When `memo_type` is `hash` or `return`, the `memo` is a base64 encoded string. When `memo_type` is `text`, the `memo` is a unicode string. However, if the original memo byte sequence in the transaction XDR is not valid unicode, Horizon will replace any invalid byte sequences with the utf-8 replacement character. Note this field is only present when `memo_type` is not `none`. |
+| memo_bytes              | string                   | A base64 encoded string of the memo bytes set in the transaction's xdr envelope. Note this field is only present when `memo_type` is `text`. |
 | signatures              | string[]                 | An array of signatures used to sign this transaction                                                                           |
 | valid_after             | RFC3339 date-time string |                                                                                                                                |
 | valid_before            | RFC3339 date-time string |                                                                                                                                |

services/horizon/internal/resourceadapter/operations.go

@@ -23,7 +23,12 @@ func NewOperation(
 ) (result hal.Pageable, err error) {
 
 	base := operations.Base{}
-	PopulateBaseOperation(ctx, &base, operationRow, transactionHash, transactionRow, ledger)
+	err = PopulateBaseOperation(
+		ctx, &base, operationRow, transactionHash, transactionRow, ledger,
+	)
+	if err != nil {
+		return
+	}
 
 	switch operationRow.Type {
 	case xdr.OperationTypeBumpSequence:
@@ -118,7 +123,7 @@ func PopulateBaseOperation(
 	transactionHash string,
 	transactionRow *history.Transaction,
 	ledger history.Ledger,
-) {
+) error {
 	dest.ID = fmt.Sprintf("%d", operationRow.ID)
 	dest.PT = operationRow.PagingToken()
 	dest.TransactionSuccessful = operationRow.TransactionSuccessful
@@ -137,8 +142,9 @@ func PopulateBaseOperation(
 
 	if transactionRow != nil {
 		dest.Transaction = new(horizon.Transaction)
-		PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow)
+		return PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow)
 	}
+	return nil
 }
 
 func populateOperationType(dest *operations.Base, row history.Operation) {

services/horizon/internal/resourceadapter/operations_test.go

@@ -25,14 +25,20 @@ func TestPopulateOperation_Successful(t *testing.T) {
 	dest = operations.Base{}
 	row = history.Operation{TransactionSuccessful: true}
 
-	PopulateBaseOperation(ctx, &dest, row, "", nil, ledger)
+	assert.NoError(
+		t,
+		PopulateBaseOperation(ctx, &dest, row, "", nil, ledger),
+	)
 	assert.True(t, dest.TransactionSuccessful)
 	assert.Nil(t, dest.Transaction)
 
 	dest = operations.Base{}
 	row = history.Operation{TransactionSuccessful: false}
 
-	PopulateBaseOperation(ctx, &dest, row, "", nil, ledger)
+	assert.NoError(
+		t,
+		PopulateBaseOperation(ctx, &dest, row, "", nil, ledger),
+	)
 	assert.False(t, dest.TransactionSuccessful)
 	assert.Nil(t, dest.Transaction)
 }
@@ -52,13 +58,16 @@ func TestPopulateOperation_WithTransaction(t *testing.T) {
 	operationsRow = history.Operation{TransactionSuccessful: true}
 	transactionRow = history.Transaction{Successful: true, MaxFee: 10000, FeeCharged: 100}
 
-	PopulateBaseOperation(
-		ctx,
-		&dest,
-		operationsRow,
-		transactionRow.TransactionHash,
-		&transactionRow,
-		ledger,
+	assert.NoError(
+		t,
+		PopulateBaseOperation(
+			ctx,
+			&dest,
+			operationsRow,
+			transactionRow.TransactionHash,
+			&transactionRow,
+			ledger,
+		),
 	)
 	assert.True(t, dest.TransactionSuccessful)
 	assert.True(t, dest.Transaction.Successful)
@@ -152,34 +161,44 @@ func TestFeeBumpOperation(t *testing.T) {
 		InnerTransactionHash: null.StringFrom("2374e99349b9ef7dba9a5db3339b78fda8f34777b1af33ba468ad5c0df946d4d"),
 	}
 
-	PopulateBaseOperation(
-		ctx,
-		&dest,
-		operationsRow,
-		transactionRow.TransactionHash,
-		nil,
-		history.Ledger{},
+	assert.NoError(
+		t,
+		PopulateBaseOperation(
+			ctx,
+			&dest,
+			operationsRow,
+			transactionRow.TransactionHash,
+			nil,
+			history.Ledger{},
+		),
 	)
 	assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash)
 
-	PopulateBaseOperation(
-		ctx,
-		&dest,
-		operationsRow,
-		transactionRow.InnerTransactionHash.String,
-		nil,
-		history.Ledger{},
+	assert.NoError(
+		t,
+		PopulateBaseOperation(
+			ctx,
+			&dest,
+			operationsRow,
+			transactionRow.InnerTransactionHash.String,
+			nil,
+			history.Ledger{},
+		),
 	)
 	assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash)
 
-	PopulateBaseOperation(
-		ctx,
-		&dest,
-		operationsRow,
-		transactionRow.TransactionHash,
-		&transactionRow,
-		history.Ledger{},
+	assert.NoError(
+		t,
+		PopulateBaseOperation(
+			ctx,
+			&dest,
+			operationsRow,
+			transactionRow.TransactionHash,
+			&transactionRow,
+			history.Ledger{},
+		),
 	)
+
 	assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash)
 	assert.Equal(t, transactionRow.TransactionHash, dest.Transaction.Hash)
 	assert.Equal(t, transactionRow.TransactionHash, dest.Transaction.ID)
@@ -194,13 +213,16 @@ func TestFeeBumpOperation(t *testing.T) {
 	assert.Equal(t, transactionRow.TransactionHash, dest.Transaction.FeeBumpTransaction.Hash)
 	assert.Equal(t, []string{"a", "b", "c"}, dest.Transaction.FeeBumpTransaction.Signatures)
 
-	PopulateBaseOperation(
-		ctx,
-		&dest,
-		operationsRow,
-		transactionRow.InnerTransactionHash.String,
-		&transactionRow,
-		history.Ledger{},
+	assert.NoError(
+		t,
+		PopulateBaseOperation(
+			ctx,
+			&dest,
+			operationsRow,
+			transactionRow.InnerTransactionHash.String,
+			&transactionRow,
+			history.Ledger{},
+		),
 	)
 	assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash)
 	assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.Transaction.Hash)

services/horizon/internal/resourceadapter/transaction.go

@@ -2,7 +2,9 @@ package resourceadapter
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
+	"github.com/stellar/go/xdr"
 	"strings"
 	"time"
 
@@ -19,7 +21,7 @@ func PopulateTransaction(
 	transactionHash string,
 	dest *protocol.Transaction,
 	row history.Transaction,
-) {
+) error {
 	dest.ID = transactionHash
 	dest.PT = row.PagingToken()
 	dest.Successful = row.Successful
@@ -38,6 +40,13 @@ func PopulateTransaction(
 	dest.FeeMetaXdr = row.TxFeeMeta
 	dest.MemoType = row.MemoType
 	dest.Memo = row.Memo.String
+	if row.MemoType == "text" {
+		if memoBytes, err := memoBytes(row.TxEnvelope); err != nil {
+			return err
+		} else {
+			dest.MemoBytes = memoBytes
+		}
+	}
 	dest.Signatures = strings.Split(row.SignatureString, ",")
 	dest.ValidBefore = timeString(dest, row.ValidBefore)
 	dest.ValidAfter = timeString(dest, row.ValidAfter)
@@ -71,6 +80,18 @@ func PopulateTransaction(
 	dest.Links.Transaction = dest.Links.Self
 	dest.Links.Succeeds = lb.Linkf("/transactions?order=desc&cursor=%s", dest.PT)
 	dest.Links.Precedes = lb.Linkf("/transactions?order=asc&cursor=%s", dest.PT)
+
+	return nil
+}
+
+func memoBytes(envelopeXDR string) (string, error) {
+	var parsedEnvelope xdr.TransactionEnvelope
+	if err := xdr.SafeUnmarshalBase64(envelopeXDR, &parsedEnvelope); err != nil {
+		return "", err
+	}
+
+	memo := *parsedEnvelope.Memo().Text
+	return base64.StdEncoding.EncodeToString([]byte(memo)), nil
 }
 
 func timeString(res *protocol.Transaction, in null.Int) string {