Add data sponsoring integration test and fix Data effects (#3060)

* Add data sponsoring integration test and fix Data effects

We were not parsing data effects properly (even if they were already
being emitted) due to missing Data effect `struct`s. That lead us to think
we didn't have to add Data Sponsorship effects (for consistency).

This commit adds:
1. An integration test for Data sponsorships
2. Effects for Data sponsorships
3. Data structures (and parsing) of Data effects.

* Appease go vet

protocols/horizon/effects/main.go

@@ -143,25 +143,34 @@ const (
 	// EffectTrustlineSponsorshipRemoved occurs when the sponsorship of a trustline ledger entry is removed
 	EffectTrustlineSponsorshipRemoved EffectType = 65 // from revoke_sponsorship
 
+	// EffectDataSponsorshipCreated occurs when a trustline ledger entry is sponsored
+	EffectDataSponsorshipCreated EffectType = 66 // from manage_data
+
+	// EffectDataSponsorshipUpdated occurs when the sponsoring of a trustline ledger entry is updated
+	EffectDataSponsorshipUpdated EffectType = 67 // from revoke_sponsorship
+
+	// EffectDataSponsorshipRemoved occurs when the sponsorship of a trustline ledger entry is removed
+	EffectDataSponsorshipRemoved EffectType = 68 // from revoke_sponsorship
+
 	// EffectClaimableBalanceSponsorshipCreated occurs when a claimable balance ledger entry is sponsored
-	EffectClaimableBalanceSponsorshipCreated EffectType = 66 // from create_claimable_balance
+	EffectClaimableBalanceSponsorshipCreated EffectType = 69 // from create_claimable_balance
 
 	// EffectClaimableBalanceSponsorshipUpdated occurs when the sponsoring of a claimable balance ledger entry
 	// is updated
-	EffectClaimableBalanceSponsorshipUpdated EffectType = 67 // from revoke_sponsorship
+	EffectClaimableBalanceSponsorshipUpdated EffectType = 70 // from revoke_sponsorship
 
 	// EffectClaimableBalanceSponsorshipRemoved occurs when the sponsorship of a claimable balance ledger entry
 	// is removed
-	EffectClaimableBalanceSponsorshipRemoved EffectType = 68 // from revoke_sponsorship
+	EffectClaimableBalanceSponsorshipRemoved EffectType = 71 // from revoke_sponsorship
 
 	// EffectSignerSponsorshipCreated occurs when the sponsorship of a signer is created
-	EffectSignerSponsorshipCreated EffectType = 69 // from set_options
+	EffectSignerSponsorshipCreated EffectType = 72 // from set_options
 
 	// EffectSignerSponsorshipUpdated occurs when the sponsorship of a signer is updated
-	EffectSignerSponsorshipUpdated EffectType = 70 // from revoke_sponsorship
+	EffectSignerSponsorshipUpdated EffectType = 73 // from revoke_sponsorship
 
 	// EffectSignerSponsorshipRemoved occurs when the sponsorship of a signer is removed
-	EffectSignerSponsorshipRemoved EffectType = 71 // from revoke_sponsorship
+	EffectSignerSponsorshipRemoved EffectType = 74 // from revoke_sponsorship
 )
 
 // Peter 30-04-2019: this is copied from the resourcadapter package
@@ -204,6 +213,9 @@ var EffectTypeNames = map[EffectType]string{
 	EffectTrustlineSponsorshipCreated:              "trustline_sponsorship_created",
 	EffectTrustlineSponsorshipUpdated:              "trustline_sponsorship_updated",
 	EffectTrustlineSponsorshipRemoved:              "trustline_sponsorship_removed",
+	EffectDataSponsorshipCreated:                   "data_sponsorship_created",
+	EffectDataSponsorshipUpdated:                   "data_sponsorship_updated",
+	EffectDataSponsorshipRemoved:                   "data_sponsorship_removed",
 	EffectClaimableBalanceSponsorshipCreated:       "claimable_balance_sponsorship_created",
 	EffectClaimableBalanceSponsorshipUpdated:       "claimable_balance_sponsorship_updated",
 	EffectClaimableBalanceSponsorshipRemoved:       "claimable_balance_sponsorship_removed",
@@ -268,6 +280,23 @@ type AccountFlagsUpdated struct {
 	AuthRevokable *bool `json:"auth_revokable_flag,omitempty"`
 }
 
+type DataCreated struct {
+	Base
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+type DataUpdated struct {
+	Base
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+type DataRemoved struct {
+	Base
+	Name string `json:"name"`
+}
+
 type SequenceBumped struct {
 	Base
 	NewSeq int64 `json:"new_seq,string"`
@@ -404,6 +433,25 @@ type TrustlineSponsorshipRemoved struct {
 	FormerSponsor string `json:"former_sponsor"`
 }
 
+type DataSponsorshipCreated struct {
+	Base
+	DataName string `json:"data_name"`
+	Sponsor  string `json:"sponsor"`
+}
+
+type DataSponsorshipUpdated struct {
+	Base
+	DataName      string `json:"data_name"`
+	FormerSponsor string `json:"former_sponsor"`
+	NewSponsor    string `json:"new_sponsor"`
+}
+
+type DataSponsorshipRemoved struct {
+	Base
+	DataName      string `json:"data_name"`
+	FormerSponsor string `json:"former_sponsor"`
+}
+
 type ClaimableBalanceSponsorshipCreated struct {
 	Base
 	BalanceID string `json:"balance_id"`
@@ -613,6 +661,24 @@ func UnmarshalEffect(effectType string, dataString []byte) (effects Effect, err
 			return
 		}
 		effects = effect
+	case EffectTypeNames[EffectDataCreated]:
+		var effect DataCreated
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
+	case EffectTypeNames[EffectDataUpdated]:
+		var effect DataUpdated
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
+	case EffectTypeNames[EffectDataRemoved]:
+		var effect DataRemoved
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
 	case EffectTypeNames[EffectClaimableBalanceCreated]:
 		var effect ClaimableBalanceCreated
 		if err = json.Unmarshal(dataString, &effect); err != nil {
@@ -667,6 +733,24 @@ func UnmarshalEffect(effectType string, dataString []byte) (effects Effect, err
 			return
 		}
 		effects = effect
+	case EffectTypeNames[EffectDataSponsorshipCreated]:
+		var effect DataSponsorshipCreated
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
+	case EffectTypeNames[EffectDataSponsorshipUpdated]:
+		var effect DataSponsorshipUpdated
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
+	case EffectTypeNames[EffectDataSponsorshipRemoved]:
+		var effect DataSponsorshipRemoved
+		if err = json.Unmarshal(dataString, &effect); err != nil {
+			return
+		}
+		effects = effect
 	case EffectTypeNames[EffectClaimableBalanceSponsorshipCreated]:
 		var effect ClaimableBalanceSponsorshipCreated
 		if err = json.Unmarshal(dataString, &effect); err != nil {

services/horizon/internal/db2/history/main.go

@@ -145,25 +145,34 @@ const (
 	// EffectTrustlineSponsorshipRemoved occurs when the sponsorship of a trustline ledger entry is removed
 	EffectTrustlineSponsorshipRemoved EffectType = 65 // from revoke_sponsorship
 
+	// EffectDataSponsorshipCreated occurs when a trustline ledger entry is sponsored
+	EffectDataSponsorshipCreated EffectType = 66 // from manage_data
+
+	// EffectDataSponsorshipUpdated occurs when the sponsoring of a trustline ledger entry is updated
+	EffectDataSponsorshipUpdated EffectType = 67 // from revoke_sponsorship
+
+	// EffectDataSponsorshipRemoved occurs when the sponsorship of a trustline ledger entry is removed
+	EffectDataSponsorshipRemoved EffectType = 68 // from revoke_sponsorship
+
 	// EffectClaimableBalanceSponsorshipCreated occurs when a claimable balance ledger entry is sponsored
-	EffectClaimableBalanceSponsorshipCreated EffectType = 66 // from create_claimable_balance
+	EffectClaimableBalanceSponsorshipCreated EffectType = 69 // from create_claimable_balance
 
 	// EffectClaimableBalanceSponsorshipUpdated occurs when the sponsoring of a claimable balance ledger entry
 	// is updated
-	EffectClaimableBalanceSponsorshipUpdated EffectType = 67 // from revoke_sponsorship
+	EffectClaimableBalanceSponsorshipUpdated EffectType = 70 // from revoke_sponsorship
 
 	// EffectClaimableBalanceSponsorshipRemoved occurs when the sponsorship of a claimable balance ledger entry
 	// is removed
-	EffectClaimableBalanceSponsorshipRemoved EffectType = 68 // from revoke_sponsorship
+	EffectClaimableBalanceSponsorshipRemoved EffectType = 71 // from revoke_sponsorship
 
 	// EffectSignerSponsorshipCreated occurs when the sponsorship of a signer is created
-	EffectSignerSponsorshipCreated EffectType = 69 // from set_options
+	EffectSignerSponsorshipCreated EffectType = 72 // from set_options
 
 	// EffectSignerSponsorshipUpdated occurs when the sponsorship of a signer is updated
-	EffectSignerSponsorshipUpdated EffectType = 70 // from revoke_sponsorship
+	EffectSignerSponsorshipUpdated EffectType = 73 // from revoke_sponsorship
 
 	// EffectSignerSponsorshipRemoved occurs when the sponsorship of a signer is removed
-	EffectSignerSponsorshipRemoved EffectType = 71 // from revoke_sponsorship
+	EffectSignerSponsorshipRemoved EffectType = 74 // from revoke_sponsorship
 )
 
 // Account is a row of data from the `history_accounts` table

services/horizon/internal/expingest/processors/effects_processor.go

@@ -250,13 +250,18 @@ var sponsoringEffectsTable = map[xdr.LedgerEntryType]struct {
 		updated: history.EffectTrustlineSponsorshipUpdated,
 		removed: history.EffectTrustlineSponsorshipRemoved,
 	},
+	xdr.LedgerEntryTypeData: {
+		created: history.EffectDataSponsorshipCreated,
+		updated: history.EffectDataSponsorshipUpdated,
+		removed: history.EffectDataSponsorshipRemoved,
+	},
 	xdr.LedgerEntryTypeClaimableBalance: {
 		created: history.EffectClaimableBalanceSponsorshipCreated,
 		updated: history.EffectClaimableBalanceSponsorshipUpdated,
 		removed: history.EffectClaimableBalanceSponsorshipRemoved,
 	},
 
-	// We intentionally don't have Sponsoring effects for Offer and Data
+	// We intentionally don't have Sponsoring effects for Offer
 	// entries because we don't generate creation effects for them.
 }
 
@@ -370,6 +375,9 @@ func (e *effectsWrapper) addLedgerEntrySponsorshipEffects(change io.Change) erro
 		aid := data.MustTrustLine().AccountId
 		accountAddress = aid.Address()
 		details["asset"] = data.MustTrustLine().Asset.StringCanonical()
+	case xdr.LedgerEntryTypeData:
+		accountAddress = e.operation.SourceAccount().Address()
+		details["data_name"] = data.MustData().DataName
 	case xdr.LedgerEntryTypeClaimableBalance:
 		accountAddress = e.operation.SourceAccount().Address()
 		var err error

services/horizon/internal/integration/protocol14_sponsorship_ops_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"encoding/base64"
 	"testing"
 	"time"
 
@@ -329,3 +330,83 @@ func TestSponsorPreAuthSigner(t *testing.T) {
 	*/
 
 }
+
+func TestSponsorData(t *testing.T) {
+	tt := assert.New(t)
+	itest := test.NewIntegrationTest(t, protocol14Config)
+	defer itest.Close()
+	sponsorPair := itest.Master()
+	sponsor := func() txnbuild.Account { return itest.MasterAccount() }
+
+	// Let's create a new account
+	pairs, _ := itest.CreateAccounts(1, "1000")
+	newAccountPair := pairs[0]
+	newAccount := func() txnbuild.Account {
+		request := sdk.AccountRequest{AccountID: newAccountPair.Address()}
+		account, err := itest.Client().AccountDetail(request)
+		tt.NoError(err)
+		return &account
+	}
+
+	// Let's add a sponsored data entry
+	//
+	// BeginSponsorship N (Source=sponsor)
+	//   ManageData "SponsoredData"="SponsoredValue" (Source=N)
+	// EndSponsorship (Source=N)
+	ops := make([]txnbuild.Operation, 3, 3)
+	ops[0] = &txnbuild.BeginSponsoringFutureReserves{
+		SponsoredID: newAccountPair.Address(),
+	}
+	ops[1] = &txnbuild.ManageData{
+		Name:          "SponsoredData",
+		Value:         []byte("SponsoredValue"),
+		SourceAccount: newAccount(),
+	}
+	ops[2] = &txnbuild.EndSponsoringFutureReserves{
+		SourceAccount: newAccount(),
+	}
+
+	signers := []*keypair.Full{sponsorPair, newAccountPair}
+	txResp, err := itest.SubmitMultiSigOperations(sponsor(), signers, ops...)
+	tt.NoError(err)
+
+	var txResult xdr.TransactionResult
+	err = xdr.SafeUnmarshalBase64(txResp.ResultXdr, &txResult)
+	tt.NoError(err)
+	tt.Equal(xdr.TransactionResultCodeTxSuccess, txResult.Result.Code)
+
+	// Verify that the data was incorporated
+	dataAdded := func() bool {
+		data := newAccount().(*protocol.Account).Data
+		if value, ok := data["SponsoredData"]; ok {
+			decoded, e := base64.StdEncoding.DecodeString(value)
+			tt.NoError(e)
+			if string(decoded) == "SponsoredValue" {
+				return true
+			}
+		}
+		return false
+	}
+	tt.Eventually(dataAdded, time.Second*10, time.Millisecond*100)
+
+	// Check effects and details of the ManageData operation
+	operationsResponse, err := itest.Client().Operations(sdk.OperationRequest{
+		ForTransaction: txResp.Hash,
+	})
+	tt.NoError(err)
+	tt.Len(operationsResponse.Embedded.Records, 3)
+	manageDataOp := operationsResponse.Embedded.Records[1].(operations.ManageData)
+	tt.Equal(sponsorPair.Address(), manageDataOp.Sponsor)
+
+	effectsResponse, err := itest.Client().Effects(sdk.EffectRequest{
+		ForOperation: manageDataOp.GetID(),
+	})
+	tt.NoError(err)
+	if tt.Len(effectsResponse.Embedded.Records, 2) {
+		dataSponsorshipEffect := effectsResponse.Embedded.Records[1].(effects.DataSponsorshipCreated)
+		tt.Equal(sponsorPair.Address(), dataSponsorshipEffect.Sponsor)
+		tt.Equal(newAccountPair.Address(), dataSponsorshipEffect.Account)
+		tt.Equal("SponsoredData", dataSponsorshipEffect.DataName)
+	}
+
+}

services/horizon/internal/resourceadapter/effects.go

@@ -45,6 +45,9 @@ var EffectTypeNames = map[history.EffectType]string{
 	history.EffectTrustlineSponsorshipCreated:              "trustline_sponsorship_created",
 	history.EffectTrustlineSponsorshipUpdated:              "trustline_sponsorship_updated",
 	history.EffectTrustlineSponsorshipRemoved:              "trustline_sponsorship_removed",
+	history.EffectDataSponsorshipCreated:                   "data_sponsorship_created",
+	history.EffectDataSponsorshipUpdated:                   "data_sponsorship_updated",
+	history.EffectDataSponsorshipRemoved:                   "data_sponsorship_removed",
 	history.EffectClaimableBalanceSponsorshipCreated:       "claimable_balance_sponsorship_created",
 	history.EffectClaimableBalanceSponsorshipUpdated:       "claimable_balance_sponsorship_updated",
 	history.EffectClaimableBalanceSponsorshipRemoved:       "claimable_balance_sponsorship_removed",
@@ -186,6 +189,18 @@ func NewEffect(
 		e := effects.TrustlineSponsorshipRemoved{Base: basev}
 		err = row.UnmarshalDetails(&e)
 		result = e
+	case history.EffectDataSponsorshipCreated:
+		e := effects.DataSponsorshipCreated{Base: basev}
+		err = row.UnmarshalDetails(&e)
+		result = e
+	case history.EffectDataSponsorshipUpdated:
+		e := effects.DataSponsorshipUpdated{Base: basev}
+		err = row.UnmarshalDetails(&e)
+		result = e
+	case history.EffectDataSponsorshipRemoved:
+		e := effects.DataSponsorshipRemoved{Base: basev}
+		err = row.UnmarshalDetails(&e)
+		result = e
 	case history.EffectClaimableBalanceSponsorshipCreated:
 		e := effects.ClaimableBalanceSponsorshipCreated{Base: basev}
 		err = row.UnmarshalDetails(&e)