Merge pull request #240 from stellar/p21-fee-updates

Add fee fields; formatting for warnings

cmd/export_ledger_entry_changes.go

@@ -57,7 +57,7 @@ be exported.`,
 		}
 
 		if allFalse {
-			for export_name, _ := range exports {
+			for export_name := range exports {
 				exports[export_name] = true
 			}
 		}

cmd/export_ledgers.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/stellar/go/historyarchive"
 	"github.com/stellar/stellar-etl/internal/input"
 	"github.com/stellar/stellar-etl/internal/transform"
 	"github.com/stellar/stellar-etl/internal/utils"
@@ -23,7 +22,7 @@ var ledgersCmd = &cobra.Command{
 		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 		env := utils.GetEnvironmentDetails(isTest, isFuture, datastoreUrl)
 
-		var ledgers []historyarchive.Ledger
+		var ledgers []utils.HistoryArchiveLedgerAndLCM
 		var err error
 
 		if useCaptiveCore {
@@ -39,8 +38,8 @@ var ledgersCmd = &cobra.Command{
 
 		numFailures := 0
 		totalNumBytes := 0
-		for i, lcm := range ledgers {
-			transformed, err := transform.TransformLedger(lcm)
+		for i, ledger := range ledgers {
+			transformed, err := transform.TransformLedger(ledger.Ledger, ledger.LCM)
 			if err != nil {
 				cmdLogger.LogError(fmt.Errorf("could not json transform ledger %d: %s", startNum+uint32(i), err))
 				numFailures += 1

cmd/export_ledgers_test.go

@@ -121,7 +121,7 @@ func indexOf(l []string, s string) int {
 	return -1
 }
 
-func sortByName(files []os.FileInfo) {
+func sortByName(files []os.DirEntry) {
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].Name() < files[j].Name()
 	})
@@ -144,14 +144,14 @@ func runCLITest(t *testing.T, test cliTest, goldenFolder string) {
 
 			// If the output arg specified is a directory, concat the contents for comparison.
 			if stat.IsDir() {
-				files, err := ioutil.ReadDir(outLocation)
+				files, err := os.ReadDir(outLocation)
 				if err != nil {
 					log.Fatal(err)
 				}
 				var buf bytes.Buffer
 				sortByName(files)
 				for _, f := range files {
-					b, err := ioutil.ReadFile(filepath.Join(outLocation, f.Name()))
+					b, err := os.ReadFile(filepath.Join(outLocation, f.Name()))
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -160,7 +160,7 @@ func runCLITest(t *testing.T, test cliTest, goldenFolder string) {
 				testOutput = buf.Bytes()
 			} else {
 				// If the output is written to a file, read the contents of the file for comparison.
-				testOutput, err = ioutil.ReadFile(outLocation)
+				testOutput, err = os.ReadFile(outLocation)
 				if err != nil {
 					log.Fatal(err)
 				}
@@ -197,16 +197,6 @@ func extractErrorMsg(loggerOutput string) string {
 	return loggerOutput[errIndex : errIndex+endIndex]
 }
 
-func removeCoreLogging(loggerOutput string) string {
-	endIndex := strings.Index(loggerOutput, "{\"")
-	// if there is no bracket, then nothing was exported except logs
-	if endIndex == -1 {
-		return ""
-	}
-
-	return loggerOutput[endIndex:]
-}
-
 func getLastSeqNum(archiveURLs []string) uint32 {
 	num, err := utils.GetLatestLedgerSequence(archiveURLs)
 	if err != nil {
@@ -218,10 +208,10 @@ func getLastSeqNum(archiveURLs []string) uint32 {
 func getGolden(t *testing.T, goldenFile string, actual string, update bool) (string, error) {
 	t.Helper()
 	f, err := os.OpenFile(goldenFile, os.O_RDWR, 0644)
-	defer f.Close()
 	if err != nil {
 		return "", err
 	}
+	defer f.Close()
 
 	// If the update flag is true, clear the current contents of the golden file and write the actual output
 	// This is useful for when new tests or added or functionality changes that breaks current tests

docker/Dockerfile

@@ -11,7 +11,7 @@ COPY . .
 RUN go build -v -o /usr/local/bin ./...
 
 # stage 2: runtime enviroment
-FROM stellar/stellar-core:20.2.0-1716.rc3.34d82fc00.focal
+FROM stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal
 
 WORKDIR /etl
 

docker/stellar-core.cfg

@@ -3,6 +3,8 @@
 # for how to properly configure your environment
 
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
+EMIT_SOROBAN_TRANSACTION_META_EXT_V1=true
+EMIT_LEDGER_CLOSE_META_EXT_V1=true
 
 #FAILURE_SAFETY is minimum number of nodes that are allowed to fail before you no longer have quorum
 FAILURE_SAFETY=1

docker/stellar-core_futurenet.cfg

@@ -7,6 +7,8 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Test SDF Future Network ; October 2022"
 
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
+EMIT_SOROBAN_TRANSACTION_META_EXT_V1=true
+EMIT_LEDGER_CLOSE_META_EXT_V1=true
 
 # DATABASE="sqlite3://stellar.db"
 PEER_PORT=11725

docker/stellar-core_testnet.cfg

@@ -6,6 +6,8 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 # DATABASE="sqlite3://stellar.db"
 
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
+EMIT_SOROBAN_TRANSACTION_META_EXT_V1=true
+EMIT_LEDGER_CLOSE_META_EXT_V1=true
 
 # Stellar Testnet validators
 [[HOME_DOMAINS]]

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.17.0
-	github.com/stellar/go v0.0.0-20240419222646-3a79646669ab
+	github.com/stellar/go v0.0.0-20240423031611-e1c5206ad1ba
 	github.com/stretchr/testify v1.9.0
 )
 

go.sum

@@ -296,10 +296,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI=
 github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI=
-github.com/stellar/go v0.0.0-20240419044405-2d7308b67c07 h1:AgxlbRmsFAc9VaOLj29K9RpsSmlmtJ0KCVnGxV7bUwo=
-github.com/stellar/go v0.0.0-20240419044405-2d7308b67c07/go.mod h1:ckzsX0B0qfTMVZQJtPELJLs7cJ6xXMYHPVLyIsReGsU=
-github.com/stellar/go v0.0.0-20240419222646-3a79646669ab h1:+uTCn/DrOc1cXugQ8PKZPAkZS3KWeHk5f2aKk9jdrDs=
-github.com/stellar/go v0.0.0-20240419222646-3a79646669ab/go.mod h1:ckzsX0B0qfTMVZQJtPELJLs7cJ6xXMYHPVLyIsReGsU=
+github.com/stellar/go v0.0.0-20240423031611-e1c5206ad1ba h1:2UPb78V6mL07B0nJ6/89nJ2cimVD3xPMCFxawwRvpJ0=
+github.com/stellar/go v0.0.0-20240423031611-e1c5206ad1ba/go.mod h1:ckzsX0B0qfTMVZQJtPELJLs7cJ6xXMYHPVLyIsReGsU=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

internal/input/ledgers.go

@@ -11,20 +11,20 @@ import (
 )
 
 // GetLedgers returns a slice of ledger close metas for the ledgers in the provided range (inclusive on both ends)
-func GetLedgers(start, end uint32, limit int64, env utils.EnvironmentDetails, useCaptiveCore bool) ([]historyarchive.Ledger, error) {
+func GetLedgers(start, end uint32, limit int64, env utils.EnvironmentDetails, useCaptiveCore bool) ([]utils.HistoryArchiveLedgerAndLCM, error) {
 	ctx := context.Background()
 	backend, err := utils.CreateLedgerBackend(ctx, useCaptiveCore, env)
 	if err != nil {
-		return []historyarchive.Ledger{}, err
+		return []utils.HistoryArchiveLedgerAndLCM{}, err
 	}
 
-	ledgerSlice := []historyarchive.Ledger{}
+	ledgerSlice := []utils.HistoryArchiveLedgerAndLCM{}
 	err = backend.PrepareRange(ctx, ledgerbackend.BoundedRange(start, end))
 	panicIf(err)
 	for seq := start; seq <= end; seq++ {
 		lcm, err := backend.GetLedger(ctx, seq)
 		if err != nil {
-			return []historyarchive.Ledger{}, err
+			return []utils.HistoryArchiveLedgerAndLCM{}, err
 		}
 
 		var ext xdr.TransactionHistoryEntryExt
@@ -68,7 +68,12 @@ func GetLedgers(start, end uint32, limit int64, env utils.EnvironmentDetails, us
 			},
 		}
 
-		ledgerSlice = append(ledgerSlice, ledger)
+		ledgerLCM := utils.HistoryArchiveLedgerAndLCM{
+			Ledger: ledger,
+			LCM:    lcm,
+		}
+
+		ledgerSlice = append(ledgerSlice, ledgerLCM)
 		if int64(len(ledgerSlice)) >= limit && limit >= 0 {
 			break
 		}

internal/input/ledgers_history_archive.go

@@ -4,26 +4,28 @@ import (
 	"context"
 
 	"github.com/stellar/stellar-etl/internal/utils"
-
-	"github.com/stellar/go/historyarchive"
 )
 
 // GetLedgers returns a slice of ledger close metas for the ledgers in the provided range (inclusive on both ends)
-func GetLedgersHistoryArchive(start, end uint32, limit int64, env utils.EnvironmentDetails, useCaptiveCore bool) ([]historyarchive.Ledger, error) {
+func GetLedgersHistoryArchive(start, end uint32, limit int64, env utils.EnvironmentDetails, useCaptiveCore bool) ([]utils.HistoryArchiveLedgerAndLCM, error) {
 	backend, err := utils.CreateBackend(start, end, env.ArchiveURLs)
 	if err != nil {
-		return []historyarchive.Ledger{}, err
+		return []utils.HistoryArchiveLedgerAndLCM{}, err
 	}
 
-	ledgerSlice := []historyarchive.Ledger{}
+	ledgerSlice := []utils.HistoryArchiveLedgerAndLCM{}
 	ctx := context.Background()
 	for seq := start; seq <= end; seq++ {
 		ledger, err := backend.GetLedgerArchive(ctx, seq)
 		if err != nil {
-			return []historyarchive.Ledger{}, err
+			return []utils.HistoryArchiveLedgerAndLCM{}, err
+		}
+
+		ledgerLCM := utils.HistoryArchiveLedgerAndLCM{
+			Ledger: ledger,
 		}
 
-		ledgerSlice = append(ledgerSlice, ledger)
+		ledgerSlice = append(ledgerSlice, ledgerLCM)
 		if int64(len(ledgerSlice)) >= limit && limit >= 0 {
 			break
 		}

internal/transform/account.go

@@ -18,7 +18,7 @@ func TransformAccount(ledgerChange ingest.Change, header xdr.LedgerHeaderHistory
 
 	accountEntry, accountFound := ledgerEntry.Data.GetAccount()
 	if !accountFound {
-		return AccountOutput{}, fmt.Errorf("Could not extract account data from ledger entry; actual type is %s", ledgerEntry.Data.Type)
+		return AccountOutput{}, fmt.Errorf("could not extract account data from ledger entry; actual type is %s", ledgerEntry.Data.Type)
 	}
 
 	outputID, err := accountEntry.AccountId.GetAddress()
@@ -28,7 +28,7 @@ func TransformAccount(ledgerChange ingest.Change, header xdr.LedgerHeaderHistory
 
 	outputBalance := accountEntry.Balance
 	if outputBalance < 0 {
-		return AccountOutput{}, fmt.Errorf("Balance is negative (%d) for account: %s", outputBalance, outputID)
+		return AccountOutput{}, fmt.Errorf("balance is negative (%d) for account: %s", outputBalance, outputID)
 	}
 
 	//The V1 struct is the first version of the extender from accountEntry. It contains information on liabilities, and in the future
@@ -39,17 +39,17 @@ func TransformAccount(ledgerChange ingest.Change, header xdr.LedgerHeaderHistory
 		liabilities := accountExtensionInfo.Liabilities
 		outputBuyingLiabilities, outputSellingLiabilities = liabilities.Buying, liabilities.Selling
 		if outputBuyingLiabilities < 0 {
-			return AccountOutput{}, fmt.Errorf("The buying liabilities count is negative (%d) for account: %s", outputBuyingLiabilities, outputID)
+			return AccountOutput{}, fmt.Errorf("the buying liabilities count is negative (%d) for account: %s", outputBuyingLiabilities, outputID)
 		}
 
 		if outputSellingLiabilities < 0 {
-			return AccountOutput{}, fmt.Errorf("The selling liabilities count is negative (%d) for account: %s", outputSellingLiabilities, outputID)
+			return AccountOutput{}, fmt.Errorf("the selling liabilities count is negative (%d) for account: %s", outputSellingLiabilities, outputID)
 		}
 	}
 
 	outputSequenceNumber := int64(accountEntry.SeqNum)
 	if outputSequenceNumber < 0 {
-		return AccountOutput{}, fmt.Errorf("Account sequence number is negative (%d) for account: %s", outputSequenceNumber, outputID)
+		return AccountOutput{}, fmt.Errorf("account sequence number is negative (%d) for account: %s", outputSequenceNumber, outputID)
 	}
 	outputSequenceLedger := accountEntry.SeqLedger()
 	outputSequenceTime := accountEntry.SeqTime()

internal/transform/account_test.go

@@ -38,15 +38,15 @@ func TestTransformAccount(t *testing.T) {
 				},
 			},
 			},
-			AccountOutput{}, fmt.Errorf("Could not extract account data from ledger entry; actual type is LedgerEntryTypeOffer"),
+			AccountOutput{}, fmt.Errorf("could not extract account data from ledger entry; actual type is LedgerEntryTypeOffer"),
 		},
 		{
 			inputStruct{wrapAccountEntry(xdr.AccountEntry{
 				AccountId: genericAccountID,
 				Balance:   -1,
 			}, 0),
 			},
-			AccountOutput{}, fmt.Errorf("Balance is negative (-1) for account: %s", genericAccountAddress),
+			AccountOutput{}, fmt.Errorf("balance is negative (-1) for account: %s", genericAccountAddress),
 		},
 		{
 			inputStruct{wrapAccountEntry(xdr.AccountEntry{
@@ -61,7 +61,7 @@ func TestTransformAccount(t *testing.T) {
 				},
 			}, 0),
 			},
-			AccountOutput{}, fmt.Errorf("The buying liabilities count is negative (-1) for account: %s", genericAccountAddress),
+			AccountOutput{}, fmt.Errorf("the buying liabilities count is negative (-1) for account: %s", genericAccountAddress),
 		},
 		{
 			inputStruct{wrapAccountEntry(xdr.AccountEntry{
@@ -76,15 +76,15 @@ func TestTransformAccount(t *testing.T) {
 				},
 			}, 0),
 			},
-			AccountOutput{}, fmt.Errorf("The selling liabilities count is negative (-2) for account: %s", genericAccountAddress),
+			AccountOutput{}, fmt.Errorf("the selling liabilities count is negative (-2) for account: %s", genericAccountAddress),
 		},
 		{
 			inputStruct{wrapAccountEntry(xdr.AccountEntry{
 				AccountId: genericAccountID,
 				SeqNum:    -3,
 			}, 0),
 			},
-			AccountOutput{}, fmt.Errorf("Account sequence number is negative (-3) for account: %s", genericAccountAddress),
+			AccountOutput{}, fmt.Errorf("account sequence number is negative (-3) for account: %s", genericAccountAddress),
 		},
 		{
 			inputStruct{

internal/transform/asset_test.go

@@ -68,7 +68,7 @@ func makeAssetTestInput() (inputTransaction ingest.LedgerTransaction, err error)
 	inputEnvelope.Tx.SourceAccount = testAccount1
 
 	inputOperations := []xdr.Operation{
-		xdr.Operation{
+		{
 			SourceAccount: nil,
 			Body: xdr.OperationBody{
 				Type: xdr.OperationTypePayment,
@@ -79,7 +79,7 @@ func makeAssetTestInput() (inputTransaction ingest.LedgerTransaction, err error)
 				},
 			},
 		},
-		xdr.Operation{
+		{
 			SourceAccount: nil,
 			Body: xdr.OperationBody{
 				Type: xdr.OperationTypePayment,
@@ -99,15 +99,15 @@ func makeAssetTestInput() (inputTransaction ingest.LedgerTransaction, err error)
 
 func makeAssetTestOutput() (transformedAssets []AssetOutput) {
 	transformedAssets = []AssetOutput{
-		AssetOutput{
+		{
 			AssetCode:   "USDT",
 			AssetIssuer: "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA",
 			AssetType:   "credit_alphanum4",
 			AssetID:     1229977787683536144,
 
 			ID: -8205667356306085451,
 		},
-		AssetOutput{
+		{
 			AssetCode:   "",
 			AssetIssuer: "",
 			AssetType:   "native",

internal/transform/claimable_balance.go

@@ -33,7 +33,7 @@ func TransformClaimableBalance(ledgerChange ingest.Change, header xdr.LedgerHead
 	}
 	balanceID, err := xdr.MarshalHex(balanceEntry.BalanceId)
 	if err != nil {
-		return ClaimableBalanceOutput{}, fmt.Errorf("Invalid balanceId in op: %d", uint32(ledgerEntry.LastModifiedLedgerSeq))
+		return ClaimableBalanceOutput{}, fmt.Errorf("invalid balanceId in op: %d", uint32(ledgerEntry.LastModifiedLedgerSeq))
 	}
 	outputFlags := uint32(balanceEntry.Flags())
 	outputAsset, err := transformSingleAsset(balanceEntry.Asset)

internal/transform/config_setting.go

@@ -18,7 +18,7 @@ func TransformConfigSetting(ledgerChange ingest.Change, header xdr.LedgerHeaderH
 
 	configSetting, ok := ledgerEntry.Data.GetConfigSetting()
 	if !ok {
-		return ConfigSettingOutput{}, fmt.Errorf("Could not extract config setting from ledger entry; actual type is %s", ledgerEntry.Data.Type)
+		return ConfigSettingOutput{}, fmt.Errorf("could not extract config setting from ledger entry; actual type is %s", ledgerEntry.Data.Type)
 	}
 
 	configSettingId := configSetting.ConfigSettingId
@@ -48,7 +48,7 @@ func TransformConfigSetting(ledgerChange ingest.Change, header xdr.LedgerHeaderH
 	writeFee1KbBucketListHigh := contractLedgerCost.WriteFee1KbBucketListHigh
 	bucketListWriteFeeGrowthFactor := contractLedgerCost.BucketListWriteFeeGrowthFactor
 
-	contractHistoricalData, ok := configSetting.GetContractHistoricalData()
+	contractHistoricalData, _ := configSetting.GetContractHistoricalData()
 	feeHistorical1Kb := contractHistoricalData.FeeHistorical1Kb
 
 	contractMetaData, _ := configSetting.GetContractEvents()
@@ -66,9 +66,9 @@ func TransformConfigSetting(ledgerChange ingest.Change, header xdr.LedgerHeaderH
 	paramsMemBytes, _ := configSetting.GetContractCostParamsMemBytes()
 	contractCostParamsMemBytes := serializeParams(paramsMemBytes)
 
-	contractDataKeySizeBytes, ok := configSetting.GetContractDataKeySizeBytes()
+	contractDataKeySizeBytes, _ := configSetting.GetContractDataKeySizeBytes()
 
-	contractDataEntrySizeBytes, ok := configSetting.GetContractDataEntrySizeBytes()
+	contractDataEntrySizeBytes, _ := configSetting.GetContractDataEntrySizeBytes()
 
 	stateArchivalSettings, _ := configSetting.GetStateArchivalSettings()
 	maxEntryTtl := stateArchivalSettings.MaxEntryTtl