From 7b197727ba90af1d7e38d375e3d27f76d9ffb866 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 13:14:20 -0600 Subject: [PATCH 01/13] export cmd utils --- cmd/export_assets_test.go | 4 ++-- cmd/export_contract_events_test.go | 4 ++-- cmd/export_effects_test.go | 4 ++-- cmd/export_ledger_entry_changes_test.go | 4 ++-- cmd/export_ledger_transaction_test.go | 4 ++-- cmd/export_ledgers_test.go | 8 ++++---- cmd/export_operations_test.go | 4 ++-- cmd/export_trades_test.go | 4 ++-- cmd/export_transactions_test.go | 4 ++-- cmd/get_ledger_range_from_times_test.go | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/export_assets_test.go b/cmd/export_assets_test.go index 73ba6e3e..60fb870b 100644 --- a/cmd/export_assets_test.go +++ b/cmd/export_assets_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportAssets(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "assets from one ledger", args: []string{"export_assets", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_assets.txt")}, @@ -33,6 +33,6 @@ func TestExportAssets(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/assets/") + RunCLITest(t, test, "testdata/assets/") } } diff --git a/cmd/export_contract_events_test.go b/cmd/export_contract_events_test.go index 29674bd4..cfc091a6 100644 --- a/cmd/export_contract_events_test.go +++ b/cmd/export_contract_events_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportContractEvents(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "contract events from multiple ledger", args: []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", gotTestDir(t, "large_range_ledger_txs.txt")}, @@ -15,6 +15,6 @@ func TestExportContractEvents(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/contract_events/") + RunCLITest(t, test, "testdata/contract_events/") } } diff --git a/cmd/export_effects_test.go b/cmd/export_effects_test.go index 78a75ed7..671794d4 100644 --- a/cmd/export_effects_test.go +++ b/cmd/export_effects_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportEffects(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "effects from one ledger", args: []string{"export_effects", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_effects.txt")}, @@ -33,6 +33,6 @@ func TestExportEffects(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/effects/") + RunCLITest(t, test, "testdata/effects/") } } diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index a341b29c..24e108dd 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -10,7 +10,7 @@ const coreConfigPath = "/etl/docker/stellar-core.cfg" func TestExportChanges(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "unbounded range with no config", args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-s", "100000"}, @@ -96,6 +96,6 @@ func TestExportChanges(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/changes/") + RunCLITest(t, test, "testdata/changes/") } } diff --git a/cmd/export_ledger_transaction_test.go b/cmd/export_ledger_transaction_test.go index d87bc596..c0217e28 100644 --- a/cmd/export_ledger_transaction_test.go +++ b/cmd/export_ledger_transaction_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportLedgerTransaction(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "Transactions from one ledger", args: []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "ledger_transactions.txt")}, @@ -15,6 +15,6 @@ func TestExportLedgerTransaction(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/ledger_transactions/") + RunCLITest(t, test, "testdata/ledger_transactions/") } } diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index 9b759185..3524cccc 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -21,7 +21,7 @@ var executableName = "stellar-etl" var update = flag.Bool("update", false, "update the golden files of this test") var gotFolder = "testdata/got/" -type cliTest struct { +type CliTest struct { name string args []string golden string @@ -52,7 +52,7 @@ func gotTestDir(t *testing.T, filename string) string { } func TestExportLedger(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "end before start", args: []string{"export_ledgers", "-s", "100", "-e", "50"}, @@ -98,7 +98,7 @@ func TestExportLedger(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/ledgers/") + RunCLITest(t, test, "testdata/ledgers/") } } @@ -117,7 +117,7 @@ func sortByName(files []os.DirEntry) { }) } -func runCLITest(t *testing.T, test cliTest, goldenFolder string) { +func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { flag.Parse() t.Run(test.name, func(t *testing.T) { dir, err := os.Getwd() diff --git a/cmd/export_operations_test.go b/cmd/export_operations_test.go index d6fdcf06..bd810d06 100644 --- a/cmd/export_operations_test.go +++ b/cmd/export_operations_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportOperations(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "operations from one ledger", args: []string{"export_operations", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_ops.txt")}, @@ -33,6 +33,6 @@ func TestExportOperations(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/operations/") + RunCLITest(t, test, "testdata/operations/") } } diff --git a/cmd/export_trades_test.go b/cmd/export_trades_test.go index 14dce90d..3110f05c 100644 --- a/cmd/export_trades_test.go +++ b/cmd/export_trades_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportTrades(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "trades from one ledger", args: []string{"export_trades", "-s", "28770265", "-e", "28770265", "-o", gotTestDir(t, "one_ledger_trades.txt")}, @@ -33,6 +33,6 @@ func TestExportTrades(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/trades/") + RunCLITest(t, test, "testdata/trades/") } } diff --git a/cmd/export_transactions_test.go b/cmd/export_transactions_test.go index 30f976ba..1cd8416d 100644 --- a/cmd/export_transactions_test.go +++ b/cmd/export_transactions_test.go @@ -5,7 +5,7 @@ import ( ) func TestExportTransactions(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "transactions from one ledger", args: []string{"export_transactions", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_txs.txt")}, @@ -33,6 +33,6 @@ func TestExportTransactions(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/transactions/") + RunCLITest(t, test, "testdata/transactions/") } } diff --git a/cmd/get_ledger_range_from_times_test.go b/cmd/get_ledger_range_from_times_test.go index 35c51931..45b1bc48 100644 --- a/cmd/get_ledger_range_from_times_test.go +++ b/cmd/get_ledger_range_from_times_test.go @@ -6,7 +6,7 @@ import ( ) func TestConvertTimes(t *testing.T) { - tests := []cliTest{ + tests := []CliTest{ { name: "wrong date format", args: []string{"get_ledger_range_from_times", "-s", "2016 01 01 4:33", "-e", "2020 03 04 12:32"}, @@ -83,6 +83,6 @@ func TestConvertTimes(t *testing.T) { } for _, test := range tests { - runCLITest(t, test, "testdata/ranges/") + RunCLITest(t, test, "testdata/ranges/") } } From 00ab9790a0a0a764992e2dacfce30f98cffbce40 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 13:34:28 -0600 Subject: [PATCH 02/13] Move common functions to command utils --- cmd/command_utils.go | 153 +++++++++++++++++++++++++++++++++++++ cmd/export_ledgers_test.go | 151 ------------------------------------ 2 files changed, 153 insertions(+), 151 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 30c4ebc7..8bace4dc 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -3,15 +3,27 @@ package cmd import ( "bytes" "encoding/json" + "flag" "fmt" + "io" + "log" "os" + "os/exec" + "path" "path/filepath" + "sort" + "strings" + "testing" "github.com/stellar/stellar-etl/internal/transform" + "github.com/stretchr/testify/assert" "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/writer" ) +var executableName = "stellar-etl" +var update = flag.Bool("update", false, "update the golden files of this test") + type CloudStorage interface { UploadTo(credentialsPath, bucket, path string) error } @@ -178,3 +190,144 @@ func writeParquet(data []transform.SchemaParquet, path string, schema interface{ } } } + +type CliTest struct { + name string + args []string + golden string + wantErr error + sortForComparison bool +} + +func indexOf(l []string, s string) int { + for idx, e := range l { + if e == s { + return idx + } + } + return -1 +} + +func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { + flag.Parse() + t.Run(test.name, func(t *testing.T) { + dir, err := os.Getwd() + assert.NoError(t, err) + + idxOfOutputArg := indexOf(test.args, "-o") + var testOutput []byte + var outLocation string + var stat os.FileInfo + if idxOfOutputArg > -1 { + outLocation = test.args[idxOfOutputArg+1] + _, err = os.Stat(outLocation) + if err != nil { + // Check if the error is due to the file not existing + if !os.IsNotExist(err) { + assert.NoError(t, err) + } + } else { + err = deleteLocalFiles(outLocation) + if err != nil { + log.Fatal(err) + } + } + } + + cmd := exec.Command(path.Join(dir, executableName), test.args...) + errOut, actualError := cmd.CombinedOutput() + if idxOfOutputArg > -1 { + stat, err = os.Stat(outLocation) + assert.NoError(t, err) + + if stat.IsDir() { + files, err := os.ReadDir(outLocation) + if err != nil { + log.Fatal(err) + } + var buf bytes.Buffer + sortByName(files) + for _, f := range files { + b, err := os.ReadFile(filepath.Join(outLocation, f.Name())) + if err != nil { + log.Fatal(err) + } + buf.Write(b) + } + testOutput = buf.Bytes() + } else { + // If the output is written to a file, read the contents of the file for comparison. + testOutput, err = os.ReadFile(outLocation) + if err != nil { + log.Fatal(err) + } + } + } + + // Since the CLI uses a logger to report errors, the final error message isn't the same as the errors thrown in code. + // Instead, it's wrapped in other os/system errors + // By reading the error text from the logger, we can extract the lower level error that the user would see + if test.golden == "" { + errorMsg := fmt.Errorf(extractErrorMsg(string(errOut))) + assert.Equal(t, test.wantErr, errorMsg) + return + } + + assert.Equal(t, test.wantErr, actualError) + actualString := string(testOutput) + if test.sortForComparison { + trimmed := strings.Trim(actualString, "\n") + lines := strings.Split(trimmed, "\n") + sort.Strings(lines) + actualString = strings.Join(lines, "\n") + actualString = fmt.Sprintf("%s\n", actualString) + } + + wantString, err := getGolden(t, goldenFolder+test.golden, actualString, *update) + assert.NoError(t, err) + assert.Equal(t, wantString, actualString) + }) +} + +func extractErrorMsg(loggerOutput string) string { + errIndex := strings.Index(loggerOutput, "msg=") + 5 + endIndex := strings.Index(loggerOutput[errIndex:], "\"") + return loggerOutput[errIndex : errIndex+endIndex] +} + +func getGolden(t *testing.T, goldenFile string, actual string, update bool) (string, error) { + t.Helper() + f, err := os.OpenFile(goldenFile, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return "", err + } + defer f.Close() + + // If the update flag is true, clear the current contents of the golden file and write the actual output + // This is useful for when new tests or added or functionality changes that breaks current tests + if update { + err := os.Truncate(goldenFile, 0) + if err != nil { + return "", err + } + + _, err = f.WriteString(actual) + if err != nil { + return "", err + } + return actual, nil + } + + wantOutput, err := io.ReadAll(f) + if err != nil { + return "", err + } + + return string(wantOutput), nil +} + +func sortByName(files []os.DirEntry) { + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) +} diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index 3524cccc..cf9f83be 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -1,34 +1,16 @@ package cmd import ( - "bytes" "flag" "fmt" - "io" - "log" "os" "os/exec" - "path" "path/filepath" - "sort" - "strings" "testing" - - "github.com/stretchr/testify/assert" ) -var executableName = "stellar-etl" -var update = flag.Bool("update", false, "update the golden files of this test") var gotFolder = "testdata/got/" -type CliTest struct { - name string - args []string - golden string - wantErr error - sortForComparison bool -} - func TestMain(m *testing.M) { if err := os.Chdir(".."); err != nil { cmdLogger.Error("could not change directory", err) @@ -101,136 +83,3 @@ func TestExportLedger(t *testing.T) { RunCLITest(t, test, "testdata/ledgers/") } } - -func indexOf(l []string, s string) int { - for idx, e := range l { - if e == s { - return idx - } - } - return -1 -} - -func sortByName(files []os.DirEntry) { - sort.Slice(files, func(i, j int) bool { - return files[i].Name() < files[j].Name() - }) -} - -func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { - flag.Parse() - t.Run(test.name, func(t *testing.T) { - dir, err := os.Getwd() - assert.NoError(t, err) - - idxOfOutputArg := indexOf(test.args, "-o") - var testOutput []byte - var outLocation string - var stat os.FileInfo - if idxOfOutputArg > -1 { - outLocation = test.args[idxOfOutputArg+1] - _, err = os.Stat(outLocation) - if err != nil { - // Check if the error is due to the file not existing - if !os.IsNotExist(err) { - assert.NoError(t, err) - } - } else { - err = deleteLocalFiles(outLocation) - if err != nil { - log.Fatal(err) - } - } - } - - cmd := exec.Command(path.Join(dir, executableName), test.args...) - errOut, actualError := cmd.CombinedOutput() - if idxOfOutputArg > -1 { - stat, err = os.Stat(outLocation) - assert.NoError(t, err) - - if stat.IsDir() { - files, err := os.ReadDir(outLocation) - if err != nil { - log.Fatal(err) - } - var buf bytes.Buffer - sortByName(files) - for _, f := range files { - b, err := os.ReadFile(filepath.Join(outLocation, f.Name())) - if err != nil { - log.Fatal(err) - } - buf.Write(b) - } - testOutput = buf.Bytes() - } else { - // If the output is written to a file, read the contents of the file for comparison. - testOutput, err = os.ReadFile(outLocation) - if err != nil { - log.Fatal(err) - } - } - } - - // Since the CLI uses a logger to report errors, the final error message isn't the same as the errors thrown in code. - // Instead, it's wrapped in other os/system errors - // By reading the error text from the logger, we can extract the lower level error that the user would see - if test.golden == "" { - errorMsg := fmt.Errorf(extractErrorMsg(string(errOut))) - assert.Equal(t, test.wantErr, errorMsg) - return - } - - assert.Equal(t, test.wantErr, actualError) - actualString := string(testOutput) - if test.sortForComparison { - trimmed := strings.Trim(actualString, "\n") - lines := strings.Split(trimmed, "\n") - sort.Strings(lines) - actualString = strings.Join(lines, "\n") - actualString = fmt.Sprintf("%s\n", actualString) - } - - wantString, err := getGolden(t, goldenFolder+test.golden, actualString, *update) - assert.NoError(t, err) - assert.Equal(t, wantString, actualString) - }) -} - -func extractErrorMsg(loggerOutput string) string { - errIndex := strings.Index(loggerOutput, "msg=") + 5 - endIndex := strings.Index(loggerOutput[errIndex:], "\"") - return loggerOutput[errIndex : errIndex+endIndex] -} - -func getGolden(t *testing.T, goldenFile string, actual string, update bool) (string, error) { - t.Helper() - f, err := os.OpenFile(goldenFile, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return "", err - } - defer f.Close() - - // If the update flag is true, clear the current contents of the golden file and write the actual output - // This is useful for when new tests or added or functionality changes that breaks current tests - if update { - err := os.Truncate(goldenFile, 0) - if err != nil { - return "", err - } - - _, err = f.WriteString(actual) - if err != nil { - return "", err - } - return actual, nil - } - - wantOutput, err := io.ReadAll(f) - if err != nil { - return "", err - } - - return string(wantOutput), nil -} From dcae12f6aa1d822479dd1a58379009acb9ccab3b Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 13:47:35 -0600 Subject: [PATCH 03/13] export more funcs --- cmd/command_utils.go | 12 ++++++------ cmd/export_assets.go | 12 ++++++------ cmd/export_contract_events.go | 12 ++++++------ cmd/export_effects.go | 12 ++++++------ cmd/export_ledger_entry_changes.go | 10 +++++----- cmd/export_ledger_transaction.go | 8 ++++---- cmd/export_ledgers.go | 12 ++++++------ cmd/export_operations.go | 12 ++++++------ cmd/export_trades.go | 12 ++++++------ cmd/export_transactions.go | 12 ++++++------ cmd/get_ledger_range_from_times.go | 2 +- 11 files changed, 58 insertions(+), 58 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 8bace4dc..582a0bbb 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -40,7 +40,7 @@ func createOutputFile(filepath string) error { return nil } -func mustOutFile(path string) *os.File { +func MustOutFile(path string) *os.File { absolutePath, err := filepath.Abs(path) if err != nil { cmdLogger.Fatal("could not get absolute filepath: ", err) @@ -64,7 +64,7 @@ func mustOutFile(path string) *os.File { return outFile } -func exportEntry(entry interface{}, outFile *os.File, extra map[string]string) (int, error) { +func ExportEntry(entry interface{}, outFile *os.File, extra map[string]string) (int, error) { // This extra marshalling/unmarshalling is silly, but it's required to properly handle the null.[String|Int*] types, and add the extra fields. m, err := json.Marshal(entry) if err != nil { @@ -99,7 +99,7 @@ func exportEntry(entry interface{}, outFile *os.File, extra map[string]string) ( } // Prints the number of attempted, failed, and successful transformations as a JSON object -func printTransformStats(attempts, failures int) { +func PrintTransformStats(attempts, failures int) { resultsMap := map[string]int{ "attempted_transforms": attempts, "failed_transforms": failures, @@ -132,7 +132,7 @@ func deleteLocalFiles(path string) error { return nil } -func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path string) { +func MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path string) { if cloudProvider == "" { cmdLogger.Info("No cloud provider specified for upload. Skipping upload.") return @@ -157,7 +157,7 @@ func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path strin } } -// writeParquet creates the parquet file and writes the exported data into it. +// WriteParquet creates the parquet file and writes the exported data into it. // // Parameters: // @@ -171,7 +171,7 @@ func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path strin // Errors: // // stellar-etl will log a Fatal error and stop in the case it cannot create or write to the parquet file -func writeParquet(data []transform.SchemaParquet, path string, schema interface{}) { +func WriteParquet(data []transform.SchemaParquet, path string, schema interface{}) { parquetFile, err := local.NewLocalFileWriter(path) if err != nil { cmdLogger.Fatal("could not create parquet file: ", err) diff --git a/cmd/export_assets.go b/cmd/export_assets.go index b24d8b66..79cc7f7a 100644 --- a/cmd/export_assets.go +++ b/cmd/export_assets.go @@ -22,7 +22,7 @@ var assetsCmd = &cobra.Command{ cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) env := utils.GetEnvironmentDetails(commonArgs) - outFile := mustOutFile(path) + outFile := MustOutFile(path) var paymentOps []input.AssetTransformInput var err error @@ -56,7 +56,7 @@ var assetsCmd = &cobra.Command{ } seenIDs[transformed.AssetID] = true - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(err) numFailures += 1 @@ -72,13 +72,13 @@ var assetsCmd = &cobra.Command{ outFile.Close() cmdLogger.Infof("%d bytes written to %s", totalNumBytes, outFile.Name()) - printTransformStats(len(paymentOps), numFailures) + PrintTransformStats(len(paymentOps), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - writeParquet(transformedAssets, parquetPath, new(transform.AssetOutputParquet)) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedAssets, parquetPath, new(transform.AssetOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) } }, } diff --git a/cmd/export_contract_events.go b/cmd/export_contract_events.go index c5ddf6e2..759172dd 100644 --- a/cmd/export_contract_events.go +++ b/cmd/export_contract_events.go @@ -27,7 +27,7 @@ var contractEventsCmd = &cobra.Command{ cmdLogger.Fatal("could not read transactions: ", err) } - outFile := mustOutFile(cmdArgs.Path) + outFile := MustOutFile(cmdArgs.Path) numFailures := 0 var transformedEvents []transform.SchemaParquet for _, transformInput := range transactions { @@ -40,7 +40,7 @@ var contractEventsCmd = &cobra.Command{ } for _, contractEvent := range transformed { - _, err := exportEntry(contractEvent, outFile, cmdArgs.Extra) + _, err := ExportEntry(contractEvent, outFile, cmdArgs.Extra) if err != nil { cmdLogger.LogError(fmt.Errorf("could not export contract event: %v", err)) numFailures += 1 @@ -56,13 +56,13 @@ var contractEventsCmd = &cobra.Command{ outFile.Close() - printTransformStats(len(transactions), numFailures) + PrintTransformStats(len(transactions), numFailures) - maybeUpload(cmdArgs.Credentials, cmdArgs.Bucket, cmdArgs.Provider, cmdArgs.Path) + MaybeUpload(cmdArgs.Credentials, cmdArgs.Bucket, cmdArgs.Provider, cmdArgs.Path) if commonArgs.WriteParquet { - writeParquet(transformedEvents, cmdArgs.ParquetPath, new(transform.ContractEventOutputParquet)) - maybeUpload(cmdArgs.Credentials, cmdArgs.Bucket, cmdArgs.Provider, cmdArgs.ParquetPath) + WriteParquet(transformedEvents, cmdArgs.ParquetPath, new(transform.ContractEventOutputParquet)) + MaybeUpload(cmdArgs.Credentials, cmdArgs.Bucket, cmdArgs.Provider, cmdArgs.ParquetPath) } }, diff --git a/cmd/export_effects.go b/cmd/export_effects.go index be372248..345f3145 100644 --- a/cmd/export_effects.go +++ b/cmd/export_effects.go @@ -27,7 +27,7 @@ var effectsCmd = &cobra.Command{ cmdLogger.Fatalf("could not read transactions in [%d, %d] (limit=%d): %v", startNum, commonArgs.EndNum, limit, err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 var transformedEffects []transform.SchemaParquet @@ -42,7 +42,7 @@ var effectsCmd = &cobra.Command{ } for _, transformed := range effects { - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(err) numFailures += 1 @@ -59,13 +59,13 @@ var effectsCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(transactions), numFailures) + PrintTransformStats(len(transactions), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - writeParquet(transformedEffects, parquetPath, new(transform.EffectOutputParquet)) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedEffects, parquetPath, new(transform.EffectOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) } }, } diff --git a/cmd/export_ledger_entry_changes.go b/cmd/export_ledger_entry_changes.go index 6779801f..eb537ce7 100644 --- a/cmd/export_ledger_entry_changes.go +++ b/cmd/export_ledger_entry_changes.go @@ -295,12 +295,12 @@ func exportTransformedData( // is included in this filename. path := filepath.Join(folderPath, exportFilename(start, end+1, resource)) parquetPath := filepath.Join(parquetFolderPath, exportParquetFilename(start, end+1, resource)) - outFile := mustOutFile(path) + outFile := MustOutFile(path) var transformedResource []transform.SchemaParquet var parquetSchema interface{} var skip bool for _, o := range output { - _, err := exportEntry(o, outFile, extra) + _, err := ExportEntry(o, outFile, extra) if err != nil { return err } @@ -352,11 +352,11 @@ func exportTransformedData( } } - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if !skip && WriteParquet { - writeParquet(transformedResource, parquetPath, parquetSchema) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedResource, parquetPath, parquetSchema) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) } } diff --git a/cmd/export_ledger_transaction.go b/cmd/export_ledger_transaction.go index 92e99046..14753ed1 100644 --- a/cmd/export_ledger_transaction.go +++ b/cmd/export_ledger_transaction.go @@ -27,7 +27,7 @@ var ledgerTransactionCmd = &cobra.Command{ cmdLogger.Fatal("could not read ledger_transaction: ", err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 for _, transformInput := range ledgerTransaction { @@ -39,7 +39,7 @@ var ledgerTransactionCmd = &cobra.Command{ continue } - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(fmt.Errorf("could not export transaction: %v", err)) numFailures += 1 @@ -51,9 +51,9 @@ var ledgerTransactionCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(ledgerTransaction), numFailures) + PrintTransformStats(len(ledgerTransaction), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) }, } diff --git a/cmd/export_ledgers.go b/cmd/export_ledgers.go index bb3b0aa5..7f4a5e46 100644 --- a/cmd/export_ledgers.go +++ b/cmd/export_ledgers.go @@ -34,7 +34,7 @@ var ledgersCmd = &cobra.Command{ cmdLogger.Fatal("could not read ledgers: ", err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 @@ -47,7 +47,7 @@ var ledgersCmd = &cobra.Command{ continue } - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(fmt.Errorf("could not export ledger %d: %s", startNum+uint32(i), err)) numFailures += 1 @@ -63,13 +63,13 @@ var ledgersCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(ledgers), numFailures) + PrintTransformStats(len(ledgers), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) - writeParquet(transformedLedgers, parquetPath, new(transform.LedgerOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedLedgers, parquetPath, new(transform.LedgerOutputParquet)) } }, } diff --git a/cmd/export_operations.go b/cmd/export_operations.go index 8c286dbd..06547962 100644 --- a/cmd/export_operations.go +++ b/cmd/export_operations.go @@ -27,7 +27,7 @@ var operationsCmd = &cobra.Command{ cmdLogger.Fatal("could not read operations: ", err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 var transformedOps []transform.SchemaParquet @@ -40,7 +40,7 @@ var operationsCmd = &cobra.Command{ continue } - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(fmt.Errorf("could not export operation: %v", err)) numFailures += 1 @@ -56,13 +56,13 @@ var operationsCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(operations), numFailures) + PrintTransformStats(len(operations), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) - writeParquet(transformedOps, parquetPath, new(transform.OperationOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedOps, parquetPath, new(transform.OperationOutputParquet)) } }, } diff --git a/cmd/export_trades.go b/cmd/export_trades.go index d6f08fd0..0b85c5a7 100644 --- a/cmd/export_trades.go +++ b/cmd/export_trades.go @@ -30,7 +30,7 @@ var tradesCmd = &cobra.Command{ cmdLogger.Fatal("could not read trades ", err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 var transformedTrades []transform.SchemaParquet @@ -44,7 +44,7 @@ var tradesCmd = &cobra.Command{ } for _, transformed := range trades { - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(err) numFailures += 1 @@ -61,13 +61,13 @@ var tradesCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(trades), numFailures) + PrintTransformStats(len(trades), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) - writeParquet(transformedTrades, parquetPath, new(transform.TradeOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedTrades, parquetPath, new(transform.TradeOutputParquet)) } }, } diff --git a/cmd/export_transactions.go b/cmd/export_transactions.go index cfb6f52e..c2750dd3 100644 --- a/cmd/export_transactions.go +++ b/cmd/export_transactions.go @@ -27,7 +27,7 @@ var transactionsCmd = &cobra.Command{ cmdLogger.Fatal("could not read transactions: ", err) } - outFile := mustOutFile(path) + outFile := MustOutFile(path) numFailures := 0 totalNumBytes := 0 var transformedTransaction []transform.SchemaParquet @@ -40,7 +40,7 @@ var transactionsCmd = &cobra.Command{ continue } - numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra) + numBytes, err := ExportEntry(transformed, outFile, commonArgs.Extra) if err != nil { cmdLogger.LogError(fmt.Errorf("could not export transaction: %v", err)) numFailures += 1 @@ -56,13 +56,13 @@ var transactionsCmd = &cobra.Command{ outFile.Close() cmdLogger.Info("Number of bytes written: ", totalNumBytes) - printTransformStats(len(transactions), numFailures) + PrintTransformStats(len(transactions), numFailures) - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) if commonArgs.WriteParquet { - maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) - writeParquet(transformedTransaction, parquetPath, new(transform.TransactionOutputParquet)) + MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) + WriteParquet(transformedTransaction, parquetPath, new(transform.TransactionOutputParquet)) } }, } diff --git a/cmd/get_ledger_range_from_times.go b/cmd/get_ledger_range_from_times.go index 07cc5041..01d10355 100644 --- a/cmd/get_ledger_range_from_times.go +++ b/cmd/get_ledger_range_from_times.go @@ -72,7 +72,7 @@ var getLedgerRangeFromTimesCmd = &cobra.Command{ } if path != "" { - outFile := mustOutFile(path) + outFile := MustOutFile(path) outFile.Write(marshalled) outFile.WriteString("\n") outFile.Close() From 323425d741f65e05ea5384a6f2701e7bd0b0784a Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 13:51:46 -0600 Subject: [PATCH 04/13] Export more funcs --- cmd/command_utils.go | 5 +++++ cmd/export_assets_test.go | 8 ++++---- cmd/export_contract_events_test.go | 2 +- cmd/export_effects_test.go | 8 ++++---- cmd/export_ledger_entry_changes_test.go | 20 ++++++++++---------- cmd/export_ledger_transaction_test.go | 2 +- cmd/export_ledgers_test.go | 15 ++++----------- cmd/export_operations_test.go | 8 ++++---- cmd/export_trades_test.go | 8 ++++---- cmd/export_transactions_test.go | 8 ++++---- cmd/get_ledger_range_from_times_test.go | 16 ++++++++-------- 11 files changed, 49 insertions(+), 51 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 582a0bbb..12dc185f 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -23,6 +23,7 @@ import ( var executableName = "stellar-etl" var update = flag.Bool("update", false, "update the golden files of this test") +var gotFolder = "testdata/got/" type CloudStorage interface { UploadTo(credentialsPath, bucket, path string) error @@ -331,3 +332,7 @@ func sortByName(files []os.DirEntry) { return files[i].Name() < files[j].Name() }) } + +func GotTestDir(t *testing.T, filename string) string { + return filepath.Join(gotFolder, t.Name(), filename) +} diff --git a/cmd/export_assets_test.go b/cmd/export_assets_test.go index 60fb870b..e6cc4c35 100644 --- a/cmd/export_assets_test.go +++ b/cmd/export_assets_test.go @@ -8,25 +8,25 @@ func TestExportAssets(t *testing.T) { tests := []CliTest{ { name: "assets from one ledger", - args: []string{"export_assets", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_assets.txt")}, + args: []string{"export_assets", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_assets.txt")}, golden: "one_ledger_assets.golden", wantErr: nil, }, { name: "assets from 10 ledgers", - args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-o", gotTestDir(t, "10_ledgers_assets.txt")}, + args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_assets.txt")}, golden: "10_ledgers_assets.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", gotTestDir(t, "large_range_assets.txt")}, + args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_assets.txt")}, golden: "large_range_assets.golden", wantErr: nil, }, { name: "ledger with no assets", - args: []string{"export_assets", "-s", "10363513", "-e", "10363513", "-o", gotTestDir(t, "ledger_no_assets.txt")}, + args: []string{"export_assets", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_assets.txt")}, golden: "ledger_no_assets.golden", wantErr: nil, }, diff --git a/cmd/export_contract_events_test.go b/cmd/export_contract_events_test.go index cfc091a6..f2dc2364 100644 --- a/cmd/export_contract_events_test.go +++ b/cmd/export_contract_events_test.go @@ -8,7 +8,7 @@ func TestExportContractEvents(t *testing.T) { tests := []CliTest{ { name: "contract events from multiple ledger", - args: []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", gotTestDir(t, "large_range_ledger_txs.txt")}, + args: []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", GotTestDir(t, "large_range_ledger_txs.txt")}, golden: "large_range_ledger_txs.golden", wantErr: nil, }, diff --git a/cmd/export_effects_test.go b/cmd/export_effects_test.go index 671794d4..3e55dbae 100644 --- a/cmd/export_effects_test.go +++ b/cmd/export_effects_test.go @@ -8,25 +8,25 @@ func TestExportEffects(t *testing.T) { tests := []CliTest{ { name: "effects from one ledger", - args: []string{"export_effects", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_effects.txt")}, + args: []string{"export_effects", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_effects.txt")}, golden: "one_ledger_effects.golden", wantErr: nil, }, { name: "effects from 10 ledgers", - args: []string{"export_effects", "-s", "30822015", "-e", "30822025", "-o", gotTestDir(t, "10_ledgers_effects.txt")}, + args: []string{"export_effects", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_effects.txt")}, golden: "10_ledgers_effects.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_effects", "-s", "25820678", "-e", "25821678", "-l", "5", "-o", gotTestDir(t, "large_range_effects.txt")}, + args: []string{"export_effects", "-s", "25820678", "-e", "25821678", "-l", "5", "-o", GotTestDir(t, "large_range_effects.txt")}, golden: "large_range_effects.golden", wantErr: nil, }, { name: "ledger with no effects", - args: []string{"export_effects", "-s", "10363513", "-e", "10363513", "-o", gotTestDir(t, "ledger_no_effects.txt")}, + args: []string{"export_effects", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_effects.txt")}, golden: "ledger_no_effects.golden", wantErr: nil, }, diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index 24e108dd..f8da3ce2 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -25,70 +25,70 @@ func TestExportChanges(t *testing.T) { }, { name: "All changes from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", gotTestDir(t, "all/")}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", GotTestDir(t, "all/")}, golden: "all.golden", wantErr: nil, sortForComparison: true, }, { name: "Account changes from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "accounts/"), "--export-accounts", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "accounts/"), "--export-accounts", "true"}, golden: "accounts.golden", wantErr: nil, sortForComparison: true, }, { name: "Claimable balance from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "claimable_balances/"), "--export-balances", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "claimable_balances/"), "--export-balances", "true"}, golden: "claimable_balances.golden", wantErr: nil, sortForComparison: true, }, { name: "trustlines from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "trustlines/"), "--export-trustlines", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "trustlines/"), "--export-trustlines", "true"}, golden: "trustlines.golden", wantErr: nil, sortForComparison: true, }, { name: "Offers from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "offers/"), "--export-offers", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "offers/"), "--export-offers", "true"}, golden: "offers.golden", wantErr: nil, sortForComparison: true, }, { name: "Pools from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "pools/"), "--export-pools", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "pools/"), "--export-pools", "true"}, golden: "pools.golden", wantErr: nil, sortForComparison: true, }, { name: "Contract code from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", gotTestDir(t, "contract_code/"), "--export-contract-code", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", GotTestDir(t, "contract_code/"), "--export-contract-code", "true"}, golden: "contract_code.golden", wantErr: nil, sortForComparison: true, }, { name: "Contract data from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", gotTestDir(t, "contract_data/"), "--export-contract-data", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", GotTestDir(t, "contract_data/"), "--export-contract-data", "true"}, golden: "contract_data.golden", wantErr: nil, sortForComparison: true, }, { name: "Config setting from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", gotTestDir(t, "config_setting/"), "--export-config-settings", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", GotTestDir(t, "config_setting/"), "--export-config-settings", "true"}, golden: "config_setting.golden", wantErr: nil, sortForComparison: true, }, { name: "ttl from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", gotTestDir(t, "ttl/"), "--export-ttl", "true"}, + args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", GotTestDir(t, "ttl/"), "--export-ttl", "true"}, golden: "ttl.golden", wantErr: nil, sortForComparison: true, diff --git a/cmd/export_ledger_transaction_test.go b/cmd/export_ledger_transaction_test.go index c0217e28..d4f66585 100644 --- a/cmd/export_ledger_transaction_test.go +++ b/cmd/export_ledger_transaction_test.go @@ -8,7 +8,7 @@ func TestExportLedgerTransaction(t *testing.T) { tests := []CliTest{ { name: "Transactions from one ledger", - args: []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "ledger_transactions.txt")}, + args: []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "ledger_transactions.txt")}, golden: "ledger_transactions.golden", wantErr: nil, }, diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index cf9f83be..f3096d75 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -5,12 +5,9 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "testing" ) -var gotFolder = "testdata/got/" - func TestMain(m *testing.M) { if err := os.Chdir(".."); err != nil { cmdLogger.Error("could not change directory", err) @@ -29,10 +26,6 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } -func gotTestDir(t *testing.T, filename string) string { - return filepath.Join(gotFolder, t.Name(), filename) -} - func TestExportLedger(t *testing.T) { tests := []CliTest{ { @@ -55,25 +48,25 @@ func TestExportLedger(t *testing.T) { }, { name: "single ledger", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822015", "-o", gotTestDir(t, "single_ledger.txt")}, + args: []string{"export_ledgers", "-s", "30822015", "-e", "30822015", "-o", GotTestDir(t, "single_ledger.txt")}, golden: "single_ledger.golden", wantErr: nil, }, { name: "10 ledgers", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-o", gotTestDir(t, "10_ledgers.txt")}, + args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers.txt")}, golden: "10_ledgers.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", gotTestDir(t, "large_range_ledgers.txt")}, + args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ledgers.txt")}, golden: "large_range_ledgers.golden", wantErr: nil, }, { name: "range from 2024", - args: []string{"export_ledgers", "-s", "52929555", "-e", "52929960", "-o", gotTestDir(t, "2024_ledgers.txt")}, + args: []string{"export_ledgers", "-s", "52929555", "-e", "52929960", "-o", GotTestDir(t, "2024_ledgers.txt")}, golden: "2024_ledgers.golden", wantErr: nil, }, diff --git a/cmd/export_operations_test.go b/cmd/export_operations_test.go index bd810d06..87b8637b 100644 --- a/cmd/export_operations_test.go +++ b/cmd/export_operations_test.go @@ -8,25 +8,25 @@ func TestExportOperations(t *testing.T) { tests := []CliTest{ { name: "operations from one ledger", - args: []string{"export_operations", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_ops.txt")}, + args: []string{"export_operations", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_ops.txt")}, golden: "one_ledger_ops.golden", wantErr: nil, }, { name: "operations from 10 ledgers", - args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-o", gotTestDir(t, "10_ledgers_ops.txt")}, + args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_ops.txt")}, golden: "10_ledgers_ops.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", gotTestDir(t, "large_range_ops.txt")}, + args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ops.txt")}, golden: "large_range_ops.golden", wantErr: nil, }, { name: "ledger with no operations", - args: []string{"export_operations", "-s", "10363513", "-e", "10363513", "-o", gotTestDir(t, "ledger_no_ops.txt")}, + args: []string{"export_operations", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_ops.txt")}, golden: "ledger_no_ops.golden", wantErr: nil, }, diff --git a/cmd/export_trades_test.go b/cmd/export_trades_test.go index 3110f05c..4788dc8c 100644 --- a/cmd/export_trades_test.go +++ b/cmd/export_trades_test.go @@ -8,25 +8,25 @@ func TestExportTrades(t *testing.T) { tests := []CliTest{ { name: "trades from one ledger", - args: []string{"export_trades", "-s", "28770265", "-e", "28770265", "-o", gotTestDir(t, "one_ledger_trades.txt")}, + args: []string{"export_trades", "-s", "28770265", "-e", "28770265", "-o", GotTestDir(t, "one_ledger_trades.txt")}, golden: "one_ledger_trades.golden", wantErr: nil, }, { name: "trades from 10 ledgers", - args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-o", gotTestDir(t, "10_ledgers_trades.txt")}, + args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-o", GotTestDir(t, "10_ledgers_trades.txt")}, golden: "10_ledgers_trades.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-l", "5", "-o", gotTestDir(t, "large_range_trades.txt")}, + args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-l", "5", "-o", GotTestDir(t, "large_range_trades.txt")}, golden: "large_range_trades.golden", wantErr: nil, }, { name: "ledger with no trades", - args: []string{"export_trades", "-s", "10363513", "-e", "10363513", "-o", gotTestDir(t, "ledger_no_trades.txt")}, + args: []string{"export_trades", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_trades.txt")}, golden: "ledger_no_trades.golden", wantErr: nil, }, diff --git a/cmd/export_transactions_test.go b/cmd/export_transactions_test.go index 1cd8416d..ce7214b7 100644 --- a/cmd/export_transactions_test.go +++ b/cmd/export_transactions_test.go @@ -8,25 +8,25 @@ func TestExportTransactions(t *testing.T) { tests := []CliTest{ { name: "transactions from one ledger", - args: []string{"export_transactions", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "one_ledger_txs.txt")}, + args: []string{"export_transactions", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_txs.txt")}, golden: "one_ledger_txs.golden", wantErr: nil, }, { name: "transactions from 10 ledgers", - args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-o", gotTestDir(t, "10_ledgers_txs.txt")}, + args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_txs.txt")}, golden: "10_ledgers_txs.golden", wantErr: nil, }, { name: "range too large", - args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", gotTestDir(t, "large_range_txs.txt")}, + args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_txs.txt")}, golden: "large_range_txs.golden", wantErr: nil, }, { name: "ledger with no transactions", - args: []string{"export_transactions", "-s", "10363513", "-e", "10363513", "-o", gotTestDir(t, "ledger_no_txs.txt")}, + args: []string{"export_transactions", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_txs.txt")}, golden: "ledger_no_txs.golden", wantErr: nil, }, diff --git a/cmd/get_ledger_range_from_times_test.go b/cmd/get_ledger_range_from_times_test.go index 45b1bc48..6234a5e6 100644 --- a/cmd/get_ledger_range_from_times_test.go +++ b/cmd/get_ledger_range_from_times_test.go @@ -15,13 +15,13 @@ func TestConvertTimes(t *testing.T) { }, { name: "normal range", - args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", gotTestDir(t, "normal_range.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "normal_range.txt")}, golden: "normal_range.golden", wantErr: nil, }, { name: "start too early", - args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", gotTestDir(t, "early_start.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "early_start.txt")}, golden: "early_start.golden", wantErr: nil, }, @@ -46,37 +46,37 @@ func TestConvertTimes(t *testing.T) { // }, { name: "end too early", - args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T12:14:32+04:00", "-e", "2006-11-10T12:14:32+04:00", "-o", gotTestDir(t, "early_end.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T12:14:32+04:00", "-e", "2006-11-10T12:14:32+04:00", "-o", GotTestDir(t, "early_end.txt")}, golden: "early_end.golden", wantErr: nil, }, { name: "same date", - args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:03:37-05:00", "-e", "2016-11-10T18:03:37-05:00", "-o", gotTestDir(t, "same_date.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:03:37-05:00", "-e", "2016-11-10T18:03:37-05:00", "-o", GotTestDir(t, "same_date.txt")}, golden: "same_date.golden", wantErr: nil, }, { name: "checkpoint range (22343680-22343743)", - args: []string{"get_ledger_range_from_times", "-s", "2019-02-06T09:14:43+00:00", "-e", "2019-02-06T09:20:23+00:00", "-o", gotTestDir(t, "checkpoint_range.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2019-02-06T09:14:43+00:00", "-e", "2019-02-06T09:20:23+00:00", "-o", GotTestDir(t, "checkpoint_range.txt")}, golden: "checkpoint_range.golden", wantErr: nil, }, { name: "checkpoint range (9341-9401)", - args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T06:20:00+00:00", "-e", "2015-10-01T06:25:00+00:00", "-o", gotTestDir(t, "checkpoint_range2.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T06:20:00+00:00", "-e", "2015-10-01T06:25:00+00:00", "-o", GotTestDir(t, "checkpoint_range2.txt")}, golden: "checkpoint_range2.golden", wantErr: nil, }, { name: "infinite loop; checkpoint range (14558-14606)", - args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T13:35:00+00:00", "-e", "2015-10-01T13:40:00+00:00", "-o", gotTestDir(t, "checkpoint_range3.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T13:35:00+00:00", "-e", "2015-10-01T13:40:00+00:00", "-o", GotTestDir(t, "checkpoint_range3.txt")}, golden: "checkpoint_range3.golden", wantErr: nil, }, { name: "early checkpoint range (3-3)", - args: []string{"get_ledger_range_from_times", "-s", "2015-09-30T16:50:00+00:00", "-e", "2015-09-30T16:55:00+00:00", "-o", gotTestDir(t, "early_checkpoint_range.txt")}, + args: []string{"get_ledger_range_from_times", "-s", "2015-09-30T16:50:00+00:00", "-e", "2015-09-30T16:55:00+00:00", "-o", GotTestDir(t, "early_checkpoint_range.txt")}, golden: "early_checkpoint_range.golden", wantErr: nil, }, From 1efbbe70be94a34ad5fd093ea4d7e496e71be6ed Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 14:05:34 -0600 Subject: [PATCH 05/13] update --- cmd/export_ledger_entry_changes.go | 6 +++--- internal/utils/main.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/export_ledger_entry_changes.go b/cmd/export_ledger_entry_changes.go index eb537ce7..df75d428 100644 --- a/cmd/export_ledger_entry_changes.go +++ b/cmd/export_ledger_entry_changes.go @@ -287,7 +287,7 @@ func exportTransformedData( transformedOutput map[string][]interface{}, cloudCredentials, cloudStorageBucket, cloudProvider string, extra map[string]string, - WriteParquet bool) error { + writeParquet bool) error { for resource, output := range transformedOutput { // Filenames are typically exclusive of end point. This processor @@ -305,7 +305,7 @@ func exportTransformedData( return err } - if WriteParquet { + if writeParquet { switch v := o.(type) { case transform.AccountOutput: transformedResource = append(transformedResource, v) @@ -354,7 +354,7 @@ func exportTransformedData( MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) - if !skip && WriteParquet { + if !skip && writeParquet { WriteParquet(transformedResource, parquetPath, parquetSchema) MaybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, parquetPath) } diff --git a/internal/utils/main.go b/internal/utils/main.go index 4456f222..1141b764 100644 --- a/internal/utils/main.go +++ b/internal/utils/main.go @@ -409,7 +409,7 @@ func MustFlags(flags *pflag.FlagSet, logger *EtlLogger) FlagValues { logger.Fatal("could not get cloud provider: ", err) } - writeParquet, err := flags.GetBool("write-parquet") + WriteParquet, err := flags.GetBool("write-parquet") if err != nil { logger.Fatal("could not get write-parquet flag: ", err) } @@ -433,7 +433,7 @@ func MustFlags(flags *pflag.FlagSet, logger *EtlLogger) FlagValues { Bucket: bucket, Credentials: credentials, Provider: provider, - WriteParquet: writeParquet, + WriteParquet: WriteParquet, } } @@ -513,7 +513,7 @@ func MustCommonFlags(flags *pflag.FlagSet, logger *EtlLogger) CommonFlagValues { logger.Fatal("could not get retry-wait uint32: ", err) } - writeParquet, err := flags.GetBool("write-parquet") + WriteParquet, err := flags.GetBool("write-parquet") if err != nil { logger.Fatal("could not get write-parquet flag: ", err) } @@ -530,7 +530,7 @@ func MustCommonFlags(flags *pflag.FlagSet, logger *EtlLogger) CommonFlagValues { NumWorkers: numWorkers, RetryLimit: retryLimit, RetryWait: retryWait, - WriteParquet: writeParquet, + WriteParquet: WriteParquet, } } From 3b3b21e85d5e6c15acc52891f3e31338a9d92c53 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 14:59:46 -0600 Subject: [PATCH 06/13] export struct vars --- cmd/command_utils.go | 40 +++++------ cmd/export_assets_test.go | 32 ++++----- cmd/export_contract_events_test.go | 8 +-- cmd/export_effects_test.go | 32 ++++----- cmd/export_ledger_entry_changes_test.go | 96 ++++++++++++------------- cmd/export_ledger_transaction_test.go | 8 +-- cmd/export_ledgers_test.go | 56 +++++++-------- cmd/export_operations_test.go | 32 ++++----- cmd/export_trades_test.go | 32 ++++----- cmd/export_transactions_test.go | 32 ++++----- cmd/get_ledger_range_from_times_test.go | 88 +++++++++++------------ 11 files changed, 228 insertions(+), 228 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 12dc185f..7849e5a9 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -22,7 +22,7 @@ import ( ) var executableName = "stellar-etl" -var update = flag.Bool("update", false, "update the golden files of this test") +var update = flag.Bool("update", false, "update the Golden files of this test") var gotFolder = "testdata/got/" type CloudStorage interface { @@ -193,11 +193,11 @@ func WriteParquet(data []transform.SchemaParquet, path string, schema interface{ } type CliTest struct { - name string - args []string - golden string - wantErr error - sortForComparison bool + Name string + Args []string + Golden string + WantErr error + SortForComparison bool } func indexOf(l []string, s string) int { @@ -209,18 +209,18 @@ func indexOf(l []string, s string) int { return -1 } -func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { +func RunCLITest(t *testing.T, test CliTest, GoldenFolder string) { flag.Parse() - t.Run(test.name, func(t *testing.T) { + t.Run(test.Name, func(t *testing.T) { dir, err := os.Getwd() assert.NoError(t, err) - idxOfOutputArg := indexOf(test.args, "-o") + idxOfOutputArg := indexOf(test.Args, "-o") var testOutput []byte var outLocation string var stat os.FileInfo if idxOfOutputArg > -1 { - outLocation = test.args[idxOfOutputArg+1] + outLocation = test.Args[idxOfOutputArg+1] _, err = os.Stat(outLocation) if err != nil { // Check if the error is due to the file not existing @@ -235,7 +235,7 @@ func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { } } - cmd := exec.Command(path.Join(dir, executableName), test.args...) + cmd := exec.Command(path.Join(dir, executableName), test.Args...) errOut, actualError := cmd.CombinedOutput() if idxOfOutputArg > -1 { stat, err = os.Stat(outLocation) @@ -268,15 +268,15 @@ func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { // Since the CLI uses a logger to report errors, the final error message isn't the same as the errors thrown in code. // Instead, it's wrapped in other os/system errors // By reading the error text from the logger, we can extract the lower level error that the user would see - if test.golden == "" { + if test.Golden == "" { errorMsg := fmt.Errorf(extractErrorMsg(string(errOut))) - assert.Equal(t, test.wantErr, errorMsg) + assert.Equal(t, test.WantErr, errorMsg) return } - assert.Equal(t, test.wantErr, actualError) + assert.Equal(t, test.WantErr, actualError) actualString := string(testOutput) - if test.sortForComparison { + if test.SortForComparison { trimmed := strings.Trim(actualString, "\n") lines := strings.Split(trimmed, "\n") sort.Strings(lines) @@ -284,7 +284,7 @@ func RunCLITest(t *testing.T, test CliTest, goldenFolder string) { actualString = fmt.Sprintf("%s\n", actualString) } - wantString, err := getGolden(t, goldenFolder+test.golden, actualString, *update) + wantString, err := getGolden(t, GoldenFolder+test.Golden, actualString, *update) assert.NoError(t, err) assert.Equal(t, wantString, actualString) }) @@ -296,18 +296,18 @@ func extractErrorMsg(loggerOutput string) string { return loggerOutput[errIndex : errIndex+endIndex] } -func getGolden(t *testing.T, goldenFile string, actual string, update bool) (string, error) { +func getGolden(t *testing.T, GoldenFile string, actual string, update bool) (string, error) { t.Helper() - f, err := os.OpenFile(goldenFile, os.O_RDWR|os.O_CREATE, 0644) + f, err := os.OpenFile(GoldenFile, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return "", err } defer f.Close() - // If the update flag is true, clear the current contents of the golden file and write the actual output + // If the update flag is true, clear the current contents of the Golden file and write the actual output // This is useful for when new tests or added or functionality changes that breaks current tests if update { - err := os.Truncate(goldenFile, 0) + err := os.Truncate(GoldenFile, 0) if err != nil { return "", err } diff --git a/cmd/export_assets_test.go b/cmd/export_assets_test.go index e6cc4c35..3b42dec1 100644 --- a/cmd/export_assets_test.go +++ b/cmd/export_assets_test.go @@ -7,28 +7,28 @@ import ( func TestExportAssets(t *testing.T) { tests := []CliTest{ { - name: "assets from one ledger", - args: []string{"export_assets", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_assets.txt")}, - golden: "one_ledger_assets.golden", - wantErr: nil, + Name: "assets from one ledger", + Args: []string{"export_assets", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_assets.txt")}, + Golden: "one_ledger_assets.golden", + WantErr: nil, }, { - name: "assets from 10 ledgers", - args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_assets.txt")}, - golden: "10_ledgers_assets.golden", - wantErr: nil, + Name: "assets from 10 ledgers", + Args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_assets.txt")}, + Golden: "10_ledgers_assets.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_assets.txt")}, - golden: "large_range_assets.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_assets", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_assets.txt")}, + Golden: "large_range_assets.golden", + WantErr: nil, }, { - name: "ledger with no assets", - args: []string{"export_assets", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_assets.txt")}, - golden: "ledger_no_assets.golden", - wantErr: nil, + Name: "ledger with no assets", + Args: []string{"export_assets", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_assets.txt")}, + Golden: "ledger_no_assets.golden", + WantErr: nil, }, } diff --git a/cmd/export_contract_events_test.go b/cmd/export_contract_events_test.go index f2dc2364..0d9e1918 100644 --- a/cmd/export_contract_events_test.go +++ b/cmd/export_contract_events_test.go @@ -7,10 +7,10 @@ import ( func TestExportContractEvents(t *testing.T) { tests := []CliTest{ { - name: "contract events from multiple ledger", - args: []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", GotTestDir(t, "large_range_ledger_txs.txt")}, - golden: "large_range_ledger_txs.golden", - wantErr: nil, + Name: "contract events from multiple ledger", + Args: []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", GotTestDir(t, "large_range_ledger_txs.txt")}, + Golden: "large_range_ledger_txs.golden", + WantErr: nil, }, } diff --git a/cmd/export_effects_test.go b/cmd/export_effects_test.go index 3e55dbae..f2840d08 100644 --- a/cmd/export_effects_test.go +++ b/cmd/export_effects_test.go @@ -7,28 +7,28 @@ import ( func TestExportEffects(t *testing.T) { tests := []CliTest{ { - name: "effects from one ledger", - args: []string{"export_effects", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_effects.txt")}, - golden: "one_ledger_effects.golden", - wantErr: nil, + Name: "effects from one ledger", + Args: []string{"export_effects", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_effects.txt")}, + Golden: "one_ledger_effects.golden", + WantErr: nil, }, { - name: "effects from 10 ledgers", - args: []string{"export_effects", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_effects.txt")}, - golden: "10_ledgers_effects.golden", - wantErr: nil, + Name: "effects from 10 ledgers", + Args: []string{"export_effects", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_effects.txt")}, + Golden: "10_ledgers_effects.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_effects", "-s", "25820678", "-e", "25821678", "-l", "5", "-o", GotTestDir(t, "large_range_effects.txt")}, - golden: "large_range_effects.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_effects", "-s", "25820678", "-e", "25821678", "-l", "5", "-o", GotTestDir(t, "large_range_effects.txt")}, + Golden: "large_range_effects.golden", + WantErr: nil, }, { - name: "ledger with no effects", - args: []string{"export_effects", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_effects.txt")}, - golden: "ledger_no_effects.golden", - wantErr: nil, + Name: "ledger with no effects", + Args: []string{"export_effects", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_effects.txt")}, + Golden: "ledger_no_effects.golden", + WantErr: nil, }, } diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index f8da3ce2..78cabe06 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -12,85 +12,85 @@ func TestExportChanges(t *testing.T) { tests := []CliTest{ { - name: "unbounded range with no config", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-s", "100000"}, - golden: "", - wantErr: fmt.Errorf("stellar-core needs a config file path when exporting ledgers continuously (endNum = 0)"), + Name: "unbounded range with no config", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-s", "100000"}, + Golden: "", + WantErr: fmt.Errorf("stellar-core needs a config file path when exporting ledgers continuously (endNum = 0)"), }, { - name: "0 batch size", - args: []string{"export_ledger_entry_changes", "-b", "0", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "100000", "-e", "164000"}, - golden: "", - wantErr: fmt.Errorf("batch-size (0) must be greater than 0"), + Name: "0 batch size", + Args: []string{"export_ledger_entry_changes", "-b", "0", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "100000", "-e", "164000"}, + Golden: "", + WantErr: fmt.Errorf("batch-size (0) must be greater than 0"), }, { - name: "All changes from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", GotTestDir(t, "all/")}, - golden: "all.golden", - wantErr: nil, + Name: "All changes from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", GotTestDir(t, "all/")}, + Golden: "all.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Account changes from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "accounts/"), "--export-accounts", "true"}, - golden: "accounts.golden", - wantErr: nil, + Name: "Account changes from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "accounts/"), "--export-accounts", "true"}, + Golden: "accounts.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Claimable balance from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "claimable_balances/"), "--export-balances", "true"}, - golden: "claimable_balances.golden", - wantErr: nil, + Name: "Claimable balance from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "claimable_balances/"), "--export-balances", "true"}, + Golden: "claimable_balances.golden", + WantErr: nil, sortForComparison: true, }, { - name: "trustlines from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "trustlines/"), "--export-trustlines", "true"}, - golden: "trustlines.golden", - wantErr: nil, + Name: "trustlines from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "trustlines/"), "--export-trustlines", "true"}, + Golden: "trustlines.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Offers from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "offers/"), "--export-offers", "true"}, - golden: "offers.golden", - wantErr: nil, + Name: "Offers from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "offers/"), "--export-offers", "true"}, + Golden: "offers.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Pools from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "pools/"), "--export-pools", "true"}, - golden: "pools.golden", - wantErr: nil, + Name: "Pools from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "pools/"), "--export-pools", "true"}, + Golden: "pools.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Contract code from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", GotTestDir(t, "contract_code/"), "--export-contract-code", "true"}, - golden: "contract_code.golden", - wantErr: nil, + Name: "Contract code from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", GotTestDir(t, "contract_code/"), "--export-contract-code", "true"}, + Golden: "contract_code.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Contract data from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", GotTestDir(t, "contract_data/"), "--export-contract-data", "true"}, - golden: "contract_data.golden", - wantErr: nil, + Name: "Contract data from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", GotTestDir(t, "contract_data/"), "--export-contract-data", "true"}, + Golden: "contract_data.golden", + WantErr: nil, sortForComparison: true, }, { - name: "Config setting from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", GotTestDir(t, "config_setting/"), "--export-config-settings", "true"}, - golden: "config_setting.golden", - wantErr: nil, + Name: "Config setting from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", GotTestDir(t, "config_setting/"), "--export-config-settings", "true"}, + Golden: "config_setting.golden", + WantErr: nil, sortForComparison: true, }, { - name: "ttl from ledger entry", - args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", GotTestDir(t, "ttl/"), "--export-ttl", "true"}, - golden: "ttl.golden", - wantErr: nil, + Name: "ttl from ledger entry", + Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", GotTestDir(t, "ttl/"), "--export-ttl", "true"}, + Golden: "ttl.golden", + WantErr: nil, sortForComparison: true, }, } diff --git a/cmd/export_ledger_transaction_test.go b/cmd/export_ledger_transaction_test.go index d4f66585..b96f8515 100644 --- a/cmd/export_ledger_transaction_test.go +++ b/cmd/export_ledger_transaction_test.go @@ -7,10 +7,10 @@ import ( func TestExportLedgerTransaction(t *testing.T) { tests := []CliTest{ { - name: "Transactions from one ledger", - args: []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "ledger_transactions.txt")}, - golden: "ledger_transactions.golden", - wantErr: nil, + Name: "Transactions from one ledger", + Args: []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "ledger_transactions.txt")}, + Golden: "ledger_transactions.golden", + WantErr: nil, }, } diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index f3096d75..46f20bc2 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -29,46 +29,46 @@ func TestMain(m *testing.M) { func TestExportLedger(t *testing.T) { tests := []CliTest{ { - name: "end before start", - args: []string{"export_ledgers", "-s", "100", "-e", "50"}, - golden: "", - wantErr: fmt.Errorf("Number of bytes written: 0"), + Name: "end before start", + Args: []string{"export_ledgers", "-s", "100", "-e", "50"}, + Golden: "", + WantErr: fmt.Errorf("Number of bytes written: 0"), }, { - name: "start is 0", - args: []string{"export_ledgers", "-s", "0", "-e", "4294967295", "-l", "4294967295"}, - golden: "", - wantErr: fmt.Errorf("could not read ledgers: LedgerCloseMeta for sequence 0 not found in the batch"), + Name: "start is 0", + Args: []string{"export_ledgers", "-s", "0", "-e", "4294967295", "-l", "4294967295"}, + Golden: "", + WantErr: fmt.Errorf("could not read ledgers: LedgerCloseMeta for sequence 0 not found in the batch"), }, { - name: "end is 0", - args: []string{"export_ledgers", "-e", "0", "-l", "4294967295"}, - golden: "", - wantErr: fmt.Errorf("Number of bytes written: 0"), + Name: "end is 0", + Args: []string{"export_ledgers", "-e", "0", "-l", "4294967295"}, + Golden: "", + WantErr: fmt.Errorf("Number of bytes written: 0"), }, { - name: "single ledger", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822015", "-o", GotTestDir(t, "single_ledger.txt")}, - golden: "single_ledger.golden", - wantErr: nil, + Name: "single ledger", + Args: []string{"export_ledgers", "-s", "30822015", "-e", "30822015", "-o", GotTestDir(t, "single_ledger.txt")}, + Golden: "single_ledger.golden", + WantErr: nil, }, { - name: "10 ledgers", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers.txt")}, - golden: "10_ledgers.golden", - wantErr: nil, + Name: "10 ledgers", + Args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers.txt")}, + Golden: "10_ledgers.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ledgers.txt")}, - golden: "large_range_ledgers.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_ledgers", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ledgers.txt")}, + Golden: "large_range_ledgers.golden", + WantErr: nil, }, { - name: "range from 2024", - args: []string{"export_ledgers", "-s", "52929555", "-e", "52929960", "-o", GotTestDir(t, "2024_ledgers.txt")}, - golden: "2024_ledgers.golden", - wantErr: nil, + Name: "range from 2024", + Args: []string{"export_ledgers", "-s", "52929555", "-e", "52929960", "-o", GotTestDir(t, "2024_ledgers.txt")}, + Golden: "2024_ledgers.golden", + WantErr: nil, }, } diff --git a/cmd/export_operations_test.go b/cmd/export_operations_test.go index 87b8637b..e8ea44f3 100644 --- a/cmd/export_operations_test.go +++ b/cmd/export_operations_test.go @@ -7,28 +7,28 @@ import ( func TestExportOperations(t *testing.T) { tests := []CliTest{ { - name: "operations from one ledger", - args: []string{"export_operations", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_ops.txt")}, - golden: "one_ledger_ops.golden", - wantErr: nil, + Name: "operations from one ledger", + Args: []string{"export_operations", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_ops.txt")}, + Golden: "one_ledger_ops.golden", + WantErr: nil, }, { - name: "operations from 10 ledgers", - args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_ops.txt")}, - golden: "10_ledgers_ops.golden", - wantErr: nil, + Name: "operations from 10 ledgers", + Args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_ops.txt")}, + Golden: "10_ledgers_ops.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ops.txt")}, - golden: "large_range_ops.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_operations", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_ops.txt")}, + Golden: "large_range_ops.golden", + WantErr: nil, }, { - name: "ledger with no operations", - args: []string{"export_operations", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_ops.txt")}, - golden: "ledger_no_ops.golden", - wantErr: nil, + Name: "ledger with no operations", + Args: []string{"export_operations", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_ops.txt")}, + Golden: "ledger_no_ops.golden", + WantErr: nil, }, } diff --git a/cmd/export_trades_test.go b/cmd/export_trades_test.go index 4788dc8c..d4fc1acc 100644 --- a/cmd/export_trades_test.go +++ b/cmd/export_trades_test.go @@ -7,28 +7,28 @@ import ( func TestExportTrades(t *testing.T) { tests := []CliTest{ { - name: "trades from one ledger", - args: []string{"export_trades", "-s", "28770265", "-e", "28770265", "-o", GotTestDir(t, "one_ledger_trades.txt")}, - golden: "one_ledger_trades.golden", - wantErr: nil, + Name: "trades from one ledger", + Args: []string{"export_trades", "-s", "28770265", "-e", "28770265", "-o", GotTestDir(t, "one_ledger_trades.txt")}, + Golden: "one_ledger_trades.golden", + WantErr: nil, }, { - name: "trades from 10 ledgers", - args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-o", GotTestDir(t, "10_ledgers_trades.txt")}, - golden: "10_ledgers_trades.golden", - wantErr: nil, + Name: "trades from 10 ledgers", + Args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-o", GotTestDir(t, "10_ledgers_trades.txt")}, + Golden: "10_ledgers_trades.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-l", "5", "-o", GotTestDir(t, "large_range_trades.txt")}, - golden: "large_range_trades.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_trades", "-s", "28770265", "-e", "28770275", "-l", "5", "-o", GotTestDir(t, "large_range_trades.txt")}, + Golden: "large_range_trades.golden", + WantErr: nil, }, { - name: "ledger with no trades", - args: []string{"export_trades", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_trades.txt")}, - golden: "ledger_no_trades.golden", - wantErr: nil, + Name: "ledger with no trades", + Args: []string{"export_trades", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_trades.txt")}, + Golden: "ledger_no_trades.golden", + WantErr: nil, }, } diff --git a/cmd/export_transactions_test.go b/cmd/export_transactions_test.go index ce7214b7..33cab911 100644 --- a/cmd/export_transactions_test.go +++ b/cmd/export_transactions_test.go @@ -7,28 +7,28 @@ import ( func TestExportTransactions(t *testing.T) { tests := []CliTest{ { - name: "transactions from one ledger", - args: []string{"export_transactions", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_txs.txt")}, - golden: "one_ledger_txs.golden", - wantErr: nil, + Name: "transactions from one ledger", + Args: []string{"export_transactions", "-s", "30820015", "-e", "30820015", "-o", GotTestDir(t, "one_ledger_txs.txt")}, + Golden: "one_ledger_txs.golden", + WantErr: nil, }, { - name: "transactions from 10 ledgers", - args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_txs.txt")}, - golden: "10_ledgers_txs.golden", - wantErr: nil, + Name: "transactions from 10 ledgers", + Args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-o", GotTestDir(t, "10_ledgers_txs.txt")}, + Golden: "10_ledgers_txs.golden", + WantErr: nil, }, { - name: "range too large", - args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_txs.txt")}, - golden: "large_range_txs.golden", - wantErr: nil, + Name: "range too large", + Args: []string{"export_transactions", "-s", "30822015", "-e", "30822025", "-l", "5", "-o", GotTestDir(t, "large_range_txs.txt")}, + Golden: "large_range_txs.golden", + WantErr: nil, }, { - name: "ledger with no transactions", - args: []string{"export_transactions", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_txs.txt")}, - golden: "ledger_no_txs.golden", - wantErr: nil, + Name: "ledger with no transactions", + Args: []string{"export_transactions", "-s", "10363513", "-e", "10363513", "-o", GotTestDir(t, "ledger_no_txs.txt")}, + Golden: "ledger_no_txs.golden", + WantErr: nil, }, } diff --git a/cmd/get_ledger_range_from_times_test.go b/cmd/get_ledger_range_from_times_test.go index 6234a5e6..6172ef1f 100644 --- a/cmd/get_ledger_range_from_times_test.go +++ b/cmd/get_ledger_range_from_times_test.go @@ -8,77 +8,77 @@ import ( func TestConvertTimes(t *testing.T) { tests := []CliTest{ { - name: "wrong date format", - args: []string{"get_ledger_range_from_times", "-s", "2016 01 01 4:33", "-e", "2020 03 04 12:32"}, - golden: "", - wantErr: fmt.Errorf("could not parse start time: parsing time \\"), + Name: "wrong date format", + Args: []string{"get_ledger_range_from_times", "-s", "2016 01 01 4:33", "-e", "2020 03 04 12:32"}, + Golden: "", + WantErr: fmt.Errorf("could not parse start time: parsing time \\"), }, { - name: "normal range", - args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "normal_range.txt")}, - golden: "normal_range.golden", - wantErr: nil, + Name: "normal range", + Args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "normal_range.txt")}, + Golden: "normal_range.golden", + WantErr: nil, }, { - name: "start too early", - args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "early_start.txt")}, - golden: "early_start.golden", - wantErr: nil, + Name: "start too early", + Args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T18:00:00-05:00", "-e", "2019-09-13T23:00:00+00:00", "-o", GotTestDir(t, "early_start.txt")}, + Golden: "early_start.golden", + WantErr: nil, }, // { - // name: "start too late", + // Name: "start too late", // // @TODO // // assertion should actually be that the start and end times equal // // since it always grabs the end ledger you cannot hardcode the expected result // // maybe grab the latest ledger through code?? - // args: []string{"get_ledger_range_from_times", "-s", "2021-09-13T23:00:00+00:00", "-e", "2021-09-13T23:30:00+00:00"}, - // golden: "late_start.golden", - // wantErr: nil, + // Args: []string{"get_ledger_range_from_times", "-s", "2021-09-13T23:00:00+00:00", "-e", "2021-09-13T23:30:00+00:00"}, + // Golden: "late_start.golden", + // WantErr: nil, // }, // { - // name: "end too late", + // Name: "end too late", // // @TODO // // Change the expected output to the max ledger time // // cannot be hardcoded in a golden ledger - // args: []string{"get_ledger_range_from_times", "-s", "2017-11-10T12:14:32+04:00", "-e", "2021-09-13T23:00:00+00:00"}, - // golden: "late_end.golden", - // wantErr: nil, + // Args: []string{"get_ledger_range_from_times", "-s", "2017-11-10T12:14:32+04:00", "-e", "2021-09-13T23:00:00+00:00"}, + // Golden: "late_end.golden", + // WantErr: nil, // }, { - name: "end too early", - args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T12:14:32+04:00", "-e", "2006-11-10T12:14:32+04:00", "-o", GotTestDir(t, "early_end.txt")}, - golden: "early_end.golden", - wantErr: nil, + Name: "end too early", + Args: []string{"get_ledger_range_from_times", "-s", "2006-11-10T12:14:32+04:00", "-e", "2006-11-10T12:14:32+04:00", "-o", GotTestDir(t, "early_end.txt")}, + Golden: "early_end.golden", + WantErr: nil, }, { - name: "same date", - args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:03:37-05:00", "-e", "2016-11-10T18:03:37-05:00", "-o", GotTestDir(t, "same_date.txt")}, - golden: "same_date.golden", - wantErr: nil, + Name: "same date", + Args: []string{"get_ledger_range_from_times", "-s", "2016-11-10T18:03:37-05:00", "-e", "2016-11-10T18:03:37-05:00", "-o", GotTestDir(t, "same_date.txt")}, + Golden: "same_date.golden", + WantErr: nil, }, { - name: "checkpoint range (22343680-22343743)", - args: []string{"get_ledger_range_from_times", "-s", "2019-02-06T09:14:43+00:00", "-e", "2019-02-06T09:20:23+00:00", "-o", GotTestDir(t, "checkpoint_range.txt")}, - golden: "checkpoint_range.golden", - wantErr: nil, + Name: "checkpoint range (22343680-22343743)", + Args: []string{"get_ledger_range_from_times", "-s", "2019-02-06T09:14:43+00:00", "-e", "2019-02-06T09:20:23+00:00", "-o", GotTestDir(t, "checkpoint_range.txt")}, + Golden: "checkpoint_range.golden", + WantErr: nil, }, { - name: "checkpoint range (9341-9401)", - args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T06:20:00+00:00", "-e", "2015-10-01T06:25:00+00:00", "-o", GotTestDir(t, "checkpoint_range2.txt")}, - golden: "checkpoint_range2.golden", - wantErr: nil, + Name: "checkpoint range (9341-9401)", + Args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T06:20:00+00:00", "-e", "2015-10-01T06:25:00+00:00", "-o", GotTestDir(t, "checkpoint_range2.txt")}, + Golden: "checkpoint_range2.golden", + WantErr: nil, }, { - name: "infinite loop; checkpoint range (14558-14606)", - args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T13:35:00+00:00", "-e", "2015-10-01T13:40:00+00:00", "-o", GotTestDir(t, "checkpoint_range3.txt")}, - golden: "checkpoint_range3.golden", - wantErr: nil, + Name: "infinite loop; checkpoint range (14558-14606)", + Args: []string{"get_ledger_range_from_times", "-s", "2015-10-01T13:35:00+00:00", "-e", "2015-10-01T13:40:00+00:00", "-o", GotTestDir(t, "checkpoint_range3.txt")}, + Golden: "checkpoint_range3.golden", + WantErr: nil, }, { - name: "early checkpoint range (3-3)", - args: []string{"get_ledger_range_from_times", "-s", "2015-09-30T16:50:00+00:00", "-e", "2015-09-30T16:55:00+00:00", "-o", GotTestDir(t, "early_checkpoint_range.txt")}, - golden: "early_checkpoint_range.golden", - wantErr: nil, + Name: "early checkpoint range (3-3)", + Args: []string{"get_ledger_range_from_times", "-s", "2015-09-30T16:50:00+00:00", "-e", "2015-09-30T16:55:00+00:00", "-o", GotTestDir(t, "early_checkpoint_range.txt")}, + Golden: "early_checkpoint_range.golden", + WantErr: nil, }, } From 8a45838610dbb1a0a5b7ac39f91be812d8272ede Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 16:08:51 -0600 Subject: [PATCH 07/13] update --- cmd/export_ledger_entry_changes_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index 78cabe06..d9b11877 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -28,70 +28,70 @@ func TestExportChanges(t *testing.T) { Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", GotTestDir(t, "all/")}, Golden: "all.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Account changes from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "accounts/"), "--export-accounts", "true"}, Golden: "accounts.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Claimable balance from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "claimable_balances/"), "--export-balances", "true"}, Golden: "claimable_balances.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "trustlines from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "trustlines/"), "--export-trustlines", "true"}, Golden: "trustlines.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Offers from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "offers/"), "--export-offers", "true"}, Golden: "offers.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Pools from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", GotTestDir(t, "pools/"), "--export-pools", "true"}, Golden: "pools.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Contract code from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", GotTestDir(t, "contract_code/"), "--export-contract-code", "true"}, Golden: "contract_code.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Contract data from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", GotTestDir(t, "contract_data/"), "--export-contract-data", "true"}, Golden: "contract_data.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "Config setting from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", GotTestDir(t, "config_setting/"), "--export-config-settings", "true"}, Golden: "config_setting.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, { Name: "ttl from ledger entry", Args: []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", GotTestDir(t, "ttl/"), "--export-ttl", "true"}, Golden: "ttl.golden", WantErr: nil, - sortForComparison: true, + SortForComparison: true, }, } From 39e63e48df5bc5d9bf31bda5a32cb7786adce405 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 16:46:13 -0600 Subject: [PATCH 08/13] make executable name arg --- cmd/command_utils.go | 8 ++++++-- cmd/export_assets_test.go | 2 +- cmd/export_contract_events_test.go | 2 +- cmd/export_effects_test.go | 2 +- cmd/export_ledger_entry_changes_test.go | 2 +- cmd/export_ledger_transaction_test.go | 2 +- cmd/export_ledgers_test.go | 4 ++-- cmd/export_operations_test.go | 2 +- cmd/export_trades_test.go | 2 +- cmd/export_transactions_test.go | 2 +- cmd/get_ledger_range_from_times_test.go | 2 +- 11 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 7849e5a9..24ca3f77 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -21,7 +21,6 @@ import ( "github.com/xitongsys/parquet-go/writer" ) -var executableName = "stellar-etl" var update = flag.Bool("update", false, "update the Golden files of this test") var gotFolder = "testdata/got/" @@ -209,7 +208,10 @@ func indexOf(l []string, s string) int { return -1 } -func RunCLITest(t *testing.T, test CliTest, GoldenFolder string) { +func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName string) { + if executableName == "" { + executableName = "stellar-etl" + } flag.Parse() t.Run(test.Name, func(t *testing.T) { dir, err := os.Getwd() @@ -236,7 +238,9 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string) { } cmd := exec.Command(path.Join(dir, executableName), test.Args...) + fmt.Printf("Command: %s %v\n", path.Join(dir, executableName), test.Args) errOut, actualError := cmd.CombinedOutput() + fmt.Println(actualError) if idxOfOutputArg > -1 { stat, err = os.Stat(outLocation) assert.NoError(t, err) diff --git a/cmd/export_assets_test.go b/cmd/export_assets_test.go index 3b42dec1..c082dfce 100644 --- a/cmd/export_assets_test.go +++ b/cmd/export_assets_test.go @@ -33,6 +33,6 @@ func TestExportAssets(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/assets/") + RunCLITest(t, test, "testdata/assets/", "") } } diff --git a/cmd/export_contract_events_test.go b/cmd/export_contract_events_test.go index 0d9e1918..658e93f2 100644 --- a/cmd/export_contract_events_test.go +++ b/cmd/export_contract_events_test.go @@ -15,6 +15,6 @@ func TestExportContractEvents(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/contract_events/") + RunCLITest(t, test, "testdata/contract_events/", "") } } diff --git a/cmd/export_effects_test.go b/cmd/export_effects_test.go index f2840d08..8c2af2c1 100644 --- a/cmd/export_effects_test.go +++ b/cmd/export_effects_test.go @@ -33,6 +33,6 @@ func TestExportEffects(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/effects/") + RunCLITest(t, test, "testdata/effects/", "") } } diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index d9b11877..89f6aad4 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -96,6 +96,6 @@ func TestExportChanges(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/changes/") + RunCLITest(t, test, "testdata/changes/", "") } } diff --git a/cmd/export_ledger_transaction_test.go b/cmd/export_ledger_transaction_test.go index b96f8515..2a38328a 100644 --- a/cmd/export_ledger_transaction_test.go +++ b/cmd/export_ledger_transaction_test.go @@ -15,6 +15,6 @@ func TestExportLedgerTransaction(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ledger_transactions/") + RunCLITest(t, test, "testdata/ledger_transactions/", "") } } diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index 46f20bc2..a466f218 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -15,7 +15,7 @@ func TestMain(m *testing.M) { } // This does the setup for further tests. It generates an executeable that can be run on the command line by other tests - buildCmd := exec.Command("go", "build", "-o", executableName) + buildCmd := exec.Command("go", "build", "-o", "stellar-etl") if err := buildCmd.Run(); err != nil { cmdLogger.Error("could not build executable", err) os.Exit(1) @@ -73,6 +73,6 @@ func TestExportLedger(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ledgers/") + RunCLITest(t, test, "testdata/ledgers/", "") } } diff --git a/cmd/export_operations_test.go b/cmd/export_operations_test.go index e8ea44f3..b1b1eab6 100644 --- a/cmd/export_operations_test.go +++ b/cmd/export_operations_test.go @@ -33,6 +33,6 @@ func TestExportOperations(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/operations/") + RunCLITest(t, test, "testdata/operations/", "") } } diff --git a/cmd/export_trades_test.go b/cmd/export_trades_test.go index d4fc1acc..8071edbc 100644 --- a/cmd/export_trades_test.go +++ b/cmd/export_trades_test.go @@ -33,6 +33,6 @@ func TestExportTrades(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/trades/") + RunCLITest(t, test, "testdata/trades/", "") } } diff --git a/cmd/export_transactions_test.go b/cmd/export_transactions_test.go index 33cab911..5c8bd14f 100644 --- a/cmd/export_transactions_test.go +++ b/cmd/export_transactions_test.go @@ -33,6 +33,6 @@ func TestExportTransactions(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/transactions/") + RunCLITest(t, test, "testdata/transactions/", "") } } diff --git a/cmd/get_ledger_range_from_times_test.go b/cmd/get_ledger_range_from_times_test.go index 6172ef1f..10895c6b 100644 --- a/cmd/get_ledger_range_from_times_test.go +++ b/cmd/get_ledger_range_from_times_test.go @@ -83,6 +83,6 @@ func TestConvertTimes(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ranges/") + RunCLITest(t, test, "testdata/ranges/", "") } } From f26bb7e4315c6bdf804afd4cde14568d54fa0cab Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 26 Nov 2024 16:59:12 -0600 Subject: [PATCH 09/13] add arg for using parent dir --- cmd/command_utils.go | 6 +++++- cmd/export_assets_test.go | 2 +- cmd/export_contract_events_test.go | 2 +- cmd/export_effects_test.go | 2 +- cmd/export_ledger_entry_changes_test.go | 2 +- cmd/export_ledger_transaction_test.go | 2 +- cmd/export_ledgers_test.go | 2 +- cmd/export_operations_test.go | 2 +- cmd/export_trades_test.go | 2 +- cmd/export_transactions_test.go | 2 +- cmd/get_ledger_range_from_times_test.go | 2 +- 11 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 24ca3f77..c53028b7 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -208,7 +208,7 @@ func indexOf(l []string, s string) int { return -1 } -func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName string) { +func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName string, useParentDir bool) { if executableName == "" { executableName = "stellar-etl" } @@ -217,6 +217,10 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName dir, err := os.Getwd() assert.NoError(t, err) + if useParentDir { + dir = filepath.Dir(dir) + } + idxOfOutputArg := indexOf(test.Args, "-o") var testOutput []byte var outLocation string diff --git a/cmd/export_assets_test.go b/cmd/export_assets_test.go index c082dfce..77ca7137 100644 --- a/cmd/export_assets_test.go +++ b/cmd/export_assets_test.go @@ -33,6 +33,6 @@ func TestExportAssets(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/assets/", "") + RunCLITest(t, test, "testdata/assets/", "", false) } } diff --git a/cmd/export_contract_events_test.go b/cmd/export_contract_events_test.go index 658e93f2..7f88b487 100644 --- a/cmd/export_contract_events_test.go +++ b/cmd/export_contract_events_test.go @@ -15,6 +15,6 @@ func TestExportContractEvents(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/contract_events/", "") + RunCLITest(t, test, "testdata/contract_events/", "", false) } } diff --git a/cmd/export_effects_test.go b/cmd/export_effects_test.go index 8c2af2c1..840d5aa3 100644 --- a/cmd/export_effects_test.go +++ b/cmd/export_effects_test.go @@ -33,6 +33,6 @@ func TestExportEffects(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/effects/", "") + RunCLITest(t, test, "testdata/effects/", "", false) } } diff --git a/cmd/export_ledger_entry_changes_test.go b/cmd/export_ledger_entry_changes_test.go index 89f6aad4..2ce64b20 100644 --- a/cmd/export_ledger_entry_changes_test.go +++ b/cmd/export_ledger_entry_changes_test.go @@ -96,6 +96,6 @@ func TestExportChanges(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/changes/", "") + RunCLITest(t, test, "testdata/changes/", "", false) } } diff --git a/cmd/export_ledger_transaction_test.go b/cmd/export_ledger_transaction_test.go index 2a38328a..b54870ef 100644 --- a/cmd/export_ledger_transaction_test.go +++ b/cmd/export_ledger_transaction_test.go @@ -15,6 +15,6 @@ func TestExportLedgerTransaction(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ledger_transactions/", "") + RunCLITest(t, test, "testdata/ledger_transactions/", "", false) } } diff --git a/cmd/export_ledgers_test.go b/cmd/export_ledgers_test.go index a466f218..f586ed79 100644 --- a/cmd/export_ledgers_test.go +++ b/cmd/export_ledgers_test.go @@ -73,6 +73,6 @@ func TestExportLedger(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ledgers/", "") + RunCLITest(t, test, "testdata/ledgers/", "", false) } } diff --git a/cmd/export_operations_test.go b/cmd/export_operations_test.go index b1b1eab6..586923ef 100644 --- a/cmd/export_operations_test.go +++ b/cmd/export_operations_test.go @@ -33,6 +33,6 @@ func TestExportOperations(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/operations/", "") + RunCLITest(t, test, "testdata/operations/", "", false) } } diff --git a/cmd/export_trades_test.go b/cmd/export_trades_test.go index 8071edbc..be1a8c42 100644 --- a/cmd/export_trades_test.go +++ b/cmd/export_trades_test.go @@ -33,6 +33,6 @@ func TestExportTrades(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/trades/", "") + RunCLITest(t, test, "testdata/trades/", "", false) } } diff --git a/cmd/export_transactions_test.go b/cmd/export_transactions_test.go index 5c8bd14f..911fffec 100644 --- a/cmd/export_transactions_test.go +++ b/cmd/export_transactions_test.go @@ -33,6 +33,6 @@ func TestExportTransactions(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/transactions/", "") + RunCLITest(t, test, "testdata/transactions/", "", false) } } diff --git a/cmd/get_ledger_range_from_times_test.go b/cmd/get_ledger_range_from_times_test.go index 10895c6b..0e24f4b5 100644 --- a/cmd/get_ledger_range_from_times_test.go +++ b/cmd/get_ledger_range_from_times_test.go @@ -83,6 +83,6 @@ func TestConvertTimes(t *testing.T) { } for _, test := range tests { - RunCLITest(t, test, "testdata/ranges/", "") + RunCLITest(t, test, "testdata/ranges/", "", false) } } From 9e5208e5242a98189afeef63d89b7b3977d63d64 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Mon, 2 Dec 2024 13:39:04 -0600 Subject: [PATCH 10/13] log --- cmd/command_utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index c53028b7..1048725e 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -228,13 +228,16 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName if idxOfOutputArg > -1 { outLocation = test.Args[idxOfOutputArg+1] _, err = os.Stat(outLocation) + fmt.Println(outLocation) if err != nil { // Check if the error is due to the file not existing if !os.IsNotExist(err) { assert.NoError(t, err) } + fmt.Println("here") } else { err = deleteLocalFiles(outLocation) + fmt.Println("deleting files") if err != nil { log.Fatal(err) } From cbe8b7c2b4a35d4b229f0e8c6d3f9c7e74859bfc Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Mon, 2 Dec 2024 13:52:17 -0600 Subject: [PATCH 11/13] run command directly --- cmd/command_utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 1048725e..bfd53108 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -9,7 +9,6 @@ import ( "log" "os" "os/exec" - "path" "path/filepath" "sort" "strings" @@ -244,10 +243,11 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName } } - cmd := exec.Command(path.Join(dir, executableName), test.Args...) - fmt.Printf("Command: %s %v\n", path.Join(dir, executableName), test.Args) + cmd := exec.Command(executableName, test.Args...) + fmt.Printf("Command: %s %v\n", executableName, test.Args) errOut, actualError := cmd.CombinedOutput() fmt.Println(actualError) + fmt.Println(errOut) if idxOfOutputArg > -1 { stat, err = os.Stat(outLocation) assert.NoError(t, err) From 0b3f25f5a9751e4e4e0272fd603c4bd27526b852 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 3 Dec 2024 13:35:25 -0600 Subject: [PATCH 12/13] Remove debug statements --- cmd/command_utils.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/command_utils.go b/cmd/command_utils.go index bfd53108..99b6f0fb 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -227,16 +227,13 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName if idxOfOutputArg > -1 { outLocation = test.Args[idxOfOutputArg+1] _, err = os.Stat(outLocation) - fmt.Println(outLocation) if err != nil { // Check if the error is due to the file not existing if !os.IsNotExist(err) { assert.NoError(t, err) } - fmt.Println("here") } else { err = deleteLocalFiles(outLocation) - fmt.Println("deleting files") if err != nil { log.Fatal(err) } @@ -246,8 +243,6 @@ func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName cmd := exec.Command(executableName, test.Args...) fmt.Printf("Command: %s %v\n", executableName, test.Args) errOut, actualError := cmd.CombinedOutput() - fmt.Println(actualError) - fmt.Println(errOut) if idxOfOutputArg > -1 { stat, err = os.Stat(outLocation) assert.NoError(t, err) From 73f77f30fb2ff93b2a55fcea47c6f517bbe036b9 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Tue, 3 Dec 2024 13:39:54 -0600 Subject: [PATCH 13/13] Separate util file for test utils --- cmd/command_utils.go | 164 ---------------------------------------- cmd/test_utils.go | 173 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 164 deletions(-) create mode 100644 cmd/test_utils.go diff --git a/cmd/command_utils.go b/cmd/command_utils.go index 99b6f0fb..f7c84d07 100644 --- a/cmd/command_utils.go +++ b/cmd/command_utils.go @@ -3,26 +3,15 @@ package cmd import ( "bytes" "encoding/json" - "flag" "fmt" - "io" - "log" "os" - "os/exec" "path/filepath" - "sort" - "strings" - "testing" "github.com/stellar/stellar-etl/internal/transform" - "github.com/stretchr/testify/assert" "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/writer" ) -var update = flag.Bool("update", false, "update the Golden files of this test") -var gotFolder = "testdata/got/" - type CloudStorage interface { UploadTo(credentialsPath, bucket, path string) error } @@ -189,156 +178,3 @@ func WriteParquet(data []transform.SchemaParquet, path string, schema interface{ } } } - -type CliTest struct { - Name string - Args []string - Golden string - WantErr error - SortForComparison bool -} - -func indexOf(l []string, s string) int { - for idx, e := range l { - if e == s { - return idx - } - } - return -1 -} - -func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName string, useParentDir bool) { - if executableName == "" { - executableName = "stellar-etl" - } - flag.Parse() - t.Run(test.Name, func(t *testing.T) { - dir, err := os.Getwd() - assert.NoError(t, err) - - if useParentDir { - dir = filepath.Dir(dir) - } - - idxOfOutputArg := indexOf(test.Args, "-o") - var testOutput []byte - var outLocation string - var stat os.FileInfo - if idxOfOutputArg > -1 { - outLocation = test.Args[idxOfOutputArg+1] - _, err = os.Stat(outLocation) - if err != nil { - // Check if the error is due to the file not existing - if !os.IsNotExist(err) { - assert.NoError(t, err) - } - } else { - err = deleteLocalFiles(outLocation) - if err != nil { - log.Fatal(err) - } - } - } - - cmd := exec.Command(executableName, test.Args...) - fmt.Printf("Command: %s %v\n", executableName, test.Args) - errOut, actualError := cmd.CombinedOutput() - if idxOfOutputArg > -1 { - stat, err = os.Stat(outLocation) - assert.NoError(t, err) - - if stat.IsDir() { - files, err := os.ReadDir(outLocation) - if err != nil { - log.Fatal(err) - } - var buf bytes.Buffer - sortByName(files) - for _, f := range files { - b, err := os.ReadFile(filepath.Join(outLocation, f.Name())) - if err != nil { - log.Fatal(err) - } - buf.Write(b) - } - testOutput = buf.Bytes() - } else { - // If the output is written to a file, read the contents of the file for comparison. - testOutput, err = os.ReadFile(outLocation) - if err != nil { - log.Fatal(err) - } - } - } - - // Since the CLI uses a logger to report errors, the final error message isn't the same as the errors thrown in code. - // Instead, it's wrapped in other os/system errors - // By reading the error text from the logger, we can extract the lower level error that the user would see - if test.Golden == "" { - errorMsg := fmt.Errorf(extractErrorMsg(string(errOut))) - assert.Equal(t, test.WantErr, errorMsg) - return - } - - assert.Equal(t, test.WantErr, actualError) - actualString := string(testOutput) - if test.SortForComparison { - trimmed := strings.Trim(actualString, "\n") - lines := strings.Split(trimmed, "\n") - sort.Strings(lines) - actualString = strings.Join(lines, "\n") - actualString = fmt.Sprintf("%s\n", actualString) - } - - wantString, err := getGolden(t, GoldenFolder+test.Golden, actualString, *update) - assert.NoError(t, err) - assert.Equal(t, wantString, actualString) - }) -} - -func extractErrorMsg(loggerOutput string) string { - errIndex := strings.Index(loggerOutput, "msg=") + 5 - endIndex := strings.Index(loggerOutput[errIndex:], "\"") - return loggerOutput[errIndex : errIndex+endIndex] -} - -func getGolden(t *testing.T, GoldenFile string, actual string, update bool) (string, error) { - t.Helper() - f, err := os.OpenFile(GoldenFile, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return "", err - } - defer f.Close() - - // If the update flag is true, clear the current contents of the Golden file and write the actual output - // This is useful for when new tests or added or functionality changes that breaks current tests - if update { - err := os.Truncate(GoldenFile, 0) - if err != nil { - return "", err - } - - _, err = f.WriteString(actual) - if err != nil { - return "", err - } - return actual, nil - } - - wantOutput, err := io.ReadAll(f) - if err != nil { - return "", err - } - - return string(wantOutput), nil -} - -func sortByName(files []os.DirEntry) { - sort.Slice(files, func(i, j int) bool { - return files[i].Name() < files[j].Name() - }) -} - -func GotTestDir(t *testing.T, filename string) string { - return filepath.Join(gotFolder, t.Name(), filename) -} diff --git a/cmd/test_utils.go b/cmd/test_utils.go new file mode 100644 index 00000000..b265f655 --- /dev/null +++ b/cmd/test_utils.go @@ -0,0 +1,173 @@ +package cmd + +import ( + "bytes" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var update = flag.Bool("update", false, "update the Golden files of this test") +var gotFolder = "testdata/got/" + +type CliTest struct { + Name string + Args []string + Golden string + WantErr error + SortForComparison bool +} + +func indexOf(l []string, s string) int { + for idx, e := range l { + if e == s { + return idx + } + } + return -1 +} + +func RunCLITest(t *testing.T, test CliTest, GoldenFolder string, executableName string, useParentDir bool) { + if executableName == "" { + executableName = "stellar-etl" + } + flag.Parse() + t.Run(test.Name, func(t *testing.T) { + dir, err := os.Getwd() + assert.NoError(t, err) + + if useParentDir { + dir = filepath.Dir(dir) + } + + idxOfOutputArg := indexOf(test.Args, "-o") + var testOutput []byte + var outLocation string + var stat os.FileInfo + if idxOfOutputArg > -1 { + outLocation = test.Args[idxOfOutputArg+1] + _, err = os.Stat(outLocation) + if err != nil { + // Check if the error is due to the file not existing + if !os.IsNotExist(err) { + assert.NoError(t, err) + } + } else { + err = deleteLocalFiles(outLocation) + if err != nil { + log.Fatal(err) + } + } + } + + cmd := exec.Command(executableName, test.Args...) + fmt.Printf("Command: %s %v\n", executableName, test.Args) + errOut, actualError := cmd.CombinedOutput() + if idxOfOutputArg > -1 { + stat, err = os.Stat(outLocation) + assert.NoError(t, err) + + if stat.IsDir() { + files, err := os.ReadDir(outLocation) + if err != nil { + log.Fatal(err) + } + var buf bytes.Buffer + sortByName(files) + for _, f := range files { + b, err := os.ReadFile(filepath.Join(outLocation, f.Name())) + if err != nil { + log.Fatal(err) + } + buf.Write(b) + } + testOutput = buf.Bytes() + } else { + // If the output is written to a file, read the contents of the file for comparison. + testOutput, err = os.ReadFile(outLocation) + if err != nil { + log.Fatal(err) + } + } + } + + // Since the CLI uses a logger to report errors, the final error message isn't the same as the errors thrown in code. + // Instead, it's wrapped in other os/system errors + // By reading the error text from the logger, we can extract the lower level error that the user would see + if test.Golden == "" { + errorMsg := fmt.Errorf(extractErrorMsg(string(errOut))) + assert.Equal(t, test.WantErr, errorMsg) + return + } + + assert.Equal(t, test.WantErr, actualError) + actualString := string(testOutput) + if test.SortForComparison { + trimmed := strings.Trim(actualString, "\n") + lines := strings.Split(trimmed, "\n") + sort.Strings(lines) + actualString = strings.Join(lines, "\n") + actualString = fmt.Sprintf("%s\n", actualString) + } + + wantString, err := getGolden(t, GoldenFolder+test.Golden, actualString, *update) + assert.NoError(t, err) + assert.Equal(t, wantString, actualString) + }) +} + +func extractErrorMsg(loggerOutput string) string { + errIndex := strings.Index(loggerOutput, "msg=") + 5 + endIndex := strings.Index(loggerOutput[errIndex:], "\"") + return loggerOutput[errIndex : errIndex+endIndex] +} + +func getGolden(t *testing.T, GoldenFile string, actual string, update bool) (string, error) { + t.Helper() + f, err := os.OpenFile(GoldenFile, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return "", err + } + defer f.Close() + + // If the update flag is true, clear the current contents of the Golden file and write the actual output + // This is useful for when new tests or added or functionality changes that breaks current tests + if update { + err := os.Truncate(GoldenFile, 0) + if err != nil { + return "", err + } + + _, err = f.WriteString(actual) + if err != nil { + return "", err + } + return actual, nil + } + + wantOutput, err := io.ReadAll(f) + if err != nil { + return "", err + } + + return string(wantOutput), nil +} + +func sortByName(files []os.DirEntry) { + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) +} + +func GotTestDir(t *testing.T, filename string) string { + return filepath.Join(gotFolder, t.Name(), filename) +}