Add bucketlist size to history_ledgers

.github/workflows/codeql.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron: "42 15 * * 6"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})

.github/workflows/integration-tests.yml

@@ -0,0 +1,67 @@
+name: Integration Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+
+      - name: Build Docker Image
+        run: |
+          docker-compose build
+
+      - name: Build and start services
+        run: |
+          docker-compose up -d
+
+      - name: Create GCP Credentials File
+        run: |
+          echo "$CREDS_TEST_HUBBLE" > ${{ runner.workspace }}/gcp-key.json
+        shell: bash
+        env:
+          CREDS_TEST_HUBBLE: ${{secrets.CREDS_TEST_HUBBLE}}
+
+      - name: Run tests
+        run: |
+          docker-compose run -v ${{ runner.workspace }}/gcp-key.json:/usr/credential.json:ro \
+            -v ${{ runner.workspace }}/coverage/:/usr/coverage/ \
+            -e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \
+            integration-tests \
+            go test -v -coverprofile=/usr/coverage/coverage.out ./cmd ./internal/transform -timeout 30m
+
+      - name: Generate Coverage Report
+        run: |
+          go tool cover -func=${{ runner.workspace }}/coverage/coverage.out
+
+      - name: Check Coverage
+        id: coverage
+        run: |
+          COVERAGE=$(go tool cover -func=${{ runner.workspace }}/coverage/coverage.out | grep total: | awk '{print $3}' | sed 's/%//')
+          echo "Coverage: $COVERAGE%"
+          if (( $(echo "$COVERAGE < 55" | bc -l) )); then
+            echo "Coverage is below the 55% threshold."
+            exit 1
+          fi
+
+      - name: Stop and remove containers
+        run: |
+          docker-compose down

.github/workflows/internal.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
     branches: [master]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/lint-tests.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - master
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
 jobs:
   pre-commit:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
     types: [closed]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

Makefile

@@ -13,3 +13,24 @@ docker-build:
 docker-push:
 	$(SUDO) docker push $(ETLHASH)
 	$(SUDO) docker push stellar/stellar-etl:latest
+
+int-test:
+	docker-compose build
+	docker-compose run \
+	-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \
+	-v $(PWD)/testdata:/usr/src/etl/testdata \
+	-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \
+	integration-tests \
+	go test -v ./cmd -timeout 30m
+
+int-test-update:
+	docker-compose build
+	docker-compose run \
+	-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \
+	-v $(PWD)/testdata:/usr/src/etl/testdata \
+	-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \
+	integration-tests \
+	go test -v ./cmd -timeout 30m -args -update=true
+
+lint:
+	pre-commit run --show-diff-on-failure --color=always --all-files

README.md

@@ -87,6 +87,46 @@ Add following to docker run command to pass gcloud credentials to docker contain
 
 <br>
 
+## **Running Tests**
+
+### Unit tests
+
+```sh
+# Running all unit tests
+go test -v -cover ./internal/transform
+
+# Running an individual test
+go test -v -run ^TestTransformAsset$ ./internal/transform
+```
+
+### Integration tests
+
+```sh
+# Running all integration tests
+make int-test
+
+# Running all integration tests and update golden files
+make int-test-update
+
+# Above essentially runs following:
+docker-compose build
+docker-compose run \
+-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \
+-v $(PWD)/testdata:/usr/src/etl/testdata \
+-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \
+integration-tests \
+go test -v ./cmd -timeout 30m -args -update=true
+
+# Running an individual test
+docker-compose build
+docker-compose run \
+-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \
+-v $(PWD)/testdata:/usr/src/etl/testdata \
+-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \
+integration-tests \
+go test -v -run ^TestExportAssets$ ./cmd -timeout 30m -args -update=true
+```
+
 ---
 
 # **Command Reference**

cmd/command_utils.go

@@ -110,13 +110,14 @@ func exportParquetFilename(start, end uint32, dataType string) string {
 	return fmt.Sprintf("%d-%d-%s.parquet", start, end-1, dataType)
 }
 
-func deleteLocalFiles(path string) {
+func deleteLocalFiles(path string) error {
 	err := os.RemoveAll(path)
 	if err != nil {
 		cmdLogger.Errorf("Unable to remove %s: %s", path, err)
-		return
+		return err
 	}
 	cmdLogger.Infof("Successfully deleted %s", path)
+	return nil
 }
 
 func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path string) {

cmd/export_contract_events_test.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestExportContractEvents(t *testing.T) {
+	tests := []cliTest{
+		{
+			name:    "contract events from multiple ledger",
+			args:    []string{"export_contract_events", "-s", "52271338", "-e", "52271350", "-o", gotTestDir(t, "large_range_ledger_txs.txt")},
+			golden:  "large_range_ledger_txs.golden",
+			wantErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		runCLITest(t, test, "testdata/contract_events/")
+	}
+}

cmd/export_ledger_entry_changes_test.go

@@ -24,16 +24,74 @@ func TestExportChanges(t *testing.T) {
 			wantErr: fmt.Errorf("batch-size (0) must be greater than 0"),
 		},
 		{
-			name:    "changes from single ledger",
-			args:    []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "140116", "-e", "140116", "-o", gotTestDir(t, "single/")},
-			golden:  "single_ledger.golden",
-			wantErr: nil,
+			name:              "All changes from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265350", "-o", gotTestDir(t, "all/")},
+			golden:            "all.golden",
+			wantErr:           nil,
+			sortForComparison: true,
 		},
 		{
-			name:    "changes from large range",
-			args:    []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "100000", "-e", "164000", "-o", gotTestDir(t, "large_range_changes/")},
-			golden:  "large_range_changes.golden",
-			wantErr: nil,
+			name:              "Account changes from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "accounts/"), "--export-accounts", "true"},
+			golden:            "accounts.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Claimable balance from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "claimable_balances/"), "--export-balances", "true"},
+			golden:            "claimable_balances.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "trustlines from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "trustlines/"), "--export-trustlines", "true"},
+			golden:            "trustlines.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Offers from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "offers/"), "--export-offers", "true"},
+			golden:            "offers.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Pools from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "49265302", "-e", "49265400", "-o", gotTestDir(t, "pools/"), "--export-pools", "true"},
+			golden:            "pools.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Contract code from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50666990", "-e", "50666999", "-o", gotTestDir(t, "contract_code/"), "--export-contract-code", "true"},
+			golden:            "contract_code.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Contract data from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "51340657", "-e", "51340757", "-o", gotTestDir(t, "contract_data/"), "--export-contract-data", "true"},
+			golden:            "contract_data.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "Config setting from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50457424", "-e", "50457440", "-o", gotTestDir(t, "config_setting/"), "--export-config-settings", "true"},
+			golden:            "config_setting.golden",
+			wantErr:           nil,
+			sortForComparison: true,
+		},
+		{
+			name:              "ttl from ledger entry",
+			args:              []string{"export_ledger_entry_changes", "-x", coreExecutablePath, "-c", coreConfigPath, "-s", "50603521", "-e", "50603621", "-o", gotTestDir(t, "ttl/"), "--export-ttl", "true"},
+			golden:            "ttl.golden",
+			wantErr:           nil,
+			sortForComparison: true,
 		},
 	}
 

cmd/export_ledger_transaction_test.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestExportLedgerTransaction(t *testing.T) {
+	tests := []cliTest{
+		{
+			name:    "Transactions from one ledger",
+			args:    []string{"export_ledger_transaction", "-s", "30820015", "-e", "30820015", "-o", gotTestDir(t, "ledger_transactions.txt")},
+			golden:  "ledger_transactions.golden",
+			wantErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		runCLITest(t, test, "testdata/ledger_transactions/")
+	}
+}