Merge pull request #280 from stellar/v22-breaking-changes

Protocol 22 Breaking changes

CHANGELOG.md

@@ -14,6 +14,8 @@
 
 ### Added
 
+- Add `Cursor` in `GetEventsResponse`. This tells the client until what ledger events are being queried. e.g.: `startLEdger` (inclusive) - `endLedger` (exclusive)
+- Limitation: getEvents are capped by 10K `LedgerScanLimit` which means you can query events for 10K ledger at maximum for a given request.
 - Add `EndLedger` in `GetEventsRequest`. This provides finer control and clarity on the range of ledgers being queried.
 - Disk-Based Event Storage: Events are now stored on disk instead of in memory. For context, storing approximately 3 million events will require around 1.5 GB of disk space.
 This change enhances the scalability and can now support a larger retention window (~7 days) for events.

Makefile

@@ -36,6 +36,10 @@ endif
 # (libpreflight.a is put at target/release-with-panic-unwind/ when not cross compiling)
 CARGO_BUILD_TARGET ?= $(shell rustc -vV | sed -n 's|host: ||p')
 
+SOROBAN_RPC_BINARY := soroban-rpc
+STELLAR_RPC_BINARY := stellar-rpc
+
+
 # update the Cargo.lock every time the Cargo.toml changes.
 Cargo.lock: Cargo.toml
 	cargo update --workspace
@@ -77,11 +81,19 @@ clean:
 	cargo clean
 	go clean ./...
 
+# DEPRECATED - please use build-stellar-rpc instead
 # the build-soroban-rpc build target is an optimized build target used by
-# https://github.com/stellar/pipelines/stellar-horizon/Jenkinsfile-soroban-rpc-package-builder
+# https://github.com/stellar/pipelines/blob/master/soroban-rpc/Jenkinsfile-soroban-rpc-package-builder
 # as part of the package building.
 build-soroban-rpc: build-libs
-	go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o soroban-rpc -trimpath -v ./cmd/soroban-rpc
+	go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o ${SOROBAN_RPC_BINARY} -trimpath -v ./cmd/soroban-rpc
+
+# the build-stellar-rpc build target is an optimized build target used by
+# https://github.com/stellar/pipelines/blob/master/soroban-rpc/Jenkinsfile-soroban-rpc-package-builder
+# as part of the package building.
+build-stellar-rpc: build-libs
+	go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o ${STELLAR_RPC_BINARY} -trimpath -v ./cmd/soroban-rpc
+
 
 go-check-branch:
 	golangci-lint run ./... --new-from-rev $$(git rev-parse origin/main)

cmd/soroban-rpc/docker/Dockerfile

@@ -1,6 +1,7 @@
 FROM golang:1.22-bullseye as build
 ARG RUST_TOOLCHAIN_VERSION=stable
 ARG REPOSITORY_VERSION
+ARG BINARY_NAME=soroban-rpc
 
 WORKDIR /go/src/github.com/stellar/soroban-rpc
 
@@ -18,11 +19,14 @@ RUN apt-get clean
 
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION
 
-RUN make REPOSITORY_VERSION=${REPOSITORY_VERSION} build-soroban-rpc
-RUN mv soroban-rpc /bin/soroban-rpc
+RUN make REPOSITORY_VERSION=${REPOSITORY_VERSION} build-${BINARY_NAME}
+
+# Move the binary to a common location
+RUN mv ${BINARY_NAME} /bin/${BINARY_NAME}
 
 FROM ubuntu:22.04
 ARG STELLAR_CORE_VERSION
+ARG BINARY_NAME=soroban-rpc
 ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*}
 ENV STELLAR_CORE_BINARY_PATH /usr/bin/stellar-core
 ENV DEBIAN_FRONTEND=noninteractive
@@ -35,5 +39,8 @@ RUN echo "deb https://apt.stellar.org focal unstable" >/etc/apt/sources.list.d/S
 RUN apt-get update && apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 RUN apt-get clean
 
-COPY --from=build /bin/soroban-rpc /app/
-ENTRYPOINT ["/app/soroban-rpc"]
+# Copy the binary from the build stage
+COPY --from=build /bin/${BINARY_NAME} /app/${BINARY_NAME}
+
+# Set the entrypoint to the specific binary
+ENTRYPOINT ["/app/${BINARY_NAME}"]

cmd/soroban-rpc/docker/Makefile

@@ -22,12 +22,22 @@ ifndef STELLAR_CORE_VERSION
         $(error STELLAR_CORE_VERSION environment variable must be set. For example 19.10.1-1310.6649f5173.focal~soroban)
 endif
 
-TAG ?= stellar/stellar-soroban-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION)
+# Set default value for BINARY_NAME if not provided
+BINARY_NAME ?= soroban-rpc
+
+# Set the TAG based on the value of BINARY_NAME
+ifeq ($(BINARY_NAME),stellar-rpc)
+    TAG := stellar/stellar-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION)
+else
+    TAG := stellar/stellar-soroban-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION)
+endif
+
 
 docker-build:
 	$(SUDO) docker build --pull --platform linux/amd64 $(DOCKER_OPTS) \
 	--label org.opencontainers.image.created="$(BUILD_DATE)" \
 	--build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION) --build-arg SOROBAN_RPC_VERSION=$(SOROBAN_RPC_VERSION_PACKAGE_VERSION) \
+	--build-arg BINARY_NAME=$(BINARY_NAME) \
 	-t $(TAG) -f Dockerfile.release .
 
 docker-push:

cmd/soroban-rpc/internal/db/transaction.go

@@ -25,6 +25,7 @@ const (
 var ErrNoTransaction = errors.New("no transaction with this hash exists")
 
 type Transaction struct {
+	TransactionHash  string
 	Result           []byte   // XDR encoded xdr.TransactionResult
 	Meta             []byte   // XDR encoded xdr.TransactionMeta
 	Envelope         []byte   // XDR encoded xdr.TransactionEnvelope
@@ -223,6 +224,7 @@ func ParseTransaction(lcm xdr.LedgerCloseMeta, ingestTx ingest.LedgerTransaction
 		Sequence:  lcm.LedgerSequence(),
 		CloseTime: lcm.LedgerCloseTime(),
 	}
+	tx.TransactionHash = ingestTx.Result.TransactionHash.HexString()
 
 	if tx.Result, err = ingestTx.Result.Result.MarshalBinary(); err != nil {
 		return tx, fmt.Errorf("couldn't encode transaction Result: %w", err)

cmd/soroban-rpc/internal/methods/get_events.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
@@ -85,12 +86,12 @@ func (e eventTypeSet) matches(event xdr.ContractEvent) bool {
 }
 
 type EventInfo struct {
-	EventType                string `json:"type"`
-	Ledger                   int32  `json:"ledger"`
-	LedgerClosedAt           string `json:"ledgerClosedAt"`
-	ContractID               string `json:"contractId"`
-	ID                       string `json:"id"`
-	PagingToken              string `json:"pagingToken"`
+	EventType      string `json:"type"`
+	Ledger         int32  `json:"ledger"`
+	LedgerClosedAt string `json:"ledgerClosedAt"`
+	ContractID     string `json:"contractId"`
+	ID             string `json:"id"`
+
 	InSuccessfulContractCall bool   `json:"inSuccessfulContractCall"`
 	TransactionHash          string `json:"txHash"`
 
@@ -336,6 +337,8 @@ type PaginationOptions struct {
 type GetEventsResponse struct {
 	Events       []EventInfo `json:"events"`
 	LatestLedger uint32      `json:"latestLedger"`
+	// Cursor represents last populated event ID if total events reach the limit or end of the search window
+	Cursor string `json:"cursor"`
 }
 
 type eventsRPCHandler struct {
@@ -439,7 +442,10 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques
 			limit = request.Pagination.Limit
 		}
 	}
-	endLedger := request.StartLedger + LedgerScanLimit
+	endLedger := start.Ledger + LedgerScanLimit
+
+	// endLedger should not exceed ledger retention window
+	endLedger = min(ledgerRange.LastLedger.Sequence+1, endLedger)
 
 	if request.EndLedger != 0 {
 		endLedger = min(request.EndLedger, endLedger)
@@ -509,9 +515,21 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques
 		results = append(results, info)
 	}
 
+	var cursor string
+	if uint(len(results)) == limit {
+		lastEvent := results[len(results)-1]
+		cursor = lastEvent.ID
+	} else {
+		// cursor represents end of the search window if events does not reach limit
+		// here endLedger is always exclusive when fetching events
+		// so search window is max Cursor value with endLedger - 1
+		cursor = db.Cursor{Ledger: endLedger - 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+	}
+
 	return GetEventsResponse{
 		LatestLedger: ledgerRange.LastLedger.Sequence,
 		Events:       results,
+		Cursor:       cursor,
 	}, nil
 }
 
@@ -535,7 +553,6 @@ func eventInfoForEvent(
 		Ledger:                   int32(cursor.Ledger),
 		LedgerClosedAt:           ledgerClosedAt,
 		ID:                       cursor.String(),
-		PagingToken:              cursor.String(),
 		InSuccessfulContractCall: event.InSuccessfulContractCall,
 		TransactionHash:          txHash,
 	}

cmd/soroban-rpc/internal/methods/get_events_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"path"
 	"strconv"
 	"strings"
@@ -648,14 +649,14 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABSC4",
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{value},
 				ValueXDR:                 value,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(i).HexString(),
 			})
 		}
-		assert.Equal(t, GetEventsResponse{expected, 1}, results)
+		cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+		assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 	})
 
 	t.Run("filtering by contract id", func(t *testing.T) {
@@ -794,14 +795,15 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABSC4",
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{counterXdr, value},
 				ValueXDR:                 value,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(4).HexString(),
 			},
 		}
-		assert.Equal(t, GetEventsResponse{expected, 1}, results)
+		cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+
+		assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 
 		results, err = handler.getEvents(ctx, GetEventsRequest{
 			StartLedger: 1,
@@ -835,7 +837,7 @@ func TestGetEvents(t *testing.T) {
 
 		expected[0].ValueJSON = valueJs
 		expected[0].TopicJSON = topicsJs
-		require.Equal(t, GetEventsResponse{expected, 1}, results)
+		require.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 	})
 
 	t.Run("filtering by both contract id and topic", func(t *testing.T) {
@@ -939,14 +941,15 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               strkey.MustEncode(strkey.VersionByteContract, contractID[:]),
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{counterXdr, value},
 				ValueXDR:                 value,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(3).HexString(),
 			},
 		}
-		assert.Equal(t, GetEventsResponse{expected, 1}, results)
+		cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+
+		assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 	})
 
 	t.Run("filtering by event type", func(t *testing.T) {
@@ -1014,14 +1017,15 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               strkey.MustEncode(strkey.VersionByteContract, contractID[:]),
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{counterXdr},
 				ValueXDR:                 counterXdr,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(0).HexString(),
 			},
 		}
-		assert.Equal(t, GetEventsResponse{expected, 1}, results)
+		cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+
+		assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 	})
 
 	t.Run("with limit", func(t *testing.T) {
@@ -1085,14 +1089,15 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABSC4",
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{value},
 				ValueXDR:                 value,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(i).HexString(),
 			})
 		}
-		assert.Equal(t, GetEventsResponse{expected, 1}, results)
+		cursor := expected[len(expected)-1].ID
+
+		assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results)
 	})
 
 	t.Run("with cursor", func(t *testing.T) {
@@ -1185,14 +1190,14 @@ func TestGetEvents(t *testing.T) {
 				LedgerClosedAt:           now.Format(time.RFC3339),
 				ContractID:               strkey.MustEncode(strkey.VersionByteContract, contractID[:]),
 				ID:                       id,
-				PagingToken:              id,
 				TopicXDR:                 []string{counterXdr},
 				ValueXDR:                 expectedXdr,
 				InSuccessfulContractCall: true,
 				TransactionHash:          ledgerCloseMeta.TransactionHash(i).HexString(),
 			})
 		}
-		assert.Equal(t, GetEventsResponse{expected, 5}, results)
+		cursor := expected[len(expected)-1].ID
+		assert.Equal(t, GetEventsResponse{expected, 5, cursor}, results)
 
 		results, err = handler.getEvents(context.TODO(), GetEventsRequest{
 			Pagination: &PaginationOptions{
@@ -1201,7 +1206,14 @@ func TestGetEvents(t *testing.T) {
 			},
 		})
 		require.NoError(t, err)
-		assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5}, results)
+
+		latestLedger := 5
+		endLedger := min(5+LedgerScanLimit, latestLedger+1)
+
+		// Note: endLedger is always exclusive when fetching events
+		// so search window is always max Cursor value with endLedger - 1
+		cursor = db.Cursor{Ledger: uint32(endLedger - 1), Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String()
+		assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5, cursor}, results)
 	})
 }
 

cmd/soroban-rpc/internal/methods/get_transactions.go

@@ -58,6 +58,9 @@ func (req GetTransactionsRequest) isValid(maxLimit uint, ledgerRange ledgerbucke
 type TransactionInfo struct {
 	// Status is one of: TransactionSuccess, TransactionFailed, TransactionNotFound.
 	Status string `json:"status"`
+	// TransactionHash is the hex encoded hash of the transaction. Note that for fee-bump transaction
+	// this will be the hash of the fee-bump transaction instead of the inner transaction hash.
+	TransactionHash string `json:"txHash"`
 	// ApplicationOrder is the index of the transaction among all the transactions
 	// for that ledger.
 	ApplicationOrder int32 `json:"applicationOrder"`
@@ -194,6 +197,7 @@ func (h transactionsRPCHandler) processTransactionsInLedger(
 		}
 
 		txInfo := TransactionInfo{
+			TransactionHash:  tx.TransactionHash,
 			ApplicationOrder: tx.ApplicationOrder,
 			FeeBump:          tx.FeeBump,
 			Ledger:           tx.Ledger.Sequence,