soroban-rpc: Calculate soroban transaction data in libpreflight (#636)

* Create Skeletton to calculate soroban transaction data in libpreflight

* Calculate transaction data a minimum fee in libpreflight

* Calculate transaction data a minimum fee in libpreflight

* Make integration tests compilable again

* Assume missing entries from ledger storage are created

* Update simulate transaction integration tests

* Add TODOs for the integration tests to pass

* Make events global

* Address review feedback

* Add TODO about taking into account the Auth size

* Add a few more TODOs and debug info

* First attempt and preflighting all transactions

* Always take into account keys when computing unmodified ledgers

* Come up with the minimum fee to add in order to make it work

* Fix transaction_resources computation

* Remove debug printout

* Increase instruction leeway

* Fix the rest of the simulate transaction test

* Fix remaining intgeration tests

* Address review feedback

* Tweak the instruction estimation further

* Address review feedback

* Fix tests again

* Obtain auth data per function

* Take auth data into account for estimating envelope size

* Split up libpreflight in multiple modules and improve the storage code

* Fix TODO

* Remove another TODO

* Remove yet another TODO

* Appease clippy

* update

* update core version

* make sure to test multiple host functions invocations.

---------

Co-authored-by: Tsachi Herman <[email protected]>

.github/workflows/e2e.yml

@@ -22,7 +22,7 @@ jobs:
 
       # core git ref should be latest commit for stable soroban functionality
       # the core bin can either be compiled in-line here as part of ci, 
-      SYSTEM_TEST_CORE_GIT_REF: https://github.com/stellar/stellar-core.git#61f76a1137b82cbf6724c54bdd325daebed28151
+      SYSTEM_TEST_CORE_GIT_REF: https://github.com/stellar/stellar-core.git#57cc8198c95c95114d60f7c0f1498933ff57bd9b
       SYSTEM_TEST_CORE_COMPILE_CONFIGURE_FLAGS: "--disable-tests --enable-next-protocol-version-unsafe-for-production"
       # or can use option to pull a pre-compiled image instead
       # SYSTEM_TEST_CORE_IMAGE: 

cmd/soroban-rpc/internal/methods/simulate_transaction.go

@@ -21,19 +21,21 @@ type SimulateTransactionCost struct {
 	MemoryBytes     uint64 `json:"memBytes,string"`
 }
 
-type SimulateTransactionResult struct {
-	Auth      []string `json:"auth"`
-	Events    []string `json:"events"`
-	Footprint string   `json:"footprint"`
-	// TODO: update documentation and review field name
-	XDRs []string `json:"xdrs"`
+// SimulateHostFunctionResult contains the simulation result of each HostFunction within the single InvokeHostFunctionOp allowed in a Transaction
+type SimulateHostFunctionResult struct {
+	Auth []string `json:"auth"`
+	XDR  string   `json:"xdr"`
 }
 
 type SimulateTransactionResponse struct {
-	Error        string                      `json:"error,omitempty"`
-	Results      []SimulateTransactionResult `json:"results,omitempty"`
-	Cost         SimulateTransactionCost     `json:"cost"`
-	LatestLedger int64                       `json:"latestLedger,string"`
+	Error string `json:"error,omitempty"`
+	// TODO: update documentation and review field names
+	TransactionData string                       `json:"transactionData"` // SorobanTransactionData XDR in base64
+	Events          []string                     `json:"events"`          // DiagnosticEvent XDR in base64
+	MinResourceFee  int64                        `json:"minResourceFee,string"`
+	Results         []SimulateHostFunctionResult `json:"results,omitempty"` // an array of the individual host function call results
+	Cost            SimulateTransactionCost      `json:"cost"`              // the effective cpu and memory cost of the invoked transaction execution.
+	LatestLedger    int64                        `json:"latestLedger,string"`
 }
 
 type PreflightGetter interface {
@@ -96,15 +98,19 @@ func NewSimulateTransactionHandler(logger *log.Entry, ledgerEntryReader db.Ledge
 			}
 		}
 
+		hostFunctionResults := make([]SimulateHostFunctionResult, len(result.Results))
+		for i := 0; i < len(hostFunctionResults); i++ {
+			hostFunctionResults[i] = SimulateHostFunctionResult{
+				Auth: result.Results[i].Auth,
+				XDR:  result.Results[i].Result,
+			}
+		}
+
 		return SimulateTransactionResponse{
-			Results: []SimulateTransactionResult{
-				{
-					Events:    result.Events,
-					Auth:      result.Auth,
-					Footprint: result.Footprint,
-					XDRs:      result.Results,
-				},
-			},
+			Results:         hostFunctionResults,
+			Events:          result.Events,
+			TransactionData: result.TransactionData,
+			MinResourceFee:  result.MinFee,
 			Cost: SimulateTransactionCost{
 				CPUInstructions: result.CPUInstructions,
 				MemoryBytes:     result.MemoryBytes,

cmd/soroban-rpc/internal/preflight/preflight.go

@@ -95,11 +95,16 @@ type PreflightParameters struct {
 	LedgerEntryReadTx  db.LedgerEntryReadTx
 }
 
+type HostFunctionPreflight struct {
+	Result string   // XDR SCVal in base64
+	Auth   []string // ContractAuths XDR in base64
+}
+
 type Preflight struct {
-	Auth            []string // ContractAuths XDR in base64
 	Events          []string // DiagnosticEvents XDR in base64
-	Footprint       string   // LedgerFootprint XDR in base64
-	Results         []string // SCVal XDRs in base64
+	TransactionData string   // SorobanTransactionData XDR in base64
+	MinFee          int64
+	Results         []HostFunctionPreflight
 	CPUInstructions uint64
 	MemoryBytes     uint64
 }
@@ -123,7 +128,6 @@ func GoNullTerminatedStringSlice(str **C.char) []string {
 }
 
 func GetPreflight(ctx context.Context, params PreflightParameters) (Preflight, error) {
-	// TODO: this will be broken until the Go XDR is updated
 	invokeHostFunctionB64, err := xdr.MarshalBase64(params.InvokeHostFunction)
 	if err != nil {
 		return Preflight{}, err
@@ -149,7 +153,7 @@ func GetPreflight(ctx context.Context, params PreflightParameters) (Preflight, e
 	sourceAccountCString := C.CString(sourceAccountB64)
 	handle := cgo.NewHandle(snapshotSourceHandle{params.LedgerEntryReadTx, params.Logger})
 	defer handle.Delete()
-	res := C.preflight_host_function(
+	res := C.preflight_invoke_hf_op(
 		C.uintptr_t(handle),
 		invokeHostFunctionCString,
 		sourceAccountCString,
@@ -163,11 +167,20 @@ func GetPreflight(ctx context.Context, params PreflightParameters) (Preflight, e
 		return Preflight{}, errors.New(C.GoString(res.error))
 	}
 
+	cHostFunctionPreflights := (*[1 << 20]C.CHostFunctionPreflight)(unsafe.Pointer(res.results))[:res.results_size:res.results_size]
+	hostFunctionPreflights := make([]HostFunctionPreflight, len(cHostFunctionPreflights))
+	for i, cHostFunctionPreflight := range cHostFunctionPreflights {
+		hostFunctionPreflights[i] = HostFunctionPreflight{
+			Result: C.GoString(cHostFunctionPreflight.result),
+			Auth:   GoNullTerminatedStringSlice(cHostFunctionPreflight.auth),
+		}
+	}
+
 	preflight := Preflight{
-		Auth:            GoNullTerminatedStringSlice(res.auth),
 		Events:          GoNullTerminatedStringSlice(res.events),
-		Footprint:       C.GoString(res.preflight),
-		Results:         GoNullTerminatedStringSlice(res.results),
+		TransactionData: C.GoString(res.transaction_data),
+		MinFee:          int64(res.min_fee),
+		Results:         hostFunctionPreflights,
 		CPUInstructions: uint64(res.cpu_instructions),
 		MemoryBytes:     uint64(res.memory_bytes),
 	}

cmd/soroban-rpc/internal/test/get_ledger_entries_test.go

@@ -79,17 +79,18 @@ func TestGetLedgerEntriesSucceeds(t *testing.T) {
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	account := txnbuild.NewSimpleAccount(kp.Address(), 0)
 
-	tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{
+	params := preflightTransactionParams(t, client, txnbuild.TransactionParams{
 		SourceAccount:        &account,
 		IncrementSequenceNum: true,
 		Operations: []txnbuild.Operation{
-			createInstallContractCodeOperation(t, account.AccountID, testContract, true),
+			createInstallContractCodeOperation(account.AccountID, testContract),
 		},
 		BaseFee: txnbuild.MinBaseFee,
 		Preconditions: txnbuild.Preconditions{
 			TimeBounds: txnbuild.NewInfiniteTimeout(),
 		},
 	})
+	tx, err := txnbuild.NewTransaction(params)
 	require.NoError(t, err)
 	tx, err = tx.Sign(StandaloneNetworkPassphrase, kp)
 	require.NoError(t, err)

cmd/soroban-rpc/internal/test/get_ledger_entry_test.go

@@ -72,17 +72,18 @@ func TestGetLedgerEntrySucceeds(t *testing.T) {
 	kp := keypair.Root(StandaloneNetworkPassphrase)
 	account := txnbuild.NewSimpleAccount(kp.Address(), 0)
 
-	tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{
+	params := preflightTransactionParams(t, client, txnbuild.TransactionParams{
 		SourceAccount:        &account,
 		IncrementSequenceNum: true,
 		Operations: []txnbuild.Operation{
-			createInstallContractCodeOperation(t, account.AccountID, testContract, true),
+			createInstallContractCodeOperation(account.AccountID, testContract),
 		},
 		BaseFee: txnbuild.MinBaseFee,
 		Preconditions: txnbuild.Preconditions{
 			TimeBounds: txnbuild.NewInfiniteTimeout(),
 		},
 	})
+	tx, err := txnbuild.NewTransaction(params)
 	assert.NoError(t, err)
 	tx, err = tx.Sign(StandaloneNetworkPassphrase, kp)
 	assert.NoError(t, err)