Add catch all panic handler in dex endblock

sei-protocol · Dec 29, 2023 · ff3b08d · ff3b08d
1 parent 3a53526
commit ff3b08d
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 9 deletions.
diff --git a/x/dex/contract/abci.go b/x/dex/contract/abci.go
@@ -228,6 +228,15 @@ func handleUnfulfilledMarketOrders(ctx context.Context, sdkCtx sdk.Context, env
 }
 
 func orderMatchingRunnable(ctx context.Context, sdkContext sdk.Context, env *environment, keeper *keeper.Keeper, contractInfo types.ContractInfoV2, tracer *otrace.Tracer) {
+	defer func() {
+		if err := recover(); err != nil {
+			msg := fmt.Sprintf("PANIC RECOVERED during order matching: %s", err)
+			sdkContext.Logger().Error(msg)
+			if env != nil {
+				env.addError(contractInfo.ContractAddr, errors.New(msg))
+			}
+		}
+	}()
 	_, span := (*tracer).Start(ctx, "orderMatchingRunnable")
 	defer span.End()
 	defer telemetry.MeasureSince(time.Now(), "dex", "order_matching_runnable")

diff --git a/x/dex/contract/execution.go b/x/dex/contract/execution.go
@@ -3,7 +3,6 @@ package contract
 import (
 	"context"
 	"fmt"
-	"sync"
 	"time"
 
 	otrace "go.opentelemetry.io/otel/trace"
@@ -130,16 +129,17 @@ func ExecutePairsInParallel(ctx sdk.Context, contractAddr string, dexkeeper *kee
 	cancelResults := []*types.Cancellation{}
 	settlements := []*types.SettlementEntry{}
 
-	mu := sync.Mutex{}
-	wg := sync.WaitGroup{}
+	// mu := sync.Mutex{}
+	// wg := sync.WaitGroup{}
 
 	for _, pair := range registeredPairs {
-		wg.Add(1)
+		// wg.Add(1)
 
 		pair := pair
 		pairCtx := ctx.WithMultiStore(multi.NewStore(ctx.MultiStore(), GetPerPairWhitelistMap(contractAddr, pair))).WithEventManager(sdk.NewEventManager())
-		go func() {
-			defer wg.Done()
+		// go func() {
+		func() {
+			// defer wg.Done()
 			pairCopy := pair
 			pairStr := types.GetPairString(&pairCopy)
 			orderbook, found := orderBooks.Load(pairStr)
@@ -150,8 +150,8 @@ func ExecutePairsInParallel(ctx sdk.Context, contractAddr string, dexkeeper *kee
 			orderIDToSettledQuantities := GetOrderIDToSettledQuantities(pairSettlements)
 			PrepareCancelUnfulfilledMarketOrders(pairCtx, typedContractAddr, pairCopy, orderIDToSettledQuantities)
 
-			mu.Lock()
-			defer mu.Unlock()
+			// mu.Lock()
+			// defer mu.Unlock()
 			orders, cancels := GetMatchResults(ctx, typedContractAddr, pairCopy)
 			orderResults = append(orderResults, orders...)
 			cancelResults = append(cancelResults, cancels...)
@@ -160,7 +160,7 @@ func ExecutePairsInParallel(ctx sdk.Context, contractAddr string, dexkeeper *kee
 			ctx.EventManager().EmitEvents(pairCtx.EventManager().Events())
 		}()
 	}
-	wg.Wait()
+	// wg.Wait()
 	dexkeeper.SetMatchResult(ctx, contractAddr, types.NewMatchResult(orderResults, cancelResults, settlements))
 
 	return settlements

diff --git a/x/dex/contract/runner.go b/x/dex/contract/runner.go
@@ -134,6 +134,12 @@ func (r *ParallelRunner) Run() {
 }
 
 func (r *ParallelRunner) wrapRunnable(contractAddr types.ContractAddress) {
+	defer func() {
+		if err := recover(); err != nil {
+			r.sdkCtx.Logger().Error(fmt.Sprintf("panic in parallel runner recovered: %s", err))
+		}
+	}()
+
 	contractInfo, _ := r.contractAddrToInfo.Load(contractAddr)
 	r.runnable(*contractInfo)
 

diff --git a/x/dex/module.go b/x/dex/module.go
@@ -304,6 +304,11 @@ func (am AppModule) getPriceToDelete(
 // EndBlock executes all ABCI EndBlock logic respective to the capability module. It
 // returns no validator updates.
 func (am AppModule) EndBlock(ctx sdk.Context, _ abci.RequestEndBlock) (ret []abci.ValidatorUpdate) {
+	defer func() {
+		if err := recover(); err != nil {
+			ctx.Logger().Error(fmt.Sprintf("panic in endblock recovered: %s", err))
+		}
+	}()
 	_, span := am.tracingInfo.Start("DexEndBlock")
 	defer span.End()
 	defer dexutils.GetMemState(ctx.Context()).Clear(ctx)