From 874124c4e7b82bdf316af14951aabc3030e0c458 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Wed, 2 Mar 2022 09:33:33 -0700 Subject: [PATCH] Handle finalize failures in fsm --- cmd/lotus-miner/info.go | 1 + extern/storage-sealing/fsm.go | 6 ++ extern/storage-sealing/sector_state.go | 130 +++++++++++++------------ 3 files changed, 73 insertions(+), 64 deletions(-) diff --git a/cmd/lotus-miner/info.go b/cmd/lotus-miner/info.go index 46070ca966c..32219e73b4a 100644 --- a/cmd/lotus-miner/info.go +++ b/cmd/lotus-miner/info.go @@ -527,6 +527,7 @@ var stateList = []stateMeta{ {col: color.FgRed, state: sealing.SnapDealsDealsExpired}, {col: color.FgRed, state: sealing.ReplicaUpdateFailed}, {col: color.FgRed, state: sealing.ReleaseSectorKeyFailed}, + {col: color.FgRed, state: sealing.FinalizeReplicaUpdateFailed}, } func init() { diff --git a/extern/storage-sealing/fsm.go b/extern/storage-sealing/fsm.go index 251d3a74aee..8678a0fe913 100644 --- a/extern/storage-sealing/fsm.go +++ b/extern/storage-sealing/fsm.go @@ -175,6 +175,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto ), FinalizeReplicaUpdate: planOne( on(SectorFinalized{}, UpdateActivating), + on(SectorFinalizeFailed{}, FinalizeReplicaUpdateFailed), ), UpdateActivating: planOne( on(SectorUpdateActive{}, ReleaseSectorKey), @@ -267,6 +268,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto ReleaseSectorKeyFailed: planOne( on(SectorUpdateActive{}, ReleaseSectorKey), ), + FinalizeReplicaUpdateFailed: planOne( + on(SectorRetryFinalize{}, FinalizeReplicaUpdate), + ), // Post-seal @@ -536,6 +540,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta return m.handleSubmitReplicaUpdateFailed, processed, nil case ReleaseSectorKeyFailed: return m.handleReleaseSectorKeyFailed, 0, err + case FinalizeReplicaUpdateFailed: + return m.handleFinalizeFailed, processed, nil case AbortUpgrade: return m.handleAbortUpgrade, processed, nil diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 5c2c56171a5..0f8228d027f 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -3,64 +3,65 @@ package sealing type SectorState string var ExistSectorStateList = map[SectorState]struct{}{ - Empty: {}, - WaitDeals: {}, - Packing: {}, - AddPiece: {}, - AddPieceFailed: {}, - GetTicket: {}, - PreCommit1: {}, - PreCommit2: {}, - PreCommitting: {}, - PreCommitWait: {}, - SubmitPreCommitBatch: {}, - PreCommitBatchWait: {}, - WaitSeed: {}, - Committing: {}, - CommitFinalize: {}, - CommitFinalizeFailed: {}, - SubmitCommit: {}, - CommitWait: {}, - SubmitCommitAggregate: {}, - CommitAggregateWait: {}, - FinalizeSector: {}, - Proving: {}, - FailedUnrecoverable: {}, - SealPreCommit1Failed: {}, - SealPreCommit2Failed: {}, - PreCommitFailed: {}, - ComputeProofFailed: {}, - CommitFailed: {}, - PackingFailed: {}, - FinalizeFailed: {}, - DealsExpired: {}, - RecoverDealIDs: {}, - Faulty: {}, - FaultReported: {}, - FaultedFinal: {}, - Terminating: {}, - TerminateWait: {}, - TerminateFinality: {}, - TerminateFailed: {}, - Removing: {}, - RemoveFailed: {}, - Removed: {}, - SnapDealsWaitDeals: {}, - SnapDealsAddPiece: {}, - SnapDealsPacking: {}, - UpdateReplica: {}, - ProveReplicaUpdate: {}, - SubmitReplicaUpdate: {}, - ReplicaUpdateWait: {}, - UpdateActivating: {}, - ReleaseSectorKey: {}, - FinalizeReplicaUpdate: {}, - SnapDealsAddPieceFailed: {}, - SnapDealsDealsExpired: {}, - SnapDealsRecoverDealIDs: {}, - ReplicaUpdateFailed: {}, - ReleaseSectorKeyFailed: {}, - AbortUpgrade: {}, + Empty: {}, + WaitDeals: {}, + Packing: {}, + AddPiece: {}, + AddPieceFailed: {}, + GetTicket: {}, + PreCommit1: {}, + PreCommit2: {}, + PreCommitting: {}, + PreCommitWait: {}, + SubmitPreCommitBatch: {}, + PreCommitBatchWait: {}, + WaitSeed: {}, + Committing: {}, + CommitFinalize: {}, + CommitFinalizeFailed: {}, + SubmitCommit: {}, + CommitWait: {}, + SubmitCommitAggregate: {}, + CommitAggregateWait: {}, + FinalizeSector: {}, + Proving: {}, + FailedUnrecoverable: {}, + SealPreCommit1Failed: {}, + SealPreCommit2Failed: {}, + PreCommitFailed: {}, + ComputeProofFailed: {}, + CommitFailed: {}, + PackingFailed: {}, + FinalizeFailed: {}, + DealsExpired: {}, + RecoverDealIDs: {}, + Faulty: {}, + FaultReported: {}, + FaultedFinal: {}, + Terminating: {}, + TerminateWait: {}, + TerminateFinality: {}, + TerminateFailed: {}, + Removing: {}, + RemoveFailed: {}, + Removed: {}, + SnapDealsWaitDeals: {}, + SnapDealsAddPiece: {}, + SnapDealsPacking: {}, + UpdateReplica: {}, + ProveReplicaUpdate: {}, + SubmitReplicaUpdate: {}, + ReplicaUpdateWait: {}, + UpdateActivating: {}, + ReleaseSectorKey: {}, + FinalizeReplicaUpdate: {}, + SnapDealsAddPieceFailed: {}, + SnapDealsDealsExpired: {}, + SnapDealsRecoverDealIDs: {}, + ReplicaUpdateFailed: {}, + ReleaseSectorKeyFailed: {}, + FinalizeReplicaUpdateFailed: {}, + AbortUpgrade: {}, } // cmd/lotus-miner/info.go defines CLI colors corresponding to these states @@ -124,12 +125,13 @@ const ( RecoverDealIDs SectorState = "RecoverDealIDs" // snap deals error modes - SnapDealsAddPieceFailed SectorState = "SnapDealsAddPieceFailed" - SnapDealsDealsExpired SectorState = "SnapDealsDealsExpired" - SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs" - AbortUpgrade SectorState = "AbortUpgrade" - ReplicaUpdateFailed SectorState = "ReplicaUpdateFailed" - ReleaseSectorKeyFailed SectorState = "ReleaseSectorKeyFailed" + SnapDealsAddPieceFailed SectorState = "SnapDealsAddPieceFailed" + SnapDealsDealsExpired SectorState = "SnapDealsDealsExpired" + SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs" + AbortUpgrade SectorState = "AbortUpgrade" + ReplicaUpdateFailed SectorState = "ReplicaUpdateFailed" + ReleaseSectorKeyFailed SectorState = "ReleaseSectorKeyFailed" + FinalizeReplicaUpdateFailed SectorState = "FinalizeReplicaUpdateFailed" Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain