Merge pull request #8014 from filecoin-project/fix/8011

fix: sealing: Stop recovery attempts after fault

extern/storage-sealing/states_failed.go

@@ -211,7 +211,7 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 
 	tok, _, err := m.Api.ChainHead(ctx.Context())
 	if err != nil {
-		log.Errorf("handleCommitting: api error, not proceeding: %+v", err)
+		log.Errorf("handleSubmitReplicaUpdateFailed: api error, not proceeding: %+v", err)
 		return nil
 	}
 
@@ -237,6 +237,17 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 		}
 	}
 
+	// Abort upgrade for sectors that went faulty since being marked for upgrade
+	active, err := sectorActive(ctx.Context(), m.Api, m.maddr, tok, sector.SectorNumber)
+	if err != nil {
+		log.Errorf("sector active check: api error, not proceeding: %+v", err)
+		return nil
+	}
+	if !active {
+		log.Errorf("sector marked for upgrade %d no longer active, aborting upgrade", sector.SectorNumber)
+		return ctx.Send(SectorAbortUpgrade{})
+	}
+
 	if err := failedCooldown(ctx, sector); err != nil {
 		return err
 	}

extern/storage-sealing/states_replica_update.go

@@ -31,6 +31,21 @@ func (m *Sealing) handleProveReplicaUpdate(ctx statemachine.Context, sector Sect
 	if sector.CommR == nil {
 		return xerrors.Errorf("invalid sector %d with nil CommR", sector.SectorNumber)
 	}
+	// Abort upgrade for sectors that went faulty since being marked for upgrade
+	tok, _, err := m.Api.ChainHead(ctx.Context())
+	if err != nil {
+		log.Errorf("handleProveReplicaUpdate: api error, not proceeding: %+v", err)
+		return nil
+	}
+	active, err := sectorActive(ctx.Context(), m.Api, m.maddr, tok, sector.SectorNumber)
+	if err != nil {
+		log.Errorf("sector active check: api error, not proceeding: %+v", err)
+		return nil
+	}
+	if !active {
+		log.Errorf("sector marked for upgrade %d no longer active, aborting upgrade", sector.SectorNumber)
+		return ctx.Send(SectorAbortUpgrade{})
+	}
 
 	vanillaProofs, err := m.sealer.ProveReplicaUpdate1(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber), *sector.CommR, *sector.UpdateSealed, *sector.UpdateUnsealed)
 	if err != nil {

extern/storage-sealing/upgrade_queue.go

@@ -3,6 +3,7 @@ package sealing
 import (
 	"context"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	market7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/market"
 
@@ -86,19 +87,11 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 		return xerrors.Errorf("failed to read sector on chain info: %w", err)
 	}
 
-	active, err := m.Api.StateMinerActiveSectors(ctx, m.maddr, tok)
+	active, err := sectorActive(ctx, m.Api, m.maddr, tok, id)
 	if err != nil {
-		return xerrors.Errorf("failed to check active sectors: %w", err)
+		return xerrors.Errorf("failed to check if sector is active")
 	}
-	// Ensure the upgraded sector is active
-	var found bool
-	for _, si := range active {
-		if si.SectorNumber == id {
-			found = true
-			break
-		}
-	}
-	if !found {
+	if !active {
 		return xerrors.Errorf("cannot mark inactive sector for upgrade")
 	}
 
@@ -110,6 +103,22 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 	return m.sectors.Send(uint64(id), SectorStartCCUpdate{})
 }
 
+func sectorActive(ctx context.Context, api SealingAPI, maddr address.Address, tok TipSetToken, sector abi.SectorNumber) (bool, error) {
+	active, err := api.StateMinerActiveSectors(ctx, maddr, tok)
+	if err != nil {
+		return false, xerrors.Errorf("failed to check active sectors: %w", err)
+	}
+	// Check if sector is among active sectors
+	var found bool
+	for _, si := range active {
+		if si.SectorNumber == sector {
+			found = true
+			break
+		}
+	}
+	return found, nil
+}
+
 func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreCommitInfo) big.Int {
 	if len(params.DealIDs) == 0 {
 		return big.Zero()