bor: fall back to eligible producers when veblop candidate set is empty

app/abci.go

@@ -961,7 +961,7 @@ func (app *HeimdallApp) checkAndAddFutureSpan(ctx sdk.Context, majorityMilestone
 			return err
 		}
 
-		err = app.BorKeeper.AddNewVeBlopSpan(ctx, currentProducer, lastSpan.EndBlock+1, endBlock, lastSpan.BorChainId, supportingValidatorIDs, uint64(ctx.BlockHeight()), borTypes.RoundRobinDefault, nil)
+		err = app.BorKeeper.AddNewVeBlopSpan(ctx, currentProducer, lastSpan.EndBlock+1, endBlock, lastSpan.BorChainId, supportingValidatorIDs, uint64(ctx.BlockHeight()), borTypes.RoundRobinDefault, nil, true)
 		if err != nil {
 			logger.Error("Error occurred while adding new veblop span", "error", err)
 			return err

app/veblop_producer_fallback_test.go

@@ -0,0 +1,217 @@
+package app
+
+import (
+	"strings"
+	"testing"
+
+	abci "github.com/cometbft/cometbft/abci/types"
+	cmtsecp256k1 "github.com/cometbft/cometbft/crypto/secp256k1"
+	cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+
+	"github.com/0xPolygon/heimdall-v2/helper"
+	"github.com/0xPolygon/heimdall-v2/sidetxs"
+	borKeeper "github.com/0xPolygon/heimdall-v2/x/bor/keeper"
+	borTypes "github.com/0xPolygon/heimdall-v2/x/bor/types"
+	milestoneTypes "github.com/0xPolygon/heimdall-v2/x/milestone/types"
+	stakeTypes "github.com/0xPolygon/heimdall-v2/x/stake/types"
+)
+
+// singletonScenario holds the staged degenerate VEBLOP state: the elected producer set
+// has collapsed to the single current producer, and a >2/3 milestone is finalized by the
+// other validators while the current producer withholds its milestone vote (so it is
+// excluded from the supporting set). Future-span creation in PreBlocker then asks
+// SelectNextSpanProducer for a replacement and the candidate set (singleton minus the
+// current producer) is empty.
+type singletonScenario struct {
+	app               *HeimdallApp
+	ctx               sdk.Context
+	height            int64
+	lastSpan          borTypes.Span
+	currentProducerID uint64
+	validators        []*stakeTypes.Validator
+	extCommitBytes    []byte
+}
+
+func stageSingletonProducer(t *testing.T) *singletonScenario {
+	t.Helper()
+	_, app, ctx, validatorPrivKeys := SetupAppWithABCICtxAndValidators(t, 4)
+
+	oldRio := helper.GetRioHeight()
+	helper.SetRioHeight(100)
+	t.Cleanup(func() { helper.SetRioHeight(oldRio) })
+
+	height := app.LastBlockHeight() + 1
+	ctx = ctx.WithBlockHeight(height)
+
+	validators := app.StakeKeeper.GetAllValidators(ctx)
+	require.Len(t, validators, 4)
+
+	currentProducerID := validators[0].ValId
+	lastSpan := borTypes.Span{
+		Id:                1,
+		StartBlock:        100,
+		EndBlock:          199,
+		BorChainId:        "1",
+		ValidatorSet:      stakeTypes.ValidatorSet{Validators: validators, Proposer: validators[0]},
+		SelectedProducers: []stakeTypes.Validator{*validators[0]},
+	}
+	require.NoError(t, app.BorKeeper.AddNewSpan(ctx, &lastSpan))
+
+	// Every validator casts a one-entry ballot for the current producer, collapsing
+	// CalculateProducerSet to the singleton [currentProducer].
+	msgServer := borKeeper.NewMsgServerImpl(app.BorKeeper)
+	for _, validator := range validators {
+		_, err := msgServer.VoteProducers(ctx, &borTypes.MsgVoteProducers{
+			Voter:   validator.Signer,
+			VoterId: validator.ValId,
+			Votes:   borTypes.ProducerVotes{Votes: []uint64{currentProducerID}},
+		})
+		require.NoError(t, err)
+	}
+	candidates, err := app.BorKeeper.CalculateProducerSet(ctx, helper.GetProducerSetLimit(ctx))
+	require.NoError(t, err)
+	require.Equal(t, []uint64{currentProducerID}, candidates)
+
+	previousMilestoneHash := common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").Bytes()
+	require.NoError(t, app.MilestoneKeeper.AddMilestone(ctx, milestoneTypes.Milestone{
+		Proposer:        validators[1].Signer,
+		Hash:            previousMilestoneHash,
+		StartBlock:      1,
+		EndBlock:        lastSpan.StartBlock - 1,
+		BorChainId:      "1",
+		MilestoneId:     "previous",
+		Timestamp:       1,
+		TotalDifficulty: 1,
+	}))
+
+	winningMilestone := &milestoneTypes.MilestoneProposition{
+		StartBlockNumber: lastSpan.StartBlock,
+		ParentHash:       previousMilestoneHash,
+		BlockHashes:      [][]byte{common.HexToHash("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").Bytes()},
+		BlockTds:         []uint64{2},
+	}
+
+	// Validators 1, 2 and 3 (75% of voting power) finalize the milestone; the current
+	// producer (validator 0) withholds, so it is absent from supportingValidatorIDs.
+	extCommitBytes := milestoneExtendedCommit(t, app.ChainID(), height, validators, validatorPrivKeys, currentProducerID, winningMilestone)
+
+	return &singletonScenario{
+		app:               app,
+		ctx:               ctx,
+		height:            height,
+		lastSpan:          lastSpan,
+		currentProducerID: currentProducerID,
+		validators:        validators,
+		extCommitBytes:    extCommitBytes,
+	}
+}
+
+// Pre-Ithaca the fallback is gated off, so the singleton-minus-current empty set still
+// errors out of PreBlocker. This pins the gate boundary: the fix changes nothing before
+// the fork activates.
+func TestVeBlopEmptyElectedSetPreIthacaErrors(t *testing.T) {
+	s := stageSingletonProducer(t)
+
+	oldIthaca := helper.GetIthacaHeight()
+	helper.SetIthacaHeight(0) // disabled: IsIthaca is always false
+	t.Cleanup(func() { helper.SetIthacaHeight(oldIthaca) })
+
+	_, err := s.app.PreBlocker(s.ctx, &abci.RequestFinalizeBlock{
+		Height:          s.height,
+		Txs:             [][]byte{s.extCommitBytes},
+		ProposerAddress: common.FromHex(s.validators[1].Signer),
+	})
+	require.Error(t, err)
+	require.True(t, strings.Contains(err.Error(), "no candidates found"), err.Error())
+
+	// Nothing committed: the failing height replays identically.
+	milestoneCount, err := s.app.MilestoneKeeper.GetMilestoneCount(s.ctx)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), milestoneCount)
+	span, err := s.app.BorKeeper.GetLastSpan(s.ctx)
+	require.NoError(t, err)
+	require.Equal(t, s.lastSpan.Id, span.Id)
+}
+
+// Post-Ithaca the fallback supplies a deterministic non-empty candidate set drawn from
+// the milestone supporters, so future-span creation succeeds, the milestone/span commit,
+// and the new producer is a supporting validator rather than the excluded incumbent.
+func TestVeBlopEmptyElectedSetPostIthacaFallsBack(t *testing.T) {
+	s := stageSingletonProducer(t)
+
+	oldIthaca := helper.GetIthacaHeight()
+	helper.SetIthacaHeight(s.height)
+	t.Cleanup(func() { helper.SetIthacaHeight(oldIthaca) })
+
+	_, err := s.app.PreBlocker(s.ctx, &abci.RequestFinalizeBlock{
+		Height:          s.height,
+		Txs:             [][]byte{s.extCommitBytes},
+		ProposerAddress: common.FromHex(s.validators[1].Signer),
+	})
+	require.NoError(t, err)
+
+	// The milestone committed and a new span was frozen.
+	milestoneCount, err := s.app.MilestoneKeeper.GetMilestoneCount(s.ctx)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), milestoneCount)
+
+	span, err := s.app.BorKeeper.GetLastSpan(s.ctx)
+	require.NoError(t, err)
+	require.Equal(t, s.lastSpan.Id+1, span.Id)
+	require.Len(t, span.SelectedProducers, 1)
+
+	// The replacement must be a supporting validator, never the excluded incumbent.
+	newProducer := span.SelectedProducers[0].ValId
+	require.NotEqual(t, s.currentProducerID, newProducer)
+	supporters := map[uint64]struct{}{
+		s.validators[1].ValId: {},
+		s.validators[2].ValId: {},
+		s.validators[3].ValId: {},
+	}
+	_, ok := supporters[newProducer]
+	require.True(t, ok, "new producer %d should be one of the milestone supporters", newProducer)
+}
+
+func milestoneExtendedCommit(t *testing.T, chainID string, height int64, validators []*stakeTypes.Validator, validatorPrivKeys []cmtsecp256k1.PrivKey, currentProducerID uint64, milestone *milestoneTypes.MilestoneProposition) []byte {
+	t.Helper()
+	require.Len(t, validatorPrivKeys, len(validators))
+
+	dummyNonRpExt, err := GetDummyNonRpVoteExtension(height-1, chainID)
+	require.NoError(t, err)
+
+	extCommit := &abci.ExtendedCommitInfo{Round: 0, Votes: make([]abci.ExtendedVoteInfo, 0, len(validators))}
+	for i, validator := range validators {
+		var prop *milestoneTypes.MilestoneProposition
+		if validator.ValId != currentProducerID {
+			prop = milestone
+		}
+
+		voteExt := sidetxs.VoteExtension{
+			BlockHash:            []byte("repro-previous-block"),
+			Height:               height - 1,
+			MilestoneProposition: prop,
+			SideTxResponses:      []sidetxs.SideTxResponse{},
+		}
+		voteExtBytes, err := voteExt.Marshal()
+		require.NoError(t, err)
+
+		voteInfo := abci.ExtendedVoteInfo{
+			Validator: abci.Validator{
+				Address: common.FromHex(validator.Signer),
+				Power:   validator.VotingPower,
+			},
+			VoteExtension:      voteExtBytes,
+			NonRpVoteExtension: dummyNonRpExt,
+			BlockIdFlag:        cmtproto.BlockIDFlagCommit,
+		}
+		createSignatureForVoteExtension(t, height-1, validatorPrivKeys[i], voteExtBytes, dummyNonRpExt, &voteInfo)
+		extCommit.Votes = append(extCommit.Votes, voteInfo)
+	}
+
+	bz, err := extCommit.Marshal()
+	require.NoError(t, err)
+	return bz
+}

x/bor/keeper/veblop.go

@@ -14,12 +14,17 @@
 	staketypes "github.com/0xPolygon/heimdall-v2/x/stake/types"
 )
 
-// AddNewVeBlopSpan adds a new veBlop (Validator-elected block producer) span
-func (k *Keeper) AddNewVeBlopSpan(ctx sdk.Context, currentProducer uint64, startBlock uint64, endBlock uint64, borChainID string, activeValidatorIDs map[uint64]struct{}, heimdallBlock uint64, targetProducerID uint64, excludedProducerIDs map[uint64]struct{}) error {
+// AddNewVeBlopSpan adds a new veBlop (Validator-elected block producer) span.
+//
+// fallbackToActiveSet opts the caller into the post-Ithaca empty-candidate recovery in
+// SelectNextSpanProducer. Only the future-span PreBlocker path passes true: there an empty
+// candidate set is a fatal chain halt. The rotation / pending-stall / side-message callers omit
+// it (default false) and keep their existing skip-and-retry behavior on an empty selection.
+func (k *Keeper) AddNewVeBlopSpan(ctx sdk.Context, currentProducer uint64, startBlock uint64, endBlock uint64, borChainID string, activeValidatorIDs map[uint64]struct{}, heimdallBlock uint64, targetProducerID uint64, excludedProducerIDs map[uint64]struct{}, fallbackToActiveSet ...bool) error {
 	logger := k.Logger(ctx)
 
 	// select next producers
-	newProducerId, err := k.SelectNextSpanProducer(ctx, currentProducer, activeValidatorIDs, helper.GetProducerSetLimit(ctx), startBlock, endBlock, targetProducerID, excludedProducerIDs)
+	newProducerId, err := k.SelectNextSpanProducer(ctx, currentProducer, activeValidatorIDs, helper.GetProducerSetLimit(ctx), startBlock, endBlock, targetProducerID, excludedProducerIDs, fallbackToActiveSet...)
 	if err != nil {
 		return err
 	}
@@ -237,7 +242,7 @@
 
 // SelectNextSpanProducer selects the next producer for a new span.
 // It calculates the candidate set, filters by active producers, and selects one.
-func (k *Keeper) SelectNextSpanProducer(ctx sdk.Context, currentProducer uint64, activeValidatorIDs map[uint64]struct{}, producerSetLimit, startBlock, endBlock uint64, targetProducerID uint64, excludedProducerIDs map[uint64]struct{}) (uint64, error) {
+func (k *Keeper) SelectNextSpanProducer(ctx sdk.Context, currentProducer uint64, activeValidatorIDs map[uint64]struct{}, producerSetLimit, startBlock, endBlock uint64, targetProducerID uint64, excludedProducerIDs map[uint64]struct{}, fallbackToActiveSet ...bool) (uint64, error) {
 	candidates, err := k.CalculateProducerSet(ctx, producerSetLimit)
 	if err != nil {
 		return 0, fmt.Errorf("failed to calculate producer set: %w", err)
@@ -268,6 +273,10 @@
 		activeCandidates = filterExcludedProducers(newCandidates, excludedProducerIDs)
 	}
 
+	// Post-Ithaca, recover a candidate so the future-span path (the only opt-in via fallbackToActiveSet) can't hand SelectProducer an empty set (see eligibleProducerFallback).
+	if len(activeCandidates) == 0 && len(fallbackToActiveSet) > 0 && fallbackToActiveSet[0] && helper.IsIthaca(ctx.BlockHeight()) {
+		activeCandidates = k.eligibleProducerFallback(ctx, currentProducer, activeValidatorIDs, excludedProducerIDs)
+	}
 	// If the declaring producer requested a specific replacement, try to honor it.
 	// The target must still be an active candidate and not down for the range.
 	// If any check fails, we fall through to round-robin rather than blocking span creation.

x/bor/keeper/veblop_producer_fallback.go

@@ -0,0 +1,55 @@
+package keeper
+
+import (
+	"slices"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// eligibleProducerFallback returns a deterministic candidate set for the degenerate case
+// where producer election leaves no selectable candidate other than the current producer
+// (the elected set has collapsed to the lone incumbent, or every alternative was filtered
+// out). It draws from the caller's active set (the milestone supporters) first, then the
+// full validator set, each sorted ascending with the current and excluded producers removed.
+//
+// Only the future-span PreBlocker path reaches this, where an empty candidate set is a fatal
+// chain halt. There it is always non-empty: a >2/3 milestone that excludes the current
+// producer leaves a non-empty supporting set, so SelectProducer never receives an empty slice
+// and the halt cannot occur. The validator-set step is defense-in-depth for that guarantee;
+// the non-fatal rotation paths do not opt into this fallback and keep their skip-and-retry
+// behavior.
+func (k *Keeper) eligibleProducerFallback(ctx sdk.Context, currentProducer uint64, activeValidatorIDs, excludedProducerIDs map[uint64]struct{}) []uint64 {
+	if c := sortedEligibleProducers(activeValidatorIDs, currentProducer, excludedProducerIDs); len(c) > 0 {
+		return c
+	}
+
+	valSet, err := k.sk.GetValidatorSet(ctx)
+	if err != nil {
+		k.Logger(ctx).Error("Failed to get validator set for producer fallback", "error", err)
+		return nil
+	}
+
+	valIDs := make(map[uint64]struct{}, len(valSet.Validators))
+	for _, v := range valSet.Validators {
+		valIDs[v.ValId] = struct{}{}
+	}
+	return sortedEligibleProducers(valIDs, currentProducer, excludedProducerIDs)
+}
+
+// sortedEligibleProducers returns the ids in set, ascending, omitting the current producer
+// and any excluded producer. Sorting makes the result deterministic across validators (Go
+// map iteration order is not).
+func sortedEligibleProducers(set map[uint64]struct{}, currentProducer uint64, excluded map[uint64]struct{}) []uint64 {
+	ids := make([]uint64, 0, len(set))
+	for id := range set {
+		if id == currentProducer {
+			continue
+		}
+		if _, isExcluded := excluded[id]; isExcluded {
+			continue
+		}
+		ids = append(ids, id)
+	}
+	slices.Sort(ids)
+	return ids
+}