fix: report jailed validators in peer-type cache and heartbeat metrics

cmd/node/node.go

@@ -278,35 +278,20 @@ func createNodesCoordinator(
 	startEpoch uint32,
 ) (sharding.NodesCoordinator, error) {
 
-	electedNodesInfo, eligibleNodesInfo, err := nodesConfig.InitialNodesInfo()
-	if err != nil {
-		return nil, err
-	}
-
-	electedValidators, err := sharding.NodesInfoToValidators(electedNodesInfo)
-	if err != nil {
-		return nil, err
-	}
-
-	eligibleValidators, err := sharding.NodesInfoToValidators(eligibleNodesInfo)
+	electedValidators, eligibleValidators, err := genesisValidators(nodesConfig)
 	if err != nil {
 		return nil, err
 	}
 
 	currentEpoch := startEpoch
 	if bootstrapParameters.NodesConfig != nil {
-		nodeRegistry := bootstrapParameters.NodesConfig
 		currentEpoch = bootstrapParameters.Epoch
-		epochsConfig, ok := nodeRegistry.EpochsConfig[fmt.Sprintf("%d", currentEpoch)]
+		epochsConfig, ok := bootstrapParameters.NodesConfig.EpochsConfig[fmt.Sprintf("%d", currentEpoch)]
 		if ok {
-			elected := epochsConfig.ElectedValidators
-			electedValidators, err = sharding.SerializableValidatorsToValidators(elected)
-			if err != nil {
-				return nil, err
-			}
-
-			eligibles := epochsConfig.EligibleValidators
-			eligibleValidators, err = sharding.SerializableValidatorsToValidators(eligibles)
+			// only elected and eligible seed the constructor arguments; the
+			// current epoch's waiting and leaving lists are restored later by
+			// LoadState during the storage bootstrap
+			electedValidators, eligibleValidators, _, _, err = registryEpochValidators(epochsConfig)
 			if err != nil {
 				return nil, err
 			}
@@ -351,34 +336,103 @@ func createNodesCoordinator(
 		return nil, err
 	}
 
-	if bootstrapParameters.NodesConfig != nil {
-		nodeRegistry := bootstrapParameters.NodesConfig
-		prevEpochsConfig, ok := nodeRegistry.EpochsConfig[fmt.Sprintf("%d", currentEpoch-1)]
-		if ok {
-			elected := prevEpochsConfig.ElectedValidators
-			electedValidators, err = sharding.SerializableValidatorsToValidators(elected)
-			if err != nil {
-				return nil, err
-			}
+	err = seedPreviousEpochFromRegistry(nodesCoordinator, bootstrapParameters.NodesConfig, currentEpoch)
+	if err != nil {
+		return nil, err
+	}
 
-			eligibles := prevEpochsConfig.EligibleValidators
-			eligibleValidators, err = sharding.SerializableValidatorsToValidators(eligibles)
-			if err != nil {
-				return nil, err
-			}
+	return nodesCoordinator, nil
+}
 
-			waiting := prevEpochsConfig.WaitingValidators
-			waitingValidators, err := sharding.SerializableValidatorsToValidators(waiting)
-			if err != nil {
-				return nil, err
-			}
+// genesisValidators converts the initial nodes setup into the elected and
+// eligible validator lists used to seed the coordinator.
+func genesisValidators(nodesConfig *sharding.NodesSetup) ([]sharding.Validator, []sharding.Validator, error) {
+	electedNodesInfo, eligibleNodesInfo, err := nodesConfig.InitialNodesInfo()
+	if err != nil {
+		return nil, nil, err
+	}
 
-			err = nodesCoordinator.SetNodes(electedValidators, eligibleValidators, waitingValidators, currentEpoch-1)
-			if err != nil {
-				return nil, err
-			}
-		}
+	electedValidators, err := sharding.NodesInfoToValidators(electedNodesInfo)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	return nodesCoordinator, nil
+	eligibleValidators, err := sharding.NodesInfoToValidators(eligibleNodesInfo)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return electedValidators, eligibleValidators, nil
+}
+
+// registryEpochValidators converts one registry epoch entry into validator
+// lists. Registries saved before the leaving list was persisted have no
+// LeavingValidators field; that list is then simply empty.
+func registryEpochValidators(
+	epochsConfig *sharding.EpochValidators,
+) (elected, eligible, waiting, leaving []sharding.Validator, err error) {
+	// a registry entry can be present but null in the stored JSON; fail
+	// explicitly instead of panicking during bootstrap
+	if epochsConfig == nil {
+		return nil, nil, nil, nil, fmt.Errorf("nil registry epoch validators")
+	}
+
+	elected, err = sharding.SerializableValidatorsToValidators(epochsConfig.ElectedValidators)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	eligible, err = sharding.SerializableValidatorsToValidators(epochsConfig.EligibleValidators)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	waiting, err = sharding.SerializableValidatorsToValidators(epochsConfig.WaitingValidators)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	leaving, err = sharding.SerializableValidatorsToValidators(epochsConfig.LeavingValidators)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	return elected, eligible, waiting, leaving, nil
+}
+
+// epochNodesSetter is the slice of the nodes coordinator needed to seed one
+// epoch's validator lists; SetNodes is not part of sharding.NodesCoordinator.
+type epochNodesSetter interface {
+	SetNodes(elected, eligible, waiting, leaving []sharding.Validator, epoch uint32) error
+}
+
+// seedPreviousEpochFromRegistry restores the previous epoch's validator lists
+// from the bootstrap registry into the freshly built coordinator, so a node
+// restarting mid-epoch can validate blocks that reference the previous epoch.
+func seedPreviousEpochFromRegistry(
+	setter epochNodesSetter,
+	nodeRegistry *sharding.NodesCoordinatorRegistry,
+	currentEpoch uint32,
+) error {
+	if nodeRegistry == nil {
+		return nil
+	}
+	// epoch 0 has no previous epoch; subtracting would wrap around and could
+	// match an unrelated registry entry
+	if currentEpoch == 0 {
+		return nil
+	}
+
+	prevEpoch := currentEpoch - 1
+	prevEpochsConfig, ok := nodeRegistry.EpochsConfig[fmt.Sprintf("%d", prevEpoch)]
+	if !ok {
+		return nil
+	}
+
+	elected, eligible, waiting, leaving, err := registryEpochValidators(prevEpochsConfig)
+	if err != nil {
+		return err
+	}
+
+	return setter.SetNodes(elected, eligible, waiting, leaving, prevEpoch)
 }

common/mock/nodesCoordinatorMock.go

@@ -24,7 +24,8 @@ type NodesCoordinatorMock struct {
 	GetAllEligibleValidatorsKeysWithEpochCalled func(uint32, bool) ([][]byte, error)
 	GetAllWaitingValidatorsKeysCalled           func() ([][]byte, error)
 	GetAllWaitingValidatorsKeysWithEpochCalled  func(uint32, bool) ([][]byte, error)
-	GetAllLeavingValidatorsPublicKeysCalled     func() ([][]byte, error)
+	GetAllLeavingValidatorsKeysCalled           func() ([][]byte, error)
+	GetAllLeavingValidatorsKeysWithEpochCalled  func(uint32, bool) ([][]byte, error)
 	CheckValidatorSlotCalled                    func(epoch uint32, slotIndex int64, pubkey []byte) bool
 	ConsensusGroupSizeCalled                    func() int
 	LoadValidatorsCalled                        func(validators []*state.ValidatorInfo) error
@@ -94,6 +95,17 @@ func (ncm *NodesCoordinatorMock) GetAllWaitingValidatorsKeys(epoch uint32, inclu
 	return nil, nil
 }
 
+// GetAllLeavingValidatorsKeys -
+func (ncm *NodesCoordinatorMock) GetAllLeavingValidatorsKeys(epoch uint32, ownerKey bool) ([][]byte, error) {
+	if ncm.GetAllLeavingValidatorsKeysWithEpochCalled != nil {
+		return ncm.GetAllLeavingValidatorsKeysWithEpochCalled(epoch, ownerKey)
+	}
+	if ncm.GetAllLeavingValidatorsKeysCalled != nil {
+		return ncm.GetAllLeavingValidatorsKeysCalled()
+	}
+	return nil, nil
+}
+
 // CheckValidatorSlot -
 func (ncm *NodesCoordinatorMock) CheckValidatorSlot(epoch uint32, slotIndex int64, pubkey []byte) bool {
 	if ncm.CheckValidatorSlotCalled != nil {
@@ -225,11 +237,6 @@ func (ncm *NodesCoordinatorMock) GetValidatorWithPublicKey(publicKey []byte) (sh
 	return nil, sharding.ErrValidatorNotFound
 }
 
-// GetAllLeavingValidatorsPublicKeys -
-func (ncm *NodesCoordinatorMock) GetAllLeavingValidatorsPublicKeys(_ uint32) ([][]byte, error) {
-	return nil, nil
-}
-
 // GetOwnPublicKey -
 func (ncm *NodesCoordinatorMock) GetOwnPublicKey() []byte {
 	return []byte("key")

core/bootstrap/disabled/disabledNodesCoordinator.go

@@ -44,6 +44,11 @@ func (n *nodesCoordinator) GetAllWaitingValidatorsKeys(_ uint32, _ bool) ([][]by
 	return nil, nil
 }
 
+// GetAllLeavingValidatorsKeys -
+func (n *nodesCoordinator) GetAllLeavingValidatorsKeys(_ uint32, _ bool) ([][]byte, error) {
+	return nil, nil
+}
+
 // CheckValidatorSlot -
 func (n *nodesCoordinator) CheckValidatorSlot(epoch uint32, slotIndex int64, pubkey []byte) bool {
 	return false

core/metrics.go

@@ -159,7 +159,8 @@ const MetricCountConsensus = "klv_count_consensus"
 // MetricNodeType is the metric for monitoring the type of the node
 const MetricNodeType = "klv_node_type"
 
-// MetricPeerType is the metric which tells the peer's type (in eligible list, in waiting list, or observer)
+// MetricPeerType is the metric which tells the peer's type (in elected, eligible,
+// waiting or jailed list, or observer)
 const MetricPeerType = "klv_peer_type"
 
 // MetricNumValidators is the metric for the number of validators
@@ -190,7 +191,7 @@ const MetricTxPoolLoad = "klv_tx_pool_load"
 const MetricIsSyncing = "klv_is_syncing"
 
 // MetricLiveValidatorNodes is the metric for monitoring live validators on the network
-// (elected, eligible or waiting)
+// (elected, eligible or waiting, excluding jailed)
 const MetricLiveValidatorNodes = "klv_live_validator_nodes"
 
 // MetricLiveConsensusValidatorNodes is the metric for monitoring live validators that can

core/process/peer/peerTypeProvider.go

@@ -122,28 +122,31 @@ func (ptp *PeerTypeProvider) createNewCache(
 ) (map[string]*peerListAndShard, bool) {
 	newCache := make(map[string]*peerListAndShard)
 
-	// waiting is seeded first so elected and eligible entries take precedence
-	// if a key ever appears in more than one list
-	nodesMapWaiting, err := ptp.nodesCoordinator.GetAllWaitingValidatorsKeys(epoch, false)
-	if err != nil {
-		log.Warn("peerTypeProvider - GetAllWaitingValidatorsKeys failed", "epoch", epoch, "error", err)
-		return nil, false
+	// the leaving list is seeded as jailed: the coordinator fills it exclusively
+	// from validators whose list is jailed (computeNodesConfigFromList), and
+	// operators should see that actionable state. Seeding order is defensive:
+	// later lists win, so working types take precedence if a key ever appears
+	// in more than one list (in production the lists are a partition, the
+	// numToStay promotion removes promoted keys from the leaving list).
+	listSources := []struct {
+		name     string
+		peerType core.PeerType
+		getKeys  func(epoch uint32, ownerKey bool) ([][]byte, error)
+	}{
+		{"GetAllLeavingValidatorsKeys", core.JailedList, ptp.nodesCoordinator.GetAllLeavingValidatorsKeys},
+		{"GetAllWaitingValidatorsKeys", core.WaitingList, ptp.nodesCoordinator.GetAllWaitingValidatorsKeys},
+		{"GetAllElectedValidatorsKeys", core.ElectedList, ptp.nodesCoordinator.GetAllElectedValidatorsKeys},
+		{"GetAllEligibleValidatorsKeys", core.EligibleList, ptp.nodesCoordinator.GetAllEligibleValidatorsKeys},
 	}
-	computePeerType(newCache, nodesMapWaiting, core.WaitingList)
 
-	nodesMapElected, err := ptp.nodesCoordinator.GetAllElectedValidatorsKeys(epoch, false)
-	if err != nil {
-		log.Warn("peerTypeProvider - GetAllElectedValidatorsKeys failed", "epoch", epoch, "error", err)
-		return nil, false
-	}
-	computePeerType(newCache, nodesMapElected, core.ElectedList)
-
-	nodesMapEligible, err := ptp.nodesCoordinator.GetAllEligibleValidatorsKeys(epoch, false)
-	if err != nil {
-		log.Warn("peerTypeProvider - GetAllEligibleValidatorsKeys failed", "epoch", epoch, "error", err)
-		return nil, false
+	for _, src := range listSources {
+		keys, err := src.getKeys(epoch, false)
+		if err != nil {
+			log.Warn("peerTypeProvider - "+src.name+" failed", "epoch", epoch, "error", err)
+			return nil, false
+		}
+		computePeerType(newCache, keys, src.peerType)
 	}
-	computePeerType(newCache, nodesMapEligible, core.EligibleList)
 
 	return newCache, true
 }