Aggressive peer discovery for direct qset peers

Author: SirTyson

src/overlay/OverlayManager.h

@@ -143,6 +143,7 @@ class OverlayManager
 
     virtual bool isPreferred(Peer* peer) const = 0;
     virtual bool isDirectQsetPeer(NodeID const& nodeID) const = 0;
+    virtual void recordProbedNonQsetAddress(PeerBareAddress const& address) = 0;
     virtual bool isPossiblyPreferred(std::string const& ip) const = 0;
     virtual bool haveSpaceForConnection(std::string const& ip) const = 0;
 

src/overlay/OverlayManagerImpl.cpp

@@ -214,6 +214,13 @@ OverlayManagerImpl::PeersList::acceptAuthenticatedPeer(Peer::pointer peer)
     ZoneScoped;
     releaseAssert(threadIsMain());
 
+    if (peer->isMutualQsetPeer())
+    {
+        // Mutual direct-qset peers are operator-bounded by QUORUM_SET and must
+        // not consume ordinary inbound/outbound authenticated capacity.
+        return moveToAuthenticated(peer);
+    }
+
     CLOG_TRACE(Overlay, "Trying to promote peer to authenticated {}",
                peer->toString());
     if (mOverlayManager.isPreferred(peer.get()))
@@ -679,6 +686,70 @@ OverlayManagerImpl::connectTo(std::vector<PeerBareAddress> const& peers,
     return count;
 }
 
+void
+OverlayManagerImpl::connectToQsetPeers(int& availablePendingSlots)
+{
+    ZoneScoped;
+    releaseAssert(availablePendingSlots >= 0);
+    if (availablePendingSlots == 0)
+    {
+        return;
+    }
+
+    auto missing = mDirectQsetPeers;
+    missing.erase(mApp.getConfig().NODE_SEED.getPublicKey());
+    for (auto const& peer : getAuthenticatedPeers())
+    {
+        missing.erase(peer.first);
+    }
+    for (auto it = missing.begin(); it != missing.end();)
+    {
+        auto info = mQuorumPeerState.getInfo(*it);
+        if (info && info->remoteRole == RemoteQsetRole::None)
+        {
+            it = missing.erase(it);
+        }
+        else
+        {
+            ++it;
+        }
+    }
+
+    if (missing.empty())
+    {
+        return;
+    }
+
+    constexpr auto QSET_PROBE_BATCH_SIZE = 4;
+    std::vector<PeerBareAddress> candidates;
+    auto appendCandidates = [&](PeerType peerType) {
+        auto peers = getPeersToConnectTo(QSET_PROBE_BATCH_SIZE, peerType);
+        candidates.insert(std::end(candidates), std::begin(peers),
+                          std::end(peers));
+    };
+    appendCandidates(PeerType::INBOUND);
+    appendCandidates(PeerType::OUTBOUND);
+
+    std::set<PeerBareAddress> tried;
+    for (auto const& address : candidates)
+    {
+        if (availablePendingSlots == 0)
+        {
+            return;
+        }
+        if (mProbedNonQset.find(address) != std::end(mProbedNonQset) ||
+            getConnectedPeer(address) || !tried.insert(address).second)
+        {
+            continue;
+        }
+
+        if (connectToImpl(address, false))
+        {
+            --availablePendingSlots;
+        }
+    }
+}
+
 void
 OverlayManagerImpl::updateTimerAndMaybeDropRandomPeer(bool shouldDrop)
 {
@@ -867,6 +938,8 @@ OverlayManagerImpl::tick()
         availablePendingSlots -= pendingUsedByOutbound;
     }
 
+    connectToQsetPeers(availablePendingSlots);
+
     // Finally, attempt to promote some inbound connections to outbound
     if (availablePendingSlots > 0)
     {
@@ -900,10 +973,16 @@ OverlayManagerImpl::availableOutboundAuthenticatedSlots() const
             ? OverlayManager::MIN_INBOUND_FACTOR
             : mApp.getConfig().TARGET_PEER_CONNECTIONS;
 
-    if (mOutboundPeers.mAuthenticated.size() < adjustedTarget)
+    auto mutualQsetCount = std::count_if(
+        std::begin(mOutboundPeers.mAuthenticated),
+        std::end(mOutboundPeers.mAuthenticated),
+        [](auto const& peer) { return peer.second->isMutualQsetPeer(); });
+    auto ordinaryOutboundCount =
+        mOutboundPeers.mAuthenticated.size() - mutualQsetCount;
+
+    if (ordinaryOutboundCount < adjustedTarget)
     {
-        return static_cast<int>(adjustedTarget -
-                                mOutboundPeers.mAuthenticated.size());
+        return static_cast<int>(adjustedTarget - ordinaryOutboundCount);
     }
     else
     {
@@ -1191,6 +1270,14 @@ OverlayManagerImpl::isDirectQsetPeer(NodeID const& nodeID) const
     return mDirectQsetPeers.count(nodeID) != 0;
 }
 
+void
+OverlayManagerImpl::recordProbedNonQsetAddress(PeerBareAddress const& address)
+{
+    releaseAssert(threadIsMain());
+    releaseAssert(!address.isEmpty());
+    mProbedNonQset.insert(address);
+}
+
 static xdr::opaque_array<32> const TX_BATCH_HASH = [] {
     xdr::opaque_array<32> bytes{};
     for (auto& b : bytes)

src/overlay/OverlayManagerImpl.h

@@ -43,6 +43,10 @@ class OverlayManagerImpl : public OverlayManager
     Application& mApp;
     std::set<PeerBareAddress> mConfigurationPreferredPeers;
     std::set<NodeID> mDirectQsetPeers;
+    // Addresses that were proactively probed while searching for direct-qset
+    // peers and authenticated as non-qset peers. This is intentionally
+    // in-memory only; next-attempt backoff limits churn until restart.
+    std::set<PeerBareAddress> mProbedNonQset;
     QuorumPeerState mQuorumPeerState;
 
     struct PeersList
@@ -137,6 +141,7 @@ class OverlayManagerImpl : public OverlayManager
     bool acceptAuthenticatedPeer(Peer::pointer peer) override;
     bool isPreferred(Peer* peer) const override;
     bool isDirectQsetPeer(NodeID const& nodeID) const override;
+    void recordProbedNonQsetAddress(PeerBareAddress const& address) override;
     std::vector<Peer::pointer> const& getInboundPendingPeers() const override;
     std::vector<Peer::pointer> const& getOutboundPendingPeers() const override;
     std::vector<Peer::pointer> getPendingPeers() const override;
@@ -208,6 +213,7 @@ class OverlayManagerImpl : public OverlayManager
     int connectTo(int maxNum, PeerType peerType);
     int connectTo(std::vector<PeerBareAddress> const& peers,
                   bool forceoutbound);
+    void connectToQsetPeers(int& availablePendingSlots);
     std::vector<PeerBareAddress> getPeersToConnectTo(int maxNum,
                                                      PeerType peerType);
 

src/overlay/Peer.cpp

@@ -1802,6 +1802,10 @@ Peer::updatePeerRecordAfterAuthentication()
                 /* preferredTypeKnown */ true);
         }
     }
+    else
+    {
+        overlayManager.recordProbedNonQsetAddress(getAddress());
+    }
 
     CLOG_DEBUG(Overlay, "successful handshake with {}@{}",
                mAppConnector.getConfig().toShortString(mPeerID), toString());