port offsets for peer mismatch

Author: JackDoan

examples/config.yml

@@ -175,7 +175,9 @@ listen:
 # listen.port+i instead of sharing one port via SO_REUSEPORT, and one extra
 # tunnel ("lane") per routine is negotiated with capable peers: lane i
 # handshakes from local port listen.port+i to the peer's advertised
-# base+(i mod peer_ports). Each lane is a full Noise session with its own
+# base+((i + pair_offset) mod peer_ports), where pair_offset is a per-pair
+# hash that spreads many small peers across a big peer's whole port range.
+# Each lane is a full Noise session with its own
 # keys, nonce counter and replay window, so flows taking different paths
 # never fight over shared replay state.
 #

handshake_manager.go

@@ -740,7 +740,11 @@ func (hm *HandshakeManager) maybeAllocLaneState(hostinfo *HostInfo, result *hand
 		// A certified-but-hostile peer doesn't get to size our state.
 		peerPorts = 256
 	}
-	hostinfo.lanes = newLaneState(hm.f.routines, uint16(peerPorts), uint16(result.PeerBasePort))
+	var offset uint16
+	if len(hm.f.myVpnAddrs) > 0 && len(hostinfo.vpnAddrs) > 0 {
+		offset = lanePortOffset(hm.f.myVpnAddrs[0], hostinfo.vpnAddrs[0], uint16(peerPorts))
+	}
+	hostinfo.lanes = newLaneState(hm.f.routines, uint16(peerPorts), uint16(result.PeerBasePort), offset)
 }
 
 // EnsureLanes starts lane handshakes for every empty, non-pending, retry-due
@@ -785,7 +789,7 @@ func (hm *HandshakeManager) startLaneHandshake(base *HostInfo, i int) {
 		ls.noteLaneFailure(i)
 		return
 	}
-	target := netip.AddrPortFrom(remote.Addr(), ls.peerBasePort+uint16(i%int(ls.peerPortCount)))
+	target := netip.AddrPortFrom(remote.Addr(), ls.laneTargetPort(i))
 
 	hostinfo := &HostInfo{
 		vpnAddrs:        slices.Clone(base.vpnAddrs),

hostmap.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"hash/fnv"
 	"log/slog"
 	"net"
 	"net/netip"
@@ -316,9 +317,11 @@ type laneState struct {
 	sync.Mutex
 
 	// peerPortCount/peerBasePort are the peer's advert from the base
-	// handshake; lane i targets peerBasePort + (i % peerPortCount).
+	// handshake; portOffset is the per-pair rotation from lanePortOffset.
+	// Lane i targets peerBasePort + ((i + portOffset) % peerPortCount).
 	peerPortCount uint16
 	peerBasePort  uint16
+	portOffset    uint16
 
 	// txLanes[i] is our established, initiator-owned lane for routine i, or
 	// nil. Index 0 is always nil — the base tunnel is lane 0. A pointer is
@@ -337,17 +340,57 @@ type laneState struct {
 	peerLanes []*HostInfo
 }
 
-func newLaneState(laneCount int, peerPortCount, peerBasePort uint16) *laneState {
+func newLaneState(laneCount int, peerPortCount, peerBasePort, portOffset uint16) *laneState {
 	return &laneState{
 		peerPortCount: peerPortCount,
 		peerBasePort:  peerBasePort,
+		portOffset:    portOffset,
 		txLanes:       make([]atomic.Pointer[HostInfo], laneCount),
 		txPending:     make([]bool, laneCount),
 		txFails:       make([]uint8, laneCount),
 		txRetryAt:     make([]time.Time, laneCount),
 	}
 }
 
+// laneTargetPort returns the peer port that owned lane i handshakes to and
+// egresses toward. The caller must ensure peerPortCount != 0.
+func (ls *laneState) laneTargetPort(i int) uint16 {
+	return ls.peerBasePort + uint16((i+int(ls.portOffset))%int(ls.peerPortCount))
+}
+
+// lanePortOffset returns the rotation applied to this pair's lane target
+// ports, in [0, peerPortCount). Without it every low-routine peer would aim
+// its few lanes at a big peer's first few ports, concentrating the big
+// peer's receive work on a couple of sockets; the hash spreads pairs across
+// the whole advertised range.
+//
+// Both sides hash the same sorted vpn-address pair and the higher address
+// negates the result, so when port counts match the two sides' rotations
+// cancel: our lane i's 4-tuple is still the reverse of a peer-owned lane's,
+// and each outbound lane handshake opens the conntrack entry its partner
+// arrives through. (The one lane a nonzero rotation lands on the peer's base
+// port has no partner lane; behind a port-restricted NAT it may not form and
+// its routine rides the base tunnel — the standard lane fallback.)
+func lanePortOffset(myAddr, peerAddr netip.Addr, peerPortCount uint16) uint16 {
+	if peerPortCount == 0 {
+		return 0
+	}
+	lo, hi := myAddr, peerAddr
+	if hi.Less(lo) {
+		lo, hi = hi, lo
+	}
+	h := fnv.New32a()
+	b := lo.As16()
+	h.Write(b[:])
+	b = hi.As16()
+	h.Write(b[:])
+	o := uint16(h.Sum32() % uint32(peerPortCount))
+	if myAddr == hi {
+		o = (peerPortCount - o) % peerPortCount
+	}
+	return o
+}
+
 const (
 	laneRetryBase = 5 * time.Second
 	laneRetryMax  = 60 * time.Second

lanes_test.go

@@ -27,7 +27,7 @@ func newTestBaseHostInfo(vpnIp netip.Addr, localIdx, remoteIdx uint32, laneCount
 		HandshakePacket: map[uint8][]byte{},
 	}
 	base.SetRemote(netip.MustParseAddrPort("192.0.2.1:4242"))
-	base.lanes = newLaneState(laneCount, uint16(laneCount), 4242)
+	base.lanes = newLaneState(laneCount, uint16(laneCount), 4242, 0)
 	return base
 }
 
@@ -47,6 +47,62 @@ func newTestLaneHostInfo(base *HostInfo, laneIndex uint16, localIdx, remoteIdx u
 	return lane
 }
 
+func TestLanePortOffset(t *testing.T) {
+	a := netip.MustParseAddr("10.0.0.1")
+	b := netip.MustParseAddr("10.0.0.2")
+
+	// Deterministic and in range.
+	for _, count := range []uint16{1, 2, 3, 4, 16, 256} {
+		o := lanePortOffset(a, b, count)
+		assert.Equal(t, o, lanePortOffset(a, b, count), "count %d not deterministic", count)
+		assert.Less(t, o, count, "count %d out of range", count)
+	}
+	assert.Equal(t, uint16(0), lanePortOffset(a, b, 0), "zero port count")
+
+	// The two sides' rotations cancel when port counts match, preserving the
+	// lane-i-reverses-lane-j conntrack pairing.
+	for _, count := range []uint16{2, 3, 4, 7, 16} {
+		for i := range 32 {
+			peer := netip.AddrFrom4([4]byte{192, 0, 2, byte(i)})
+			oA := lanePortOffset(a, peer, count)
+			oB := lanePortOffset(peer, a, count)
+			assert.Equal(t, uint16(0), (oA+oB)%count,
+				"offsets don't cancel for peer %s count %d", peer, count)
+		}
+	}
+
+	// Distinct small peers land on distinct rotations of a big peer's range,
+	// not all on the same first ports.
+	const bigPeerPorts = 16
+	distinct := map[uint16]struct{}{}
+	for i := range 64 {
+		client := netip.AddrFrom4([4]byte{192, 0, 2, byte(i)})
+		distinct[lanePortOffset(client, a, bigPeerPorts)] = struct{}{}
+	}
+	assert.GreaterOrEqual(t, len(distinct), 8, "64 clients only produced %d distinct offsets", len(distinct))
+}
+
+func TestLaneTargetPort(t *testing.T) {
+	// No rotation: lane i targets base+i, wrapping past the peer's range.
+	ls := newLaneState(4, 4, 4242, 0)
+	for i, want := range map[int]uint16{1: 4243, 2: 4244, 3: 4245, 5: 4243} {
+		assert.Equal(t, want, ls.laneTargetPort(i), "lane %d", i)
+	}
+
+	// Rotation shifts the whole mapping; the wrapped lane lands on the base
+	// port itself, which is a valid distinct 4-tuple (our source port differs).
+	ls = newLaneState(4, 4, 4242, 3)
+	for i, want := range map[int]uint16{1: 4242, 2: 4243, 3: 4244} {
+		assert.Equal(t, want, ls.laneTargetPort(i), "rotated lane %d", i)
+	}
+
+	// Fewer peer ports than local lanes: rotation still spreads across all of
+	// the peer's ports.
+	ls = newLaneState(16, 2, 4242, 1)
+	assert.Equal(t, uint16(4242), ls.laneTargetPort(1))
+	assert.Equal(t, uint16(4243), ls.laneTargetPort(2))
+}
+
 func TestLaneHostmapLifecycle(t *testing.T) {
 	l := test.NewLogger()
 	hostMap := newHostMap(l)