Skip to content

Commit 18f83c0

Browse files
nbrownusnbrownus
committed
Fix e2e test races when looking at hostmap counts
1 parent 1b84bd0 commit 18f83c0

3 files changed

Lines changed: 48 additions & 36 deletions

File tree

‎control_tester.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,14 @@ func (c *Control) GetHostmap() *HostMap {
125125
return c.f.hostMap
126126
}
127127

128+
// GetHostmapIndexCount returns the number of entries in the main hostmap Indexes table, holding
129+
// the hostmap read lock so tests can poll it while connection manager churns tunnels.
130+
func (c *Control) GetHostmapIndexCount() int {
131+
c.f.hostMap.RLock()
132+
defer c.f.hostMap.RUnlock()
133+
return len(c.f.hostMap.Indexes)
134+
}
135+
128136
func (c *Control) GetF() *Interface {
129137
return c.f
130138
}

‎e2e/handshakes_test.go‎

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ func TestStage1Race(t *testing.T) {
405405

406406
r.Log("Spin until connection manager tears down a tunnel")
407407

408-
for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
408+
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 {
409409
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
410410
t.Log("Connection manager hasn't ticked yet")
411411
time.Sleep(time.Second)
@@ -453,9 +453,11 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
453453

454454
r.Log("Nuke my hostmap")
455455
myHostmap := myControl.GetHostmap()
456+
myHostmap.Lock()
456457
myHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{}
457458
myHostmap.Indexes = map[uint32]*nebula.HostInfo{}
458459
myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
460+
myHostmap.Unlock()
459461

460462
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again")))
461463
p = r.RouteForAllUntilTxTun(theirControl)
@@ -465,10 +467,10 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
465467
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
466468

467469
r.Log("Wait for the dead index to go away")
468-
start := len(theirControl.GetHostmap().Indexes)
470+
start := theirControl.GetHostmapIndexCount()
469471
for {
470472
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
471-
if len(theirControl.GetHostmap().Indexes) < start {
473+
if theirControl.GetHostmapIndexCount() < start {
472474
break
473475
}
474476
time.Sleep(time.Second)
@@ -504,9 +506,11 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
504506

505507
r.Log("Nuke my hostmap")
506508
theirHostmap := theirControl.GetHostmap()
509+
theirHostmap.Lock()
507510
theirHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{}
508511
theirHostmap.Indexes = map[uint32]*nebula.HostInfo{}
509512
theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
513+
theirHostmap.Unlock()
510514

511515
theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again")))
512516
p = r.RouteForAllUntilTxTun(myControl)
@@ -517,10 +521,10 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
517521
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
518522

519523
r.Log("Wait for the dead index to go away")
520-
start := len(myControl.GetHostmap().Indexes)
524+
start := myControl.GetHostmapIndexCount()
521525
for {
522526
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
523-
if len(myControl.GetHostmap().Indexes) < start {
527+
if myControl.GetHostmapIndexCount() < start {
524528
break
525529
}
526530
time.Sleep(time.Second)
@@ -628,10 +632,10 @@ func TestReestablishRelays(t *testing.T) {
628632
r.Log("Close the tunnel")
629633
relayControl.CloseTunnel(theirVpnIpNet[0].Addr(), true)
630634

631-
start := len(myControl.GetHostmap().Indexes)
632-
curIndexes := len(myControl.GetHostmap().Indexes)
635+
start := myControl.GetHostmapIndexCount()
636+
curIndexes := myControl.GetHostmapIndexCount()
633637
for curIndexes >= start {
634-
curIndexes = len(myControl.GetHostmap().Indexes)
638+
curIndexes = myControl.GetHostmapIndexCount()
635639
r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
636640
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail")))
637641

@@ -819,18 +823,18 @@ func TestStage1RaceRelays2(t *testing.T) {
819823

820824
t.Log("Wait until we remove extra tunnels")
821825
t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
822-
len(myControl.GetHostmap().Indexes),
823-
len(theirControl.GetHostmap().Indexes),
824-
len(relayControl.GetHostmap().Indexes),
826+
myControl.GetHostmapIndexCount(),
827+
theirControl.GetHostmapIndexCount(),
828+
relayControl.GetHostmapIndexCount(),
825829
)
826-
hostInfos := len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
830+
hostInfos := myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount()
827831
retries := 60
828832
for hostInfos > 6 && retries > 0 {
829-
hostInfos = len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
833+
hostInfos = myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount()
830834
t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
831-
len(myControl.GetHostmap().Indexes),
832-
len(theirControl.GetHostmap().Indexes),
833-
len(relayControl.GetHostmap().Indexes),
835+
myControl.GetHostmapIndexCount(),
836+
theirControl.GetHostmapIndexCount(),
837+
relayControl.GetHostmapIndexCount(),
834838
)
835839
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
836840
t.Log("Connection manager hasn't ticked yet")
@@ -924,24 +928,24 @@ func TestRehandshakingRelays(t *testing.T) {
924928
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
925929
r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl)
926930
// We should have two hostinfos on all sides
927-
for len(myControl.GetHostmap().Indexes) != 2 {
928-
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes))
931+
for myControl.GetHostmapIndexCount() != 2 {
932+
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount())
929933
r.Log("Assert the relay tunnel still works")
930934
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
931935
r.Log("yupitdoes")
932936
time.Sleep(time.Second)
933937
}
934938
t.Logf("myControl hostinfos got cleaned up!")
935-
for len(theirControl.GetHostmap().Indexes) != 2 {
936-
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes))
939+
for theirControl.GetHostmapIndexCount() != 2 {
940+
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount())
937941
r.Log("Assert the relay tunnel still works")
938942
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
939943
r.Log("yupitdoes")
940944
time.Sleep(time.Second)
941945
}
942946
t.Logf("theirControl hostinfos got cleaned up!")
943-
for len(relayControl.GetHostmap().Indexes) != 2 {
944-
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes))
947+
for relayControl.GetHostmapIndexCount() != 2 {
948+
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount())
945949
r.Log("Assert the relay tunnel still works")
946950
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
947951
r.Log("yupitdoes")
@@ -1029,24 +1033,24 @@ func TestRehandshakingRelaysPrimary(t *testing.T) {
10291033
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
10301034
r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl)
10311035
// We should have two hostinfos on all sides
1032-
for len(myControl.GetHostmap().Indexes) != 2 {
1033-
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes))
1036+
for myControl.GetHostmapIndexCount() != 2 {
1037+
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount())
10341038
r.Log("Assert the relay tunnel still works")
10351039
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
10361040
r.Log("yupitdoes")
10371041
time.Sleep(time.Second)
10381042
}
10391043
t.Logf("myControl hostinfos got cleaned up!")
1040-
for len(theirControl.GetHostmap().Indexes) != 2 {
1041-
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes))
1044+
for theirControl.GetHostmapIndexCount() != 2 {
1045+
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount())
10421046
r.Log("Assert the relay tunnel still works")
10431047
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
10441048
r.Log("yupitdoes")
10451049
time.Sleep(time.Second)
10461050
}
10471051
t.Logf("theirControl hostinfos got cleaned up!")
1048-
for len(relayControl.GetHostmap().Indexes) != 2 {
1049-
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes))
1052+
for relayControl.GetHostmapIndexCount() != 2 {
1053+
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount())
10501054
r.Log("Assert the relay tunnel still works")
10511055
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
10521056
r.Log("yupitdoes")
@@ -1123,7 +1127,7 @@ func TestRehandshaking(t *testing.T) {
11231127
theirConfig.ReloadConfigString(string(rc))
11241128

11251129
r.Log("Spin until there is only 1 tunnel")
1126-
for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
1130+
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 {
11271131
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
11281132
t.Log("Connection manager hasn't ticked yet")
11291133
time.Sleep(time.Second)
@@ -1223,7 +1227,7 @@ func TestRehandshakingLoser(t *testing.T) {
12231227
myConfig.ReloadConfigString(string(rc))
12241228

12251229
r.Log("Spin until there is only 1 tunnel")
1226-
for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
1230+
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 {
12271231
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
12281232
t.Log("Connection manager hasn't ticked yet")
12291233
time.Sleep(time.Second)

‎e2e/tunnels_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ func TestDropInactiveTunnels(t *testing.T) {
4343
r.Log("Go inactive and wait for the tunnels to get dropped")
4444
waitStart := time.Now()
4545
for {
46-
myIndexes := len(myControl.GetHostmap().Indexes)
47-
theirIndexes := len(theirControl.GetHostmap().Indexes)
46+
myIndexes := myControl.GetHostmapIndexCount()
47+
theirIndexes := theirControl.GetHostmapIndexCount()
4848
if myIndexes == 0 && theirIndexes == 0 {
4949
break
5050
}
@@ -493,8 +493,8 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
493493

494494
waitStart := time.Now()
495495
for {
496-
myIndexes := len(myControl.GetHostmap().Indexes)
497-
theirIndexes := len(theirControl.GetHostmap().Indexes)
496+
myIndexes := myControl.GetHostmapIndexCount()
497+
theirIndexes := theirControl.GetHostmapIndexCount()
498498
if myIndexes == 0 && theirIndexes == 0 {
499499
break
500500
}
@@ -548,8 +548,8 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
548548
r.Log("Injected bogus close tunnel. Let's see!")
549549
waitStart = time.Now()
550550
for {
551-
myIndexes := len(myControl.GetHostmap().Indexes)
552-
theirIndexes := len(theirControl.GetHostmap().Indexes)
551+
myIndexes := myControl.GetHostmapIndexCount()
552+
theirIndexes := theirControl.GetHostmapIndexCount()
553553
if myIndexes == 0 {
554554
t.Fatal("myIndexes should not be 0")
555555
}

0 commit comments

Comments
 (0)