Skip to content

Commit 3d34cc9

Browse files
nbrownusnbrownus
authored
Try to make smoke less flakey (#1663)
1 parent e80b983 commit 3d34cc9

5 files changed

Lines changed: 155 additions & 29 deletions

File tree

‎.github/workflows/smoke/build-relay.sh‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,18 @@ relay:
1616
am_relay: true
1717
EOF
1818

19-
export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
20-
export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
19+
# TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
20+
# Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
21+
export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
22+
export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
2123

2224
HOST="host2" ../genconfig.sh >host2.yml <<EOF
2325
relay:
2426
relays:
2527
- 192.168.100.1
2628
EOF
2729

28-
export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
30+
export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
2931

3032
HOST="host3" ../genconfig.sh >host3.yml
3133

‎.github/workflows/smoke/build.sh‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@ set -e -x
55
rm -rf ./build
66
mkdir ./build
77

8-
# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
9-
# - We could make this better by launching the lighthouse first and then fetching what IP it is.
10-
NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
8+
# Smoke containers run on a dedicated docker network whose subnet is allocated
9+
# at smoke time, not known at build time. Configs are written with TEST-NET-3
10+
# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
11+
# sed the real container IPs in before starting nebula.
12+
#
13+
# Placeholder mapping (last octet == fixed container slot):
14+
# 203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
15+
# 203.0.113.4 -> host3, 203.0.113.5 -> host4.
16+
LIGHTHOUSE_IP="203.0.113.2"
1117

1218
(
1319
cd build
@@ -25,16 +31,16 @@ NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{
2531
../genconfig.sh >lighthouse1.yml
2632

2733
HOST="host2" \
28-
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
34+
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
2935
../genconfig.sh >host2.yml
3036

3137
HOST="host3" \
32-
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
38+
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
3339
INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
3440
../genconfig.sh >host3.yml
3541

3642
HOST="host4" \
37-
LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
43+
LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
3844
OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
3945
../genconfig.sh >host4.yml
4046

‎.github/workflows/smoke/smoke-relay.sh‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ set -o pipefail
66

77
mkdir -p logs
88

9+
NETWORK="nebula-smoke-relay"
10+
911
cleanup() {
1012
echo
1113
echo " *** cleanup"
@@ -16,22 +18,53 @@ cleanup() {
1618
then
1719
docker kill lighthouse1 host2 host3 host4
1820
fi
21+
docker network rm "$NETWORK" >/dev/null 2>&1
1922
}
2023

2124
trap cleanup EXIT
2225

26+
# Create a dedicated smoke network with an explicit subnet (required for --ip
27+
# below). Probe a short list of candidates so a locally-used range doesn't
28+
# fail the whole test — we only need one to be free.
29+
docker network rm "$NETWORK" >/dev/null 2>&1 || true
30+
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
31+
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
32+
break
33+
fi
34+
done
35+
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
36+
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
37+
exit 1
38+
fi
39+
40+
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
41+
# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
42+
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
43+
PREFIX="${SUBNET%/*}"
44+
PREFIX="${PREFIX%.*}"
45+
LIGHTHOUSE_IP="$PREFIX.2"
46+
HOST2_IP="$PREFIX.3"
47+
HOST3_IP="$PREFIX.4"
48+
HOST4_IP="$PREFIX.5"
49+
50+
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
51+
for f in build/host2.yml build/host3.yml build/host4.yml; do
52+
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
53+
mv "$f.tmp" "$f"
54+
done
55+
2356
docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
24-
docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
25-
docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
26-
docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
57+
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
58+
docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
59+
docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
2760

28-
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
61+
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
2962
sleep 1
30-
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
63+
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
3164
sleep 1
32-
docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
65+
docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
3366
sleep 1
34-
docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
67+
docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
3568
sleep 1
3669

3770
set +x
@@ -76,7 +109,13 @@ docker exec host4 sh -c 'kill 1'
76109
docker exec host3 sh -c 'kill 1'
77110
docker exec host2 sh -c 'kill 1'
78111
docker exec lighthouse1 sh -c 'kill 1'
79-
sleep 5
112+
113+
# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
114+
# fixed sleep.
115+
for _ in $(seq 1 30); do
116+
[ -z "$(jobs -r)" ] && break
117+
sleep 1
118+
done
80119

81120
if [ "$(jobs -r)" ]
82121
then

‎.github/workflows/smoke/smoke-vagrant.sh‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
88

99
mkdir -p logs
1010

11+
NETWORK="nebula-smoke"
12+
1113
cleanup() {
1214
echo
1315
echo " *** cleanup"
@@ -19,21 +21,51 @@ cleanup() {
1921
docker kill lighthouse1 host2
2022
fi
2123
vagrant destroy -f
24+
docker network rm "$NETWORK" >/dev/null 2>&1
2225
}
2326

2427
trap cleanup EXIT
2528

29+
# Create a dedicated smoke network with an explicit subnet (required for --ip
30+
# below). Probe a short list of candidates so a locally-used range doesn't
31+
# fail the whole test — we only need one to be free.
32+
docker network rm "$NETWORK" >/dev/null 2>&1 || true
33+
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
34+
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
35+
break
36+
fi
37+
done
38+
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
39+
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
40+
exit 1
41+
fi
42+
43+
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
44+
# .3 host2 — matches the placeholders in build.sh.
45+
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
46+
PREFIX="${SUBNET%/*}"
47+
PREFIX="${PREFIX%.*}"
48+
LIGHTHOUSE_IP="$PREFIX.2"
49+
HOST2_IP="$PREFIX.3"
50+
51+
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
52+
# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
53+
for f in build/host2.yml build/host3.yml; do
54+
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
55+
mv "$f.tmp" "$f"
56+
done
57+
2658
CONTAINER="nebula:${NAME:-smoke}"
2759

2860
docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
29-
docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
61+
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
3062

3163
vagrant up
3264
vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
3365

34-
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
66+
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
3567
sleep 1
36-
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
68+
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
3769
sleep 1
3870
vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/ [host3] /' &
3971
sleep 15
@@ -96,7 +128,14 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
96128
vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
97129
docker exec host2 sh -c 'kill 1'
98130
docker exec lighthouse1 sh -c 'kill 1'
99-
sleep 1
131+
132+
# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
133+
# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
134+
# racy.
135+
for _ in $(seq 1 30); do
136+
[ -z "$(jobs -r)" ] && break
137+
sleep 1
138+
done
100139

101140
if [ "$(jobs -r)" ]
102141
then

‎.github/workflows/smoke/smoke.sh‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ set -o pipefail
66

77
mkdir -p logs
88

9+
NETWORK="nebula-smoke"
10+
911
cleanup() {
1012
echo
1113
echo " *** cleanup"
@@ -16,24 +18,56 @@ cleanup() {
1618
then
1719
docker kill lighthouse1 host2 host3 host4
1820
fi
21+
docker network rm "$NETWORK" >/dev/null 2>&1
1922
}
2023

2124
trap cleanup EXIT
2225

26+
# Create a dedicated smoke network with an explicit subnet (required for --ip
27+
# below). Probe a short list of candidates so a locally-used range doesn't
28+
# fail the whole test — we only need one to be free.
29+
docker network rm "$NETWORK" >/dev/null 2>&1 || true
30+
for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
31+
if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
32+
break
33+
fi
34+
done
35+
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
36+
echo "failed to create $NETWORK: every candidate subnet is in use" >&2
37+
exit 1
38+
fi
39+
40+
# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
41+
# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
42+
SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
43+
PREFIX="${SUBNET%/*}"
44+
PREFIX="${PREFIX%.*}"
45+
LIGHTHOUSE_IP="$PREFIX.2"
46+
HOST2_IP="$PREFIX.3"
47+
HOST3_IP="$PREFIX.4"
48+
HOST4_IP="$PREFIX.5"
49+
50+
# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
51+
# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
52+
for f in build/host2.yml build/host3.yml build/host4.yml; do
53+
sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
54+
mv "$f.tmp" "$f"
55+
done
56+
2357
CONTAINER="nebula:${NAME:-smoke}"
2458

2559
docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
26-
docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
27-
docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
28-
docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
60+
docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
61+
docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
62+
docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
2963

30-
docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
64+
docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/ [lighthouse1] /' &
3165
sleep 1
32-
docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
66+
docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/ [host2] /' &
3367
sleep 1
34-
docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
68+
docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/ [host3] /' &
3569
sleep 1
36-
docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
70+
docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/ [host4] /' &
3771
sleep 1
3872

3973
# grab tcpdump pcaps for debugging
@@ -131,7 +165,13 @@ docker exec host4 sh -c 'kill 1'
131165
docker exec host3 sh -c 'kill 1'
132166
docker exec host2 sh -c 'kill 1'
133167
docker exec lighthouse1 sh -c 'kill 1'
134-
sleep 5
168+
169+
# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
170+
# fixed sleep.
171+
for _ in $(seq 1 30); do
172+
[ -z "$(jobs -r)" ] && break
173+
sleep 1
174+
done
135175

136176
if [ "$(jobs -r)" ]
137177
then

0 commit comments

Comments
 (0)