Skip to content

Commit 328483e

Browse files
LillecarlLillecarl
committed
CI work
1 parent 8f932bf commit 328483e

2 files changed

Lines changed: 169 additions & 162 deletions

File tree

‎tests/nixos/cluster-module.nix‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,20 +47,20 @@
4747
};
4848
};
4949

50-
# -- CNI bridge config --
51-
# Simple bridge CNI — no cluster-level setup needed, just this config file.
52-
environment.etc."cni/net.d/10-bridge.conflist".text = builtins.toJSON {
50+
# -- CNI ptp config --
51+
# PTP (point-to-point) CNI: each pod gets a veth pair, no shared bridge.
52+
# Traffic between pods is routed at L3 on the host, which lets kube-proxy
53+
# iptables/ipvs rules see and rewrite service traffic correctly.
54+
environment.etc."cni/net.d/10-ptp.conflist".text = builtins.toJSON {
5355
cniVersion = "1.0.0";
54-
name = "bridge";
56+
name = "ptp";
5557
plugins = [
5658
{
57-
type = "bridge";
58-
bridge = "cni0";
59-
isGateway = true;
59+
type = "ptp";
6060
ipMasq = true;
6161
ipam = {
6262
type = "host-local";
63-
ranges = [ [ { subnet = "10.244.0.0/16"; } ] ];
63+
subnet = "10.113.37.0/24";
6464
routes = [ { dst = "0.0.0.0/0"; } ];
6565
};
6666
}

‎tests/nixos/integration.nix‎

Lines changed: 161 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -49,158 +49,165 @@ pkgs.testers.nixosTest {
4949
# Skip lint checks that fail on test environments
5050
skipLint = true;
5151

52-
testScript = ''
53-
import json
54-
55-
control.start()
56-
57-
# Debug: Check if networking is available
58-
with control.nested("check networking"):
59-
control.wait_for_unit("network-online.target", timeout=30)
60-
61-
# Capture VM network state
62-
print("\n=== IP Addresses ===")
63-
print(control.succeed("ip addr"))
64-
65-
print("\n=== Routing Table ===")
66-
print(control.succeed("ip route"))
67-
68-
print("\n=== Default Route ===")
69-
print(control.succeed("ip route | grep default"))
70-
71-
print("\n=== Kernel IP Forwarding ===")
72-
print(control.succeed("cat /proc/sys/net/ipv4/ip_forward"))
73-
74-
# Capture DNS state
75-
print("\n=== /etc/resolv.conf ===")
76-
print(control.succeed("cat /etc/resolv.conf || echo 'No resolv.conf'"))
77-
78-
print("\n=== DNS Lookup Test ===")
79-
print(control.succeed("nslookup google.com || dig google.com || echo 'DNS lookup failed'"))
80-
81-
# Try pings
82-
print("\n=== Ping Test ===")
83-
print(control.succeed("ping -c 1 1.1.1.1 || ping -c 1 9.9.9.9 || echo 'WARNING: No external ping response'"))
84-
85-
control.wait_for_unit("containerd.service")
86-
87-
# ── Phase 1: Bootstrap kubeadm cluster ──────────────────────────────
88-
89-
with control.nested("kubeadm init"):
90-
control.succeed(
91-
"kubeadm init"
92-
" --pod-network-cidr=10.244.0.0/16"
93-
" --node-name=control"
94-
" --v=5"
95-
" 2>&1 | tee /tmp/kubeadm-init.log",
96-
timeout=300,
97-
)
98-
99-
control.succeed("mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config")
100-
101-
# Remove control-plane taint so all pods (including test workloads) can schedule
102-
control.succeed(
103-
"kubectl taint nodes control node-role.kubernetes.io/control-plane:NoSchedule-"
104-
)
105-
106-
# Wait for node to reach Ready (CNI + kubelet + kube-proxy must all converge)
107-
control.wait_until_succeeds(
108-
"kubectl wait --for=condition=Ready node/control --timeout=5s",
109-
timeout=120,
110-
)
111-
112-
# Wait for CoreDNS to be available (needed for pod DNS resolution)
113-
control.wait_until_succeeds(
114-
"kubectl -n kube-system wait --for=condition=Available deployment/coredns --timeout=5s",
115-
timeout=120,
116-
)
117-
118-
# ── Phase 2: Deploy nixkube manifests ───────────────────────────────
119-
120-
with control.nested("deploy nixkube"):
121-
control.succeed(
122-
"kubectl apply --server-side -f ${manifests} 2>&1 | tee /tmp/kubectl-apply.log"
123-
)
124-
125-
# Wait for the init Job to create SSH secrets
126-
control.wait_until_succeeds(
127-
"kubectl -n nixkube get secret ssh-key",
128-
timeout=120,
129-
)
130-
131-
# Wait for the DaemonSet node pod to be ready
132-
control.wait_until_succeeds(
133-
"kubectl -n nixkube wait --for=condition=Ready"
134-
" pod -l app.kubernetes.io/component=node"
135-
" --timeout=5s",
136-
timeout=600,
137-
)
138-
139-
# Verify CSI driver is registered
140-
control.wait_until_succeeds(
141-
"kubectl get csidriver nixkube",
142-
timeout=30,
143-
)
144-
145-
# ── Phase 3: Wait for test workloads ────────────────────────────────
146-
147-
# CSI test Jobs are included in the manifest (from kubenix/ci/).
148-
# They have backoffLimit=6, so they'll retry if the CSI driver isn't
149-
# ready immediately. Wait for at least one CSI test to complete.
150-
151-
with control.nested("wait for CSI test jobs"):
152-
# path-hello is a simple storePath CSI test
153-
control.wait_until_succeeds(
154-
"kubectl -n nixkube wait --for=condition=Complete"
155-
" job/path-hello"
156-
" --timeout=5s",
157-
timeout=600,
158-
)
159-
160-
with control.nested("wait for NRI test jobs"):
161-
# nri-hello-ro tests NRI read-only mount
162-
control.wait_until_succeeds(
163-
"kubectl -n nixkube wait --for=condition=Complete"
164-
" job/nri-hello-ro"
165-
" --timeout=5s",
166-
timeout=600,
167-
)
168-
169-
# ── Phase 4: Verify ────────────────────────────────────────────────
170-
171-
with control.nested("verify results"):
172-
# Check that the CSI test pod ran successfully
173-
result = control.succeed(
174-
"kubectl -n nixkube get job path-hello -o jsonpath='{.status.succeeded}'"
175-
)
176-
assert result.strip().strip("'") == "1", f"path-hello job did not succeed: {result}"
177-
178-
# Check that the NRI test pod ran successfully
179-
result = control.succeed(
180-
"kubectl -n nixkube get job nri-hello-ro -o jsonpath='{.status.succeeded}'"
181-
)
182-
assert result.strip().strip("'") == "1", f"nri-hello-ro job did not succeed: {result}"
183-
184-
# Verify DaemonSet is healthy
185-
result = control.succeed(
186-
"kubectl -n nixkube get daemonset nix-node"
187-
" -o jsonpath='{.status.numberReady}'"
188-
)
189-
assert result.strip().strip("'") == "1", f"DaemonSet not ready: {result}"
190-
191-
# Check for NixVolumeMount events (CSI driver reports these)
192-
events = control.succeed(
193-
"kubectl -n nixkube get events --field-selector reason=NixVolumeMount -o json"
194-
)
195-
event_data = json.loads(events)
196-
assert len(event_data.get("items", [])) >= 1, (
197-
f"Expected NixVolumeMount events, got {len(event_data.get('items', []))}"
198-
)
199-
200-
# ── Dump state on success (useful for debugging) ───────────────────
201-
202-
control.succeed("kubectl get nodes -o wide")
203-
control.succeed("kubectl -n nixkube get all")
204-
control.succeed("kubectl -n nixkube get events --sort-by=.lastTimestamp | tail -30")
205-
'';
52+
testScript = # python
53+
''
54+
import json
55+
56+
control.start()
57+
58+
# Debug: Check if networking is available
59+
with control.nested("check networking"):
60+
control.wait_for_unit("network-online.target", timeout=30)
61+
62+
# Capture VM network state
63+
print("\n=== IP Addresses ===")
64+
print(control.succeed("ip addr"))
65+
66+
print("\n=== Routing Table ===")
67+
print(control.succeed("ip route"))
68+
69+
print("\n=== Default Route ===")
70+
print(control.succeed("ip route | grep default"))
71+
72+
print("\n=== Kernel IP Forwarding ===")
73+
print(control.succeed("cat /proc/sys/net/ipv4/ip_forward"))
74+
75+
# Capture DNS state
76+
print("\n=== /etc/resolv.conf ===")
77+
print(control.succeed("cat /etc/resolv.conf || echo 'No resolv.conf'"))
78+
79+
print("\n=== DNS Lookup Test ===")
80+
print(control.succeed("nslookup google.com || dig google.com || echo 'DNS lookup failed'"))
81+
82+
# Try pings
83+
print("\n=== Ping Test ===")
84+
print(control.succeed("ping -c 1 1.1.1.1 || ping -c 1 9.9.9.9 || echo 'WARNING: No external ping response'"))
85+
86+
control.wait_for_unit("containerd.service")
87+
88+
# ── Phase 1: Bootstrap kubeadm cluster ──────────────────────────────
89+
90+
with control.nested("kubeadm init"):
91+
control.succeed(
92+
"kubeadm init"
93+
" --pod-network-cidr=10.113.37.0/24"
94+
" --service-cidr=10.113.38.0/24"
95+
" --node-name=control"
96+
" --v=5"
97+
" 2>&1 | tee /tmp/kubeadm-init.log",
98+
timeout=300,
99+
)
100+
101+
control.succeed("mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config")
102+
103+
# Remove control-plane taint so all pods (including test workloads) can schedule
104+
control.succeed(
105+
"kubectl taint nodes control node-role.kubernetes.io/control-plane:NoSchedule-"
106+
)
107+
108+
# Wait for node to reach Ready (CNI + kubelet + kube-proxy must all converge)
109+
control.wait_until_succeeds(
110+
"kubectl wait --for=condition=Ready node/control --timeout=60s",
111+
)
112+
113+
# Wait for CoreDNS to be available (needed for pod DNS resolution)
114+
control.wait_until_succeeds(
115+
"kubectl -n kube-system wait --for=condition=Available deployment/coredns --timeout=60s",
116+
)
117+
118+
# Wait for kube-proxy to be ready (needed for Service ClusterIP routing)
119+
control.wait_until_succeeds(
120+
"kubectl -n kube-system wait --for=condition=Ready"
121+
" pod -l k8s-app=kube-proxy"
122+
" --timeout=60s",
123+
)
124+
125+
# ── Phase 2: Deploy nixkube manifests ───────────────────────────────
126+
127+
with control.nested("deploy nixkube"):
128+
control.succeed(
129+
"kubectl apply --server-side -f ${manifests} 2>&1 | tee /tmp/kubectl-apply.log"
130+
)
131+
132+
# Wait for the init Job to create SSH secrets
133+
control.wait_until_succeeds(
134+
"kubectl -n nixkube get secret ssh-key",
135+
timeout=120,
136+
)
137+
138+
# Wait for the DaemonSet node pod to be ready
139+
control.wait_until_succeeds(
140+
"kubectl -n nixkube wait --for=condition=Ready"
141+
" pod -l app.kubernetes.io/component=node"
142+
" --timeout=5s",
143+
timeout=600,
144+
)
145+
146+
# Verify CSI driver is registered
147+
control.wait_until_succeeds(
148+
"kubectl get csidriver nixkube",
149+
timeout=30,
150+
)
151+
152+
# ── Phase 3: Wait for test workloads ────────────────────────────────
153+
154+
# CSI test Jobs are included in the manifest (from kubenix/ci/).
155+
# They have backoffLimit=6, so they'll retry if the CSI driver isn't
156+
# ready immediately. Wait for at least one CSI test to complete.
157+
158+
with control.nested("wait for CSI test jobs"):
159+
# path-hello is a simple storePath CSI test
160+
control.wait_until_succeeds(
161+
"kubectl -n nixkube wait --for=condition=Complete"
162+
" job/path-hello"
163+
" --timeout=5s",
164+
timeout=600,
165+
)
166+
167+
with control.nested("wait for NRI test jobs"):
168+
# nri-hello-ro tests NRI read-only mount
169+
control.wait_until_succeeds(
170+
"kubectl -n nixkube wait --for=condition=Complete"
171+
" job/nri-hello-ro"
172+
" --timeout=5s",
173+
timeout=600,
174+
)
175+
176+
# ── Phase 4: Verify ────────────────────────────────────────────────
177+
178+
with control.nested("verify results"):
179+
# Check that the CSI test pod ran successfully
180+
result = control.succeed(
181+
"kubectl -n nixkube get job path-hello -o jsonpath='{.status.succeeded}'"
182+
)
183+
assert result.strip().strip("'") == "1", f"path-hello job did not succeed: {result}"
184+
185+
# Check that the NRI test pod ran successfully
186+
result = control.succeed(
187+
"kubectl -n nixkube get job nri-hello-ro -o jsonpath='{.status.succeeded}'"
188+
)
189+
assert result.strip().strip("'") == "1", f"nri-hello-ro job did not succeed: {result}"
190+
191+
# Verify DaemonSet is healthy
192+
result = control.succeed(
193+
"kubectl -n nixkube get daemonset nix-node"
194+
" -o jsonpath='{.status.numberReady}'"
195+
)
196+
assert result.strip().strip("'") == "1", f"DaemonSet not ready: {result}"
197+
198+
# Check for NixVolumeMount events (CSI driver reports these)
199+
events = control.succeed(
200+
"kubectl -n nixkube get events --field-selector reason=NixVolumeMount -o json"
201+
)
202+
event_data = json.loads(events)
203+
assert len(event_data.get("items", [])) >= 1, (
204+
f"Expected NixVolumeMount events, got {len(event_data.get('items', []))}"
205+
)
206+
207+
# ── Dump state on success (useful for debugging) ───────────────────
208+
209+
control.succeed("kubectl get nodes -o wide")
210+
control.succeed("kubectl -n nixkube get all")
211+
control.succeed("kubectl -n nixkube get events --sort-by=.lastTimestamp | tail -30")
212+
'';
206213
}

0 commit comments

Comments
 (0)