Skip to content

Commit 524f200

Browse files
web-flowclaude
web-flow
and
claude
committed
feat(crucible): add fun-bedbug as storage consumer, fix mount unit restart
- Add fun-bedbug to HOSTS array (ports 3860-3862) - Fix setup-nbd-client.sh to restart mount unit after NBD reconnect - Update runbook with all 5 hosts - Add new crucible scripts: build, deploy, HA, templates, k8s job Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent c6a8409 commit 524f200

15 files changed

Lines changed: 1444 additions & 515 deletions

‎docs/crucible-deployment-runbook.md‎

Lines changed: 250 additions & 452 deletions
Large diffs are not rendered by default.

‎scripts/crucible/attach-volumes-to-proxmox.sh‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,68 @@
11
#!/bin/bash
22
# Attach Crucible volumes to all Proxmox hosts
3+
#
4+
# Usage: ./attach-volumes-to-proxmox.sh [CRUCIBLE_IP]
5+
#
6+
# Deploys crucible-nbd-server and setup script to each Proxmox host,
7+
# configuring NBD connections to Crucible downstairs processes.
38
set -e
49

5-
CRUCIBLE_IP="192.168.4.189"
10+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
11+
CRUCIBLE_IP="${1:-192.168.4.189}"
12+
SETUP_SCRIPT="${SCRIPT_DIR}/setup-nbd-client.sh"
613

7-
# Copy binary once
8-
echo "=== Copying crucible-nbd-server binary ==="
9-
scp ubuntu@${CRUCIBLE_IP}:/home/ubuntu/crucible-nbd-server /tmp/crucible-nbd-server
10-
11-
# Host:ports mapping
14+
# Host:ports mapping - each host gets dedicated downstairs ports
1215
declare -A HOSTS=(
1316
["pve"]="3820 3821 3822"
1417
["still-fawn"]="3830 3831 3832"
1518
["pumped-piglet"]="3840 3841 3842"
1619
["chief-horse"]="3850 3851 3852"
20+
["fun-bedbug"]="3860 3861 3862"
1721
)
1822

19-
for host in "${!HOSTS[@]}"; do
20-
ports=(${HOSTS[$host]})
21-
echo ""
22-
echo "=== $host (ports ${ports[*]}) ==="
23-
24-
# pve doesn't need .maas suffix
25-
if [ "$host" == "pve" ]; then
26-
ssh_host="pve"
23+
# Resolve SSH hostname (pve is special - no .maas suffix)
24+
resolve_ssh_host() {
25+
local host="$1"
26+
if [[ "$host" == "pve" ]]; then
27+
echo "pve"
2728
else
28-
ssh_host="${host}.maas"
29+
echo "${host}.maas"
2930
fi
31+
}
32+
33+
# Deploy to a single host
34+
deploy_to_host() {
35+
local host="$1"
36+
local ports="$2"
37+
local ssh_host
38+
ssh_host=$(resolve_ssh_host "$host")
39+
40+
read -ra port_array <<< "$ports"
41+
echo ""
42+
echo "=== $host (ports ${port_array[*]}) ==="
3043

3144
# Copy binary (force overwrite)
32-
ssh root@${ssh_host} "rm -f /usr/local/bin/crucible-nbd-server"
33-
scp /tmp/crucible-nbd-server root@${ssh_host}:/usr/local/bin/
45+
ssh "root@${ssh_host}" "rm -f /usr/local/bin/crucible-nbd-server"
46+
scp /tmp/crucible-nbd-server "root@${ssh_host}:/usr/local/bin/"
47+
48+
# Copy and run setup script
49+
scp "$SETUP_SCRIPT" "root@${ssh_host}:/tmp/"
50+
ssh "root@${ssh_host}" "chmod +x /tmp/setup-nbd-client.sh && /tmp/setup-nbd-client.sh ${CRUCIBLE_IP} ${port_array[0]} ${port_array[1]} ${port_array[2]}"
51+
}
3452

35-
# Copy setup script
36-
scp scripts/crucible/setup-nbd-client.sh root@${ssh_host}:/tmp/
53+
# Validate setup script exists
54+
if [[ ! -f "$SETUP_SCRIPT" ]]; then
55+
echo "ERROR: Setup script not found: $SETUP_SCRIPT" >&2
56+
exit 1
57+
fi
3758

38-
# Run it
39-
ssh root@${ssh_host} "chmod +x /tmp/setup-nbd-client.sh && /tmp/setup-nbd-client.sh ${CRUCIBLE_IP} ${ports[0]} ${ports[1]} ${ports[2]}"
59+
# Fetch binary from Crucible host
60+
echo "=== Fetching crucible-nbd-server from ${CRUCIBLE_IP} ==="
61+
scp "ubuntu@${CRUCIBLE_IP}:/home/ubuntu/crucible-nbd-server" /tmp/crucible-nbd-server
62+
63+
# Deploy to all hosts
64+
for host in "${!HOSTS[@]}"; do
65+
deploy_to_host "$host" "${HOSTS[$host]}"
4066
done
4167

4268
echo ""

‎scripts/crucible/build-nbd-server.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
# Build patched crucible-nbd-server with --address support on proper-raptor
3+
#
4+
# This patches the NBD server to support a custom listen address, required
5+
# for per-VM volumes where each VM needs its own NBD port.
6+
#
7+
# After building, the binary is copied to /home/ubuntu/crucible-nbd-server
8+
#
9+
set -e
10+
11+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
12+
CRUCIBLE_IP="192.168.4.189"
13+
CRUCIBLE_HOST="ubuntu@${CRUCIBLE_IP}"
14+
15+
echo "=== Building Patched crucible-nbd-server ==="
16+
echo ""
17+
18+
# Check SSH access
19+
if ! ssh -o ConnectTimeout=5 -o BatchMode=yes "$CRUCIBLE_HOST" "echo 'SSH OK'" 2>/dev/null; then
20+
echo "ERROR: Cannot SSH to $CRUCIBLE_HOST"
21+
exit 1
22+
fi
23+
24+
# Check if Crucible source exists
25+
if ! ssh "$CRUCIBLE_HOST" "test -d /home/ubuntu/crucible"; then
26+
echo "ERROR: Crucible source not found at /home/ubuntu/crucible"
27+
echo "Clone it first: git clone https://github.com/oxidecomputer/crucible"
28+
exit 1
29+
fi
30+
31+
# Apply the patch
32+
echo "Applying --address patch to nbd_server/src/main.rs..."
33+
34+
ssh "$CRUCIBLE_HOST" "cat > /tmp/patch-nbd-server.sh" << 'REMOTE_SCRIPT'
35+
#!/bin/bash
36+
set -e
37+
38+
cd /home/ubuntu/crucible
39+
40+
# Check if already patched
41+
if grep -q '"127.0.0.1:10809"' nbd_server/src/main.rs && \
42+
grep -q 'opt.address' nbd_server/src/main.rs; then
43+
echo "Already patched (opt.address found)"
44+
else
45+
echo "Applying patch..."
46+
47+
# Backup original
48+
cp nbd_server/src/main.rs nbd_server/src/main.rs.bak
49+
50+
# Add --address argument to Opt struct (after target)
51+
# Find the target line and add address after it
52+
sed -i '/target: Vec<SocketAddr>,/a\
53+
\
54+
/// Address to bind the NBD server (default: 127.0.0.1:10809)\
55+
#[clap(short = '\''a'\'', long, default_value = "127.0.0.1:10809")]\
56+
address: String,' nbd_server/src/main.rs
57+
58+
# Replace hardcoded bind address with opt.address
59+
# The original line is: let listener = TcpListener::bind("127.0.0.1:10809").await?;
60+
sed -i 's/TcpListener::bind("127.0.0.1:10809")/TcpListener::bind(\&opt.address)/' nbd_server/src/main.rs
61+
62+
echo "Patch applied"
63+
fi
64+
65+
# Verify changes
66+
echo ""
67+
echo "Verifying patch..."
68+
if grep -q 'address: String' nbd_server/src/main.rs && \
69+
grep -q 'bind(&opt.address)' nbd_server/src/main.rs; then
70+
echo "Patch verified OK"
71+
else
72+
echo "ERROR: Patch verification failed"
73+
echo "Restoring backup..."
74+
mv nbd_server/src/main.rs.bak nbd_server/src/main.rs
75+
exit 1
76+
fi
77+
REMOTE_SCRIPT
78+
79+
ssh "$CRUCIBLE_HOST" "chmod +x /tmp/patch-nbd-server.sh && /tmp/patch-nbd-server.sh"
80+
81+
# Build
82+
echo ""
83+
echo "Building crucible-nbd-server (this may take a while)..."
84+
ssh "$CRUCIBLE_HOST" "cd /home/ubuntu/crucible && cargo build --release -p nbd_server 2>&1" | tail -20
85+
86+
# Copy to home directory
87+
echo ""
88+
echo "Copying binary to /home/ubuntu/crucible-nbd-server..."
89+
ssh "$CRUCIBLE_HOST" "cp /home/ubuntu/crucible/target/release/crucible-nbd-server /home/ubuntu/"
90+
91+
# Verify
92+
echo ""
93+
echo "Verifying build..."
94+
VERSION=$(ssh "$CRUCIBLE_HOST" "/home/ubuntu/crucible-nbd-server --help 2>&1 | head -5" || true)
95+
echo "$VERSION"
96+
97+
if echo "$VERSION" | grep -q "address"; then
98+
echo ""
99+
echo "=== Build Successful ==="
100+
echo "Binary at: /home/ubuntu/crucible-nbd-server"
101+
echo ""
102+
echo "New flag: --address <IP:PORT> (default: 127.0.0.1:10809)"
103+
echo ""
104+
echo "Next step: Run deploy-ha-storage.sh to deploy to Proxmox hosts"
105+
else
106+
echo ""
107+
echo "WARNING: --address flag not found in help output"
108+
echo "Build may have failed or patch not applied correctly"
109+
fi

‎scripts/crucible/create-vm-disk-volumes.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ declare -A VOLUMES=(
2121
["still-fawn"]=3830
2222
["pumped-piglet"]=3840
2323
["chief-horse"]=3850
24+
["fun-bedbug"]=3860
2425
)
2526

2627
# Volume size: 100 extents * 32768 blocks * 4096 bytes = ~12.5 GB per volume

0 commit comments

Comments
 (0)