Skip to content

Commit 6d64058

Browse files
bhumi46bhumi46
authored
Refactor RKE2 setup script with timeout and logging
Signed-off-by: bhumi46 <111699703+bhumi46@users.noreply.github.com>
1 parent 9795354 commit 6d64058

1 file changed

Lines changed: 74 additions & 96 deletions

File tree

  • terraform/modules/aws/rke2-cluster

‎terraform/modules/aws/rke2-cluster/main.tf‎

Lines changed: 74 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -174,46 +174,36 @@ resource "null_resource" "rke2-primary-cluster-setup" {
174174
on_failure = fail
175175
}
176176

177-
# Execute setup script with retry logic
177+
# Execute setup script with timeout and better process management
178178
provisioner "remote-exec" {
179179
inline = [
180-
<<-EOF
181-
set -e
182-
echo "Starting RKE2 setup script..."
183-
184-
# Create a wrapper script with retry logic
185-
cat > /tmp/rke2-setup-wrapper.sh << 'WRAPPER_EOF'
186-
#!/bin/bash
187-
set -e
188-
189-
MAX_ATTEMPTS=3
190-
ATTEMPT=1
191-
192-
while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
193-
echo "RKE2 setup attempt $ATTEMPT/$MAX_ATTEMPTS"
194-
195-
if sudo timeout 600 bash /tmp/rke2-setup.sh; then
196-
echo "RKE2 setup completed successfully"
197-
exit 0
198-
else
199-
EXIT_CODE=$?
200-
echo "RKE2 setup failed with exit code $EXIT_CODE on attempt $ATTEMPT"
201-
202-
if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
203-
echo "All attempts failed. Exiting."
204-
exit $EXIT_CODE
205-
else
206-
echo "Waiting 30 seconds before retry..."
207-
sleep 30
208-
ATTEMPT=$((ATTEMPT + 1))
209-
fi
210-
fi
211-
done
212-
WRAPPER_EOF
213-
214-
chmod +x /tmp/rke2-setup-wrapper.sh
215-
sudo bash /tmp/rke2-setup-wrapper.sh
216-
EOF
180+
"echo 'Starting RKE2 setup script with timeout...'",
181+
"nohup sudo timeout 900 bash /tmp/rke2-setup.sh > /tmp/rke2-setup.log 2>&1 &",
182+
"SETUP_PID=$!",
183+
"echo \"Setup process started with PID: $SETUP_PID\"",
184+
"",
185+
"# Monitor the process",
186+
"while kill -0 $SETUP_PID 2>/dev/null; do",
187+
" echo 'RKE2 setup still running...'",
188+
" sleep 30",
189+
"done",
190+
"",
191+
"# Wait for the process to complete",
192+
"wait $SETUP_PID",
193+
"EXIT_CODE=$?",
194+
"",
195+
"# Show the log output",
196+
"echo 'Setup completed. Log output:'",
197+
"tail -50 /tmp/rke2-setup.log",
198+
"",
199+
"if [ $EXIT_CODE -eq 0 ]; then",
200+
" echo 'RKE2 setup completed successfully'",
201+
"else",
202+
" echo \"RKE2 setup failed with exit code: $EXIT_CODE\"",
203+
" echo 'Full log:'",
204+
" cat /tmp/rke2-setup.log",
205+
" exit $EXIT_CODE",
206+
"fi"
217207
]
218208

219209
on_failure = fail
@@ -269,66 +259,54 @@ resource "null_resource" "rke2-cluster-setup" {
269259
)
270260
}
271261

272-
# Execute setup with retry
262+
# Execute setup with timeout and better monitoring
273263
provisioner "remote-exec" {
274264
inline = [
275-
<<-EOF
276-
set -e
277-
echo "Starting RKE2 setup for ${each.key}..."
278-
279-
# Wait for control plane to be ready (if this is not the control plane)
280-
if [ "${each.value}" != "${local.CONTROL_PLANE_NODE_1}" ]; then
281-
echo "Waiting for control plane to be ready..."
282-
max_wait=300
283-
wait_time=0
284-
285-
while [ $wait_time -lt $max_wait ]; do
286-
if nc -z ${local.CONTROL_PLANE_NODE_1} 6443; then
287-
echo "Control plane is ready"
288-
break
289-
fi
290-
echo "Waiting for control plane... ($wait_time/$max_wait seconds)"
291-
sleep 10
292-
wait_time=$((wait_time + 10))
293-
done
294-
fi
295-
296-
# Create retry wrapper
297-
cat > /tmp/rke2-setup-wrapper.sh << 'WRAPPER_EOF'
298-
#!/bin/bash
299-
set -e
300-
301-
MAX_ATTEMPTS=3
302-
ATTEMPT=1
303-
304-
while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
305-
echo "RKE2 setup attempt $ATTEMPT/$MAX_ATTEMPTS for ${each.key}"
306-
307-
if sudo timeout 600 bash /tmp/rke2-setup.sh; then
308-
echo "RKE2 setup completed successfully for ${each.key}"
309-
exit 0
310-
else
311-
EXIT_CODE=$?
312-
echo "Setup failed with exit code $EXIT_CODE on attempt $ATTEMPT"
313-
314-
if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
315-
echo "All attempts failed for ${each.key}. Exiting."
316-
exit $EXIT_CODE
317-
else
318-
echo "Cleaning up and retrying in 30 seconds..."
319-
# Clean up any partial installation
320-
sudo systemctl stop rke2-server.service || true
321-
sudo systemctl stop rke2-agent.service || true
322-
sleep 30
323-
ATTEMPT=$((ATTEMPT + 1))
324-
fi
325-
fi
326-
done
327-
WRAPPER_EOF
328-
329-
chmod +x /tmp/rke2-setup-wrapper.sh
330-
sudo bash /tmp/rke2-setup-wrapper.sh
331-
EOF
265+
"echo 'Starting RKE2 setup for ${each.key} with timeout...'",
266+
"",
267+
"# Wait for control plane if this is not the control plane node",
268+
"if [ '${each.value}' != '${local.CONTROL_PLANE_NODE_1}' ]; then",
269+
" echo 'Waiting for control plane to be ready...'",
270+
" max_wait=300",
271+
" wait_time=0",
272+
" while [ $wait_time -lt $max_wait ]; do",
273+
" if nc -z ${local.CONTROL_PLANE_NODE_1} 6443; then",
274+
" echo 'Control plane is ready'",
275+
" break",
276+
" fi",
277+
" echo \"Waiting for control plane... ($wait_time/$max_wait seconds)\"",
278+
" sleep 10",
279+
" wait_time=$((wait_time + 10))",
280+
" done",
281+
"fi",
282+
"",
283+
"# Run setup with timeout and logging",
284+
"nohup sudo timeout 900 bash /tmp/rke2-setup.sh > /tmp/rke2-setup.log 2>&1 &",
285+
"SETUP_PID=$!",
286+
"echo \"Setup process started with PID: $SETUP_PID\"",
287+
"",
288+
"# Monitor the process",
289+
"while kill -0 $SETUP_PID 2>/dev/null; do",
290+
" echo 'RKE2 setup still running for ${each.key}...'",
291+
" sleep 30",
292+
"done",
293+
"",
294+
"# Wait for completion and get exit code",
295+
"wait $SETUP_PID",
296+
"EXIT_CODE=$?",
297+
"",
298+
"# Show log output",
299+
"echo 'Setup completed for ${each.key}. Recent log output:'",
300+
"tail -50 /tmp/rke2-setup.log",
301+
"",
302+
"if [ $EXIT_CODE -eq 0 ]; then",
303+
" echo 'RKE2 setup completed successfully for ${each.key}'",
304+
"else",
305+
" echo \"RKE2 setup failed for ${each.key} with exit code: $EXIT_CODE\"",
306+
" echo 'Full log:'",
307+
" cat /tmp/rke2-setup.log",
308+
" exit $EXIT_CODE",
309+
"fi"
332310
]
333311
}
334312
}

0 commit comments

Comments
 (0)