Skip to content

Commit 380aa4b

Browse files
authored
feat(recovery): add ENOSPC disk-full detection with auto-retry using doubled disk size (#12511)
1 parent aca721e commit 380aa4b

1 file changed

Lines changed: 60 additions & 1 deletion

File tree

misc/build.func

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4222,6 +4222,7 @@ EOF'
42224222
local is_network_issue=false
42234223
local is_apt_issue=false
42244224
local is_cmd_not_found=false
4225+
local is_disk_full=false
42254226
local error_explanation=""
42264227
if declare -f explain_exit_code >/dev/null 2>&1; then
42274228
error_explanation="$(explain_exit_code "$install_exit_code")"
@@ -4242,6 +4243,14 @@ EOF'
42424243
;;
42434244
esac
42444245

4246+
# Disk full / ENOSPC detection: errno -28 (ENOSPC), exit 228 (custom handler), exit 23 (curl write error)
4247+
if [[ $install_exit_code -eq 228 || $install_exit_code -eq 23 ]]; then
4248+
is_disk_full=true
4249+
fi
4250+
if [[ -f "$combined_log" ]] && grep -qiE 'ENOSPC|no space left on device|No space left on device|Disk quota exceeded|errno -28' "$combined_log"; then
4251+
is_disk_full=true
4252+
fi
4253+
42454254
# Command not found detection
42464255
if [[ $install_exit_code -eq 127 ]]; then
42474256
is_cmd_not_found=true
@@ -4278,6 +4287,9 @@ EOF'
42784287
if grep -qiE ': command not found|No such file or directory.*/s?bin/' "$combined_log"; then
42794288
is_cmd_not_found=true
42804289
fi
4290+
if grep -qiE 'ENOSPC|no space left on device|Disk quota exceeded|errno -28' "$combined_log"; then
4291+
is_disk_full=true
4292+
fi
42814293
fi
42824294

42834295
# Show error explanation if available
@@ -4299,6 +4311,12 @@ EOF'
42994311
echo ""
43004312
fi
43014313

4314+
if [[ "$is_disk_full" == true ]]; then
4315+
echo -e "${TAB}${INFO} The container ran out of disk space during installation (${GN}ENOSPC${CL})."
4316+
echo -e "${TAB}${INFO} Current disk size: ${GN}${DISK_SIZE} GB${CL}. A rebuild with doubled disk may resolve this."
4317+
echo ""
4318+
fi
4319+
43024320
if [[ "$is_cmd_not_found" == true ]]; then
43034321
local missing_cmd=""
43044322
if [[ -f "$combined_log" ]]; then
@@ -4318,7 +4336,7 @@ EOF'
43184336
echo -e " ${GN}3)${CL} Retry with verbose mode (full rebuild)"
43194337

43204338
local next_option=4
4321-
local APT_OPTION="" OOM_OPTION="" DNS_OPTION=""
4339+
local APT_OPTION="" OOM_OPTION="" DNS_OPTION="" DISK_OPTION=""
43224340

43234341
if [[ "$is_apt_issue" == true ]]; then
43244342
if [[ "$var_os" == "alpine" ]]; then
@@ -4343,6 +4361,18 @@ EOF'
43434361
fi
43444362
fi
43454363

4364+
if [[ "$is_disk_full" == true ]]; then
4365+
local disk_recovery_attempt="${DISK_RECOVERY_ATTEMPT:-0}"
4366+
if [[ $disk_recovery_attempt -lt 2 ]]; then
4367+
local new_disk=$((DISK_SIZE * 2))
4368+
echo -e " ${GN}${next_option})${CL} Retry with more disk space (Disk: ${DISK_SIZE}${new_disk} GB)"
4369+
DISK_OPTION=$next_option
4370+
next_option=$((next_option + 1))
4371+
else
4372+
echo -e " ${DGN}-)${CL} ${DGN}Disk resize retry exhausted (already retried ${disk_recovery_attempt}x)${CL}"
4373+
fi
4374+
fi
4375+
43464376
if [[ "$is_network_issue" == true ]]; then
43474377
echo -e " ${GN}${next_option})${CL} Retry with DNS override in LXC (8.8.8.8 / 1.1.1.1)"
43484378
DNS_OPTION=$next_option
@@ -4503,6 +4533,35 @@ EOF'
45034533
return $?
45044534
fi
45054535

4536+
if [[ -n "${DISK_OPTION}" && "${response}" == "${DISK_OPTION}" ]]; then
4537+
# Retry with doubled disk size
4538+
handled=true
4539+
echo -e "\n${TAB}${HOLD}${YW}Removing container ${CTID} for rebuild with more disk space...${CL}"
4540+
pct stop "$CTID" &>/dev/null || true
4541+
pct destroy "$CTID" &>/dev/null || true
4542+
echo -e "${BFR}${CM}${GN}Container ${CTID} removed${CL}"
4543+
echo ""
4544+
local old_ctid="$CTID"
4545+
local old_disk="$DISK_SIZE"
4546+
export CTID=$(get_valid_container_id "$CTID")
4547+
export DISK_SIZE=$((DISK_SIZE * 2))
4548+
export var_disk="$DISK_SIZE"
4549+
export VERBOSE="yes"
4550+
export var_verbose="yes"
4551+
export DISK_RECOVERY_ATTEMPT=$((${DISK_RECOVERY_ATTEMPT:-0} + 1))
4552+
4553+
echo -e "${YW}Rebuilding with increased disk space (attempt ${DISK_RECOVERY_ATTEMPT}/2):${CL}"
4554+
echo -e " Container ID: ${old_ctid}${CTID}"
4555+
echo -e " Disk: ${old_disk}${GN}${DISK_SIZE}${CL} GB (x2)"
4556+
echo -e " RAM: ${RAM_SIZE} MiB | CPU: ${CORE_COUNT} cores"
4557+
echo -e " Network: ${NET:-dhcp} | Bridge: ${BRG:-vmbr0}"
4558+
echo -e " Verbose: ${GN}enabled${CL}"
4559+
echo ""
4560+
msg_info "Restarting installation..."
4561+
build_container
4562+
return $?
4563+
fi
4564+
45064565
if [[ -n "${DNS_OPTION}" && "${response}" == "${DNS_OPTION}" ]]; then
45074566
# Retry with DNS override in LXC
45084567
handled=true

0 commit comments

Comments
 (0)