test: make soak retry delay configurable with validation

Add RETRY_DELAY_SECONDS as a validated integer override and use it in operator pod retry logging while keeping the existing 15-second default behavior.

Made-with: Cursor

Author: yinkscss

scripts/soak-test.sh

@@ -17,6 +17,9 @@ THRESHOLD_KB=5120                         # 5 MB growth limit
 NODE_COUNT=100
 TEST_NAMESPACE="${TEST_NAMESPACE:-soak-test}"
 RESULTS_FILE="${RESULTS_FILE:-/tmp/soak-memory.log}"
+CLEANUP_DONE=false
+CLEANUP_TIMEOUT_SECONDS="${CLEANUP_TIMEOUT_SECONDS:-120}"
+RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-15}"
 
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -26,6 +29,36 @@ get_operator_pid() {
     -o jsonpath='{.items[0].metadata.name}' 2>/dev/null
 }
 
+validate_positive_integer() {
+  local name="$1"
+  local value="$2"
+  if ! [[ "$value" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: ${name} must be an integer, got '${value}'"
+    exit 1
+  fi
+  if [[ "$value" -lt 1 ]]; then
+    echo "ERROR: ${name} must be >= 1, got '${value}'"
+    exit 1
+  fi
+}
+
+get_operator_pid_with_retry() {
+  local max_attempts="${1:-5}"
+  local attempt=1
+  local pod_name=""
+  while [[ "$attempt" -le "$max_attempts" ]]; do
+    pod_name=$(get_operator_pid)
+    if [[ -n "$pod_name" ]]; then
+      echo "$pod_name"
+      return 0
+    fi
+    echo "Operator pod not found (attempt ${attempt}/${max_attempts}); retrying in ${RETRY_DELAY_SECONDS}s..."
+    sleep "$RETRY_DELAY_SECONDS"
+    attempt=$(( attempt + 1 ))
+  done
+  return 1
+}
+
 get_rss_kb() {
   local pod="$1"
   # Read /proc/1/status from inside the container (PID 1 = operator process)
@@ -58,24 +91,77 @@ delete_nodes() {
     kubectl delete stellarnode "soak-node-${i}" -n "$TEST_NAMESPACE" \
       --ignore-not-found --wait=false
   done
-  # Wait for all to be gone before the next wave
-  kubectl wait stellarnode \
+  # Wait for all to be gone before the next wave.
+  if kubectl wait stellarnode \
     --for=delete \
     --all \
     -n "$TEST_NAMESPACE" \
-    --timeout=120s 2>/dev/null || true
+    --timeout="${CLEANUP_TIMEOUT_SECONDS}s" 2>/dev/null; then
+    echo "Cleanup finished within ${CLEANUP_TIMEOUT_SECONDS}s"
+    return 0
+  fi
+
+  local remaining
+  remaining=$(kubectl get stellarnode -n "$TEST_NAMESPACE" --no-headers 2>/dev/null | wc -l | tr -d ' ')
+  echo "WARN: Cleanup timeout reached after ${CLEANUP_TIMEOUT_SECONDS}s with ${remaining} resource(s) still present."
+  echo "Aborting soak loop because it is unsafe to proceed with leftover resources."
+  return 1
 }
 
+cleanup_resources() {
+  local reason="${1:-exit}"
+  if [[ "$CLEANUP_DONE" == "true" ]]; then
+    echo "[cleanup] Already completed (reason: ${reason})"
+    return
+  fi
+  CLEANUP_DONE=true
+
+  echo "[cleanup] Starting cleanup (reason: ${reason})..."
+  delete_nodes || true
+  kubectl delete namespace "$TEST_NAMESPACE" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  echo "[cleanup] Cleanup finished for namespace: ${TEST_NAMESPACE}"
+}
+
+handle_exit() {
+  local exit_code=$?
+  cleanup_resources "exit"
+  if [[ $exit_code -ne 0 ]]; then
+    echo "[cleanup] Exiting with failure code: ${exit_code}"
+  else
+    echo "[cleanup] Exiting successfully"
+  fi
+  exit "$exit_code"
+}
+
+handle_signal() {
+  local signal_name="$1"
+  local signal_code=1
+  case "$signal_name" in
+    INT) signal_code=130 ;;
+    TERM) signal_code=143 ;;
+  esac
+  echo "[cleanup] Received ${signal_name}; requesting graceful shutdown..."
+  exit "$signal_code"
+}
+
+trap 'handle_signal INT' INT
+trap 'handle_signal TERM' TERM
+trap 'handle_exit' EXIT
+
 # ── Setup ─────────────────────────────────────────────────────────────────────
 
 kubectl create namespace "$TEST_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
 
-OPERATOR_POD=$(get_operator_pid)
+validate_positive_integer "RETRY_DELAY_SECONDS" "$RETRY_DELAY_SECONDS"
+echo "Retry delay: ${RETRY_DELAY_SECONDS}s"
+
+OPERATOR_POD=$(get_operator_pid_with_retry 5)
 if [[ -z "$OPERATOR_POD" ]]; then
   echo "ERROR: No stellar-operator pod found in namespace $OPERATOR_NAMESPACE"
   exit 1
 fi
 echo "Operator pod: $OPERATOR_POD"
+echo "Cleanup timeout: ${CLEANUP_TIMEOUT_SECONDS}s"
 
 # Baseline — let the operator settle for one sample interval first
 sleep "$SAMPLE_INTERVAL"
@@ -99,7 +185,7 @@ while [[ $ELAPSED -lt $SOAK_DURATION ]]; do
   delete_nodes
 
   # Sample memory after each wave (and on the fixed interval)
-  OPERATOR_POD=$(get_operator_pid)
+  OPERATOR_POD=$(get_operator_pid_with_retry 5)
   CURRENT_KB=$(get_rss_kb "$OPERATOR_POD")
   GROWTH_KB=$(( CURRENT_KB - BASELINE_KB ))
   NOW=$(date +%s)