Merge branch 'main' into fix/474-make-soak-cleanup-timeout-configurable-and-visible

Author: OtowoSamuel

CONTRIBUTING.md

@@ -117,6 +117,7 @@ Please do not delete the template sections. PRs with empty descriptions or unche
 
 - **Formatting**: Always run `cargo fmt` before committing.
 - **Linting**: We use Clippy. Ensure `cargo clippy --all-targets --all-features -- -D warnings` passes.
+- **Shell scripts**: We lint scripts under `scripts/` with ShellCheck. Run `find scripts -type f -name "*.sh" -print0 | xargs -0 shellcheck -S error` locally.
 - **Security**: All dependencies must be audited. We resolve all `RUSTSEC` advisories immediately.
 - **Error Handling**: Prefer the `Result<T>` type defined in `src/error.rs` using `thiserror`.
 

scripts/soak-test.sh

@@ -19,6 +19,7 @@ TEST_NAMESPACE="${TEST_NAMESPACE:-soak-test}"
 RESULTS_FILE="${RESULTS_FILE:-/tmp/soak-memory.log}"
 CLEANUP_DONE=false
 CLEANUP_TIMEOUT_SECONDS="${CLEANUP_TIMEOUT_SECONDS:-120}"
+RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-15}"
 
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -28,6 +29,36 @@ get_operator_pid() {
     -o jsonpath='{.items[0].metadata.name}' 2>/dev/null
 }
 
+validate_positive_integer() {
+  local name="$1"
+  local value="$2"
+  if ! [[ "$value" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: ${name} must be an integer, got '${value}'"
+    exit 1
+  fi
+  if [[ "$value" -lt 1 ]]; then
+    echo "ERROR: ${name} must be >= 1, got '${value}'"
+    exit 1
+  fi
+}
+
+get_operator_pid_with_retry() {
+  local max_attempts="${1:-5}"
+  local attempt=1
+  local pod_name=""
+  while [[ "$attempt" -le "$max_attempts" ]]; do
+    pod_name=$(get_operator_pid)
+    if [[ -n "$pod_name" ]]; then
+      echo "$pod_name"
+      return 0
+    fi
+    echo "Operator pod not found (attempt ${attempt}/${max_attempts}); retrying in ${RETRY_DELAY_SECONDS}s..."
+    sleep "$RETRY_DELAY_SECONDS"
+    attempt=$(( attempt + 1 ))
+  done
+  return 1
+}
+
 get_rss_kb() {
   local pod="$1"
   # Read /proc/1/status from inside the container (PID 1 = operator process)
@@ -121,7 +152,10 @@ trap 'handle_exit' EXIT
 
 kubectl create namespace "$TEST_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
 
-OPERATOR_POD=$(get_operator_pid)
+validate_positive_integer "RETRY_DELAY_SECONDS" "$RETRY_DELAY_SECONDS"
+echo "Retry delay: ${RETRY_DELAY_SECONDS}s"
+
+OPERATOR_POD=$(get_operator_pid_with_retry 5)
 if [[ -z "$OPERATOR_POD" ]]; then
   echo "ERROR: No stellar-operator pod found in namespace $OPERATOR_NAMESPACE"
   exit 1
@@ -151,7 +185,7 @@ while [[ $ELAPSED -lt $SOAK_DURATION ]]; do
   delete_nodes
 
   # Sample memory after each wave (and on the fixed interval)
-  OPERATOR_POD=$(get_operator_pid)
+  OPERATOR_POD=$(get_operator_pid_with_retry 5)
   CURRENT_KB=$(get_rss_kb "$OPERATOR_POD")
   GROWTH_KB=$(( CURRENT_KB - BASELINE_KB ))
   NOW=$(date +%s)