🧪 Test script for event-based rev-reg management

app/exceptions/handle_acapy_call.py

@@ -1,7 +1,7 @@
 from collections.abc import Callable, Coroutine
 from typing import Any
 
-from aiohttp import ClientConnectionResetError
+import aiohttp
 from aries_cloudcontroller.exceptions import (
     ApiException,
     BadRequestException,
@@ -73,11 +73,14 @@ async def handle_acapy_call[T](
             # Handle other / 500 errors:
             logger.warning("Error during {}: {}", method_identifier, error_msg)
             raise CloudApiException(status_code=status, detail=error_msg) from e
-    except ClientConnectionResetError as e:  # pragma: no cover
-        logger.error("Client connection reset error")
+    except aiohttp.ClientConnectionResetError as e:  # pragma: no cover
+        logger.error("Client connection reset error: {}", e)
         raise CloudApiException(
             status_code=500, detail="Client connection reset error"
         ) from e
+    except aiohttp.ClientConnectionError as e:  # pragma: no cover
+        logger.error("Client connection error: {}", e)
+        raise CloudApiException(status_code=503, detail="Service unavailable") from e
     except Exception as e:
         # General exceptions:
         logger.exception("Unexpected exception from ACA-Py call")

app/services/revocation_registry.py

@@ -389,7 +389,7 @@ async def get_created_active_registries(
     except CloudApiException as e:
         detail = (
             "Error while creating credential definition: "
-            + f"Could not retrieve active revocation registries `{e.detail}`."
+            f"Could not retrieve active revocation registries `{e.detail}`."
         )
         raise CloudApiException(detail=detail, status_code=e.status_code) from e
 

app/tests/e2e/experimental/rev_reg_resilience_testing/log_pattern_monitor.py

@@ -0,0 +1,83 @@
+"""Utility functions for resilience testing
+
+This module provides helper functions for:
+- Pod monitoring and kubectl operations
+- Agent status checking and health monitoring
+- Enhanced error handling and logging
+"""
+
+import subprocess
+
+from shared.log_config import get_logger
+
+LOGGER = get_logger(__name__)
+
+
+class LogPatternMonitor:
+    """Monitor for specific log patterns that should trigger pod kills"""
+
+    def __init__(self, monitor_script_path: str = "./monitor_and_kill_pod.sh") -> None:
+        """Initialize the log pattern monitor"""
+        self.monitor_script_path = monitor_script_path
+        self.monitor_process = None
+
+    def start_monitoring(self, log_pattern: str | None = None) -> subprocess.Popen:
+        """Start log pattern monitoring
+
+        Args:
+            log_pattern: Custom log pattern to monitor for. If None, uses script default.
+
+        Example patterns:
+            "Registering revocation registry definition"
+            "Uploading tails file"
+            "Emitting store revocation registry definition event"
+
+        """
+        LOGGER.info("Starting log pattern monitoring...")
+
+        # Build command with optional log pattern
+        cmd = [self.monitor_script_path]
+        if log_pattern:
+            cmd.append(log_pattern)
+            LOGGER.info(f"Using custom log pattern: '{log_pattern}'")
+        else:
+            LOGGER.info("Using default log pattern from script")
+
+        self.monitor_process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+            universal_newlines=True,
+        )
+
+        LOGGER.info(f"Log monitor started (PID: {self.monitor_process.pid})")
+        return self.monitor_process
+
+    def is_monitoring_active(self) -> bool:
+        """Check if monitoring process is still active"""
+        return self.monitor_process and self.monitor_process.poll() is None
+
+    def stop_monitoring(self):
+        """Stop the monitoring process"""
+        if self.monitor_process and self.monitor_process.poll() is None:
+            LOGGER.info("Stopping log monitor...")
+            self.monitor_process.terminate()
+            try:
+                self.monitor_process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.monitor_process.kill()
+                self.monitor_process.wait()
+            LOGGER.info("Log monitor stopped")
+
+    def get_monitor_output(self) -> tuple[str, str]:
+        """Get the output from the monitoring process"""
+        if not self.monitor_process:
+            return "", ""
+
+        try:
+            stdout, stderr = self.monitor_process.communicate(timeout=1)
+            return stdout, stderr
+        except subprocess.TimeoutExpired:
+            return "", ""

app/tests/e2e/experimental/rev_reg_resilience_testing/monitor_and_kill_pod.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+# Default log pattern if none provided
+DEFAULT_PATTERN="Publishing revocation registry definition resource"
+LOG_PATTERN="${1:-$DEFAULT_PATTERN}"
+
+echo "🔍 Looking for multitenant-agent pod..."
+echo "🎯 Monitoring for pattern: '$LOG_PATTERN'"
+
+# Get the multitenant-agent pod name
+POD_NAME=$(kubectl get pods --no-headers | grep "^multitenant-agent-" | awk '{print $1}' | head -1)
+
+if [ -z "$POD_NAME" ]; then
+    echo "❌ No multitenant-agent pod found!"
+    exit 1
+fi
+
+echo "✅ Found pod: $POD_NAME"
+echo "👀 Following logs and waiting for pattern match..."
+
+# Follow logs from now onwards and look for the specific pattern
+kubectl logs -f --tail=0 "$POD_NAME" | while IFS= read -r line; do
+    # Check if the line matches our pattern
+    if echo "$line" | grep -q "$LOG_PATTERN"; then
+        echo ""
+        echo "🎯 Found pattern match! Deleting pod..."
+
+        # Kill the pod immediately
+        kubectl delete pod "$POD_NAME" --force --grace-period=0
+
+        echo "💀 Pod $POD_NAME has been deleted!"
+        break
+    fi
+done
+
+echo "✨ Script completed!"