[PROMETHEUS] Add prometheus to remaining services (#105)

* [PROMETHEUS] Add prometheus to remaining services

* [PROMETHEUS] Add missing EOF's

Author: Subfly

client/nginx.conf

@@ -4,6 +4,20 @@ server {
     root /usr/share/nginx/html;
     index index.html;
 
+    # Health check endpoint
+    location /health {
+        alias /usr/share/nginx/html/health.json;
+        add_header Content-Type application/json;
+        add_header Cache-Control "no-cache, no-store, must-revalidate";
+    }
+
+    # Basic metrics endpoint (nginx status)
+    location /metrics {
+        stub_status on;
+        access_log off;
+        add_header Content-Type text/plain;
+    }
+
     # Handle client-side routing
     location / {
         try_files $uri $uri/ /index.html;

client/public/health.json

@@ -0,0 +1,6 @@
+{
+    "status": "healthy",
+    "timestamp": "2024-01-01T00:00:00Z",
+    "service": "client",
+    "version": "1.0.0"
+}

genai/main.py

@@ -8,10 +8,21 @@
 import logging
 from typing import Callable
 
+# Prometheus metrics imports
+from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
+
 from request_models import ChatRequest, RecipeIndexRequest, RecipeDeleteRequest, RecipeSuggestionRequest
 from response_models import ChatResponse, RecipeIndexResponse, RecipeDeleteResponse, RecipeSuggestionResponse, HealthResponse
 from llm import RecipeLLM
 
+# Prometheus metrics
+REQUEST_COUNT = Counter('genai_requests_total', 'Total number of requests', ['method', 'endpoint', 'status'])
+REQUEST_DURATION = Histogram('genai_request_duration_seconds', 'Request duration in seconds', ['method', 'endpoint'])
+ACTIVE_REQUESTS = Gauge('genai_active_requests', 'Number of active requests')
+LLM_OPERATIONS = Counter('genai_llm_operations_total', 'Total number of LLM operations', ['operation_type', 'status'])
+VECTOR_OPERATIONS = Counter('genai_vector_operations_total', 'Total number of vector operations', ['operation_type', 'status'])
+SERVICE_HEALTH = Gauge('genai_service_health', 'Service health status (1=healthy, 0=unhealthy)')
+
 # Enhanced logging configuration
 class StructuredFormatter(logging.Formatter):
     """Custom formatter for structured logging"""
@@ -60,12 +71,15 @@ def format(self, record):
 
 logger = logging.getLogger(__name__)
 
-# Add request ID tracking middleware
+# Add request ID tracking and Prometheus metrics middleware
 async def add_request_id_middleware(request: Request, call_next: Callable) -> Response:
-    """Add request ID to all requests for tracing"""
+    """Add request ID to all requests for tracing and collect Prometheus metrics"""
     request_id = str(uuid.uuid4())
     request.state.request_id = request_id
 
+    # Track active requests
+    ACTIVE_REQUESTS.inc()
+    
     # Log incoming request
     start_time = time.time()
     structured_logger.info(
@@ -87,6 +101,18 @@ async def add_request_id_middleware(request: Request, call_next: Callable) -> Re
 
         # Calculate request duration
         duration_ms = round((time.time() - start_time) * 1000, 2)
+        duration_seconds = (time.time() - start_time)
+        
+        # Update Prometheus metrics
+        REQUEST_COUNT.labels(
+            method=request.method, 
+            endpoint=request.url.path, 
+            status=str(response.status_code)
+        ).inc()
+        REQUEST_DURATION.labels(
+            method=request.method, 
+            endpoint=request.url.path
+        ).observe(duration_seconds)
 
         # Log response
         structured_logger.info(
@@ -108,6 +134,19 @@ async def add_request_id_middleware(request: Request, call_next: Callable) -> Re
 
     except Exception as e:
         duration_ms = round((time.time() - start_time) * 1000, 2)
+        duration_seconds = (time.time() - start_time)
+        
+        # Update Prometheus metrics for failed requests
+        REQUEST_COUNT.labels(
+            method=request.method, 
+            endpoint=request.url.path, 
+            status="500"
+        ).inc()
+        REQUEST_DURATION.labels(
+            method=request.method, 
+            endpoint=request.url.path
+        ).observe(duration_seconds)
+        
         structured_logger.error(
             f"Request failed: {request.method} {request.url.path} - {str(e)}",
             extra={
@@ -120,6 +159,9 @@ async def add_request_id_middleware(request: Request, call_next: Callable) -> Re
             }
         )
         raise
+    finally:
+        # Decrement active requests
+        ACTIVE_REQUESTS.dec()
 
 # Global LLM instance
 llm_instance: RecipeLLM = None
@@ -264,6 +306,22 @@ async def health_check(request: Request):
             services={"error": str(e)}
         )
 
+@app.get("/metrics")
+async def metrics():
+    """Prometheus metrics endpoint"""
+    # Update service health metric based on current status
+    try:
+        if llm_instance:
+            services_status = llm_instance.get_health_status()
+            overall_health = 1 if all("healthy" in status for status in services_status.values()) else 0
+        else:
+            overall_health = 0
+        SERVICE_HEALTH.set(overall_health)
+    except Exception:
+        SERVICE_HEALTH.set(0)
+    
+    return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST)
+
 @app.post("/genai/chat", response_model=ChatResponse)
 async def chat(request: ChatRequest, http_request: Request):
     """Chat with the AI assistant for recipe search and creation"""
@@ -296,6 +354,9 @@ async def chat(request: ChatRequest, http_request: Request):
         response = llm_instance.chat(request.message)
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update LLM operation metrics
+        LLM_OPERATIONS.labels(operation_type='chat', status='success').inc()
+        
         structured_logger.info(
             "Chat request completed successfully",
             extra={
@@ -315,6 +376,9 @@ async def chat(request: ChatRequest, http_request: Request):
     except Exception as e:
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update LLM operation metrics for failures
+        LLM_OPERATIONS.labels(operation_type='chat', status='error').inc()
+        
         logger.error(f"Error in chat: {e}", exc_info=True)
         structured_logger.error(
             f"Chat request failed: {str(e)}",
@@ -366,6 +430,9 @@ async def index_recipe(request: RecipeIndexRequest, http_request: Request):
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
         if success:
+            # Update vector operation metrics for success
+            VECTOR_OPERATIONS.labels(operation_type='index', status='success').inc()
+            
             structured_logger.info(
                 f"Recipe indexed successfully: {request.recipe.metadata.title}",
                 extra={
@@ -384,6 +451,9 @@ async def index_recipe(request: RecipeIndexRequest, http_request: Request):
                 recipe_id=request.recipe.metadata.id
             )
         else:
+            # Update vector operation metrics for failure
+            VECTOR_OPERATIONS.labels(operation_type='index', status='error').inc()
+            
             structured_logger.error(
                 f"Recipe indexing failed: {request.recipe.metadata.title}",
                 extra={
@@ -402,6 +472,9 @@ async def index_recipe(request: RecipeIndexRequest, http_request: Request):
     except Exception as e:
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update vector operation metrics for exceptions
+        VECTOR_OPERATIONS.labels(operation_type='index', status='error').inc()
+        
         logger.error(f"Error indexing recipe: {e}", exc_info=True)
         structured_logger.error(
             f"Recipe indexing failed: {str(e)}",
@@ -451,6 +524,9 @@ async def delete_recipe(recipe_id: str, http_request: Request):
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
         if success:
+            # Update vector operation metrics for success
+            VECTOR_OPERATIONS.labels(operation_type='delete', status='success').inc()
+            
             structured_logger.info(
                 f"Recipe deleted successfully: {recipe_id}",
                 extra={
@@ -468,6 +544,9 @@ async def delete_recipe(recipe_id: str, http_request: Request):
                 recipe_id=recipe_id
             )
         else:
+            # Update vector operation metrics for failure
+            VECTOR_OPERATIONS.labels(operation_type='delete', status='error').inc()
+            
             structured_logger.error(
                 f"Recipe deletion failed: {recipe_id}",
                 extra={
@@ -485,6 +564,9 @@ async def delete_recipe(recipe_id: str, http_request: Request):
     except Exception as e:
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update vector operation metrics for exceptions
+        VECTOR_OPERATIONS.labels(operation_type='delete', status='error').inc()
+        
         logger.error(f"Error deleting recipe: {e}", exc_info=True)
         structured_logger.error(
             f"Recipe deletion failed: {str(e)}",
@@ -533,6 +615,9 @@ async def suggest_recipe(request: RecipeSuggestionRequest, http_request: Request
         response = llm_instance.suggest_recipe(request.query)
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update LLM operation metrics for success
+        LLM_OPERATIONS.labels(operation_type='suggest', status='success').inc()
+        
         structured_logger.info(
             "Recipe suggestion completed successfully",
             extra={
@@ -551,6 +636,9 @@ async def suggest_recipe(request: RecipeSuggestionRequest, http_request: Request
     except Exception as e:
         duration_ms = round((time.time() - start_time) * 1000, 2)
 
+        # Update LLM operation metrics for failures
+        LLM_OPERATIONS.labels(operation_type='suggest', status='error').inc()
+        
         logger.error(f"Error in recipe suggestion: {e}", exc_info=True)
         structured_logger.error(
             f"Recipe suggestion failed: {str(e)}",

genai/requirements.txt

@@ -10,3 +10,4 @@ langchain-community
 langchain-text-splitters
 langchain-huggingface
 sentence-transformers
+prometheus_client

helm/client/values.yaml

@@ -20,8 +20,8 @@ container:
 
 podAnnotations:
   prometheus.io/scrape: "true"
-  prometheus.io/path: "/actuator/prometheus"
-  prometheus.io/port: "8080"
+  prometheus.io/path: "/health"
+  prometheus.io/port: "3000"
   cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
   app.kubernetes.io/version: "1.0.0"
   sidecar.istio.io/inject: "true"

server/api-gateway/build.gradle.kts

@@ -27,6 +27,10 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-gateway")
     implementation("org.springframework.boot:spring-boot-starter-oauth2-resource-server")
     implementation("org.springframework.boot:spring-boot-starter-actuator")
+    implementation("io.micrometer:micrometer-registry-prometheus")
+    implementation("org.springframework.boot:spring-boot-starter-logging")
+    implementation("ch.qos.logback:logback-classic")
+    implementation("net.logstash.logback:logstash-logback-encoder:7.4")
 
     implementation("com.nimbusds:oauth2-oidc-sdk:11.24")
     implementation("org.springdoc:springdoc-openapi-starter-webflux-ui:2.3.0")

server/api-gateway/src/main/resources/application.yml

@@ -12,6 +12,18 @@ spring:
           uri: http://localhost:8082
           predicates:
             - Path=/version
+
+logging:
+  level:
+    root: INFO
+    com.recipefy.gateway: DEBUG
+    com.recipefy.gateway.config: DEBUG
+    com.recipefy.gateway.controller: DEBUG
+    org.springframework.web: DEBUG
+    org.springframework.cloud.gateway: DEBUG
+  pattern:
+    console: "%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"
+
 app:
   security:
     cors:
@@ -31,7 +43,16 @@ management:
       probes:
         enabled: true
       show-details: always
+    prometheus:
+      enabled: true
+  metrics:
+    export:
+      prometheus:
+        enabled: true
+    distribution:
+      percentiles-histogram:
+        http.server.requests: true
   endpoints:
     web:
       exposure:
-        include: health,info
+        include: health,info,prometheus

server/api-gateway/src/main/resources/logback-spring.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <!-- JSON Logging for production -->
+    <springProfile name="prod">
+        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+            <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+                <providers>
+                    <timestamp/>
+                    <logLevel/>
+                    <loggerName/>
+                    <message/>
+                    <mdc/>
+                    <stackTrace/>
+                </providers>
+            </encoder>
+        </appender>
+    </springProfile>
+
+    <!-- Development profile - standard console logging with MDC -->
+    <springProfile name="!prod">
+        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+            <encoder>
+                <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{requestId},%X{userId}] - %msg%n</pattern>
+            </encoder>
+        </appender>
+    </springProfile>
+
+    <!-- Log levels -->
+    <logger name="com.recipefy.gateway" level="DEBUG"/>
+    <logger name="com.recipefy.gateway.config" level="DEBUG"/>
+    <logger name="com.recipefy.gateway.controller" level="DEBUG"/>
+    <logger name="org.springframework.web" level="DEBUG"/>
+    <logger name="org.springframework.cloud.gateway" level="DEBUG"/>
+
+    <!-- Root logger - only console output for containerized deployment -->
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+</configuration> 

server/keycloak/build.gradle

@@ -22,7 +22,10 @@ dependencies {
 	implementation 'org.springframework.boot:spring-boot-starter-webflux'
 	implementation 'org.springframework.boot:spring-boot-starter-validation'
 	implementation 'org.springframework.boot:spring-boot-starter-actuator'
-
+	implementation 'io.micrometer:micrometer-registry-prometheus'
+	implementation 'org.springframework.boot:spring-boot-starter-logging'
+	implementation 'ch.qos.logback:logback-classic'
+	implementation 'net.logstash.logback:logstash-logback-encoder:7.4'
 
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
 	testRuntimeOnly 'org.junit.platform:junit-platform-launcher'

server/keycloak/src/main/resources/application.yml

@@ -2,6 +2,17 @@ spring:
   application:
     name: keycloak
 
+logging:
+  level:
+    root: INFO
+    com.recipefy.keycloak: DEBUG
+    com.recipefy.keycloak.controller: DEBUG
+    com.recipefy.keycloak.service: DEBUG
+    com.recipefy.keycloak.config: DEBUG
+    org.springframework.web: DEBUG
+  pattern:
+    console: "%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"
+
 keycloak:
   baseUrl: https://keycloak.team-breaking-build.student.k8s.aet.cit.tum.de
   realm: recipefy
@@ -21,8 +32,17 @@ management:
       probes:
         enabled: true
       show-details: always
+    prometheus:
+      enabled: true
+  metrics:
+    export:
+      prometheus:
+        enabled: true
+    distribution:
+      percentiles-histogram:
+        http.server.requests: true
   endpoints:
     web:
       exposure:
-        include: health,info
+        include: health,info,prometheus