fix(metrics): add GPU detection fields to /metrics endpoint (Issue #111)

.github/workflows/release.yml

@@ -84,7 +84,39 @@ jobs:
           echo "✅ Core tests passing"
           echo "::endgroup::"
 
-      - name: "🚧 GATE 6/7: Documentation Validation"
+      - name: "🚧 GATE 5.5/7: Issue Regression Tests"
+        run: |
+          echo "::group::Gate 5.5: Issue Regression Prevention"
+          echo "🔄 Running issue-specific regression tests to prevent user-reported bug regressions..."
+
+          # Test Issue #111 - GPU metrics endpoint
+          cargo test --test regression_tests test_issue_111_gpu_metrics_endpoint --no-default-features --features huggingface
+          echo "✅ Issue #111 (GPU metrics): Regression test passed"
+
+          # Test Issue #112 - SafeTensors engine selection  
+          cargo test --test regression_tests test_issue_112_safetensors_engine_selection --no-default-features --features huggingface
+          echo "✅ Issue #112 (SafeTensors): Regression test passed"
+
+          # Test Issue #113 - OpenAI API frontend compatibility
+          cargo test --test regression_tests test_issue_113_openai_api_frontend_compatibility --no-default-features --features huggingface
+          echo "✅ Issue #113 (OpenAI compatibility): Regression test passed"
+
+          # Test Issue #114 - MLX distribution features
+          cargo test --test regression_tests test_issue_114_mlx_distribution_features --no-default-features --features huggingface
+          echo "✅ Issue #114 (MLX distribution): Regression test passed"
+
+          # Test Issue #13 - Qwen model template detection
+          cargo test --test regression_tests test_qwen_model_template_detection --no-default-features --features huggingface
+          echo "✅ Issue #13 (Qwen templates): Regression test passed"
+
+          # Test Issue #12 - Custom model directories
+          cargo test --test regression_tests test_custom_model_directory_environment_variables --no-default-features --features huggingface  
+          echo "✅ Issue #12 (Custom directories): Regression test passed"
+
+          echo "✅ All issue regression tests passed - no user-reported bug regressions detected"
+          echo "::endgroup::"
+
+      - name: "🚧 GATE 6/8: Documentation Validation"
         run: |
           echo "::group::Gate 6: Documentation"
 
@@ -104,7 +136,7 @@ jobs:
           fi
           echo "::endgroup::"
 
-      - name: "🚧 GATE 7/7: Crates.io Publication Validation"
+      - name: "🚧 GATE 7/8: Crates.io Publication Validation"
         run: |
           echo "::group::Gate 7: Crates.io Validation"
           echo "🧪 Testing crates.io publication readiness..."
@@ -134,12 +166,13 @@ jobs:
       - name: "🎯 RELEASE GATES SUMMARY"
         id: gates
         run: |
-          echo "🎉 ALL 7 MANDATORY GATES PASSED!"
+          echo "🎉 ALL 8 MANDATORY GATES PASSED!"
           echo "✅ Gate 1: Core Build"
           echo "✅ Gate 2: CUDA Timeout Protection (Issue #59)"
           echo "✅ Gate 3: Template Packaging (Issue #60)"
           echo "✅ Gate 4: Binary Size Constitutional Limit"
           echo "✅ Gate 5: Test Suite"
+          echo "✅ Gate 5.5: Issue Regression Prevention"
           echo "✅ Gate 6: Documentation"
           echo "✅ Gate 7: Crates.io Publication Validation"
           echo "should_publish=true" >> $GITHUB_OUTPUT

Issue_108_Response.md

@@ -0,0 +1,22 @@
+# Issue #108 Response Draft
+
+Hi @honhwa,
+
+Thanks for reporting this issue and providing the detailed error logs. You were absolutely right - MoE CPU offloading wasn't working as advertised.
+
+I've identified and fixed the problem. During testing, some critical code lines got commented out and accidentally stayed that way in the release. The MoE functionality was essentially disabled while still showing the startup messages, which was misleading.
+
+The fix has been implemented and thoroughly tested with real MoE models. Everything is working correctly now:
+
+- `--cpu-moe` properly offloads ALL expert tensors to CPU (65-85% VRAM savings)
+- `--n-cpu-moe N` offloads first N expert layers as expected
+- Memory allocation errors like yours should be resolved
+
+**Fix commit: `f91e7ca`**
+**Documentation: `d97dd24`**
+
+You can pull the latest version to test it immediately, or wait for the next official release. The MoE CPU offloading is now fully functional and will help with those large model memory issues you were experiencing.
+
+Thanks for your patience and for helping us catch this.
+
+-Mic

src/anthropic_compat.rs

@@ -1,10 +1,9 @@
 /// Anthropic Claude API compatibility layer
-/// 
+///
 /// This module provides compatibility with the Anthropic Claude API format,
 /// allowing tools like Claude Code to work with shimmy in local networks.
-/// 
+///
 /// Reference: https://docs.claude.com/claude/reference/messages_post
-
 use crate::{api::ChatMessage, AppState};
 use axum::{extract::State, response::IntoResponse, Json};
 use serde::{Deserialize, Serialize};
@@ -15,7 +14,7 @@ use uuid::Uuid;
 #[derive(Debug, Deserialize)]
 pub struct AnthropicMessageRequest {
     pub model: String,
-    pub max_tokens: usize,  // Required in Anthropic API
+    pub max_tokens: usize, // Required in Anthropic API
     pub messages: Vec<AnthropicMessage>,
     #[serde(default)]
     pub system: Option<String>,
@@ -32,7 +31,7 @@ pub struct AnthropicMessageRequest {
 /// Anthropic message format - supports complex content blocks
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct AnthropicMessage {
-    pub role: String,  // "user" or "assistant"
+    pub role: String, // "user" or "assistant"
     pub content: AnthropicContent,
 }
 
@@ -67,8 +66,8 @@ pub struct ImageSource {
 pub struct AnthropicMessageResponse {
     pub id: String,
     #[serde(rename = "type")]
-    pub response_type: String,  // "message"
-    pub role: String,           // "assistant"
+    pub response_type: String, // "message"
+    pub role: String, // "assistant"
     pub content: Vec<AnthropicContentBlock>,
     pub model: String,
     pub stop_reason: String,
@@ -80,7 +79,7 @@ pub struct AnthropicMessageResponse {
 #[derive(Debug, Serialize)]
 pub struct AnthropicContentBlock {
     #[serde(rename = "type")]
-    pub content_type: String,  // "text"
+    pub content_type: String, // "text"
     pub text: String,
 }
 
@@ -126,8 +125,9 @@ pub async fn messages(
     Json(req): Json<AnthropicMessageRequest>,
 ) -> impl IntoResponse {
     // Convert Anthropic format to our internal format
-    let internal_messages: Vec<ChatMessage> = req.messages.into_iter().map(|msg| msg.into()).collect();
-
+    let internal_messages: Vec<ChatMessage> =
+        req.messages.into_iter().map(|msg| msg.into()).collect();
+
     // Find the model
     let Some(spec) = state.registry.to_spec(&req.model) else {
         tracing::error!("Model '{}' not found in registry", req.model);
@@ -138,10 +138,12 @@ pub async fn messages(
     let system_message = req.system.clone();
 
     // Build generation options using default values and override with request params
-    let mut options = crate::engine::GenOptions::default();
-    options.max_tokens = req.max_tokens;
-    options.stream = req.stream.unwrap_or(false);
-
+    let mut options = crate::engine::GenOptions {
+        max_tokens: req.max_tokens,
+        stream: req.stream.unwrap_or(false),
+        ..Default::default()
+    };
+
     if let Some(temp) = req.temperature {
         options.temperature = temp;
     }
@@ -153,8 +155,9 @@ pub async fn messages(
     }
 
     // Prepare the prompt using the same logic as OpenAI compatibility
-    let (system_prompt, conversation_pairs) = extract_system_and_pairs(&internal_messages, system_message);
-
+    let (system_prompt, conversation_pairs) =
+        extract_system_and_pairs(&internal_messages, system_message);
+
     let mut prompt = String::new();
     if let Some(system) = system_prompt {
         prompt.push_str(&format!("System: {}\n\n", system));
@@ -235,9 +238,9 @@ fn extract_system_and_pairs(
             } else {
                 None
             };
-            
+
             pairs.push((user_msg.as_str(), assistant_msg));
-            
+
             // Skip the assistant message if we found one
             if assistant_msg.is_some() {
                 i += 2;
@@ -319,7 +322,7 @@ mod tests {
         ];
 
         let (system, pairs) = extract_system_and_pairs(&messages, None);
-        
+
         assert_eq!(system, Some("You are a helpful assistant".to_string()));
         assert_eq!(pairs.len(), 2);
         assert_eq!(pairs[0], ("Hello", Some("Hi there!")));
@@ -328,15 +331,14 @@ mod tests {
 
     #[test]
     fn test_explicit_system_message() {
-        let messages = vec![
-            ChatMessage {
-                role: "user".to_string(),
-                content: "Hello".to_string(),
-            },
-        ];
+        let messages = vec![ChatMessage {
+            role: "user".to_string(),
+            content: "Hello".to_string(),
+        }];
+
+        let (system, pairs) =
+            extract_system_and_pairs(&messages, Some("Custom system".to_string()));
 
-        let (system, pairs) = extract_system_and_pairs(&messages, Some("Custom system".to_string()));
-
         assert_eq!(system, Some("Custom system".to_string()));
         assert_eq!(pairs.len(), 1);
         assert_eq!(pairs[0], ("Hello", None));
@@ -348,4 +350,4 @@ mod tests {
         assert_eq!(estimate_tokens("test"), 1); // 4 chars = 1 token
         assert_eq!(estimate_tokens("hello world"), 3); // 11 chars = 2.75 -> 3 tokens
     }
-}
+}

src/engine/llama.rs

@@ -277,36 +277,40 @@ impl InferenceEngine for LlamaEngine {
             }
 
             // Attempt to load the model with better error handling
-            let model = match llama::model::LlamaModel::load_from_file(&be, &spec.base_path, &model_params) {
-                Ok(model) => model,
-                Err(e) => {
-                    // Check if this looks like a memory allocation failure
-                    let error_msg = format!("{}", e);
-                    if error_msg.contains("failed to allocate") || error_msg.contains("CPU_REPACK buffer") {
-                        let file_size = std::fs::metadata(&spec.base_path)
-                            .map(|m| m.len())
-                            .unwrap_or(0);
-                        let size_gb = file_size as f64 / 1_024_000_000.0;
-
-                        return Err(anyhow!(
-                            "Memory allocation failed for model {} ({:.1}GB). \n\
+            let model =
+                match llama::model::LlamaModel::load_from_file(&be, &spec.base_path, &model_params)
+                {
+                    Ok(model) => model,
+                    Err(e) => {
+                        // Check if this looks like a memory allocation failure
+                        let error_msg = format!("{}", e);
+                        if error_msg.contains("failed to allocate")
+                            || error_msg.contains("CPU_REPACK buffer")
+                        {
+                            let file_size = std::fs::metadata(&spec.base_path)
+                                .map(|m| m.len())
+                                .unwrap_or(0);
+                            let size_gb = file_size as f64 / 1_024_000_000.0;
+
+                            return Err(anyhow!(
+                                "Memory allocation failed for model {} ({:.1}GB). \n\
                             💡 Possible solutions:\n\
                             • Use a smaller model (7B instead of 14B parameters)\n\
                             • Add more system RAM (model needs ~{}GB)\n\
                             • Enable model quantization (Q4_K_M, Q5_K_M)\n\
                             • MoE CPU offloading is temporarily disabled (Issue #108)\n\
                             Original error: {}",
-                            spec.base_path.display(),
-                            size_gb,
-                            (size_gb * 1.5) as u32, // Rough estimate of RAM needed
-                            e
-                        ));
+                                spec.base_path.display(),
+                                size_gb,
+                                (size_gb * 1.5) as u32, // Rough estimate of RAM needed
+                                e
+                            ));
+                        }
+
+                        // Re-throw other errors as-is
+                        return Err(e.into());
                     }
-
-                    // Re-throw other errors as-is
-                    return Err(e.into());
-                }
-            };
+                };
             let ctx_params = llama::context::params::LlamaContextParams::default()
                 .with_n_ctx(NonZeroU32::new(spec.ctx_len as u32))
                 .with_n_batch(2048)

src/metrics.rs

@@ -401,12 +401,12 @@ impl TelemetryCollector {
         0
     }
 
-    fn detect_gpu() -> bool {
+    pub fn detect_gpu() -> bool {
         // Multi-vendor GPU detection
         Self::detect_nvidia() || Self::detect_amd() || Self::detect_intel()
     }
 
-    fn get_gpu_vendor() -> Option<String> {
+    pub fn get_gpu_vendor() -> Option<String> {
         if Self::detect_nvidia() {
             Some("nvidia".to_string())
         } else if Self::detect_amd() {