fix: 收口 GeminiAnalyzer 主分析链路的 LLM prompt/response 日志… (#877) #897
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,7 @@ | |
| import json | ||
| import logging | ||
| import math | ||
| import re | ||
| import time | ||
| from dataclasses import dataclass | ||
| from typing import Optional, Dict, Any, List, Tuple | ||
|
|
@@ -45,9 +46,144 @@ | |
| ) | ||
| from src.schemas.report_schema import AnalysisReportSchema | ||
| from src.market_context import get_market_role, get_market_guidelines | ||
| from src.logging_config import is_sensitive_log_preview_enabled | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| _LLM_PREVIEW_MAX_CHARS = 240 | ||
| _LLM_AUTHORIZATION_SAFE_SCHEMES = { | ||
| "aws4-hmac-sha256", | ||
| "basic", | ||
| "bearer", | ||
| "digest", | ||
| "dpop", | ||
| "hoba", | ||
| "mutual", | ||
| "negotiate", | ||
| "ntlm", | ||
| "pop", | ||
| "signature", | ||
| "token", | ||
| "vapid", | ||
| } | ||
| _LLM_SENSITIVE_ASSIGNMENT_VALUE_PATTERN = ( | ||
| r"[^\s]+(?:\s+(?!(?:[\w.-]+|\"[^\"]+\"|'[^']+')\s*[:=])\S+)*" | ||
| ) | ||
| _LLM_SENSITIVE_FIELD_NAME_PATTERN = ( | ||
| r"(?:api[_-]?keys?|tokens?|secrets?|passwords?|passwd|passphrase|credentials?|session[_-]?id" | ||
| r"|[\w.-]+(?:api[_-]?keys?|tokens?|secrets?|passwords?|passwd|passphrase|credentials?|session[_-]?id|keys?))" | ||
| ) | ||
|
Comment on lines
+72
to
+75
There was a problem hiding this comment.
The sensitive-field matcher only covers exact names like Useful? React with 👍 / 👎.
Comment on lines
+72
to
+75
There was a problem hiding this comment.
The sensitive-key matcher only recognizes Useful? React with 👍 / 👎.
Comment on lines
+72
to
+75
There was a problem hiding this comment. * The sensitive-name regex only treats generic Useful? React with 👍 / 👎. |
||
|
|
||
|
|
||
| def _redact_authorization_preview_value(value: str) -> str: | ||
| parts = str(value or "").strip().split(None, 1) | ||
| if len(parts) == 2 and parts[0] and parts[0].lower() in _LLM_AUTHORIZATION_SAFE_SCHEMES: | ||
| return f"{parts[0]} [REDACTED]" | ||
|
Comment on lines
+79
to
+81
There was a problem hiding this comment.
Useful? React with 👍 / 👎. |
||
| return "[REDACTED]" | ||
|
|
||
|
|
||
| def _replace_quoted_authorization_preview(match) -> str: | ||
| return ( | ||
| f"{match.group(1)}{match.group(2)}{match.group(1)}" | ||
| f"{match.group(3)}{match.group(4)}" | ||
| f"{_redact_authorization_preview_value(match.group('value'))}" | ||
| f"{match.group(4)}" | ||
| ) | ||
|
|
||
|
|
||
| def _replace_authorization_preview(match) -> str: | ||
| return f"{match.group(1)}={_redact_authorization_preview_value(match.group('value'))}" | ||
|
|
||
|
|
||
| _LLM_RAW_LINE_SENSITIVE_PATTERNS = ( | ||
| ( | ||
| re.compile(r"(?im)\b(authorization)\s*[:=]\s*(?P<value>[^\n\r]*)"), | ||
| _replace_authorization_preview, | ||
| ), | ||
| ) | ||
|
|
||
|
|
||
| _LLM_SENSITIVE_PATTERNS = ( | ||
| ( | ||
| re.compile(r'(?i)(["\'])(authorization)\1\s*([:=])\s*(["\'])(?P<value>(?:\\.|(?!\4).)*)\4'), | ||
| _replace_quoted_authorization_preview, | ||
|
Comment on lines
+108
to
+109
There was a problem hiding this comment.
The quoted-header sanitizer only matches the exact key Useful? React with 👍 / 👎. |
||
| ), | ||
| ( | ||
| re.compile(rf"(?i)\b(authorization)\s*[:=]\s*(?P<value>{_LLM_SENSITIVE_ASSIGNMENT_VALUE_PATTERN})"), | ||
| _replace_authorization_preview, | ||
|
Comment on lines
+112
to
+113
There was a problem hiding this comment.
The unquoted Useful? React with 👍 / 👎.
Comment on lines
+112
to
+113
There was a problem hiding this comment.
The unquoted authorization matcher only recognizes a standalone Useful? React with 👍 / 👎. |
||
| ), | ||
| ( | ||
| re.compile(r'(?i)(["\'])(set-cookie|cookie)\1\s*([:=])\s*(["\'])(?:\\.|(?!\4).)*\4'), | ||
| r"\1\2\1\3\4[REDACTED]\4", | ||
| ), | ||
| ( | ||
| re.compile( | ||
| r"(?i)\b(set-cookie|cookie)\s*[:=]\s*[^;=\s]+(?:\s*=\s*[^;\s]+)?(?:\s*;\s*[^;\n\r]+(?:\s*=\s*[^;\n\r]+)?)*" | ||
| ), | ||
| r"\1=[REDACTED]", | ||
| ), | ||
| ( | ||
| re.compile( | ||
| rf'(?i)(["\'])({_LLM_SENSITIVE_FIELD_NAME_PATTERN})\1\s*:\s*(["\'])(?:\\.|(?!\3).)*\3' | ||
| ), | ||
|
Comment on lines
+126
to
+128
There was a problem hiding this comment.
The sanitizer only has a quoted-key pattern for double-quoted JSON, so Python-style dict payloads with single quotes are not redacted; for example Useful? React with 👍 / 👎. |
||
| r"\1\2\1:\3[REDACTED]\3", | ||
|
Comment on lines
+126
to
+129
There was a problem hiding this comment.
The new sanitizer only redacts quoted-key fields when the value is also quoted (see the Useful? React with 👍 / 👎. |
||
| ), | ||
| ( | ||
| re.compile( | ||
| rf'(?i)(["\'])({_LLM_SENSITIVE_FIELD_NAME_PATTERN})\1\s*:\s*(?!["\'])(-?[\w.+\-]+)' | ||
| ), | ||
|
Comment on lines
+132
to
+134
There was a problem hiding this comment.
The quoted-key sanitizer only handles string values and primitive tokens ( Useful? React with 👍 / 👎. |
||
| r"\1\2\1:[REDACTED]", | ||
| ), | ||
| ( | ||
| re.compile( | ||
| rf"(?i)\b({_LLM_SENSITIVE_FIELD_NAME_PATTERN})\b\s*[:=]\s*(['\"])(?:\\.|(?!\2).)*\2" | ||
| ), | ||
|
Comment on lines
+138
to
+140
There was a problem hiding this comment.
The new redaction patterns only match unquoted assignments (for example Useful? React with 👍 / 👎. |
||
| r"\1=\2[REDACTED]\2", | ||
| ), | ||
| ( | ||
| re.compile( | ||
| rf"(?i)\b({_LLM_SENSITIVE_FIELD_NAME_PATTERN})\b\s*[:=]\s*(?!['\"])({_LLM_SENSITIVE_ASSIGNMENT_VALUE_PATTERN})" | ||
| ), | ||
| r"\1=[REDACTED]", | ||
| ), | ||
| ( | ||
| re.compile(r"(?i)\b[\w.+-]+@[\w.-]+\.[a-z]{2,}\b"), | ||
| "[REDACTED_EMAIL]", | ||
| ), | ||
| ) | ||
|
|
||
|
|
||
| def _should_log_llm_content_preview(config: Optional[Config] = None) -> bool: | ||
| """Allow LLM content preview only under explicit debug switches.""" | ||
| if is_sensitive_log_preview_enabled(): | ||
| return True | ||
|
Comment on lines
+158
to
+159
There was a problem hiding this comment.
The early return on Useful? React with 👍 / 👎. |
||
| runtime_config = config if config is not None else get_config() | ||
| return bool( | ||
| getattr(runtime_config, "debug", False) | ||
| or str(getattr(runtime_config, "log_level", "INFO") or "INFO").upper() == "DEBUG" | ||
| ) | ||
|
|
||
|
|
||
| def _sanitize_llm_log_preview(content: str, max_chars: int = _LLM_PREVIEW_MAX_CHARS) -> str: | ||
| """Normalize, redact, and truncate preview text for logs.""" | ||
| sanitized_raw = str(content or "") | ||
| for pattern, replacement in _LLM_RAW_LINE_SENSITIVE_PATTERNS: | ||
| sanitized_raw = pattern.sub(replacement, sanitized_raw) | ||
| normalized = re.sub(r"\s+", " ", sanitized_raw).strip() | ||
| if not normalized: | ||
| return "[empty]" | ||
| sanitized = normalized | ||
| for pattern, replacement in _LLM_SENSITIVE_PATTERNS: | ||
| sanitized = pattern.sub(replacement, sanitized) | ||
| if len(sanitized) <= max_chars: | ||
| return sanitized | ||
| return sanitized[:max_chars].rstrip() + "..." | ||
|
|
||
|
|
||
| def _build_llm_log_preview(label: str, content: str, max_chars: int = _LLM_PREVIEW_MAX_CHARS) -> str: | ||
| """Build a safe one-line preview entry for debug logs.""" | ||
| return f"[{label}] len={len(content or '')} preview={_sanitize_llm_log_preview(content, max_chars=max_chars)}" | ||
|
|
||
|
|
||
| def check_content_integrity(result: "AnalysisResult") -> Tuple[bool, List[str]]: | ||
| """ | ||
|
|
@@ -1175,16 +1311,15 @@ def analyze( | |
| prompt = self._format_prompt(context, name, news_context, report_language=report_language) | ||
|
|
||
| config = self._get_runtime_config() | ||
| allow_content_preview = _should_log_llm_content_preview(config) | ||
| model_name = config.litellm_model or "unknown" | ||
| logger.info(f"========== AI 分析 {name}({code}) ==========") | ||
| logger.info(f"[LLM配置] 模型: {model_name}") | ||
| logger.info(f"[LLM配置] Prompt 长度: {len(prompt)} 字符") | ||
| logger.info(f"[LLM配置] 是否包含新闻: {'是' if news_context else '否'}") | ||
|
|
||
| if allow_content_preview: | ||
| logger.debug(_build_llm_log_preview("LLM Prompt 调试预览", prompt)) | ||
|
|
||
| # 设置生成配置 | ||
| generation_config = { | ||
|
|
@@ -1209,14 +1344,12 @@ def analyze( | |
| elapsed = time.time() - start_time | ||
|
|
||
| # 记录响应信息 | ||
| actual_model_name = model_used or model_name | ||
| logger.info( | ||
| f"[LLM返回] {actual_model_name} 响应成功, 耗时 {elapsed:.2f}s, 响应长度 {len(response_text)} 字符" | ||
| ) | ||
|
Comment on lines
1348
to
1350
There was a problem hiding this comment. [LLM返回] 这一行使用的是 |
||
| if allow_content_preview: | ||
| logger.debug(_build_llm_log_preview("LLM返回 调试预览", response_text)) | ||
|
|
||
| # 解析响应 | ||
| result = self._parse_response(response_text, code, name) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This commit adds a second
## [3.11.0] - 2026-03-27section, duplicating that release block. Our desktop release workflow extracts notes withawkfrom the first matching version header to the next## [...]header (.github/workflows/desktop-release.ymlline 180), so duplicate headings for the same version produce inconsistent/truncated release body content. Keep a single header per version and append the #877 note once.Useful? React with 👍 / 👎.