feat(text_processor): 优化文本规范化功能，增强对缩写和特殊格式的处理，修复edge-tts因版本过低生成语音失败问题

Author: Pi3-l22

requirements.txt

@@ -2,7 +2,7 @@ fastapi>=0.68.0,<1.0.0
 uvicorn>=0.35.0,<1.0.0
 jinja2>=3.0.0,<4.0.0
 python-multipart>=0.0.20,<1.0.0
-edge-tts>=7.0.0,<8.0.0
+edge-tts>=7.2.7,<8.0.0
 PyMuPDF>=1.26.0,<2.0.0
 nltk>=3.9.0,<4.0.0
 translators>=6.0.0,<7.0.0

tests/test_text_processor.py

@@ -8,6 +8,13 @@ def test_normalize_text(self):
         text = "This   is a  test sentence。Some people may wonder，“How long does it take to form a habit？”This is a another test sentence！"
         normalized_text = normalize_text(text)
         self.assertEqual(normalized_text, 'This is a test sentence. Some people may wonder, "How long does it take to form a habit?" This is a another test sentence!')
+
+        # 回归测试：专有缩写和小数不应被错误插入空格
+        edge_text = "by C. Liu, et al., we compare TLS 1.3 in detail."
+        normalized_edge_text = normalize_text(edge_text)
+        self.assertIn("et al.,", normalized_edge_text)
+        self.assertIn("TLS 1.3", normalized_edge_text)
+
         ja_text = '今日はいい天気だ...でも“急に雨が降り出した!”と彼は叫んだ. 明日の予定は? (キャンプを中止する) みんなで“楽しみにしていた”イベントだったのに…本当に残念ですね。'
         normalized_text = normalize_text(ja_text, 'japanese')
         self.assertEqual(normalized_text, '今日はいい天気だ…でも『急に雨が降り出した！』と彼は叫んだ。明日の予定は？（キャンプを中止する）みんなで『楽しみにしていた』イベントだったのに…本当に残念ですね。')
@@ -47,6 +54,43 @@ def test_get_sentences_lang(self):
         for lang, text in texts.items():
             sentences = get_sentences(text, lang)
             self.assertEqual(len(sentences), 2)
+
+    def test_get_sentences_with_abbreviation_and_decimal(self):
+        """回归测试：缩写 + 小数场景不应导致误拆分"""
+        init_nltk()
+        text = "This was reported by C. Liu, et al., in TLS 1.3 research. The result is stable."
+        sentences = get_sentences(text, 'english')
+        self.assertEqual(len(sentences), 2)
+        self.assertIn("et al.,", sentences[0])
+        self.assertIn("TLS 1.3", sentences[0])
+
+    def test_normalize_text_special_cases(self):
+        """更多特殊情况：缩写链、版本号、URL、邮箱、数字格式"""
+        text = (
+            "In the U.S.A.we use v3.10 and TLS 1.3.This is common!"
+            "Contact me at test@example.com,and check https://example.com/docs."
+            "The budget is 1,000.25 at 12:30."
+        )
+        normalized = normalize_text(text)
+
+        self.assertIn("U.S.A.", normalized)
+        self.assertIn("v3.10", normalized)
+        self.assertIn("TLS 1.3", normalized)
+        self.assertIn("test@example.com", normalized)
+        self.assertIn("https://example.com/docs", normalized)
+        self.assertIn("1,000.25", normalized)
+        self.assertIn("12:30", normalized)
+        self.assertIn("This is common! Contact", normalized)
+
+    def test_get_sentences_with_acronym_and_url(self):
+        """回归测试：缩写链与URL不应导致误拆分"""
+        init_nltk()
+        text = "In the U.S.A. we test TLS 1.3. Read more at https://example.com/docs. This is final."
+        sentences = get_sentences(text, 'english')
+        self.assertEqual(len(sentences), 3)
+        self.assertIn("U.S.A.", sentences[0])
+        self.assertIn("TLS 1.3", sentences[0])
+        self.assertIn("https://example.com/docs.", sentences[1])
 
 
 if __name__ == '__main__':

utils/text_processor.py

@@ -145,10 +145,29 @@ def normalize_text(text: str, lang: str = 'english') -> str:
         # 中文符号替换为英文符号
         for chinese_punct, english_punct in chinese_to_english_punctuation.items():
             text = text.replace(chinese_punct, english_punct)
+
+        # 先保护高风险片段，避免后续正则误插空格导致语义破坏
+        protected_tokens = []
+
+        def _protect(pattern: str):
+            nonlocal text
+
+            def _repl(match: re.Match) -> str:
+                protected_tokens.append(match.group(0))
+                return f"__PROTECTED_TOKEN_{len(protected_tokens)-1}__"
+
+            text = re.sub(pattern, _repl, text)
+
+        _protect(r'https?://[^\s<>"\']+|www\.[^\s<>"\']+')
+        _protect(r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}')
+        _protect(r'\b(?:[A-Za-z]\.){2,}')
+        _protect(r'\b\d+(?:\.\d+){1,}\b')
 
-    # 在标点符号后添加空格（除了引号和括号类符号）
-    # 处理句号、感叹号、问号后需要空格的情况
-    text = re.sub(r'([.!?])([^\s"])', r'\1 \2', text)
+    # 在句末符号后按需补空格：
+    # 1) 仅在更像“句子边界”时补空格（例如后续是大写开头或引号/括号）
+    # 2) 避免打断缩写链（U.S.A.）、版本号/小数（3.10 / 1.3）
+    text = re.sub(r'([!?])(?=[^\s"\'\)\]\},.;:!?])', r'\1 ', text)
+    text = re.sub(r'(?<!\d)\.(?!\d)(?=[A-Z"\'\(\[])', r'. ', text)
 
     # 处理句号、感叹号、问号在引号内的情况，在引号后添加空格
     text = re.sub(r'([.!?])(["\'])', r'\1\2', text)  # 先确保标点和引号紧挨着
@@ -157,8 +176,12 @@ def normalize_text(text: str, lang: str = 'english') -> str:
     # 处理括号类符号
     text = re.sub(r'([(){}\[\]<>])', r' \1 ', text)
 
-    # 处理逗号、分号、冒号等符号
-    text = re.sub(r'([,;:])', r'\1 ', text)
+    # 处理逗号、分号、冒号等符号：
+    # 1) 千分位数字（1,000）不加空格
+    # 2) 时间格式（12:30）不加空格
+    text = re.sub(r'(?<=\D),(?=\S)', r', ', text)
+    text = re.sub(r';(?=\S)', r'; ', text)
+    text = re.sub(r'(?<!\d):(?=\S)', r': ', text)
 
     # 处理开头引号前的空格
     text = re.sub(r'\s+"\s+', r' "', text)
@@ -171,5 +194,10 @@ def normalize_text(text: str, lang: str = 'english') -> str:
 
     # 去除前后空白符
     text = text.strip()
+
+    # 还原被保护片段
+    if lang != 'japanese':
+        for i, token in enumerate(protected_tokens):
+            text = text.replace(f"__PROTECTED_TOKEN_{i}__", token)
 
     return text