Add repeating character preprocessor

chatterbot/preprocessors.py

@@ -3,7 +3,7 @@
 """
 from chatterbot.conversation import Statement
 from unicodedata import normalize
-from re import sub as re_sub
+from re import sub as re_sub, compile as re_compile
 from html import unescape
 
 
@@ -45,3 +45,27 @@ def convert_to_ascii(statement: Statement) -> Statement:
 
     statement.text = str(text)
     return statement
+
+# Matches a single letter that is immediately repeated three or more times.
+# Digits, punctuation, and whitespace are intentionally excluded so that
+# values such as "1000000" or "!!!" are left unchanged.
+_REPEATING_CHARACTER_PATTERN = re_compile(r'([^\W\d_])\1{2,}')
+
+
+def normalize_repeating_characters(statement: Statement) -> Statement:
+    """
+    Reduce runs of three or more repeated letters down to two.
+
+    Elongated words are common in conversational text (for example
+    "I am sooooo happy"). Collapsing the repeated characters maps these
+    variations to a single, consistent form ("I am soo happy") which helps
+    the chat bot match input against statements it has been trained on.
+
+    Letter pairs that occur naturally (such as the "oo" in "cool") are
+    preserved, and repeated digits or punctuation are left unchanged.
+    """
+    statement.text = _REPEATING_CHARACTER_PATTERN.sub(
+        lambda match: match.group(1) * 2, statement.text
+    )
+
+    return statement

docs/preprocessors.rst

@@ -28,6 +28,8 @@ ChatterBot comes with several built-in preprocessors.
 
 .. autofunction:: chatterbot.preprocessors.convert_to_ascii
 
+.. autofunction:: chatterbot.preprocessors.normalize_repeating_characters
+
 
 Creating new preprocessors
 ==========================

tests/test_preprocessors.py

@@ -78,3 +78,33 @@ def test_convert_to_ascii(self):
         normal_text = 'Kluft skrams infor pa federal electoral groe'
 
         self.assertEqual(cleaned.text, normal_text)
+
+class NormalizeRepeatingCharactersPreprocessorTestCase(ChatBotTestCase):
+    """
+    Make sure that ChatterBot's repeating-character preprocessor works as expected.
+    """
+
+    def test_elongated_word_is_reduced(self):
+        statement = Statement(text='I am sooooo happy')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+        self.assertEqual(cleaned.text, 'I am soo happy')
+
+    def test_multiple_elongated_words(self):
+        statement = Statement(text='Yesss that was greaaaat')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+        self.assertEqual(cleaned.text, 'Yess that was greaat')
+
+    def test_natural_double_letters_preserved(self):
+        statement = Statement(text='That book looks really cool')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+        self.assertEqual(cleaned.text, 'That book looks really cool')
+
+    def test_repeating_digits_preserved(self):
+        statement = Statement(text='I have 1000000 dollars')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+        self.assertEqual(cleaned.text, 'I have 1000000 dollars')
+
+    def test_repeating_punctuation_preserved(self):
+        statement = Statement(text='Wow!!!')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+        self.assertEqual(cleaned.text, 'Wow!!!')