feat: add word-characters language option

refactor
languages: add word-characters

Author: dmun

book/src/languages.md

@@ -72,6 +72,7 @@ These configuration keys are available:
 | `rulers`              | Overrides the `editor.rulers` config key for the language. |
 | `path-completion`     | Overrides the `editor.path-completion` config key for the language. |
 | `word-completion`     | Overrides the [`editor.word-completion`](./editor.md#editorword-completion-section) configuration for the language. |
+| `extra-word-characters` | Extra characters treated as word characters. |
 | `workspace-lsp-roots`     | Directories (relative to the workspace root) that stop the upward root search early. Meant for project-specific hard overrides in a local `.helix/config.toml`; |
 | `persistent-diagnostic-sources` | An array of LSP diagnostic sources assumed unchanged when the language server resends the same set of diagnostics. Helix can track the position for these diagnostics internally instead. Useful for diagnostics that are recomputed on save.
 | `rainbow-brackets` | Overrides the `editor.rainbow-brackets` config key for the language |

helix-core/src/chars.rs

@@ -1,7 +1,52 @@
 //! Utility functions to categorize a `char`.
 
+use std::borrow::Cow;
+
 use crate::LineEnding;
 
+pub static DEFAULT_WORD_CHARS: WordChars = WordChars::new("");
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WordChars(Cow<'static, str>);
+
+impl WordChars {
+    pub const fn new(chars: &'static str) -> Self {
+        Self(Cow::Borrowed(chars))
+    }
+
+    #[inline]
+    pub fn is_word(&self, ch: char) -> bool {
+        char_is_word(ch) || self.0.contains(ch)
+    }
+
+    #[inline]
+    pub fn categorize(&self, ch: char) -> CharCategory {
+        if char_is_line_ending(ch) {
+            CharCategory::Eol
+        } else if ch.is_whitespace() {
+            CharCategory::Whitespace
+        } else if self.is_word(ch) {
+            CharCategory::Word
+        } else if char_is_punctuation(ch) {
+            CharCategory::Punctuation
+        } else {
+            CharCategory::Unknown
+        }
+    }
+}
+
+impl Default for WordChars {
+    fn default() -> Self {
+        Self::new("")
+    }
+}
+
+impl From<String> for WordChars {
+    fn from(value: String) -> Self {
+        Self(Cow::Owned(value))
+    }
+}
+
 #[derive(Debug, Eq, PartialEq)]
 pub enum CharCategory {
     Whitespace,
@@ -13,17 +58,7 @@ pub enum CharCategory {
 
 #[inline]
 pub fn categorize_char(ch: char) -> CharCategory {
-    if char_is_line_ending(ch) {
-        CharCategory::Eol
-    } else if ch.is_whitespace() {
-        CharCategory::Whitespace
-    } else if char_is_word(ch) {
-        CharCategory::Word
-    } else if char_is_punctuation(ch) {
-        CharCategory::Punctuation
-    } else {
-        CharCategory::Unknown
-    }
+    DEFAULT_WORD_CHARS.categorize(ch)
 }
 
 /// Determine whether a character is a line ending.

helix-core/src/movement.rs

@@ -4,7 +4,7 @@ use ropey::iter::Chars;
 
 use crate::{
     char_idx_at_visual_offset,
-    chars::{categorize_char, char_is_line_ending, CharCategory},
+    chars::{categorize_char, char_is_line_ending, CharCategory, WordChars},
     doc_formatter::TextFormat,
     graphemes::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
@@ -165,20 +165,20 @@ pub fn move_vertically(
     new_range
 }
 
-pub fn move_next_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {
-    word_move(slice, range, count, WordMotionTarget::NextWordStart)
+pub fn move_next_word_start(slice: RopeSlice, range: Range, count: usize, word_chars: &WordChars) -> Range {
+    word_move(slice, range, count, WordMotionTarget::NextWordStart(word_chars))
 }
 
-pub fn move_next_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {
-    word_move(slice, range, count, WordMotionTarget::NextWordEnd)
+pub fn move_next_word_end(slice: RopeSlice, range: Range, count: usize, word_chars: &WordChars) -> Range {
+    word_move(slice, range, count, WordMotionTarget::NextWordEnd(word_chars))
 }
 
-pub fn move_prev_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {
-    word_move(slice, range, count, WordMotionTarget::PrevWordStart)
+pub fn move_prev_word_start(slice: RopeSlice, range: Range, count: usize, word_chars: &WordChars) -> Range {
+    word_move(slice, range, count, WordMotionTarget::PrevWordStart(word_chars))
 }
 
-pub fn move_prev_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {
-    word_move(slice, range, count, WordMotionTarget::PrevWordEnd)
+pub fn move_prev_word_end(slice: RopeSlice, range: Range, count: usize, word_chars: &WordChars) -> Range {
+    word_move(slice, range, count, WordMotionTarget::PrevWordEnd(word_chars))
 }
 
 pub fn move_next_long_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {
@@ -216,10 +216,10 @@ pub fn move_prev_sub_word_end(slice: RopeSlice, range: Range, count: usize) -> R
 fn word_move(slice: RopeSlice, range: Range, count: usize, target: WordMotionTarget) -> Range {
     let is_prev = matches!(
         target,
-        WordMotionTarget::PrevWordStart
+        WordMotionTarget::PrevWordStart(_)
             | WordMotionTarget::PrevLongWordStart
             | WordMotionTarget::PrevSubWordStart
-            | WordMotionTarget::PrevWordEnd
+            | WordMotionTarget::PrevWordEnd(_)
             | WordMotionTarget::PrevLongWordEnd
             | WordMotionTarget::PrevSubWordEnd
     );
@@ -389,11 +389,11 @@ where
 
 /// Possible targets of a word motion
 #[derive(Copy, Clone, Debug)]
-pub enum WordMotionTarget {
-    NextWordStart,
-    NextWordEnd,
-    PrevWordStart,
-    PrevWordEnd,
+pub enum WordMotionTarget<'a> {
+    NextWordStart(&'a WordChars),
+    NextWordEnd(&'a WordChars),
+    PrevWordStart(&'a WordChars),
+    PrevWordEnd(&'a WordChars),
     // A "Long word" (also known as a WORD in Vim/Kakoune) is strictly
     // delimited by whitespace, and can consist of punctuation as well
     // as alphanumerics.
@@ -420,10 +420,10 @@ impl CharHelpers for Chars<'_> {
     fn range_to_target(&mut self, target: WordMotionTarget, origin: Range) -> Range {
         let is_prev = matches!(
             target,
-            WordMotionTarget::PrevWordStart
+            WordMotionTarget::PrevWordStart(_)
                 | WordMotionTarget::PrevLongWordStart
                 | WordMotionTarget::PrevSubWordStart
-                | WordMotionTarget::PrevWordEnd
+                | WordMotionTarget::PrevWordEnd(_)
                 | WordMotionTarget::PrevLongWordEnd
                 | WordMotionTarget::PrevSubWordEnd
         );
@@ -489,8 +489,8 @@ impl CharHelpers for Chars<'_> {
     }
 }
 
-fn is_word_boundary(a: char, b: char) -> bool {
-    categorize_char(a) != categorize_char(b)
+fn is_word_boundary(a: char, b: char, word_chars: &WordChars) -> bool {
+    word_chars.categorize(a) != word_chars.categorize(b)
 }
 
 fn is_long_word_boundary(a: char, b: char) -> bool {
@@ -523,12 +523,12 @@ fn is_sub_word_boundary(a: char, b: char, dir: Direction) -> bool {
 
 fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> bool {
     match target {
-        WordMotionTarget::NextWordStart | WordMotionTarget::PrevWordEnd => {
-            is_word_boundary(prev_ch, next_ch)
+        WordMotionTarget::NextWordStart(word_chars) | WordMotionTarget::PrevWordEnd(word_chars) => {
+            is_word_boundary(prev_ch, next_ch, word_chars)
                 && (char_is_line_ending(next_ch) || !next_ch.is_whitespace())
         }
-        WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
-            is_word_boundary(prev_ch, next_ch)
+        WordMotionTarget::NextWordEnd(word_chars) | WordMotionTarget::PrevWordStart(word_chars) => {
+            is_word_boundary(prev_ch, next_ch, word_chars)
                 && (!prev_ch.is_whitespace() || char_is_line_ending(next_ch))
         }
         WordMotionTarget::NextLongWordStart | WordMotionTarget::PrevLongWordEnd => {
@@ -979,19 +979,22 @@ mod test {
     #[test]
     #[should_panic]
     fn nonsensical_ranges_panic_on_forward_movement_attempt_in_debug_mode() {
-        move_next_word_start(Rope::from("Sample").slice(..), Range::point(99999999), 1);
+        let word_chars = WordChars::default();
+        move_next_word_start(Rope::from("Sample").slice(..), Range::point(99999999), 1, &word_chars);
     }
 
     #[test]
     #[should_panic]
     fn nonsensical_ranges_panic_on_forward_to_end_movement_attempt_in_debug_mode() {
-        move_next_word_end(Rope::from("Sample").slice(..), Range::point(99999999), 1);
+        let word_chars = WordChars::default();
+        move_next_word_end(Rope::from("Sample").slice(..), Range::point(99999999), 1, &word_chars);
     }
 
     #[test]
     #[should_panic]
     fn nonsensical_ranges_panic_on_backwards_movement_attempt_in_debug_mode() {
-        move_prev_word_start(Rope::from("Sample").slice(..), Range::point(99999999), 1);
+        let word_chars = WordChars::default();
+        move_prev_word_start(Rope::from("Sample").slice(..), Range::point(99999999), 1, &word_chars);
     }
 
     #[test]
@@ -1071,10 +1074,11 @@ mod test {
                     (1, Range::new(0, 0), Range::new(0, 6)),
                 ]),
         ];
+        let word_chars = WordChars::default();
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
-                let range = move_next_word_start(Rope::from(sample).slice(..), begin, count);
+                let range = move_next_word_start(Rope::from(sample).slice(..), begin, count, &word_chars);
                 assert_eq!(range, expected_end, "Case failed: [{}]", sample);
             }
         }
@@ -1412,10 +1416,11 @@ mod test {
                     (1, Range::new(0, 6), Range::new(6, 0)),
                 ]),
         ];
+        let word_chars = WordChars::default();
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
-                let range = move_prev_word_start(Rope::from(sample).slice(..), begin, count);
+                let range = move_prev_word_start(Rope::from(sample).slice(..), begin, count, &word_chars);
                 assert_eq!(range, expected_end, "Case failed: [{}]", sample);
             }
         }
@@ -1679,10 +1684,11 @@ mod test {
                     (1, Range::new(0, 0), Range::new(0, 5)),
                 ]),
         ];
+        let word_chars = WordChars::default();
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
-                let range = move_next_word_end(Rope::from(sample).slice(..), begin, count);
+                let range = move_next_word_end(Rope::from(sample).slice(..), begin, count, &word_chars);
                 assert_eq!(range, expected_end, "Case failed: [{}]", sample);
             }
         }
@@ -1761,10 +1767,11 @@ mod test {
                     (1, Range::new(0, 10), Range::new(10, 4)),
                 ]),
         ];
+        let word_chars = WordChars::default();
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
-                let range = move_prev_word_end(Rope::from(sample).slice(..), begin, count);
+                let range = move_prev_word_end(Rope::from(sample).slice(..), begin, count, &word_chars);
                 assert_eq!(range, expected_end, "Case failed: [{}]", sample);
             }
         }

helix-core/src/syntax/config.rs

@@ -1,4 +1,4 @@
-use crate::{auto_pairs::AutoPairs, diagnostic::Severity, Language};
+use crate::{auto_pairs::AutoPairs, chars::WordChars, diagnostic::Severity, Language};
 
 use helix_stdx::rope;
 use serde::{ser::SerializeSeq as _, Deserialize, Serialize};
@@ -67,6 +67,9 @@ pub struct LanguageConfiguration {
     /// If set, overrides `editor.word-completion`.
     pub word_completion: Option<WordCompletion>,
 
+    #[serde(default, rename = "extra-word-characters", skip_serializing, deserialize_with = "deserialize_word_chars")]
+    pub word_chars: WordChars,
+
     #[serde(default)]
     pub diagnostic_severity: Severity,
 
@@ -650,6 +653,13 @@ where
     Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))
 }
 
+pub fn deserialize_word_chars<'de, D>(deserializer: D) -> Result<WordChars, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    String::deserialize(deserializer).map(WordChars::from)
+}
+
 fn default_timeout() -> u64 {
     20
 }

helix-core/src/textobject.rs

@@ -2,15 +2,21 @@ use std::fmt::Display;
 
 use ropey::RopeSlice;
 
-use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
+use crate::chars::{char_is_whitespace, CharCategory, WordChars};
 use crate::graphemes::{next_grapheme_boundary, prev_grapheme_boundary};
 use crate::line_ending::rope_is_line_ending;
 use crate::movement::Direction;
 use crate::syntax;
 use crate::Range;
 use crate::{surround, Syntax};
 
-fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, long: bool) -> usize {
+fn find_word_boundary(
+    slice: RopeSlice,
+    mut pos: usize,
+    direction: Direction,
+    long: bool,
+    word_chars: &WordChars,
+) -> usize {
     use CharCategory::{Eol, Whitespace};
 
     let iter = match direction {
@@ -24,13 +30,13 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, lo
 
     let mut prev_category = match direction {
         Direction::Forward if pos == 0 => Whitespace,
-        Direction::Forward => categorize_char(slice.char(pos - 1)),
+        Direction::Forward => word_chars.categorize(slice.char(pos - 1)),
         Direction::Backward if pos == slice.len_chars() => Whitespace,
-        Direction::Backward => categorize_char(slice.char(pos)),
+        Direction::Backward => word_chars.categorize(slice.char(pos)),
     };
 
     for ch in iter {
-        match categorize_char(ch) {
+        match word_chars.categorize(ch) {
             Eol | Whitespace => return pos,
             category => {
                 if !long && category != prev_category && pos != 0 && pos != slice.len_chars() {
@@ -74,13 +80,14 @@ pub fn textobject_word(
     textobject: TextObject,
     _count: usize,
     long: bool,
+    word_chars: &WordChars,
 ) -> Range {
     let pos = range.cursor(slice);
 
-    let word_start = find_word_boundary(slice, pos, Direction::Backward, long);
-    let word_end = match slice.get_char(pos).map(categorize_char) {
+    let word_start = find_word_boundary(slice, pos, Direction::Backward, long, word_chars);
+    let word_end = match slice.get_char(pos).map(|ch| word_chars.categorize(ch)) {
         None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,
-        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long),
+        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long, word_chars),
     };
 
     // Special case.
@@ -395,6 +402,7 @@ mod test {
                 vec![(19, Inside, (17, 20)), (19, Around, (16, 20))],
             ),
         ];
+        let word_chars = WordChars::default();
 
         for (sample, scenario) in tests {
             let doc = Rope::from(*sample);
@@ -403,7 +411,7 @@ mod test {
                 let (pos, objtype, expected_range) = case;
                 // cursor is a single width selection
                 let range = Range::new(pos, pos + 1);
-                let result = textobject_word(slice, range, objtype, 1, false);
+                let result = textobject_word(slice, range, objtype, 1, false, &word_chars);
                 assert_eq!(
                     result,
                     expected_range.into(),