Skip to content

Commit 2c6d4dd

Browse files
claudesinelaw
authored andcommitted
fix(render): preserve past-EOF tokenisation in segmented build_base_tokens
Phase 5b's segmentation introduced two early-exit paths that broke tokenisation when `top_byte` lands at or past `buffer.len()`: - A `cursor >= buffer_len` outer-loop break that returned an empty token vector. The original code instead let `LineIterator::new` clamp `top_byte` internally and use a backward scan to locate the line containing the (clamped) position, so a viewport scrolled past EOF on a single very long wrapped line still produced tokens for the final source line. - An inner-loop `line_start >= segment_end` break that fired even when `segment_end` was just `buffer_len` (no fold ahead). For an EOF-empty trailing line (e.g. buffer ends in `\n`), `LineIterator` emits a `(buffer_len, "")` line that was being dropped. Both checks now only trigger when there's an actual upcoming fold. With an empty `fold_skip`, `LineIterator` runs to natural exhaustion — matching the pre-Phase-5b behaviour exactly. Reproduced and verified by `test_mouse_wheel_scrolls_wrapped_content` (60-col viewport, single 1600-char line, 20 mouse-wheel scrolls): viewport now correctly shows later wrapped sections after scroll. https://claude.ai/code/session_014WZ8ca3tvhrQ9NcVHrV53d
1 parent e1492ac commit 2c6d4dd

1 file changed

Lines changed: 24 additions & 15 deletions

File tree

crates/fresh-editor/src/view/ui/split_rendering/base_tokens.rs

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ pub(super) fn build_base_tokens(
3131
let max_lines = visible_count.saturating_add(4);
3232
let mut lines_seen = 0usize;
3333
let buffer_len = buffer.len();
34-
let mut cursor = top_byte.min(buffer_len);
34+
// Don't clamp `cursor` to buffer_len: `LineIterator::new` clamps
35+
// internally and uses a backward scan to locate the line containing
36+
// `top_byte`, so a `top_byte >= buffer_len` (post-scroll past EOF on a
37+
// single very long line) still produces tokens for that final line.
38+
let mut cursor = top_byte;
3539
let mut fold_idx = 0usize;
3640
// Fast-forward past folds already ending at/before the cursor.
3741
while fold_idx < fold_skip.len() && fold_skip[fold_idx].end <= cursor {
@@ -40,7 +44,7 @@ pub(super) fn build_base_tokens(
4044
// If the cursor landed inside a fold, jump past it before reading anything.
4145
if let Some(r) = fold_skip.get(fold_idx) {
4246
if r.start <= cursor && cursor < r.end {
43-
cursor = r.end.min(buffer_len);
47+
cursor = r.end;
4448
fold_idx += 1;
4549
}
4650
}
@@ -49,30 +53,35 @@ pub(super) fn build_base_tokens(
4953
// `LineIterator` is constructed per segment so source bytes covered by
5054
// a collapsed fold are never read, never decoded, and never tokenised.
5155
'segments: loop {
52-
if lines_seen >= max_lines || cursor >= buffer_len {
56+
if lines_seen >= max_lines {
5357
break;
5458
}
55-
let segment_end = fold_skip
56-
.get(fold_idx)
57-
.map(|r| r.start)
58-
.unwrap_or(buffer_len);
59-
// Zero-length segment (adjacent folds, or fold starting exactly at
60-
// cursor): skip the fold and continue.
61-
if cursor >= segment_end {
62-
if let Some(r) = fold_skip.get(fold_idx) {
63-
cursor = r.end.min(buffer_len);
59+
let next_fold_start = fold_skip.get(fold_idx).map(|r| r.start);
60+
let segment_end = next_fold_start.unwrap_or(buffer_len);
61+
// Zero-length segment between adjacent folds (or fold starting
62+
// exactly at cursor): jump past the fold and try again. Only fires
63+
// when there's actually a fold ahead — without one, segment_end
64+
// is `buffer_len`, but `cursor >= buffer_len` is fine: `LineIterator`
65+
// handles the past-EOF case via internal clamping.
66+
if next_fold_start.is_some() {
67+
if cursor >= segment_end {
68+
let r = &fold_skip[fold_idx];
69+
cursor = r.end;
6470
fold_idx += 1;
6571
continue;
6672
}
67-
break;
6873
}
6974

7075
let mut iter = buffer.line_iterator(cursor, estimated_line_length);
7176
while lines_seen < max_lines {
7277
let Some((line_start, line_content)) = iter.next_line() else {
7378
break 'segments;
7479
};
75-
if line_start >= segment_end {
80+
// Stop the inner loop when the next line crosses into the
81+
// upcoming fold. Without a fold ahead, `next_fold_start` is
82+
// `None` and we keep tokenising until the iterator reports EOF
83+
// — preserving the trailing-empty-line behaviour at buffer end.
84+
if next_fold_start.is_some_and(|s| line_start >= s) {
7685
break;
7786
}
7887
let mut byte_offset = 0usize;
@@ -183,7 +192,7 @@ pub(super) fn build_base_tokens(
183192
// Jump past the fold at fold_idx (which drove segment_end). If we
184193
// ran out of folds, we've finished the last segment.
185194
if let Some(r) = fold_skip.get(fold_idx) {
186-
cursor = r.end.min(buffer_len);
195+
cursor = r.end;
187196
fold_idx += 1;
188197
} else {
189198
break;

0 commit comments

Comments
 (0)