Skip to content

Commit c3a00bb

Browse files
Peter Stengerclaude
andcommitted
Fix embedded tokenizer: extract content from text/html_entity children
The tree-sitter parser produces separate `text` and `html_entity` children (not a single `html_text` node) for entity-encoded content inside regular HTML elements. Fix content extraction to slice the full source text between start and end tags instead of looking for specific child node types. Bump LSP extension version to 0.0.26. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent dcd0be8 commit c3a00bb

3 files changed

Lines changed: 89 additions & 7 deletions

File tree

lsp/package.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

lsp/server/src/server.ts

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -342,23 +342,38 @@ function findCustomCodeTagContent(
342342
if (config) {
343343
const languageId = resolveCustomCodeLanguage(node, config);
344344
if (languageId) {
345-
// For html_raw_element, get html_raw_text child
346-
// For html_element, collect text content between start and end tags
345+
// Extract content between start and end tags.
346+
// We use the node's full text and slice between the start tag end
347+
// and end tag start, because tree-sitter produces separate `text`
348+
// and `html_entity` children with gaps between them (whitespace).
349+
let startTag: SyntaxNode | null = null;
350+
let endTag: SyntaxNode | null = null;
347351
for (let i = 0; i < node.childCount; i++) {
348352
const child = node.child(i);
353+
if (child?.type === 'html_start_tag') startTag = child;
354+
if (child?.type === 'html_end_tag') endTag = child;
355+
// For raw elements, grab raw text directly
349356
if (child?.type === 'html_raw_text') {
350357
results.push({
351358
text: child.text,
352359
languageId,
353360
startRow: child.startPosition.row,
354361
startCol: child.startPosition.column,
355362
});
356-
} else if (child?.type === 'html_text') {
363+
}
364+
}
365+
366+
// For regular html_element, extract text between start and end tags
367+
if (startTag && node.type === 'html_element') {
368+
const contentStartIndex = startTag.endIndex;
369+
const contentEndIndex = endTag ? endTag.startIndex : node.endIndex;
370+
const contentText = node.tree.rootNode.text.slice(contentStartIndex, contentEndIndex);
371+
if (contentText.length > 0) {
357372
results.push({
358-
text: child.text,
373+
text: contentText,
359374
languageId,
360-
startRow: child.startPosition.row,
361-
startCol: child.startPosition.column,
375+
startRow: startTag.endPosition.row,
376+
startCol: startTag.endPosition.column,
362377
});
363378
}
364379
}

lsp/server/test/embeddedTokenizer.test.ts

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,3 +530,70 @@ describe('tokenizeEmbeddedContent (integration)', () => {
530530
expect(tokens.length).toBe(0);
531531
});
532532
});
533+
534+
describe('tree-sitter content extraction', () => {
535+
// These tests verify that entity-encoded content inside custom code tags
536+
// is correctly extracted from the tree-sitter parse tree, where the parser
537+
// produces separate `text` and `html_entity` children (not a single text node).
538+
539+
it('parses <pl-code> content as text and html_entity children', () => {
540+
const tree = parseText('<pl-code language="c">x &lt; 2</pl-code>');
541+
const root = tree.rootNode;
542+
543+
// The pl-code element should be an html_element
544+
const element = root.child(0);
545+
expect(element?.type).toBe('html_element');
546+
547+
// Collect child types between start/end tags
548+
const childTypes: string[] = [];
549+
for (let i = 0; i < element!.childCount; i++) {
550+
childTypes.push(element!.child(i)!.type);
551+
}
552+
553+
// Should have start_tag, content nodes (text/html_entity), and end_tag
554+
expect(childTypes).toContain('html_start_tag');
555+
expect(childTypes).toContain('html_end_tag');
556+
// Content is split into text and html_entity nodes
557+
expect(childTypes).toContain('text');
558+
expect(childTypes).toContain('html_entity');
559+
});
560+
561+
it('text between start/end tags includes entities as literal text', () => {
562+
const tree = parseText('<pl-code language="c">x &lt; 2</pl-code>');
563+
const element = tree.rootNode.child(0)!;
564+
565+
// Extract text between start and end tags using character indices
566+
let startTag: { endIndex: number } | null = null;
567+
let endTag: { startIndex: number } | null = null;
568+
for (let i = 0; i < element.childCount; i++) {
569+
const child = element.child(i)!;
570+
if (child.type === 'html_start_tag') startTag = child;
571+
if (child.type === 'html_end_tag') endTag = child;
572+
}
573+
574+
const fullText = tree.rootNode.text;
575+
const content = fullText.slice(startTag!.endIndex, endTag!.startIndex);
576+
expect(content).toBe('x &lt; 2');
577+
});
578+
579+
it('multi-line entity-encoded content is fully captured', () => {
580+
const input = `<pl-code language="c">
581+
tax = price * 100;
582+
cout &lt;&lt; fixed &lt;&lt; endl;
583+
</pl-code>`;
584+
const tree = parseText(input);
585+
const element = tree.rootNode.child(0)!;
586+
587+
let startTag: { endIndex: number } | null = null;
588+
let endTag: { startIndex: number } | null = null;
589+
for (let i = 0; i < element.childCount; i++) {
590+
const child = element.child(i)!;
591+
if (child.type === 'html_start_tag') startTag = child;
592+
if (child.type === 'html_end_tag') endTag = child;
593+
}
594+
595+
const fullText = tree.rootNode.text;
596+
const content = fullText.slice(startTag!.endIndex, endTag!.startIndex);
597+
expect(content).toBe('\ntax = price * 100;\ncout &lt;&lt; fixed &lt;&lt; endl;\n');
598+
});
599+
});

0 commit comments

Comments
 (0)