Skip to content

Commit f4fc30f

Browse files
jmcollinjmcollin
committed
feat(lexer): implement CommonMark tab expansion (§2.1)
Expand tabs to 4-column stops before indent detection so mixed space+tab prefixes correctly trigger indented code blocks (spec ex 1–2). Fix PATTERN_HORIZONTAL_RULE to allow tab-separated thematic break chars (spec ex 11). Preserve internal tabs in code block content via stripLeadingColumns. Fix blockquote inner content to keep trailing spaces so hard line breaks survive re-tokenisation.
1 parent eb7acaf commit f4fc30f

5 files changed

Lines changed: 313 additions & 32 deletions

File tree

‎src/Lexer/Lexer.php‎

Lines changed: 89 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ final class Lexer
1717
private const PATTERN_BLOCKQUOTE = '/^((?:>[ \t]*)++)(.*)/';
1818
private const PATTERN_UNORDERED_LIST = '/^( *)[-*+]\s+(.+)/';
1919
private const PATTERN_ORDERED_LIST = '/^( *)\d+\.\s+(.+)/';
20-
private const PATTERN_HORIZONTAL_RULE = '/^(-{3,}|\*{3,}|_{3,})\s*$/';
20+
private const PATTERN_HORIZONTAL_RULE = '/^[ \t]{0,3}([-*_])([ \t]*\1){2,}[ \t]*$/';
2121
private const PATTERN_LINK_DEFINITION = '/^\[([^\]\[]+)\]:\s+(?:<((?:[^<>\\\\\n]|\\\\.)*)>|(\S+))(?:\s+(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\)))?$/';
2222
/** Matches a standalone title line (CommonMark §4.7 multiline link ref definition). */
2323
private const PATTERN_STANDALONE_TITLE = '/^(?:"((?:[^"\\\\]|\\\\.)*)"|\'((?:[^\'\\\\]|\\\\.)*)\'|\(((?:[^()\\\\]|\\\\.)*)\))\s*$/';
@@ -28,7 +28,6 @@ final class Lexer
2828
private const PATTERN_COLUMNS_OPEN = '/^:::\s*columns\s*$/i';
2929
private const PATTERN_COLUMNS_CLOSE = '/^:::$/';
3030
private const PATTERN_COLUMNS_SEP = '/^\|\|\|$/';
31-
private const PATTERN_INDENTED_CODE = '/^( |\t)(.*)/s';
3231
private const PATTERN_FOOTNOTE_DEF = '/^\[\^([A-Za-z0-9_-]{1,50})\]:\s+(.+)$/';
3332

3433
/**
@@ -105,7 +104,8 @@ public function tokenize(string $markdown): array
105104
$footnoteBodyLines = [];
106105

107106
foreach ($lines as $raw) {
108-
$line = rtrim($raw, "\r");
107+
$line = rtrim($raw, "\r");
108+
$expanded = $this->expandTabs($line);
109109

110110
// Collect lines for an in-progress HTML block.
111111
// The block ends on the first blank line (CommonMark §4.6 type 6/7).
@@ -316,11 +316,11 @@ public function tokenize(string $markdown): array
316316

317317
// Indented code block drain (continuation).
318318
if ($inIndentedBlock) {
319-
if (preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
319+
if (preg_match('/^ (.*)$/s', $expanded, $m)
320320
&& !preg_match(self::PATTERN_UNORDERED_LIST, $line)
321321
&& !preg_match(self::PATTERN_ORDERED_LIST, $line)
322322
) {
323-
$indentedLines = [...$indentedLines, ...$pendingBlanks, $m[2]];
323+
$indentedLines = [...$indentedLines, ...$pendingBlanks, $this->stripLeadingColumns($line, 4)];
324324
$pendingBlanks = [];
325325
continue;
326326
}
@@ -343,12 +343,12 @@ public function tokenize(string $markdown): array
343343
// and only when the line is not a list item — list items with leading spaces
344344
// are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST).
345345
if (!$hadPendingLines
346-
&& preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
346+
&& preg_match('/^ (.*)$/s', $expanded, $m)
347347
&& !preg_match(self::PATTERN_UNORDERED_LIST, $line)
348348
&& !preg_match(self::PATTERN_ORDERED_LIST, $line)
349349
) {
350350
$inIndentedBlock = true;
351-
$indentedLines = [$m[2]];
351+
$indentedLines = [$this->stripLeadingColumns($line, 4)];
352352
$pendingBlanks = [];
353353
continue;
354354
}
@@ -457,6 +457,80 @@ private function extractTaskChecked(string $content): array
457457
return [$content, null];
458458
}
459459

460+
/**
461+
* Expand tab characters to spaces using 4-column tab stops (CommonMark §2.1).
462+
*
463+
* @param int $startCol Column position of the first character of $line (default 0).
464+
*/
465+
private function expandTabs(string $line, int $startCol = 0): string
466+
{
467+
$out = '';
468+
$col = $startCol;
469+
$len = strlen($line);
470+
for ($i = 0; $i < $len; $i++) {
471+
$ch = $line[$i];
472+
if ($ch === "\t") {
473+
$spaces = 4 - ($col % 4);
474+
$out .= str_repeat(' ', $spaces);
475+
$col += $spaces;
476+
} else {
477+
$out .= $ch;
478+
$col++;
479+
}
480+
}
481+
return $out;
482+
}
483+
484+
/**
485+
* Return the suffix of $line after consuming exactly $cols columns,
486+
* prepending any overshoot spaces when a tab spans the boundary.
487+
*/
488+
private function stripLeadingColumns(string $line, int $cols): string
489+
{
490+
$col = 0;
491+
$len = strlen($line);
492+
for ($i = 0; $i < $len; $i++) {
493+
if ($col >= $cols) {
494+
return substr($line, $i);
495+
}
496+
$ch = $line[$i];
497+
if ($ch === "\t") {
498+
$tabStop = 4 - ($col % 4);
499+
$newCol = $col + $tabStop;
500+
if ($newCol > $cols) {
501+
$surplus = $newCol - $cols;
502+
return str_repeat(' ', $surplus) . substr($line, $i + 1);
503+
}
504+
$col = $newCol;
505+
} else {
506+
$col++;
507+
}
508+
}
509+
return '';
510+
}
511+
512+
/**
513+
* Strip blockquote markers from an already-expanded line.
514+
*
515+
* Implements CommonMark §5.1: each `>` consumes one mandatory character plus one
516+
* optional space. Operates on the expanded form so tab overshoot is already resolved.
517+
*/
518+
private function stripBlockquoteMarkers(string $expandedLine, int $level): string
519+
{
520+
$pos = 0;
521+
$len = strlen($expandedLine);
522+
for ($l = 0; $l < $level; $l++) {
523+
if ($pos < $len && $expandedLine[$pos] === '>') {
524+
$pos++;
525+
}
526+
// Consume at most one optional space following the marker.
527+
if ($pos < $len && $expandedLine[$pos] === ' ') {
528+
$pos++;
529+
}
530+
}
531+
return substr($expandedLine, $pos);
532+
}
533+
460534
private function matchLine(string $line): Token
461535
{
462536
if ($line === '' || ctype_space($line)) {
@@ -477,10 +551,16 @@ private function matchLine(string $line): Token
477551
}
478552

479553
if (preg_match(self::PATTERN_BLOCKQUOTE, $line, $m)) {
554+
// Compute inner content from the expanded line using the CommonMark §5.1 rule:
555+
// each '>' marker consumes the '>' character and optionally one space.
556+
// Operating on the expanded form ensures tabs in the prefix are correctly handled.
557+
$level = substr_count($m[1], '>');
558+
$expandedLine = $this->expandTabs($line);
559+
$innerContent = $this->stripBlockquoteMarkers($expandedLine, $level);
480560
return new Token(
481561
TokenType::BLOCKQUOTE,
482-
trim($m[2]),
483-
['level' => substr_count($m[1], '>')],
562+
$innerContent,
563+
['level' => $level],
484564
);
485565
}
486566

‎src/Parser/Parser.php‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -492,17 +492,14 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
492492
if ($level < $minLevel) {
493493
// Level decrease: flush buffer and yield cursor to caller.
494494
if ($buffer !== []) {
495-
$children[] = new ParagraphNode(
496-
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
497-
);
495+
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
496+
$buffer = [];
498497
}
499498
return new BlockquoteNode(children: $children);
500499
}
501500

502501
if ($level === $minLevel) {
503-
if ($tokens[$i]->content !== '') {
504-
$buffer[] = $tokens[$i]->content;
505-
}
502+
$buffer[] = $tokens[$i]->content;
506503
$i++;
507504

508505
// Flush buffer when the next token changes level or ends the blockquote run.
@@ -511,19 +508,15 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
511508
&& $tokens[$i]->meta['level'] === $minLevel;
512509

513510
if (!$nextIsCurrentLevel && $buffer !== []) {
514-
$children[] = new ParagraphNode(
515-
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
516-
);
511+
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
517512
$buffer = [];
518513
}
519514
continue;
520515
}
521516

522517
// $level > $minLevel: flush buffer then recurse.
523518
if ($buffer !== []) {
524-
$children[] = new ParagraphNode(
525-
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
526-
);
519+
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
527520
$buffer = [];
528521
}
529522

@@ -534,20 +527,35 @@ private function buildBlockquote(array $tokens, int &$i, int $minLevel = 1): Blo
534527
// are intentionally rendered as content inside the level-32 blockquote rather
535528
// than triggering unbounded recursion. The extra ">" markers are consumed and
536529
// discarded; only the text content is preserved.
537-
if ($tokens[$i]->content !== '') {
538-
$buffer[] = $tokens[$i]->content;
539-
}
530+
$buffer[] = $tokens[$i]->content;
540531
$i++;
541532
}
542533
}
543534

544535
// Flush any remaining buffer at end of token stream.
545536
if ($buffer !== []) {
546-
$children[] = new ParagraphNode(
547-
children: $this->inlineParser->parse(implode(' ', $buffer), $this->linkRefs, $this->footnoteDefs),
548-
);
537+
array_push($children, ...$this->parseBlockquoteBuffer($buffer));
549538
}
550539

551540
return new BlockquoteNode(children: $children);
552541
}
542+
543+
/**
544+
* Re-tokenize a collected blockquote content buffer and parse it into block nodes.
545+
*
546+
* Each entry in $buffer is the raw content string of one BLOCKQUOTE token (one line).
547+
* Empty strings (blank lines) become blank-line separators between blocks.
548+
* The content is joined with newlines and passed through the Lexer so that
549+
* indented code blocks, headings, etc. within a blockquote are correctly detected.
550+
*
551+
* @param string[] $buffer
552+
* @return array<int, \PhpMarkdown\Node\BlockNodeInterface>
553+
*/
554+
private function parseBlockquoteBuffer(array $buffer): array
555+
{
556+
$lexer = new Lexer();
557+
$raw = implode("\n", $buffer);
558+
$innerTokens = $lexer->tokenize($raw);
559+
return $this->parseBlocks($innerTokens);
560+
}
553561
}

‎tests/Integration/MarkdownParserTest.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -373,13 +373,14 @@ public function testDepthGuardAbove32ContentPreserved(): void
373373
$this->assertStringContainsString('text', $html);
374374
}
375375

376-
public function testMultiLineSameLevelJoinedWithSpace(): void
376+
public function testMultiLineSameLevelJoinedWithSoftBreak(): void
377377
{
378-
// Two consecutive level-1 tokens accumulate into one ParagraphNode joined by space.
378+
// Two consecutive level-1 tokens re-tokenize as a single paragraph with a soft line break.
379+
// CommonMark §2.3: a soft line break is a newline that is not a hard line break.
379380
$md = "> line one\n> line two";
380381
$html = $this->parser->parse($md);
381382

382-
$this->assertSame("<blockquote>\n<p>line one line two</p>\n</blockquote>\n", $html);
383+
$this->assertSame("<blockquote>\n<p>line one\nline two</p>\n</blockquote>\n", $html);
383384
}
384385

385386
public function testLevelSkipOneToThree(): void

‎tests/Unit/LexerTest.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,15 @@ public function testBlockquoteNoSpaceAfterInnerMarker(): void
163163
$this->assertSame('text', $tokens[0]->content);
164164
}
165165

166-
public function testBlockquoteTrailingSpacesInContentAreTrimmed(): void
166+
public function testBlockquoteLeadingSurplusSpacesKeptTrailingSpacesPreserved(): void
167167
{
168+
// CommonMark §5.1: '>' strips one optional space; extra leading spaces are content.
169+
// The two extra spaces in '> ' remain as leading content spaces.
170+
// Trailing spaces are preserved — the inner re-tokenisation detects hard line breaks.
168171
$tokens = $this->lexer->tokenize('> lots of spaces ');
169172

170173
$this->assertSame(TokenType::BLOCKQUOTE, $tokens[0]->type);
171-
$this->assertSame('lots of spaces', $tokens[0]->content);
174+
$this->assertSame(' lots of spaces ', $tokens[0]->content);
172175
}
173176

174177
public function testHorizontalRuleVariants(): void

0 commit comments

Comments
 (0)