Skip to content

Commit b2e720a

Browse files
jmcollinjmcollin
authored
Merge pull request #35 from jmcollin/feat/setext-heading-multiline
feat(lexer): support multiline setext heading content
2 parents 4a74271 + ab946ce commit b2e720a

4 files changed

Lines changed: 205 additions & 35 deletions

File tree

‎src/Lexer/Lexer.php‎

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public function tokenize(string $markdown): array
8484
$fencedLines = [];
8585
$fenceChar = '';
8686
$fenceLength = 0;
87-
$pendingToken = null;
87+
$pendingLines = [];
8888
$pendingLinkDef = null; // LINK_DEFINITION token awaiting a possible next-line title
8989

9090
$inHtmlBlock = false;
@@ -149,10 +149,10 @@ public function tokenize(string $markdown): array
149149

150150
if ($inColumnsBlock) {
151151
if (preg_match(self::PATTERN_COLUMNS_CLOSE, $line)) {
152-
if ($pendingToken !== null) {
153-
$tokens[] = $pendingToken;
154-
$pendingToken = null;
152+
foreach ($pendingLines as $pt) {
153+
$tokens[] = $pt;
155154
}
155+
$pendingLines = [];
156156
$tokens[] = new Token(
157157
TokenType::COLUMNS_CONTAINER,
158158
'',
@@ -176,10 +176,10 @@ public function tokenize(string $markdown): array
176176
}
177177

178178
if (preg_match(self::PATTERN_COLUMNS_OPEN, $line)) {
179-
if ($pendingToken !== null) {
180-
$tokens[] = $pendingToken;
181-
$pendingToken = null;
179+
foreach ($pendingLines as $pt) {
180+
$tokens[] = $pt;
182181
}
182+
$pendingLines = [];
183183
$inColumnsBlock = true;
184184
$columnsSepFound = false;
185185
$columnsLeftLines = [];
@@ -189,10 +189,10 @@ public function tokenize(string $markdown): array
189189

190190
if ($inFencedBlock) {
191191
if (preg_match('/^ {0,3}' . preg_quote($fenceChar, '/') . '{' . $fenceLength . ',}\s*$/', $line)) {
192-
if ($pendingToken !== null) {
193-
$tokens[] = $pendingToken;
194-
$pendingToken = null;
192+
foreach ($pendingLines as $pt) {
193+
$tokens[] = $pt;
195194
}
195+
$pendingLines = [];
196196
$tokens[] = new Token(
197197
TokenType::FENCED_CODE,
198198
implode("\n", $fencedLines),
@@ -210,10 +210,10 @@ public function tokenize(string $markdown): array
210210
}
211211

212212
if (preg_match(self::PATTERN_FENCED_OPEN, $line, $m)) {
213-
if ($pendingToken !== null) {
214-
$tokens[] = $pendingToken;
215-
$pendingToken = null;
213+
foreach ($pendingLines as $pt) {
214+
$tokens[] = $pt;
216215
}
216+
$pendingLines = [];
217217
$inFencedBlock = true;
218218
$fencedLanguage = $m[2];
219219
$fencedLines = [];
@@ -226,10 +226,10 @@ public function tokenize(string $markdown): array
226226
// Must run before setext/paragraph logic so that block-level HTML tags
227227
// are not consumed as paragraphs.
228228
if (preg_match($this->patternHtmlBlockStart, $line)) {
229-
if ($pendingToken !== null) {
230-
$tokens[] = $pendingToken;
231-
$pendingToken = null;
229+
foreach ($pendingLines as $pt) {
230+
$tokens[] = $pt;
232231
}
232+
$pendingLines = [];
233233
// Detect whether this is an HTML comment opener.
234234
$isCommentStart = str_starts_with(ltrim($line), '<!--');
235235
$isCommentClosed = $isCommentStart && str_contains($line, '-->');
@@ -269,22 +269,22 @@ public function tokenize(string $markdown): array
269269
}
270270

271271
// Capture paragraph-active state before setext promotion may clear it.
272-
$hadPendingToken = $pendingToken !== null;
272+
$hadPendingLines = $pendingLines !== [];
273273

274-
// Setext heading detection: a pending text line followed by === or ---
275-
if ($pendingToken !== null) {
274+
// Setext heading detection: pending text lines followed by === or ---
275+
if ($pendingLines !== []) {
276276
if (preg_match(self::PATTERN_SETEXT_H1, $line)) {
277-
$tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 1]);
278-
$pendingToken = null;
277+
$content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines));
278+
$tokens[] = new Token(TokenType::HEADING, $content, ['level' => 1]);
279+
$pendingLines = [];
279280
continue;
280281
}
281282
if (preg_match(self::PATTERN_SETEXT_H2, $line)) {
282-
$tokens[] = new Token(TokenType::HEADING, $pendingToken->content, ['level' => 2]);
283-
$pendingToken = null;
283+
$content = implode(' ', array_map(fn(Token $t) => $t->content, $pendingLines));
284+
$tokens[] = new Token(TokenType::HEADING, $content, ['level' => 2]);
285+
$pendingLines = [];
284286
continue;
285287
}
286-
$tokens[] = $pendingToken;
287-
$pendingToken = null;
288288
}
289289

290290
// Multiline link ref title (CommonMark §4.7): a LINK_DEFINITION with no title
@@ -342,7 +342,7 @@ public function tokenize(string $markdown): array
342342
// Start new indented code block (only when no paragraph was active,
343343
// and only when the line is not a list item — list items with leading spaces
344344
// are handled by matchLine() via PATTERN_UNORDERED_LIST / PATTERN_ORDERED_LIST).
345-
if (!$hadPendingToken
345+
if (!$hadPendingLines
346346
&& preg_match(self::PATTERN_INDENTED_CODE, $line, $m)
347347
&& !preg_match(self::PATTERN_UNORDERED_LIST, $line)
348348
&& !preg_match(self::PATTERN_ORDERED_LIST, $line)
@@ -364,11 +364,10 @@ public function tokenize(string $markdown): array
364364
// A FOOTNOTE_DEFINITION token opens multi-line body accumulation.
365365
// Flush any pending setext heading candidate and pending link def first.
366366
if ($token->type === TokenType::FOOTNOTE_DEFINITION) {
367-
/** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */
368-
if ($pendingToken !== null) {
369-
$tokens[] = $pendingToken;
370-
$pendingToken = null;
367+
foreach ($pendingLines as $pt) {
368+
$tokens[] = $pt;
371369
}
370+
$pendingLines = [];
372371
/** @psalm-suppress TypeDoesNotContainType @phpstan-ignore notIdentical.alwaysFalse */
373372
if ($pendingLinkDef !== null) {
374373
$tokens[] = $pendingLinkDef;
@@ -382,10 +381,14 @@ public function tokenize(string $markdown): array
382381

383382
// A plain paragraph line is held as pending to allow setext promotion on the next line.
384383
if ($token->type === TokenType::PARAGRAPH && ($line !== '' && !ctype_space($line))) {
385-
$pendingToken = $token;
384+
$pendingLines[] = $token;
386385
continue;
387386
}
388387

388+
foreach ($pendingLines as $pt) {
389+
$tokens[] = $pt;
390+
}
391+
$pendingLines = [];
389392
$tokens[] = $token;
390393
}
391394

@@ -394,9 +397,9 @@ public function tokenize(string $markdown): array
394397
$tokens[] = $pendingLinkDef;
395398
}
396399

397-
// Flush any remaining pending token
398-
if ($pendingToken !== null) {
399-
$tokens[] = $pendingToken;
400+
// Flush any remaining pending lines
401+
foreach ($pendingLines as $pt) {
402+
$tokens[] = $pt;
400403
}
401404

402405
// Flush any in-progress footnote definition body

‎tests/Integration/fixtures/setext-headings.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
<h1>My Title</h1><h2>Sub Title</h2><h1><strong>bold</strong> title</h1><h2>text</h2><h1>Title</h1>
1+
<h1>My Title</h1><h2>Sub Title</h2><h1><strong>bold</strong> title</h1><h2>text</h2><h1>Title</h1><h1>foo bar</h1><h2>line one line two line three</h2><h1><em>foo bar</em></h1>

‎tests/Integration/fixtures/setext-headings.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,13 @@ text
88
---
99
Title
1010
=
11+
foo
12+
bar
13+
===
14+
line one
15+
line two
16+
line three
17+
---
18+
*foo
19+
bar*
20+
===

‎tests/Unit/SetextHeadingMultilineTest.php‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace PhpMarkdown\Tests\Unit;
6+
7+
use PhpMarkdown\Lexer\Lexer;
8+
use PhpMarkdown\Lexer\TokenType;
9+
use PhpMarkdown\Parser\Parser;
10+
use PhpMarkdown\Renderer\HtmlRenderer;
11+
use PHPUnit\Framework\TestCase;
12+
13+
/**
14+
* Tests for setext headings with multiple preceding paragraph lines.
15+
*/
16+
final class SetextHeadingMultilineTest extends TestCase
17+
{
18+
// -------------------------------------------------------------------------
19+
// Token-level tests
20+
// -------------------------------------------------------------------------
21+
22+
public function testTwoLineH1Token(): void
23+
{
24+
$lexer = new Lexer();
25+
$tokens = $lexer->tokenize("foo\nbar\n===");
26+
27+
$headingTokens = array_values(array_filter(
28+
$tokens,
29+
static fn($t) => $t->type === TokenType::HEADING,
30+
));
31+
32+
self::assertCount(1, $headingTokens);
33+
self::assertSame(1, $headingTokens[0]->meta['level']);
34+
self::assertSame('foo bar', $headingTokens[0]->content);
35+
}
36+
37+
public function testThreeLineH2Token(): void
38+
{
39+
$lexer = new Lexer();
40+
$tokens = $lexer->tokenize("line one\nline two\nline three\n---");
41+
42+
$headingTokens = array_values(array_filter(
43+
$tokens,
44+
static fn($t) => $t->type === TokenType::HEADING,
45+
));
46+
47+
self::assertCount(1, $headingTokens);
48+
self::assertSame(2, $headingTokens[0]->meta['level']);
49+
self::assertSame('line one line two line three', $headingTokens[0]->content);
50+
}
51+
52+
public function testSingleLineRegressionToken(): void
53+
{
54+
$lexer = new Lexer();
55+
$tokens = $lexer->tokenize("foo\n===");
56+
57+
$headingTokens = array_values(array_filter(
58+
$tokens,
59+
static fn($t) => $t->type === TokenType::HEADING,
60+
));
61+
62+
self::assertCount(1, $headingTokens);
63+
self::assertSame(1, $headingTokens[0]->meta['level']);
64+
self::assertSame('foo', $headingTokens[0]->content);
65+
}
66+
67+
public function testBlankBreaksAccumulationTokens(): void
68+
{
69+
$lexer = new Lexer();
70+
$tokens = $lexer->tokenize("foo\n\nbar\n===");
71+
72+
$types = array_map(static fn($t) => $t->type, $tokens);
73+
74+
self::assertContains(TokenType::PARAGRAPH, $types);
75+
self::assertContains(TokenType::BLANK, $types);
76+
self::assertContains(TokenType::HEADING, $types);
77+
78+
$paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH));
79+
$headings = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING));
80+
81+
self::assertCount(1, $paragraphs);
82+
self::assertSame('foo', $paragraphs[0]->content);
83+
84+
self::assertCount(1, $headings);
85+
self::assertSame(1, $headings[0]->meta['level']);
86+
self::assertSame('bar', $headings[0]->content);
87+
}
88+
89+
public function testInlineMarkupSpansLinesToken(): void
90+
{
91+
$lexer = new Lexer();
92+
$tokens = $lexer->tokenize("*foo\nbar*\n===");
93+
94+
$headingTokens = array_values(array_filter(
95+
$tokens,
96+
static fn($t) => $t->type === TokenType::HEADING,
97+
));
98+
99+
self::assertCount(1, $headingTokens);
100+
self::assertSame(1, $headingTokens[0]->meta['level']);
101+
self::assertSame('*foo bar*', $headingTokens[0]->content);
102+
}
103+
104+
public function testAtxHeadingInterruptsPendingLines(): void
105+
{
106+
$lexer = new Lexer();
107+
$tokens = $lexer->tokenize("foo\n## bar");
108+
109+
$paragraphs = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::PARAGRAPH));
110+
$headings = array_values(array_filter($tokens, static fn($t) => $t->type === TokenType::HEADING));
111+
112+
self::assertCount(1, $paragraphs);
113+
self::assertSame('foo', $paragraphs[0]->content);
114+
115+
self::assertCount(1, $headings);
116+
self::assertSame(2, $headings[0]->meta['level']);
117+
self::assertSame('bar', $headings[0]->content);
118+
}
119+
120+
// -------------------------------------------------------------------------
121+
// Render-level tests
122+
// -------------------------------------------------------------------------
123+
124+
private function render(string $markdown): string
125+
{
126+
$lexer = new Lexer();
127+
$parser = new Parser();
128+
$renderer = new HtmlRenderer();
129+
130+
return trim($renderer->render($parser->parse($lexer->tokenize($markdown))));
131+
}
132+
133+
public function testTwoLineH1Renders(): void
134+
{
135+
self::assertSame('<h1>foo bar</h1>', $this->render("foo\nbar\n==="));
136+
}
137+
138+
public function testThreeLineH2Renders(): void
139+
{
140+
self::assertSame('<h2>line one line two line three</h2>', $this->render("line one\nline two\nline three\n---"));
141+
}
142+
143+
public function testSingleLineRegressionRenders(): void
144+
{
145+
self::assertSame('<h1>foo</h1>', $this->render("foo\n==="));
146+
}
147+
148+
public function testBlankBreaksAccumulationRenders(): void
149+
{
150+
self::assertSame('<p>foo</p><h1>bar</h1>', $this->render("foo\n\nbar\n==="));
151+
}
152+
153+
public function testInlineMarkupSpansLinesRenders(): void
154+
{
155+
self::assertSame('<h1><em>foo bar</em></h1>', $this->render("*foo\nbar*\n==="));
156+
}
157+
}

0 commit comments

Comments
 (0)