Skip to content

Commit 23d040e

Browse files
varmeshvarmesh
committed
fix(tools): track fences inside list containers
Related to #2144 Signed-off-by: Varun Ramesh <varamesh@nvidia.com>
1 parent e791007 commit 23d040e

2 files changed

Lines changed: 646 additions & 69 deletions

File tree

‎tools/check-docs-mdx‎

Lines changed: 190 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,10 @@
3333
#
3434
# Code handling (applies to checks 2-6; check 1 greps the fence-stripped line
3535
# WITHOUT inline-span stripping): fenced code blocks in BOTH backtick
36-
# (```) and tilde (~~~) styles are skipped, following CommonMark fence rules —
37-
# a fence opens on a run of >= 3 of either character, indented by up to three
38-
# spaces, and closes only on a later line whose leading run is the SAME
36+
# (```) and tilde (~~~) styles are skipped, following the pinned MDX parser's
37+
# fence behavior. Unlike CommonMark, MDX has no indented code blocks, so a
38+
# fence may have any leading spaces or tabs. It opens on a run of >= 3 of either
39+
# character and closes only on a later line whose leading run is the SAME
3940
# character, at least as long, and followed by nothing but whitespace (so an
4041
# info string such as ```yaml opens a block and never closes one). Inline
4142
# code spans are stripped before scanning; a span opens on a run of N backticks
@@ -119,6 +120,190 @@ frontmatter_end() {
119120
echo "${end:-0}"
120121
}
121122

123+
# Shared awk prologue: frontmatter skip + the fence tracker. Check 1 and
124+
# checks 2-6 run separate awk programs but must agree exactly on what counts
125+
# as code, so the tracker is defined once here and concatenated into both.
126+
# Keeping two hand-maintained copies in sync is what let earlier fixes land
127+
# in one tracker and not the other.
128+
readonly FENCE_AWK='
129+
function indent_width(s, i, ch, width) {
130+
width = 0
131+
for (i = 1; i <= length(s); i++) {
132+
ch = substr(s, i, 1)
133+
if (ch == " ") width++
134+
else if (ch == "\t") width += 4 - (width % 4)
135+
else break
136+
}
137+
return width
138+
}
139+
function column_width(s, i, ch, width) {
140+
width = 0
141+
for (i = 1; i <= length(s); i++) {
142+
ch = substr(s, i, 1)
143+
if (ch == "\t") width += 4 - (width % 4)
144+
else width++
145+
}
146+
return width
147+
}
148+
function thematic_break(s, text) {
149+
text = s
150+
sub(/\r$/, "", text)
151+
return text ~ /^[ \t]*(-[ \t]*){3,}$/ ||
152+
text ~ /^[ \t]*(\*[ \t]*){3,}$/ ||
153+
text ~ /^[ \t]*(_[ \t]*){3,}$/
154+
}
155+
function setext_underline(s, text) {
156+
text = s
157+
sub(/^[ \t]+/, "", text); sub(/\r$/, "", text); sub(/[ \t]+$/, "", text)
158+
return text ~ /^(=+|-+)$/
159+
}
160+
function table_delimiter(s, text, outer) {
161+
text = s
162+
sub(/^[ \t]+/, "", text); sub(/\r$/, "", text); sub(/[ \t]+$/, "", text)
163+
outer = (text ~ /^\|.*\|$/)
164+
gsub(/[ \t]/, "", text)
165+
if (text !~ /\|/ && !outer) return 0
166+
sub(/^\|/, "", text); sub(/\|$/, "", text)
167+
return text ~ /^:?-+:?(\|:?-+:?)*$/
168+
}
169+
function lazy_text(s, text, fence_rest) {
170+
text = s
171+
sub(/^[ \t]+/, "", text)
172+
sub(/\r$/, "", text)
173+
if (text !~ /[^ \t]/) return 0
174+
if (thematic_break(text)) return 0
175+
if (text ~ /^(>|#{1,6}([ \t]|$)|~{3,})/) return 0
176+
if (match(text, /^`{3,}/)) {
177+
fence_rest = substr(text, RSTART + RLENGTH)
178+
if (index(fence_rest, "`") == 0) return 0
179+
}
180+
if (text ~ /^\{/ || text ~ /^<[^ \t]/) return 0
181+
if (text ~ /^([-+*]|[0-9]{1,9}[.)])[ \t]+/) return 0
182+
return 1
183+
}
184+
NR <= fm_end { next }
185+
# Fence rules shared by both checker passes. Getting any of these wrong
186+
# leaves the tracker open and silently skips every later hazard.
187+
# indent: any leading spaces or tabs, matching the MDX parser
188+
# open: a backtick in the info string makes the line prose, not a
189+
# fence (ambiguous with an inline code span); tildes are exempt
190+
# close: same char, at least as long, nothing after but whitespace, so
191+
# ```yaml only ever opens; the \r admits a CRLF closer
192+
# scope: a fence opened inside a list item ends implicitly when a
193+
# nonblank line leaves that item. list_content/fscope carry
194+
# container indents; list_lazy/top_lazy preserve CommonMark
195+
# paragraph continuation lines, and thematic breaks never
196+
# create lists.
197+
{
198+
blank = ($0 ~ /^[ \t]*\r?$/)
199+
ind = indent_width($0)
200+
fence_in_marker = 0
201+
is_fence = match($0, /^[ \t]*(`{3,}|~{3,})/)
202+
if (is_fence) {
203+
m = substr($0, RSTART, RLENGTH); sub(/^[ \t]*/, "", m)
204+
ch = substr(m, 1, 1); len = length(m)
205+
rest = substr($0, RSTART + RLENGTH)
206+
}
207+
208+
# A nonblank outdent ends both the list item and any unclosed fence it
209+
# contains. Re-process the boundary line below: it may be prose, a new
210+
# list item, or the opener for a following top-level fence.
211+
if (in_fence && fscope > 0 && !blank && ind < fscope) {
212+
in_fence = 0; fscope = 0
213+
}
214+
215+
if (in_fence) {
216+
if (is_fence && ch == fch && len >= flen && rest ~ /^[ \t]*\r?$/) { in_fence = 0; next }
217+
next
218+
}
219+
220+
# Maintain just enough list-container state to distinguish an
221+
# outdented list-fence boundary from a valid less-indented top-level
222+
# closer. CommonMark lazy paragraph lines keep the item open. A blank
223+
# ends laziness and also ends an item whose marker had no content.
224+
table_continuation = top_table && $0 ~ /\|/
225+
if (top_table && !table_continuation) top_table = 0
226+
setext_line = top_lazy && setext_underline($0)
227+
rejected_marker = 0
228+
list_candidate = !setext_line && !table_continuation && !thematic_break($0) && match($0, /^[ \t]*([-+*]|[0-9]{1,9}[.)])([ \t]+|[ \t]*\r?$)/)
229+
if (list_candidate) {
230+
marker_prefix_len = RLENGTH
231+
marker_indent = ind
232+
content = substr($0, RSTART + marker_prefix_len)
233+
marker = substr($0, 1, marker_prefix_len)
234+
sub(/^[ \t]*/, "", marker)
235+
match(marker, /^([-+*]|[0-9]{1,9}[.)])/)
236+
marker = substr(marker, RSTART, RLENGTH)
237+
content_empty = (content ~ /^\r?$/)
238+
if (content_empty) {
239+
content_indent = marker_indent + length(marker) + 1
240+
} else {
241+
content_indent = column_width(substr($0, 1, marker_prefix_len))
242+
}
243+
can_interrupt = !content_empty && (marker !~ /^[0-9]/ || marker + 0 == 1)
244+
target_depth = list_depth
245+
while (target_depth > 0 && marker_indent < list_content[target_depth] && marker_indent != list_marker[target_depth]) target_depth--
246+
continues_list = target_depth > 0 && marker_indent == list_marker[target_depth]
247+
paragraph_open = target_depth > 0 ? list_lazy[target_depth] : top_lazy
248+
if (!continues_list && paragraph_open && !can_interrupt) {
249+
list_candidate = 0; rejected_marker = 1; list_depth = target_depth
250+
}
251+
}
252+
if (list_candidate) {
253+
if (match(content, /^(`{3,}|~{3,})/)) {
254+
fence_in_marker = 1; is_fence = 1
255+
m = substr(content, RSTART, RLENGTH)
256+
ch = substr(m, 1, 1); len = length(m)
257+
rest = substr(content, RSTART + RLENGTH)
258+
}
259+
list_depth = target_depth
260+
if (list_depth == 0 || marker_indent != list_marker[list_depth]) {
261+
if (list_depth > 0) { list_lazy[list_depth] = 0; list_empty[list_depth] = 0 }
262+
list_depth++
263+
}
264+
list_marker[list_depth] = marker_indent
265+
list_content[list_depth] = content_indent
266+
list_lazy[list_depth] = lazy_text(content)
267+
list_empty[list_depth] = content_empty
268+
top_lazy = 0; top_pipe = 0; top_table = 0
269+
} else if (blank) {
270+
while (list_depth > 0 && list_empty[list_depth]) list_depth--
271+
for (i = 1; i <= list_depth; i++) list_lazy[i] = 0
272+
top_lazy = 0; top_pipe = 0; top_table = 0
273+
} else if (rejected_marker) {
274+
if (list_depth > 0) {
275+
list_empty[list_depth] = 0; list_lazy[list_depth] = 1
276+
top_lazy = 0
277+
} else {
278+
top_lazy = 1; top_pipe = ($0 ~ /\|/)
279+
}
280+
} else {
281+
while (list_depth > 0 && ind < list_content[list_depth] && !(list_lazy[list_depth] && lazy_text($0))) list_depth--
282+
if (list_depth > 0) {
283+
list_empty[list_depth] = 0
284+
if (is_fence && (ch != "`" || index(rest, "`") == 0)) list_lazy[list_depth] = 0
285+
else list_lazy[list_depth] = lazy_text($0)
286+
top_lazy = 0
287+
} else if (table_continuation) {
288+
top_lazy = 0; top_pipe = 0
289+
} else if (setext_line) {
290+
top_lazy = 0; top_pipe = 0
291+
} else if (top_lazy && top_pipe && table_delimiter($0)) {
292+
top_lazy = 0; top_pipe = 0; top_table = 1
293+
} else {
294+
top_lazy = lazy_text($0)
295+
top_pipe = top_lazy && $0 ~ /\|/
296+
}
297+
}
298+
299+
if (is_fence && (ch != "`" || index(rest, "`") == 0)) {
300+
in_fence = 1; fch = ch; flen = len
301+
fscope = fence_in_marker ? content_indent : ((list_depth > 0 && ind >= list_content[list_depth]) ? list_content[list_depth] : 0)
302+
next
303+
}
304+
}
305+
'
306+
122307
# --- Check 1: non-self-closing void HTML elements (outside fenced code blocks) ---
123308
# These must use <tag /> in MDX, not <tag> or <tag attr>.
124309
VOID_ELEMENTS='img|br|hr|input|source|meta|link|area|base|col|embed|param|track|wbr'
@@ -128,22 +313,7 @@ for file in "${FILES[@]}"; do
128313
[[ -z "${match}" ]] && continue
129314
echo "MDX: non-self-closing void element: ${file}:${match}"
130315
ERRORS=$((ERRORS + 1))
131-
done < <(awk -v fm_end="${FM_END}" '
132-
NR <= fm_end { next }
133-
match($0, /^ {0,3}(`{3,}|~{3,})/) {
134-
m = substr($0, RSTART, RLENGTH); sub(/^ +/, "", m)
135-
ch = substr(m, 1, 1); len = length(m)
136-
rest = substr($0, RSTART + RLENGTH)
137-
if (!in_fence) { in_fence = 1; fch = ch; flen = len; next }
138-
# A closer is the same char, at least as long, and followed only by
139-
# whitespace: ```yaml opens a new block, it never closes one. The
140-
# optional \r keeps a CRLF-terminated closer working; without it the
141-
# block never closes and later hazards are silently skipped.
142-
else if (ch == fch && len >= flen && rest ~ /^[ \t]*\r?$/) { in_fence = 0; next }
143-
# else: fence-like line inside a fence of a different char/shorter run
144-
# is content -> fall through to the in_fence guard below.
145-
}
146-
in_fence { next }
316+
done < <(awk -v fm_end="${FM_END}" "${FENCE_AWK}"'
147317
{ print NR": "$0 }
148318
' "$file" | grep -E "<(${VOID_ELEMENTS})([[:space:]][^>]*)?" | grep -Ev "<(${VOID_ELEMENTS})([[:space:]][^>]*)?[[:space:]]*/>" || true)
149319
done
@@ -157,22 +327,7 @@ for file in "${FILES[@]}"; do
157327
# awk exits 1 when it finds a hazard; keep it inside the `if` condition so
158328
# `set -e` does not abort the whole scan at the first offending file
159329
# (otherwise violations in later files are silently skipped).
160-
if ! awk -v fm_end="${FM_END}" '
161-
NR <= fm_end { next }
162-
match($0, /^ {0,3}(`{3,}|~{3,})/) {
163-
m = substr($0, RSTART, RLENGTH); sub(/^ +/, "", m)
164-
ch = substr(m, 1, 1); len = length(m)
165-
rest = substr($0, RSTART + RLENGTH)
166-
if (!in_fence) { in_fence = 1; fch = ch; flen = len; next }
167-
# A closer is the same char, at least as long, and followed only by
168-
# whitespace: ```yaml opens a new block, it never closes one. The
169-
# optional \r keeps a CRLF-terminated closer working; without it the
170-
# block never closes and later hazards are silently skipped.
171-
else if (ch == fch && len >= flen && rest ~ /^[ \t]*\r?$/) { in_fence = 0; next }
172-
# else: fence-like line inside a fence of a different char/shorter run
173-
# is content -> fall through to the in_fence guard below.
174-
}
175-
in_fence { next }
330+
if ! awk -v fm_end="${FM_END}" "${FENCE_AWK}"'
176331
{
177332
line = $0
178333
# Strip inline code spans FIRST so backticked tokens — including a

0 commit comments

Comments
 (0)