fix(tools): narrow fence tracking to the direct-list case

Related to #2144

Signed-off-by: Varun Ramesh <varamesh@nvidia.com>

Author: varmesh

tools/check-docs-mdx

@@ -33,36 +33,32 @@
 #
 # Code handling (applies to checks 2-6; check 1 greps the fence-stripped line
 # WITHOUT inline-span stripping): fenced code blocks in BOTH backtick
-# (```) and tilde (~~~) styles are skipped, following the pinned MDX parser's
-# fence behavior. Unlike CommonMark, MDX has no indented code blocks, so a
-# fence may have any leading spaces or tabs. It opens on a run of >= 3 of either
-# character and closes only on a later line whose leading run is the SAME
-# character, at least as long, and followed by nothing but whitespace (so an
-# info string such as ```yaml opens a block and never closes one). Inline
-# code spans are stripped before scanning; a span opens on a run of N backticks
-# and closes at the next run of exactly N backticks, so spans of any
-# backtick-run length are honored (not just single-backtick spans).
+# (```) and tilde (~~~) styles are skipped by a lexical fence tracker. An
+# explicit fence opens on a run of >= 3 of either character after 0-3 leading
+# spaces, and closes only on an explicit later line whose leading run is the
+# SAME character, at least as long, and followed by nothing but whitespace.
+# Thus ```yaml starts a fence when outside one and is content, not a closer,
+# when inside one. For an immediate direct top-level bullet or 1. item whose
+# content indent is at most three, a nonblank outdent implicitly ends its fence.
+# Inline code spans are stripped before scanning; a span opens on a run of N
+# backticks and closes at the next run of exactly N backticks.
 #
 # NOTE: this checker is a fast, dependency-free APPROXIMATION of the MDX
 # parser, not a reimplementation of it. tools/check-docs-mdx-parse runs the
 # real parser (@mdx-js/mdx, locked in tools/mdx/) and is the authoritative
 # gate — it is what the merge-gate 'docs-mdx' job blocks on. A green run HERE
 # is not parser-level validation; it means no known hazard pattern matched.
 #
-# The rules above are deliberately a strict SUBSET of what the parser rejects.
-# Erring toward under-reporting is the correct direction: a miss is caught by
-# the blocking parse gate, whereas a false positive would force contributors to
-# mangle prose the publish step would have accepted. Known misses left to the
-# parser: a stray closing tag ('</div>'), an unclosed fragment ('<>'), a
-# placeholder sharing a line with well-formed JSX, unbalanced expression braces
-# spanning lines, and acorn-level syntax errors.
+# The hazard rules deliberately cover common failures rather than every parser
+# rejection. Outside the lexical code-handling subset documented above, this
+# line-based approximation can disagree with the parser in either direction.
+# Known misses left to the parser include a stray closing tag ('</div>'), an
+# unclosed fragment ('<>'), a placeholder sharing a line with well-formed JSX,
+# unbalanced expression braces spanning lines, and acorn-level syntax errors.
 #
-# Known code-handling limit: a code span is matched within one line, even
-# though CommonMark lets a span cross a line break (an opening backtick run on
-# one line closing on a later one). Content between such a break is scanned as
-# prose and can surface as a false positive here. This is a deliberate
-# limitation of a line-based scanner, not an oversight: the parse gate
-# (check-docs-mdx-parse) is the arbiter and will accept the file.
+# Known code-handling limits: later list items, lazy continuations, nested lists,
+# and mixed containers are parser-owned. Code spans are matched within one line;
+# the parser handles multi-line spans.
 
 set -euo pipefail
 
@@ -136,169 +132,48 @@ readonly FENCE_AWK='
         }
         return width
       }
-      function column_width(s,    i, ch, width) {
-        width = 0
-        for (i = 1; i <= length(s); i++) {
-          ch = substr(s, i, 1)
-          if (ch == "\t") width += 4 - (width % 4)
-          else width++
-        }
-        return width
-      }
-      function thematic_break(s,    text) {
-        text = s
-        sub(/\r$/, "", text)
-        return text ~ /^[ \t]*(-[ \t]*){3,}$/ ||
-               text ~ /^[ \t]*(\*[ \t]*){3,}$/ ||
-               text ~ /^[ \t]*(_[ \t]*){3,}$/
-      }
-      function setext_underline(s,    text) {
-        text = s
-        sub(/^[ \t]+/, "", text); sub(/\r$/, "", text); sub(/[ \t]+$/, "", text)
-        return text ~ /^(=+|-+)$/
-      }
-      function table_delimiter(s,    text, outer) {
-        text = s
-        sub(/^[ \t]+/, "", text); sub(/\r$/, "", text); sub(/[ \t]+$/, "", text)
-        outer = (text ~ /^\|.*\|$/)
-        gsub(/[ \t]/, "", text)
-        if (text !~ /\|/ && !outer) return 0
-        sub(/^\|/, "", text); sub(/\|$/, "", text)
-        return text ~ /^:?-+:?(\|:?-+:?)*$/
-      }
-      function lazy_text(s,    text, fence_rest) {
-        text = s
-        sub(/^[ \t]+/, "", text)
-        sub(/\r$/, "", text)
-        if (text !~ /[^ \t]/) return 0
-        if (thematic_break(text)) return 0
-        if (text ~ /^(>|#{1,6}([ \t]|$)|~{3,})/) return 0
-        if (match(text, /^`{3,}/)) {
-          fence_rest = substr(text, RSTART + RLENGTH)
-          if (index(fence_rest, "`") == 0) return 0
-        }
-        if (text ~ /^\{/ || text ~ /^<[^ \t]/) return 0
-        if (text ~ /^([-+*]|[0-9]{1,9}[.)])[ \t]+/) return 0
-        return 1
-      }
       NR <= fm_end { next }
-      # Fence rules shared by both checker passes. Getting any of these wrong
-      # leaves the tracker open and silently skips every later hazard.
-      #   indent: any leading spaces or tabs, matching the MDX parser
+      # Lexical fence rules shared by both checker passes:
+      #   indent: 0-3 leading spaces; tabs are not fence indentation
       #   open:   a backtick in the info string makes the line prose, not a
       #           fence (ambiguous with an inline code span); tildes are exempt
-      #   close:  same char, at least as long, nothing after but whitespace, so
-      #           ```yaml only ever opens; the \r admits a CRLF closer
-      #   scope:  a fence opened inside a list item ends implicitly when a
-      #           nonblank line leaves that item. list_content/fscope carry
-      #           container indents; list_lazy/top_lazy preserve CommonMark
-      #           paragraph continuation lines, and thematic breaks never
-      #           create lists.
+      #   close:  same char, at least as long, nothing after but whitespace;
+      #           the optional \r admits a CRLF closer
+      #   scope:  an immediate direct top-level bullet or 1. item may own an
+      #           indented fence; any nonblank outdent ends that scope
       {
         blank = ($0 ~ /^[ \t]*\r?$/)
         ind = indent_width($0)
-        fence_in_marker = 0
-        is_fence = match($0, /^[ \t]*(`{3,}|~{3,})/)
+        is_fence = match($0, /^ {0,3}(`{3,}|~{3,})/)
         if (is_fence) {
-          m = substr($0, RSTART, RLENGTH); sub(/^[ \t]*/, "", m)
+          m = substr($0, RSTART, RLENGTH); sub(/^ +/, "", m)
           ch = substr(m, 1, 1); len = length(m)
           rest = substr($0, RSTART + RLENGTH)
         }
 
-        # A nonblank outdent ends both the list item and any unclosed fence it
-        # contains. Re-process the boundary line below: it may be prose, a new
-        # list item, or the opener for a following top-level fence.
+        # Re-process a direct list boundary as prose or as a new outer fence.
         if (in_fence && fscope > 0 && !blank && ind < fscope) {
-          in_fence = 0; fscope = 0
+          in_fence = 0; fscope = 0; list_indent = 0
         }
 
         if (in_fence) {
-          if (is_fence && ch == fch && len >= flen && rest ~ /^[ \t]*\r?$/) { in_fence = 0; next }
+          if (is_fence && ch == fch && len >= flen && rest ~ /^[ \t]*\r?$/) { in_fence = 0; fscope = 0; next }
           next
         }
 
-        # Maintain just enough list-container state to distinguish an
-        # outdented list-fence boundary from a valid less-indented top-level
-        # closer. CommonMark lazy paragraph lines keep the item open. A blank
-        # ends laziness and also ends an item whose marker had no content.
-        table_continuation = top_table && $0 ~ /\|/
-        if (top_table && !table_continuation) top_table = 0
-        setext_line = top_lazy && setext_underline($0)
-        rejected_marker = 0
-        list_candidate = !setext_line && !table_continuation && !thematic_break($0) && match($0, /^[ \t]*([-+*]|[0-9]{1,9}[.)])([ \t]+|[ \t]*\r?$)/)
-        if (list_candidate) {
-          marker_prefix_len = RLENGTH
-          marker_indent = ind
-          content = substr($0, RSTART + marker_prefix_len)
-          marker = substr($0, 1, marker_prefix_len)
-          sub(/^[ \t]*/, "", marker)
-          match(marker, /^([-+*]|[0-9]{1,9}[.)])/)
-          marker = substr(marker, RSTART, RLENGTH)
-          content_empty = (content ~ /^\r?$/)
-          if (content_empty) {
-            content_indent = marker_indent + length(marker) + 1
-          } else {
-            content_indent = column_width(substr($0, 1, marker_prefix_len))
-          }
-          can_interrupt = !content_empty && (marker !~ /^[0-9]/ || marker + 0 == 1)
-          target_depth = list_depth
-          while (target_depth > 0 && marker_indent < list_content[target_depth] && marker_indent != list_marker[target_depth]) target_depth--
-          continues_list = target_depth > 0 && marker_indent == list_marker[target_depth]
-          paragraph_open = target_depth > 0 ? list_lazy[target_depth] : top_lazy
-          if (!continues_list && paragraph_open && !can_interrupt) {
-            list_candidate = 0; rejected_marker = 1; list_depth = target_depth
-          }
-        }
-        if (list_candidate) {
-          if (match(content, /^(`{3,}|~{3,})/)) {
-            fence_in_marker = 1; is_fence = 1
-            m = substr(content, RSTART, RLENGTH)
-            ch = substr(m, 1, 1); len = length(m)
-            rest = substr(content, RSTART + RLENGTH)
-          }
-          list_depth = target_depth
-          if (list_depth == 0 || marker_indent != list_marker[list_depth]) {
-            if (list_depth > 0) { list_lazy[list_depth] = 0; list_empty[list_depth] = 0 }
-            list_depth++
-          }
-          list_marker[list_depth] = marker_indent
-          list_content[list_depth] = content_indent
-          list_lazy[list_depth] = lazy_text(content)
-          list_empty[list_depth] = content_empty
-          top_lazy = 0; top_pipe = 0; top_table = 0
-        } else if (blank) {
-          while (list_depth > 0 && list_empty[list_depth]) list_depth--
-          for (i = 1; i <= list_depth; i++) list_lazy[i] = 0
-          top_lazy = 0; top_pipe = 0; top_table = 0
-        } else if (rejected_marker) {
-          if (list_depth > 0) {
-            list_empty[list_depth] = 0; list_lazy[list_depth] = 1
-            top_lazy = 0
-          } else {
-            top_lazy = 1; top_pipe = ($0 ~ /\|/)
-          }
-        } else {
-          while (list_depth > 0 && ind < list_content[list_depth] && !(list_lazy[list_depth] && lazy_text($0))) list_depth--
-          if (list_depth > 0) {
-            list_empty[list_depth] = 0
-            if (is_fence && (ch != "`" || index(rest, "`") == 0)) list_lazy[list_depth] = 0
-            else list_lazy[list_depth] = lazy_text($0)
-            top_lazy = 0
-          } else if (table_continuation) {
-            top_lazy = 0; top_pipe = 0
-          } else if (setext_line) {
-            top_lazy = 0; top_pipe = 0
-          } else if (top_lazy && top_pipe && table_delimiter($0)) {
-            top_lazy = 0; top_pipe = 0; top_table = 1
-          } else {
-            top_lazy = lazy_text($0)
-            top_pipe = top_lazy && $0 ~ /\|/
-          }
+        # Deliberately narrow list recognition: direct top-level bullets and
+        # ordered items starting at 1, with a normal content indent <= 3.
+        # Exclude thematic breaks, which otherwise resemble bullet items.
+        marker = !($0 ~ /^(-[ \t]*){3,}\r?$/ || $0 ~ /^(\*[ \t]*){3,}\r?$/) && match($0, /^([-+*]|1[.)]) +/)
+        if (marker && RLENGTH <= 3 && substr($0, RLENGTH + 1, 1) ~ /[^ \t\r]/) {
+          list_indent = RLENGTH
+        } else if (!blank && list_indent > 0 && ind < list_indent) {
+          list_indent = 0
         }
 
         if (is_fence && (ch != "`" || index(rest, "`") == 0)) {
           in_fence = 1; fch = ch; flen = len
-          fscope = fence_in_marker ? content_indent : ((list_depth > 0 && ind >= list_content[list_depth]) ? list_content[list_depth] : 0)
+          fscope = (list_indent > 0 && ind >= list_indent) ? list_indent : 0
           next
         }
       }
@@ -424,8 +299,7 @@ for file in "${FILES[@]}"; do
       # valid content. Deciding whether tags actually balance needs parser
       # state, which this script does not have; the trade is that a placeholder
       # sharing a line with real JSX is caught by tools/check-docs-mdx-parse
-      # instead of here. That is the safe direction — see the strict-subset
-      # note in the header.
+      # instead of here, as described by the parser-ownership note above.
       line ~ /<[a-zA-Z]/ && line !~ /\/>/ && line !~ /<\// && line !~ /<(img|br|hr|input|source|meta|link|area|base|col|embed|param|track|wbr)([ \t>\/])/ && line !~ /<https?:\/\// {
         printf "MDX: bare <word> tag outside code fence: %s:%d: %s\n", FILENAME, NR, $0
         errors++
@@ -439,8 +313,7 @@ for file in "${FILES[@]}"; do
       # parse cleanly through @mdx-js/mdx — micromark only enters JSX-tag mode
       # when a name-ish character follows immediately, so `<` + space stays
       # literal text. Flagging them forced contributors to backtick prose the
-      # publish step accepts, which is the false-positive direction this script
-      # must avoid (see the strict-subset note in the header).
+      # publish step accepts, which this approximation must avoid.
       #
       # `</div>` and `<>` are NOT covered here: both are real MDX errors, but
       # deciding whether a tag is balanced needs parser state. They are caught