LaTeX reader: support LaTeX3 (xparse) document commands.

Support the features described in the LaTeX usrguide
(https://www.latex-project.org/help/documentation/usrguide.pdf):

- `\New/Renew/Provide/DeclareDocumentCommand` and the corresponding
  `...DocumentEnvironment` commands, with argument specifiers
  `m o O d D s t r R v e E b` and argument processors (`>{...}`:
  `\TrimSpaces`, `\ReverseBoolean`, `\SplitArgument`, `\SplitList`).
  Environment end code can refer to the environment's arguments.
- `\ProcessList`, `\UseName`, and `\ExpandArgs` (c, n, N letters).
- The conditionals `\IfNoValueTF`, `\IfValueTF`, `\IfBooleanTF`,
  `IfBlankTF` (and variants with just `T` or `F`).
- `\New/Renew/DeclareCommandCopy` and `...EnvironmentCopy`.
- Case changing: `\MakeUppercase`, `\MakeLowercase`, and
  `\MakeTitlecase`, `\NoCaseChange`,
  `\Declare{Upper,Lower,Title}caseExclusions`; `\CaseSwitch` and
  `\Declare*caseMapping`.
- The expandable evaluators `\inteval` and `\fpeval` (a practical
  arithmetic subset); `\dimeval` and `\skipeval` substitute their
  expression unevaluated.
- `\expandableinput`.
- `\ExplSyntaxOn/Off` regions are tokenized with `:` and `_` as
  letters, so expl3 code can be skipped cleanly; `\ShowCommand` and
  the ltkeys commands (`\DeclareKeys`, `\SetKeys`,
  `\ProcessKeyOptions`, ...) are parsed and ignored.
- Some constants, such as `c_ampersand_str`, are supported in
  expl3 regions.

Adds new `ArgSpec` constructors to non-exported module Text.Pandoc.TeX.

New tests in `test/command/latex3-commands.md` and
`test/command/latex3-usrguide.md`. (The latter correspond to the
examples in the user guide and were confirmed against LaTeX output
when possible.)

Closes #7540.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: jgm

src/Text/Pandoc/Readers/LaTeX.hs

@@ -407,22 +407,30 @@ inlineCommands = M.unions
     , ("sl", extractSpaces emph <$> inlines)
     , ("bf", extractSpaces strong <$> inlines)
     , ("tt", formatCode nullAttr <$> inlines)
+    , ("ttfamily", extractSpaces (formatCode nullAttr) <$> inlines)
     , ("rm", inlines)
     , ("itshape", extractSpaces emph <$> inlines)
     , ("slshape", extractSpaces emph <$> inlines)
     , ("scshape", extractSpaces smallcaps <$> inlines)
     , ("bfseries", extractSpaces strong <$> inlines)
-    , ("MakeUppercase", makeUppercase <$> tok)
-    , ("MakeTextUppercase", makeUppercase <$> tok) -- textcase
-    , ("uppercase", makeUppercase <$> tok)
-    , ("MakeLowercase", makeLowercase <$> tok)
-    , ("MakeTextLowercase", makeLowercase <$> tok)
-    , ("lowercase", makeLowercase <$> tok)
+    , ("MakeUppercase", caseTransform "upper" T.toUpper)
+    , ("MakeTextUppercase", caseTransform "upper" T.toUpper) -- textcase
+    , ("uppercase", caseTransform "upper" T.toUpper)
+    , ("MakeLowercase", caseTransform "lower" T.toLower)
+    , ("MakeTextLowercase", caseTransform "lower" T.toLower)
+    , ("lowercase", caseTransform "lower" T.toLower)
+    , ("MakeTitlecase", makeTitlecaseCommand)
+    , ("NoCaseChange", spanWith ("",["nocasechange"],[]) <$> tok)
+    , ("CaseSwitch", tok <* tok <* tok <* tok)
     , ("thanks", skipopts >> note <$> grouped block)
     , ("footnote", skipopts >> footnote)
     , ("footnotemark", footnotemark)
     , ("footnotetext", footnotetext)
     , ("newline", pure B.linebreak)
+    -- xparse argument markers, in case they leak into the document:
+    , ("NoValue", pure (B.str "-NoValue-"))
+    , ("BooleanTrue", pure mempty)
+    , ("BooleanFalse", pure mempty)
     , ("passthrough", fixPassthroughEscapes <$> tok)
     -- \passthrough macro used by latex writer
                            -- for listings
@@ -457,6 +465,7 @@ inlineCommands = M.unions
     , ("iftoggle", try $ ifToggle >> inline)
     -- include
     , ("input", rawInlineOr "input" $ include "input")
+    , ("expandableinput", rawInlineOr "expandableinput" $ include "input")
     -- soul package
     , ("st", extractSpaces strikeout <$> tok)
     , ("ul", underline <$> tok)
@@ -471,6 +480,19 @@ inlineCommands = M.unions
     -- this is used internally by pandoc but the definition is too complicated
     -- for pandoc to handle (see #11140):
     , ("pandocbounded", tok)
+    -- LaTeX3 constants
+    , ("c_ampersand_str", pure (str "&"))
+    , ("c_atsign_str", pure (str "@"))
+    , ("c_backslash_str", pure (str "\\"))
+    , ("c_left_brace_str", pure (str "{"))
+    , ("c_right_brace_str", pure (str "}"))
+    , ("c_circumflex_str", pure (str "^"))
+    , ("c_colon_str", pure (str ":"))
+    , ("c_dollar_str", pure (str "$"))
+    , ("c_hash_str", pure (str "#"))
+    , ("c_percent_str", pure (str "%"))
+    , ("c_tilde_str", pure (str "~"))
+    , ("c_underscore_str", pure (str "_"))
     ]
 
 bracedFilename :: PandocMonad m => LP m Text
@@ -547,15 +569,97 @@ ifdim = do
   contents <- manyTill anyTok (controlSeq "fi")
   return $ rawInline "latex" $ "\\ifdim" <> untokenize contents <> "\\fi"
 
-makeUppercase :: Inlines -> Inlines
-makeUppercase = fromList . walk (alterStr T.toUpper) . toList
-
-makeLowercase :: Inlines -> Inlines
-makeLowercase = fromList . walk (alterStr T.toLower) . toList
+-- | Parse the argument of a case-changing command (\MakeUppercase,
+-- \MakeLowercase) and apply the case transformation, honoring
+-- exclusions declared with \Declare*caseExclusions.
+caseTransform :: PandocMonad m => Text -> (Text -> Text) -> LP m Inlines
+caseTransform kind f = do
+  void $ option [] keyvals -- locale options, ignored
+  excl <- M.findWithDefault Set.empty kind . sCaseExclusions <$> getState
+  caseTransformWith excl f <$> tok
+
+-- | Apply a case transformation to text, leaving math, code and
+-- citations untouched, unwrapping (and skipping) \NoCaseChange
+-- content, and skipping excluded words.
+caseTransformWith :: Set.Set Text -> (Text -> Text) -> Inlines -> Inlines
+caseTransformWith excl f = fromList . go . toList
+  where
+    go = concatMap goInline
+    goInline (Span ("",["nocasechange"],[]) ils) = ils
+    goInline (Str t)
+      | t `Set.member` excl = [Str t]
+      | otherwise = [Str (f t)]
+    goInline (Emph ils) = [Emph (go ils)]
+    goInline (Strong ils) = [Strong (go ils)]
+    goInline (Underline ils) = [Underline (go ils)]
+    goInline (Strikeout ils) = [Strikeout (go ils)]
+    goInline (Superscript ils) = [Superscript (go ils)]
+    goInline (Subscript ils) = [Subscript (go ils)]
+    goInline (SmallCaps ils) = [SmallCaps (go ils)]
+    goInline (Quoted qt ils) = [Quoted qt (go ils)]
+    goInline (Span attr ils) = [Span attr (go ils)]
+    goInline (Link attr ils target) = [Link attr (go ils) target]
+    goInline x = [x] -- Math, Code, Cite, Space, etc.
+
+-- | Parse the arguments of \MakeTitlecase, supporting the
+-- @words=all@ option and title-case exclusions.
+makeTitlecaseCommand :: PandocMonad m => LP m Inlines
+makeTitlecaseCommand = do
+  options <- option [] keyvals
+  excl <- M.findWithDefault Set.empty "title" . sCaseExclusions <$> getState
+  (if lookup "words" options == Just "all"
+      then makeTitlecaseAll excl
+      else makeTitlecase) <$> tok
+
+-- | Titlecase the first letter of each word (\MakeTitlecase with
+-- @words=all@), skipping excluded words.
+makeTitlecaseAll :: Set.Set Text -> Inlines -> Inlines
+makeTitlecaseAll excl = fromList . go . toList
+  where
+    go [] = []
+    go xs =
+      let (w, rest) = break isSep xs
+          (seps, rest') = span isSep rest
+       in tcWord w ++ seps ++ go rest'
+    isSep Space = True
+    isSep SoftBreak = True
+    isSep LineBreak = True
+    isSep _ = False
+    tcWord w
+      | stringify w `Set.member` excl = w
+      | otherwise = toList (makeTitlecase (fromList w))
+
+-- | Handle \DeclareUppercaseExclusions and friends: store a
+-- comma-separated list of words excluded from case changing.
+declareCaseExclusions :: PandocMonad m => Text -> LP m Blocks
+declareCaseExclusions kind = do
+  ws <- map T.strip . T.splitOn "," . untokenize <$> braced
+  updateState $ \st -> st{ sCaseExclusions =
+      M.insertWith Set.union kind (Set.fromList ws) (sCaseExclusions st) }
+  return mempty
 
-alterStr :: (Text -> Text) -> Inline -> Inline
-alterStr f (Str xs) = Str (f xs)
-alterStr _ x = x
+-- | Uppercase the first character of the first string (LaTeX3
+-- \MakeTitlecase, which title-cases only the first word by default).
+makeTitlecase :: Inlines -> Inlines
+makeTitlecase = fromList . snd . go . toList
+  where
+    go :: [Inline] -> (Bool, [Inline])
+    go (x : xs) =
+      case goInline x of
+        (True, x')  -> (True, x' : xs)
+        (False, x') -> (x' :) <$> go xs
+    go [] = (False, [])
+    goInline (Str t) | not (T.null t) =
+      (True, Str (T.toTitle (T.take 1 t) <> T.drop 1 t))
+    goInline (Emph ils) = Emph <$> go ils
+    goInline (Strong ils) = Strong <$> go ils
+    goInline (Underline ils) = Underline <$> go ils
+    goInline (SmallCaps ils) = SmallCaps <$> go ils
+    goInline (Span attr ils) = Span attr <$> go ils
+    goInline (Link attr ils target) =
+      (\ils' -> Link attr ils' target) <$> go ils
+    goInline (Quoted qt ils) = Quoted qt <$> go ils
+    goInline x = (False, x)
 
 fixPassthroughEscapes :: Inlines -> Inlines
 fixPassthroughEscapes = walk go
@@ -1062,10 +1166,30 @@ blockCommands = M.fromList
    -- include
    , ("include", rawBlockOr "include" $ include "include")
    , ("input", rawBlockOr "input" $ include "input")
+   , ("expandableinput", rawBlockOr "expandableinput" $ include "input")
    , ("subfile", rawBlockOr "subfile" doSubfile)
    , ("usepackage", rawBlockOr "usepackage" usepackage)
    -- preamble
    , ("PackageError", mempty <$ (braced >> braced >> braced))
+   -- LaTeX3 conveniences, parsed and ignored:
+   , ("ExplSyntaxOn", pure mempty)
+   , ("ExplSyntaxOff", pure mempty)
+   , ("ShowCommand", mempty <$ withVerbatimMode (spaces *> anyControlSeq))
+   , ("ShowEnvironment", mempty <$ braced)
+   , ("DeclareKeys", mempty <$ (skipopts *> braced))
+   , ("DeclareUnknownKeyHandler", mempty <$ (skipopts *> braced))
+   , ("ProcessKeyOptions", mempty <$ skipopts)
+   , ("SetKeys", mempty <$ (skipopts *> braced))
+   -- LaTeX3 case changing
+   , ("DeclareUppercaseExclusions", declareCaseExclusions "upper")
+   , ("DeclareLowercaseExclusions", declareCaseExclusions "lower")
+   , ("DeclareTitlecaseExclusions", declareCaseExclusions "title")
+   , ("AddToNoCaseChangeList", mempty <$ braced)
+   , ("DeclareCaseChangeEquivalent", mempty <$
+        (withVerbatimMode (spaces *> anyControlSeq) *> braced))
+   , ("DeclareUppercaseMapping", mempty <$ (skipopts *> braced *> braced))
+   , ("DeclareLowercaseMapping", mempty <$ (skipopts *> braced *> braced))
+   , ("DeclareTitlecaseMapping", mempty <$ (skipopts *> braced *> braced))
    -- epigraph package
    , ("epigraph", epigraph)
    -- alignment

src/Text/Pandoc/Readers/LaTeX/Inline.hs

@@ -36,8 +36,7 @@ import Text.Pandoc.Translations (translateTerm)
 import Text.Pandoc.Readers.LaTeX.Parsing
 import Text.Pandoc.Extensions (extensionEnabled, Extension(..))
 import Text.Pandoc.Parsing (getOption, updateState, getState, notFollowedBy,
-                            manyTill, getInput, setInput, incSourceColumn,
-                            option, many1)
+                            manyTill, option, many1)
 import Data.Char (isDigit)
 import Text.Pandoc.Highlighting (fromListingsLanguage,)
 import Data.Maybe (maybeToList, fromMaybe)
@@ -89,19 +88,6 @@ doverb = do
     code . untokenize <$>
       manyTill (notFollowedBy newlineTok >> verbTok marker) (symbol marker)
 
-verbTok :: PandocMonad m => Char -> LP m Tok
-verbTok stopchar = do
-  t@(Tok pos toktype txt) <- anyTok
-  case T.findIndex (== stopchar) txt of
-       Nothing -> return t
-       Just i  -> do
-         let (t1, t2) = T.splitAt i txt
-         TokStream macrosExpanded inp <- getInput
-         setInput $ TokStream macrosExpanded
-                  $ Tok (incSourceColumn pos i) Symbol (T.singleton stopchar)
-                  : tokenize (incSourceColumn pos (i + 1)) (T.drop 1 t2) ++ inp
-         return $ Tok pos toktype t1
-
 listingsLanguage :: [(Text, Text)] -> Maybe Text
 listingsLanguage opts =
   case lookup "language" opts of