Fix grammars depending on empty lines

Author: wooorm

lib/parse.js

@@ -8,11 +8,13 @@
 import {transparent, classes, grandparents} from './theme.js'
 
 // Source: <https://github.com/microsoft/vscode-textmate/blob/9157c7f/src/metadata.ts#L33-L35>
+//   Also: <https://github.com/microsoft/vscode/blob/8ca37ee/src/vs/editor/common/encodedTokenAttributes.ts#L71>
 const FONT_STYLE_MASK = 0b0000_0000_0000_0000_0111_1000_0000_0000
 const FOREGROUND_MASK = 0b0000_0000_1111_1111_1000_0000_0000_0000
 const BACKGROUND_MASK = 0b1111_1111_0000_0000_0000_0000_0000_0000
 
 // Source: <https://github.com/microsoft/vscode-textmate/blob/9157c7f/src/metadata.ts#L37-L42>
+//   Also: <https://github.com/microsoft/vscode/blob/8ca37ee/src/vs/editor/common/encodedTokenAttributes.ts#L92-L94>
 const FONT_STYLE_OFFSET = 11
 const FOREGROUND_OFFSET = 15
 const BACKGROUND_OFFSET = 24
@@ -28,42 +30,41 @@ export function parse(value, grammar, colors) {
   /** @type {Root} */
   const tree = {type: 'root', children: []}
   const search = /\r?\n|\r/g
-  /** @type {StateStack|null} */
+  /** @type {StateStack | null} */
   let stack = null
   let start = 0
 
   while (start < value.length) {
     const match = search.exec(value)
     const end = match ? match.index : value.length
 
-    // Empty lines could still match `source` and be turned into a span.
-    // Ignore those.
-    if (start !== end) {
-      const {tokens, ruleStack} = grammar.tokenizeLine2(
-        value.slice(start, end),
-        stack
-      )
-      let index = 0
-
-      while (index < tokens.length) {
-        const tokenStart = start + tokens[index++]
-        const metadata = tokens[index++]
-        const tokenEnd = index < tokens.length ? start + tokens[index] : end
-        // Source: <https://github.com/microsoft/vscode-textmate/blob/9157c7f/src/metadata.ts#L71-L93>
-        const fg = (metadata & FOREGROUND_MASK) >>> FOREGROUND_OFFSET
-        const bg = (metadata & BACKGROUND_MASK) >>> BACKGROUND_OFFSET
-        const fs = (metadata & FONT_STYLE_MASK) >>> FONT_STYLE_OFFSET
-        /** @type {Root|Element} */
-        let scope = tree
-        scope = delveIfClassName(scope, fontStyleToClass(fs))
-        scope = delveIfClassName(scope, colorToClass(colors[bg]))
-        scope = delveIfClassName(scope, colorToClass(colors[fg]))
-        appendText(scope, value.slice(tokenStart, tokenEnd))
-      }
-
-      stack = ruleStack
+    // > 👉 **Important**: empty lines have to be tokenized, as some patterns
+    // > look for them.
+    const {tokens, ruleStack} = grammar.tokenizeLine2(
+      value.slice(start, end),
+      stack
+    )
+    let index = 0
+
+    while (index < tokens.length) {
+      const tokenStart = start + tokens[index++]
+      const metadata = tokens[index++]
+      const tokenEnd = index < tokens.length ? start + tokens[index] : end
+      // Source: <https://github.com/microsoft/vscode-textmate/blob/9157c7f/src/metadata.ts#L71-L93>
+      const fg = (metadata & FOREGROUND_MASK) >>> FOREGROUND_OFFSET
+      const bg = (metadata & BACKGROUND_MASK) >>> BACKGROUND_OFFSET
+      const fs = (metadata & FONT_STYLE_MASK) >>> FONT_STYLE_OFFSET
+
+      /** @type {Root|Element} */
+      let scope = tree
+      scope = delveIfClassName(scope, fontStyleToClass(fs))
+      scope = delveIfClassName(scope, colorToClass(colors[bg]))
+      scope = delveIfClassName(scope, colorToClass(colors[fg]))
+      appendText(scope, value.slice(tokenStart, tokenEnd))
     }
 
+    stack = ruleStack
+
     start = end
 
     if (match) {

test.js

@@ -168,4 +168,20 @@ class Country(val name : String)`,
     '<span class="pl-k">package</span> <span class="pl-en">addressbook</span>\n\n<span class="pl-k">class</span> <span class="pl-en">Country</span>(<span class="pl-k">val</span> <span class="pl-smi">name</span> <span class="pl-k">:</span> <span class="pl-c1">String</span>)',
     'should highlight some kotlin'
   )
+
+  // Real world example of this test case:
+  // <https://github.com/microsoft/vscode-markdown-tm-grammar/blob/eed230887a39da1ecf5bfc914e00a1e1813c0fdb/markdown.tmLanguage.base.yaml#L125>
+  const starryNightBlankLines = await createStarryNight([
+    {
+      names: [],
+      extensions: [],
+      scopeName: 'x',
+      patterns: [{begin: '^a$', end: '^[\\t ]*$', name: 'invalid.illegal'}]
+    }
+  ])
+  assert.equal(
+    toHtml(starryNightBlankLines.highlight('a\n\nb', 'x')),
+    '<span class="pl-ii">a</span>\n\nb',
+    'should be able to match on empty lines'
+  )
 })