fix(typst): use HeadingLine single token to fix inline element highlighting

The two-token approach (HeadingMark + HeadingTitle) caused LALR state
merging: the parser state waiting for HeadingTitle after HeadingMark was
merged into body-text item* states. In those merged states the
headingTitleTokenizer fired for every paragraph line, swallowing bold,
italic, math and inline function tokens — leaving body text black.

Fix: collapse the heading into a single HeadingLine external token that
covers the entire heading line (= prefix + title). A single-token Heading
rule leaves no post-token parser state waiting for a second token, so no
LALR merging can occur. The ViewPlugin and all HeadingMark/HeadingTitle
infrastructure are removed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

services/web/frontend/js/features/source-editor/languages/typst/index.ts

@@ -5,17 +5,8 @@ import {
   foldInside,
   HighlightStyle,
   syntaxHighlighting,
-  syntaxTree,
 } from '@codemirror/language'
 import { styleTags, tags as t } from '@lezer/highlight'
-import { RangeSetBuilder } from '@codemirror/state'
-import {
-  Decoration,
-  DecorationSet,
-  EditorView,
-  ViewPlugin,
-  ViewUpdate,
-} from '@codemirror/view'
 import { parser } from '../../lezer-typst/typst.mjs'
 import { typstCompletions } from './complete'
 import { typstDocumentOutline } from './document-outline'
@@ -39,9 +30,8 @@ export const TypstLanguage = LRLanguage.define({
         CodeArgs: foldInside,
       }),
       styleTags({
-        // HeadingMark covers the "=+ " prefix; heading line content is handled
-        // by the headingLinePlugin ViewPlugin below (not a grammar token).
-        HeadingMark: t.heading,
+        // HeadingLine is the entire heading line (prefix + title) as one token.
+        HeadingLine: t.heading,
 
         // Comments
         'LineComment LineCommentContent': t.comment,
@@ -110,50 +100,10 @@ const typstHighlightStyle = HighlightStyle.define([
   { tag: t.emphasis, fontStyle: 'italic' },
 ])
 
-// Heading title decoration: HeadingTitle exists in the grammar but is NOT in
-// styleTags, so it never applies heading colour on its own. This plugin walks
-// the syntax tree, finds each HeadingMark, and extends a heading-style
-// decoration to end-of-line. Belt-and-suspenders: even if HeadingTitle fires
-// spuriously (LALR state merging), no bleed happens because styleTags ignores
-// it; and the ViewPlugin only decorates lines that start with a HeadingMark.
-const headingTitleMark = Decoration.mark({
-  class: 'tok-heading',
-  attributes: { style: 'font-weight:bold' },
-})
-
-function buildHeadingDecorations(view: EditorView): DecorationSet {
-  const builder = new RangeSetBuilder<Decoration>()
-  syntaxTree(view.state).cursor().iterate(node => {
-    if (node.name === 'HeadingMark') {
-      const line = view.state.doc.lineAt(node.from)
-      if (node.to < line.to) {
-        builder.add(node.to, line.to, headingTitleMark)
-      }
-    }
-  })
-  return builder.finish()
-}
-
-const headingLinePlugin = ViewPlugin.fromClass(
-  class {
-    decorations: DecorationSet
-    constructor(view: EditorView) {
-      this.decorations = buildHeadingDecorations(view)
-    }
-    update(update: ViewUpdate) {
-      if (update.docChanged || update.viewportChanged) {
-        this.decorations = buildHeadingDecorations(update.view)
-      }
-    }
-  },
-  { decorations: v => v.decorations }
-)
-
 export const typst = () => {
   return new LanguageSupport(TypstLanguage, [
     TypstLanguage.data.of({ autocomplete: typstCompletions }),
     typstDocumentOutline,
     syntaxHighlighting(typstHighlightStyle),
-    headingLinePlugin,
   ])
 }

services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs

@@ -2,8 +2,7 @@
 
 import { ExternalTokenizer } from '@lezer/lr'
 import {
-  HeadingMark,
-  HeadingTitle,
+  HeadingLine,
   RawBlockOpen,
   RawBlockBody,
   RawBlockClose,
@@ -25,54 +24,44 @@ const DOLLAR    = 36  // $
 const OPEN_BRACE  = 123 // {
 const CLOSE_BRACE = 125 // }
 
-// ── headingTokenizer ────────────────────────────────────────────────────
-// Emits HeadingMark when '=+' appears at the start of a line, followed by
-// at least one space.  The emitted token covers the '=' chars + the space(s).
-export const headingTokenizer = new ExternalTokenizer(
+// ── headingLineTokenizer ────────────────────────────────────────────────
+// Emits HeadingLine — the entire heading line: "=+" markers, trailing
+// space, and title text, all as one token.
+//
+// Using a single token per heading line eliminates the two-step
+// (HeadingMark + HeadingTitle) pattern that caused LALR state-merging
+// problems: having a separate HeadingTitle token created a parser state
+// that "waited" after HeadingMark, and LALR merged that state into
+// body-text item* states.  In those merged states headingTitleTokenizer
+// fired for every paragraph line, swallowing inline markup tokens.
+export const headingLineTokenizer = new ExternalTokenizer(
   (input, _stack) => {
-    // Start-of-line check: position 0 or previous char is newline.
-    // (With @skip { spaces }, @skip only removes horizontal whitespace, so
-    // newlines remain visible and input.peek(-1) reliably finds them.)
+    // Only fire at the start of a line.
     if (input.pos > 0 && input.peek(-1) !== NEWLINE) return
 
     if (input.next !== EQUALS) return
 
-    // Consume one or more '=' heading level markers.
+    // Require one or more '=' heading markers.
     while (input.next === EQUALS) input.advance()
 
-    // Must be immediately followed by whitespace.
+    // Must be followed by whitespace.
     if (input.next !== SPACE && input.next !== TAB) return
 
-    // Consume the whitespace — it's part of the marker.
+    // Consume the whitespace.
     while (input.next === SPACE || input.next === TAB) input.advance()
 
-    input.acceptToken(HeadingMark)
+    // Consume the title text to end of line (stop before line comment).
+    while (input.next !== -1 && input.next !== NEWLINE) {
+      if (input.next === SLASH &&
+          (input.peek(1) === SLASH || input.peek(1) === STAR)) break
+      input.advance()
+    }
+
+    input.acceptToken(HeadingLine)
   },
   { contextual: false }
 )
 
-// ── headingTitleTokenizer ───────────────────────────────────────────────
-// Emits HeadingTitle — the rest of the line after HeadingMark.
-// contextual: true + canShift ensures this only fires when the parser is
-// in the state immediately after accepting HeadingMark, preventing the
-// token from being matched in body-text states due to LALR state merging.
-// HeadingTitle is intentionally absent from styleTags; the ViewPlugin in
-// index.ts decorates heading title text by extending from HeadingMark.to
-// to line end, which means even spurious HeadingTitle tokens (if the
-// contextual guard ever fails) cannot bleed heading style into body text.
-export const headingTitleTokenizer = new ExternalTokenizer(
-  (input, stack) => {
-    if (!stack.canShift(HeadingTitle)) return
-    let hasContent = false
-    while (input.next !== -1 && input.next !== NEWLINE) {
-      input.advance()
-      hasContent = true
-    }
-    if (hasContent) input.acceptToken(HeadingTitle)
-  },
-  { contextual: true }
-)
-
 // ── rawTokenizer ────────────────────────────────────────────────────────
 // Handles all three raw-block tokens (contextual: uses stack.canShift).
 //

services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar

@@ -1,12 +1,14 @@
 // typst.grammar — Lezer LR grammar for the Typst typesetting language.
 // Covers markup mode (top-level), code mode (#expr) and math mode ($...$).
 // External tokenizers handle constructs requiring context-sensitive lexing:
-//   headingTokenizer      — start-of-line detection for heading markers (=+ )
+//   headingLineTokenizer  — entire heading line (=+ prefix + title) as one token
 //   rawTokenizer          — triple-backtick raw block open/body/close
 //   rawInlineTokenizer    — single-backtick raw inline content
 //   codeBlockTokenizer    — brace-depth tracking inside #{ ... }
 //   blockCommentTokenizer — depth-tracked nested /* ... */ comments
-// headingTitleTokenizer — the heading title text (rest of line after HeadingMark)
+// Using one token for the whole line avoids LALR state-merge issues that arose
+// when HeadingTitle (a separate post-HeadingMark token) was accepted by
+// merged body-text parser states, causing body text to be swallowed.
 
 @top Document { item* }
 
@@ -29,11 +31,11 @@ item {
 }
 
 // ── Headings ──────────────────────────────────────────────────────────────
-// HeadingMark is produced by an external tokenizer that enforces the
-// start-of-line constraint and captures the "=+" prefix + trailing space.
-// HeadingTitle is an external token that reads the rest of the line to EOL,
-// avoiding LALR(1) conflicts with document-level items.
-Heading { HeadingMark HeadingTitle? }
+// HeadingLine covers the entire heading line: the "=+" prefix, trailing
+// space, and title text.  One token per line means there is no LALR state
+// that "waits" for a second heading token, so no merged body-text state
+// can accidentally accept HeadingLine.
+Heading { HeadingLine }
 
 // ── Comments ──────────────────────────────────────────────────────────────
 LineComment { "//" LineCommentContent? }
@@ -123,12 +125,8 @@ Ref   { "@" RefName  }
 Escape { "\\" EscapeChar }
 
 // ── External tokenizer declarations ──────────────────────────────────────
-@external tokens headingTokenizer from "./tokens.mjs" {
-  HeadingMark
-}
-
-@external tokens headingTitleTokenizer from "./tokens.mjs" {
-  HeadingTitle
+@external tokens headingLineTokenizer from "./tokens.mjs" {
+  HeadingLine
 }
 
 @external tokens rawTokenizer from "./tokens.mjs" {