From f9d46aabeb2d2afeb1ccfe6021f8c76432f2986b Mon Sep 17 00:00:00 2001
From: claude <claude@verso>
Date: Mon, 8 Jun 2026 21:50:15 +0000
Subject: [PATCH] fix: revert mathContentTokenizer regression (contextual +
 no-newline-stop)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous change switched mathContentTokenizer to contextual:true with no
newline stop, intending to support multi-line Typst block math.  However,
LALR state merging causes canShift(MathContent) to spuriously return true in
body-text positions (e.g. after a RawInline backtick close), so the tokenizer
consumed everything until the next '$' — turning a full paragraph orange.

Revert to contextual:false with newline stop.  This correctly handles both
inline ($x^2$) and single-line block ($ integral ... $) math.  Multi-line
block math ($ formula\n continuation $) remains a separate issue for later.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../source-editor/lezer-typst/tokens.mjs        | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs
index a91dfb01b7..90fb687339 100644
--- a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs
+++ b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs
@@ -238,26 +238,19 @@ export const lineCommentContentTokenizer = new ExternalTokenizer(
 )
 
 // ── mathContentTokenizer ────────────────────────────────────────────────
-// Emits MathContent — everything between the $...$ delimiters.
-//
-// contextual: true — only fires when MathContent is in the valid set, i.e.
-// inside InlineMath → "$" . MathContent? "$".  This prevents the tokenizer
-// from consuming body text in LALR-merged states.
-//
-// No newline restriction: Typst block math spans multiple lines
-// ($ formula \n continuation $), so we must read until the closing '$'
-// regardless of newlines.  The grammar's InlineMath rule handles both inline
-// ($x^2$) and block ($ ... $) forms with the same production.
+// Emits MathContent — everything between the $...$ delimiters (no newlines).
+// External rather than a @tokens rule for the same reason as LineCommentContent:
+// ![$\n]+ overlaps with spaces, '<', '@', and other literals in merged states.
 export const mathContentTokenizer = new ExternalTokenizer(
   (input, _stack) => {
     let hasContent = false
-    while (input.next !== -1 && input.next !== DOLLAR) {
+    while (input.next !== -1 && input.next !== DOLLAR && input.next !== NEWLINE) {
       input.advance()
       hasContent = true
     }
     if (hasContent) input.acceptToken(MathContent)
   },
-  { contextual: true }
+  { contextual: false }
 )
 
 // ── codeKeywordTokenizer ─────────────────────────────────────────────────