From 1b773fdda0f3fbc75de48da0c61dec5291d9e541 Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 9 Jun 2026 08:47:47 +0000 Subject: [PATCH] fix(typst): add newlines to @skip so multi-line code args parse cleanly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit '\n' inside CodeArgs was an invalid token, triggering Lezer error recovery and resetting parser state before codeIdentTokenizer could fire. Heading detection is unaffected — headingTokenizer uses raw input.peek(-1) char reads which see the '\n' byte regardless of what @skip consumes. Co-Authored-By: Claude Sonnet 4.6 --- .../source-editor/lezer-typst/typst.grammar | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar index 3473f5006f..b22373b547 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar +++ b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar @@ -27,7 +27,6 @@ item { Label | Ref | Escape | - Newline | MarkupContent } @@ -184,10 +183,12 @@ Escape { "\\" EscapeChar } // ── Regular tokens ──────────────────────────────────────────────────────── @tokens { - // Horizontal whitespace only. Newlines are kept as explicit Newline items - // so that HeadingMark (which checks start-of-line via input.peek(-1)) can - // reliably detect newlines in the raw input stream. - spaces { $[ \t]+ } + // All whitespace including newlines. Heading detection still works because + // headingTokenizer uses input.peek(-1) on the raw character stream — it sees + // the '\n' byte regardless of what @skip consumes at the token level. + // Including '\n' here lets multi-line code expressions (e.g. #figure(\n ...\n)) + // parse without error instead of triggering Lezer error recovery. + spaces { $[ \t\n\r]+ } // Boolean / null literals — distinct from keywords for highlighting. CodeBool { "true" | "false" | "none" | "auto" } @@ -213,9 +214,6 @@ Escape { "\\" EscapeChar } // Escape: any single character after backslash. EscapeChar { _ } - // Newline item — kept out of @skip so heading detection works. - Newline { "\n" } - // Resolve ambiguities in merged states: // EscapeChar > spaces: after '\', EscapeChar must win over the skip token. // "(" > "." > "]": callSuffix delimiters must win over MarkupContent after