From f5a94c0ced1e4671070dac7ec1687e8460694f2c Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 8 Jun 2026 19:44:05 +0000 Subject: [PATCH] fix(typst): guard RawBlockBody against LALR-merged body-text states canShift(RawBlockBody) returns true in states LALR-merged with the post-RawBlockOpen state, causing the tokenizer to consume all remaining body text as one giant RawBlockBody. Add a backward character scan: require newline immediately before input.pos, then walk back past any lang tag (A-Za-z0-9) and verify three backticks precede it. Body-text positions never have backtick-backtick-backtick there, so the guard rejects them. This was the root cause of everything after the first heading being black: RawBlockBody swallowed the entire document from the user-name line onward, making headings, bold, italic and math invisible. Co-Authored-By: Claude Sonnet 4.6 --- .../features/source-editor/lezer-typst/tokens.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs index 557614d5ec..186b9945c4 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs +++ b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs @@ -105,6 +105,20 @@ export const rawTokenizer = new ExternalTokenizer( } if (stack.canShift(RawBlockBody)) { + // Guard: must genuinely follow a RawBlockOpen (which ends with \n). + // Walk backward past any lang tag (A-Za-z0-9) and require ```. + // This blocks spurious LALR-merged states from consuming body text. + if (input.peek(-1) !== NEWLINE) return + let back = -2 + while ( + (input.peek(back) >= 65 && input.peek(back) <= 90) || + (input.peek(back) >= 97 && input.peek(back) <= 122) || + (input.peek(back) >= 48 && input.peek(back) <= 57) + ) { back-- } + if (input.peek(back) !== BACKTICK || + input.peek(back - 1) !== BACKTICK || + input.peek(back - 2) !== BACKTICK) return + let hasContent = false while (input.next !== -1) { if (