From 0099672015d782ad44d57a1035cda742803e36b9 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 8 Jun 2026 15:38:38 +0000 Subject: [PATCH] fix(typst): restore HeadingTitle token to fix broken syntax highlighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing HeadingTitle from the grammar left HeadingTitle? undeclared, causing the Lezer grammar compiler to fail and producing no parser output — hence everything rendered as unstyled black text. Dual approach to prevent heading style bleed: - HeadingTitle exists in grammar with contextual: true + canShift guard (prevents it from matching in body-text LALR states) - HeadingTitle is intentionally absent from styleTags so even spurious matches cannot apply heading colour to body text - ViewPlugin styles heading titles by finding HeadingMark nodes and extending tok-heading decoration to end-of-line Co-Authored-By: Claude Sonnet 4.6 --- .../source-editor/languages/typst/index.ts | 12 +++++----- .../source-editor/lezer-typst/tokens.mjs | 23 +++++++++++++++++++ .../source-editor/lezer-typst/typst.grammar | 13 +++++++---- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/services/web/frontend/js/features/source-editor/languages/typst/index.ts b/services/web/frontend/js/features/source-editor/languages/typst/index.ts index dd8fa90cf1..86b0bd35e1 100644 --- a/services/web/frontend/js/features/source-editor/languages/typst/index.ts +++ b/services/web/frontend/js/features/source-editor/languages/typst/index.ts @@ -110,12 +110,12 @@ const typstHighlightStyle = HighlightStyle.define([ { tag: t.emphasis, fontStyle: 'italic' }, ]) -// Heading title decoration: the grammar only has a HeadingMark token (the "=+ " -// prefix). This plugin walks the syntax tree, finds each HeadingMark, and -// extends a heading-style decoration to the end of the line so the title text -// gets the same blue + bold treatment without needing a HeadingTitle token -// (which had LALR state-merging issues causing body text to bleed into heading -// style in the previous implementation). +// Heading title decoration: HeadingTitle exists in the grammar but is NOT in +// styleTags, so it never applies heading colour on its own. This plugin walks +// the syntax tree, finds each HeadingMark, and extends a heading-style +// decoration to end-of-line. Belt-and-suspenders: even if HeadingTitle fires +// spuriously (LALR state merging), no bleed happens because styleTags ignores +// it; and the ViewPlugin only decorates lines that start with a HeadingMark. const headingTitleMark = Decoration.mark({ class: 'tok-heading', attributes: { style: 'font-weight:bold' }, diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs index 57f1cfa2ec..0f772e348a 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs +++ b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs @@ -3,6 +3,7 @@ import { ExternalTokenizer } from '@lezer/lr' import { HeadingMark, + HeadingTitle, RawBlockOpen, RawBlockBody, RawBlockClose, @@ -50,6 +51,28 @@ export const headingTokenizer = new ExternalTokenizer( { contextual: false } ) +// ── headingTitleTokenizer ─────────────────────────────────────────────── +// Emits HeadingTitle — the rest of the line after HeadingMark. +// contextual: true + canShift ensures this only fires when the parser is +// in the state immediately after accepting HeadingMark, preventing the +// token from being matched in body-text states due to LALR state merging. +// HeadingTitle is intentionally absent from styleTags; the ViewPlugin in +// index.ts decorates heading title text by extending from HeadingMark.to +// to line end, which means even spurious HeadingTitle tokens (if the +// contextual guard ever fails) cannot bleed heading style into body text. +export const headingTitleTokenizer = new ExternalTokenizer( + (input, stack) => { + if (!stack.canShift(HeadingTitle)) return + let hasContent = false + while (input.next !== -1 && input.next !== NEWLINE) { + input.advance() + hasContent = true + } + if (hasContent) input.acceptToken(HeadingTitle) + }, + { contextual: true } +) + // ── rawTokenizer ──────────────────────────────────────────────────────── // Handles all three raw-block tokens (contextual: uses stack.canShift). // diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar index 5c01db7a00..56997790d1 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar +++ b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar @@ -6,8 +6,7 @@ // rawInlineTokenizer — single-backtick raw inline content // codeBlockTokenizer — brace-depth tracking inside #{ ... } // blockCommentTokenizer — depth-tracked nested /* ... */ comments -// Note: heading title text is NOT a grammar token. The ViewPlugin in -// languages/typst/index.ts decorates heading lines via the syntax tree. +// headingTitleTokenizer — the heading title text (rest of line after HeadingMark) @top Document { item* } @@ -32,9 +31,9 @@ item { // ── Headings ────────────────────────────────────────────────────────────── // HeadingMark is produced by an external tokenizer that enforces the // start-of-line constraint and captures the "=+" prefix + trailing space. -// The heading title (text after the marker) is NOT a grammar token; a -// ViewPlugin in index.ts decorates the rest of the line via the syntax tree. -Heading { HeadingMark } +// HeadingTitle is an external token that reads the rest of the line to EOL, +// avoiding LALR(1) conflicts with document-level items. +Heading { HeadingMark HeadingTitle? } // ── Comments ────────────────────────────────────────────────────────────── LineComment { "//" LineCommentContent? } @@ -128,6 +127,10 @@ Escape { "\\" EscapeChar } HeadingMark } +@external tokens headingTitleTokenizer from "./tokens.mjs" { + HeadingTitle +} + @external tokens rawTokenizer from "./tokens.mjs" { RawBlockOpen, RawBlockBody,