From 34025dc084143e2d4ff170174a86090641b66fdf Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 8 Jun 2026 16:55:09 +0000 Subject: [PATCH] fix(typst): use HeadingLine single token to fix inline element highlighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-token approach (HeadingMark + HeadingTitle) caused LALR state merging: the parser state waiting for HeadingTitle after HeadingMark was merged into body-text item* states. In those merged states the headingTitleTokenizer fired for every paragraph line, swallowing bold, italic, math and inline function tokens — leaving body text black. Fix: collapse the heading into a single HeadingLine external token that covers the entire heading line (= prefix + title). A single-token Heading rule leaves no post-token parser state waiting for a second token, so no LALR merging can occur. The ViewPlugin and all HeadingMark/HeadingTitle infrastructure are removed. Co-Authored-By: Claude Sonnet 4.6 --- .../source-editor/languages/typst/index.ts | 54 +---------------- .../source-editor/lezer-typst/tokens.mjs | 59 ++++++++----------- .../source-editor/lezer-typst/typst.grammar | 24 ++++---- 3 files changed, 37 insertions(+), 100 deletions(-) diff --git a/services/web/frontend/js/features/source-editor/languages/typst/index.ts b/services/web/frontend/js/features/source-editor/languages/typst/index.ts index 86b0bd35e1..a9da188e62 100644 --- a/services/web/frontend/js/features/source-editor/languages/typst/index.ts +++ b/services/web/frontend/js/features/source-editor/languages/typst/index.ts @@ -5,17 +5,8 @@ import { foldInside, HighlightStyle, syntaxHighlighting, - syntaxTree, } from '@codemirror/language' import { styleTags, tags as t } from '@lezer/highlight' -import { RangeSetBuilder } from '@codemirror/state' -import { - Decoration, - DecorationSet, - EditorView, - ViewPlugin, - ViewUpdate, -} from '@codemirror/view' import { parser } from '../../lezer-typst/typst.mjs' import { typstCompletions } from './complete' import { typstDocumentOutline } from './document-outline' @@ -39,9 +30,8 @@ export const TypstLanguage = LRLanguage.define({ CodeArgs: foldInside, }), styleTags({ - // HeadingMark covers the "=+ " prefix; heading line content is handled - // by the headingLinePlugin ViewPlugin below (not a grammar token). - HeadingMark: t.heading, + // HeadingLine is the entire heading line (prefix + title) as one token. + HeadingLine: t.heading, // Comments 'LineComment LineCommentContent': t.comment, @@ -110,50 +100,10 @@ const typstHighlightStyle = HighlightStyle.define([ { tag: t.emphasis, fontStyle: 'italic' }, ]) -// Heading title decoration: HeadingTitle exists in the grammar but is NOT in -// styleTags, so it never applies heading colour on its own. This plugin walks -// the syntax tree, finds each HeadingMark, and extends a heading-style -// decoration to end-of-line. Belt-and-suspenders: even if HeadingTitle fires -// spuriously (LALR state merging), no bleed happens because styleTags ignores -// it; and the ViewPlugin only decorates lines that start with a HeadingMark. -const headingTitleMark = Decoration.mark({ - class: 'tok-heading', - attributes: { style: 'font-weight:bold' }, -}) - -function buildHeadingDecorations(view: EditorView): DecorationSet { - const builder = new RangeSetBuilder() - syntaxTree(view.state).cursor().iterate(node => { - if (node.name === 'HeadingMark') { - const line = view.state.doc.lineAt(node.from) - if (node.to < line.to) { - builder.add(node.to, line.to, headingTitleMark) - } - } - }) - return builder.finish() -} - -const headingLinePlugin = ViewPlugin.fromClass( - class { - decorations: DecorationSet - constructor(view: EditorView) { - this.decorations = buildHeadingDecorations(view) - } - update(update: ViewUpdate) { - if (update.docChanged || update.viewportChanged) { - this.decorations = buildHeadingDecorations(update.view) - } - } - }, - { decorations: v => v.decorations } -) - export const typst = () => { return new LanguageSupport(TypstLanguage, [ TypstLanguage.data.of({ autocomplete: typstCompletions }), typstDocumentOutline, syntaxHighlighting(typstHighlightStyle), - headingLinePlugin, ]) } diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs index 0f772e348a..652dff574a 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs +++ b/services/web/frontend/js/features/source-editor/lezer-typst/tokens.mjs @@ -2,8 +2,7 @@ import { ExternalTokenizer } from '@lezer/lr' import { - HeadingMark, - HeadingTitle, + HeadingLine, RawBlockOpen, RawBlockBody, RawBlockClose, @@ -25,54 +24,44 @@ const DOLLAR = 36 // $ const OPEN_BRACE = 123 // { const CLOSE_BRACE = 125 // } -// ── headingTokenizer ──────────────────────────────────────────────────── -// Emits HeadingMark when '=+' appears at the start of a line, followed by -// at least one space. The emitted token covers the '=' chars + the space(s). -export const headingTokenizer = new ExternalTokenizer( +// ── headingLineTokenizer ──────────────────────────────────────────────── +// Emits HeadingLine — the entire heading line: "=+" markers, trailing +// space, and title text, all as one token. +// +// Using a single token per heading line eliminates the two-step +// (HeadingMark + HeadingTitle) pattern that caused LALR state-merging +// problems: having a separate HeadingTitle token created a parser state +// that "waited" after HeadingMark, and LALR merged that state into +// body-text item* states. In those merged states headingTitleTokenizer +// fired for every paragraph line, swallowing inline markup tokens. +export const headingLineTokenizer = new ExternalTokenizer( (input, _stack) => { - // Start-of-line check: position 0 or previous char is newline. - // (With @skip { spaces }, @skip only removes horizontal whitespace, so - // newlines remain visible and input.peek(-1) reliably finds them.) + // Only fire at the start of a line. if (input.pos > 0 && input.peek(-1) !== NEWLINE) return if (input.next !== EQUALS) return - // Consume one or more '=' heading level markers. + // Require one or more '=' heading markers. while (input.next === EQUALS) input.advance() - // Must be immediately followed by whitespace. + // Must be followed by whitespace. if (input.next !== SPACE && input.next !== TAB) return - // Consume the whitespace — it's part of the marker. + // Consume the whitespace. while (input.next === SPACE || input.next === TAB) input.advance() - input.acceptToken(HeadingMark) + // Consume the title text to end of line (stop before line comment). + while (input.next !== -1 && input.next !== NEWLINE) { + if (input.next === SLASH && + (input.peek(1) === SLASH || input.peek(1) === STAR)) break + input.advance() + } + + input.acceptToken(HeadingLine) }, { contextual: false } ) -// ── headingTitleTokenizer ─────────────────────────────────────────────── -// Emits HeadingTitle — the rest of the line after HeadingMark. -// contextual: true + canShift ensures this only fires when the parser is -// in the state immediately after accepting HeadingMark, preventing the -// token from being matched in body-text states due to LALR state merging. -// HeadingTitle is intentionally absent from styleTags; the ViewPlugin in -// index.ts decorates heading title text by extending from HeadingMark.to -// to line end, which means even spurious HeadingTitle tokens (if the -// contextual guard ever fails) cannot bleed heading style into body text. -export const headingTitleTokenizer = new ExternalTokenizer( - (input, stack) => { - if (!stack.canShift(HeadingTitle)) return - let hasContent = false - while (input.next !== -1 && input.next !== NEWLINE) { - input.advance() - hasContent = true - } - if (hasContent) input.acceptToken(HeadingTitle) - }, - { contextual: true } -) - // ── rawTokenizer ──────────────────────────────────────────────────────── // Handles all three raw-block tokens (contextual: uses stack.canShift). // diff --git a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar index 56997790d1..f28c78dd26 100644 --- a/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar +++ b/services/web/frontend/js/features/source-editor/lezer-typst/typst.grammar @@ -1,12 +1,14 @@ // typst.grammar — Lezer LR grammar for the Typst typesetting language. // Covers markup mode (top-level), code mode (#expr) and math mode ($...$). // External tokenizers handle constructs requiring context-sensitive lexing: -// headingTokenizer — start-of-line detection for heading markers (=+ ) +// headingLineTokenizer — entire heading line (=+ prefix + title) as one token // rawTokenizer — triple-backtick raw block open/body/close // rawInlineTokenizer — single-backtick raw inline content // codeBlockTokenizer — brace-depth tracking inside #{ ... } // blockCommentTokenizer — depth-tracked nested /* ... */ comments -// headingTitleTokenizer — the heading title text (rest of line after HeadingMark) +// Using one token for the whole line avoids LALR state-merge issues that arose +// when HeadingTitle (a separate post-HeadingMark token) was accepted by +// merged body-text parser states, causing body text to be swallowed. @top Document { item* } @@ -29,11 +31,11 @@ item { } // ── Headings ────────────────────────────────────────────────────────────── -// HeadingMark is produced by an external tokenizer that enforces the -// start-of-line constraint and captures the "=+" prefix + trailing space. -// HeadingTitle is an external token that reads the rest of the line to EOL, -// avoiding LALR(1) conflicts with document-level items. -Heading { HeadingMark HeadingTitle? } +// HeadingLine covers the entire heading line: the "=+" prefix, trailing +// space, and title text. One token per line means there is no LALR state +// that "waits" for a second heading token, so no merged body-text state +// can accidentally accept HeadingLine. +Heading { HeadingLine } // ── Comments ────────────────────────────────────────────────────────────── LineComment { "//" LineCommentContent? } @@ -123,12 +125,8 @@ Ref { "@" RefName } Escape { "\\" EscapeChar } // ── External tokenizer declarations ────────────────────────────────────── -@external tokens headingTokenizer from "./tokens.mjs" { - HeadingMark -} - -@external tokens headingTitleTokenizer from "./tokens.mjs" { - HeadingTitle +@external tokens headingLineTokenizer from "./tokens.mjs" { + HeadingLine } @external tokens rawTokenizer from "./tokens.mjs" {