feat(editor): native Lezer grammar for Typst syntax highlighting
Build and Deploy Verso / deploy (push) Has been cancelled

Replace the StreamLanguage tokenizer with a full LR grammar compiled by
@lezer/generator, giving Typst the same parse-tree infrastructure that
LaTeX and BibTeX already use.

Grammar features:
- Headings (=, ==, …) via SOL-detecting external tokenizer
- Code expressions (#keyword, #func(args), #ident.method, #{…}, #[…])
- Named argument highlighting (key: value in function calls)
- Inline and display math ($…$)
- Strong (*…*) and emphasis (_…_) with bold/italic formatting
- Raw blocks (```lang…```) and inline raw (`…`)
- Nested block comments (/* /* */ */) via depth-tracking external tokenizer
- Labels (<name>) and references (@name)
- Backslash escapes

Infrastructure changes:
- lezer-typst/typst.grammar — new Lezer grammar
- lezer-typst/tokens.mjs — external tokenizers for context-sensitive lexing
- scripts/lezer-latex/generate.mjs — Typst added to grammars array so the
  existing lezer-latex:generate script (and Dockerfile step) compile it
- .gitignore — generated typst.mjs / typst.terms.mjs excluded from git

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
claude
2026-06-07 19:49:17 +00:00
co-authored by Claude Sonnet 4.6
parent 2c0f387cef
commit 045d458875
6 changed files with 468 additions and 148 deletions
+2
View File
@@ -31,6 +31,8 @@ frontend/js/features/source-editor/lezer-latex/latex.mjs
frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
frontend/js/features/source-editor/lezer-typst/typst.mjs
frontend/js/features/source-editor/lezer-typst/typst.terms.mjs
!**/fixtures/**/*.log
@@ -51,9 +51,8 @@ function computeOutline(
return { items, status: ProjectionStatus.Complete }
}
// The Typst language uses a StreamLanguage, whose syntax tree has no structural
// heading nodes to project from, so we scan the document text directly. Typst
// files are small, so a full rescan on each edit is cheap.
// Scan document text for headings rather than traversing the syntax tree,
// since a full text scan is simpler and cheap for typical Typst file sizes.
export const typstDocumentOutline = StateField.define<
ProjectionResult<FlatOutlineItem>
>({
@@ -1,179 +1,105 @@
import {
StreamLanguage,
StreamParser,
LRLanguage,
LanguageSupport,
foldNodeProp,
foldInside,
HighlightStyle,
syntaxHighlighting,
} from '@codemirror/language'
import { tags as t } from '@lezer/highlight'
import { styleTags, tags as t } from '@lezer/highlight'
import { parser } from '../../lezer-typst/typst.mjs'
import { typstCompletions } from './complete'
import { typstDocumentOutline } from './document-outline'
const keywords = new Set([
'let',
'set',
'show',
'import',
'include',
'if',
'else',
'for',
'while',
'return',
'break',
'continue',
'in',
'as',
'and',
'or',
'not',
'context',
])
// Note on tree structure: rules starting with a lowercase letter in the grammar
// are inline (no tree node), so their children are promoted to the parent.
// E.g. codeArgItem, codeValue, callSuffix, codeArgList are all inline.
// Therefore:
// - The named-argument key "CodeIdent" is a *direct* child of CodeArgs.
// - Positional arguments that are identifiers are wrapped in CallExpr.
const atoms = new Set(['none', 'auto', 'true', 'false'])
export const TypstLanguage = LRLanguage.define({
name: 'typst',
parser: parser.configure({
props: [
foldNodeProp.add({
RawBlock: foldInside,
BlockComment: foldInside,
CodeBlock: foldInside,
ContentBlock: foldInside,
CodeArgs: foldInside,
}),
styleTags({
// Headings
'HeadingMark HeadingText': t.heading,
type TypstState = {
inBlockComment: boolean
inHeading: boolean
}
// Comments
'LineComment LineCommentContent': t.comment,
'BlockComment BlockCommentBody': t.comment,
'"/*" "*/"': t.comment,
// A lightweight stream tokenizer for Typst. It is not a full grammar, but it
// gives sensible highlighting for the common constructs: comments, strings,
// headings, code (#... functions and keywords), math delimiters, labels,
// references, numbers with units and markup bold/italic markers. Token names
// are mapped to standard highlight tags via `tokenTable`, so the editor's
// global class highlighter themes them automatically (light + dark).
const parser: StreamParser<TypstState> = {
startState() {
return { inBlockComment: false, inHeading: false }
},
// Raw content
'RawBlockOpen RawBlockBody RawBlockClose': t.monospace,
'RawInlineContent': t.monospace,
'RawInline/"`"': t.monospace,
token(stream, state) {
if (state.inBlockComment) {
if (stream.match(/.*?\*\//)) {
state.inBlockComment = false
} else {
stream.skipToEnd()
}
return 'comment'
}
// The '#' sigil that enters code mode
'CodeExpr/"#"': t.processingInstruction,
// Color the heading title text (everything after the '= ' prefix)
if (state.inHeading) {
stream.skipToEnd()
state.inHeading = false
return 'heading'
}
// Code keywords and boolean/null literals
CodeKeyword: t.keyword,
CodeBool: t.atom,
if (stream.eatSpace()) {
return null
}
// Identifiers:
// - direct child of CallExpr → function/method name
// - direct child of CodeArgs → named argument key (key: value syntax)
// - everywhere else → plain variable
'CallExpr/CodeIdent': t.function(t.variableName),
'CodeArgs/CodeIdent': t.attributeName,
CodeIdent: t.variableName,
// Comments
if (stream.match('/*')) {
state.inBlockComment = true
return 'comment'
}
if (stream.match('//')) {
stream.skipToEnd()
return 'comment'
}
// Literals in code mode
CodeString: t.string,
CodeNumber: t.number,
// Strings
if (stream.match(/"(?:[^"\\]|\\.)*"/)) {
return 'string'
}
// Code punctuation
'CodeBlock/"{"': t.brace,
'CodeBlock/"}"': t.brace,
'ContentBlock/"["': t.squareBracket,
'ContentBlock/"]"': t.squareBracket,
'CodeArgs/"(" CodeArgs/")"': t.paren,
// Headings: one or more '=' at the start of a line followed by a space.
// Set inHeading so the title text on the same line is also colored.
if (stream.sol() && stream.match(/=+\s/)) {
state.inHeading = true
return 'heading'
}
// Math ($...$) — coloured like a string in most themes
'InlineMath/"$"': t.string,
MathContent: t.string,
// Labels <name> and references @name
if (stream.match(/<[\w-]+>/)) {
return 'label'
}
if (stream.match(/@[\w-]+/)) {
return 'ref'
}
// Markup emphasis
'Strong/"*" Strong/StrongText': t.strong,
'Emphasis/"_" Emphasis/EmphText': t.emphasis,
// Math delimiter
if (stream.eat('$')) {
return 'operator'
}
// Code mode: '#' introduces a function call or keyword
if (stream.match(/#[A-Za-z_][\w-]*/)) {
const word = stream.current().slice(1)
return keywords.has(word) ? 'keyword' : 'function'
}
if (stream.eat('#')) {
return 'operator'
}
// Identifiers, keywords and atoms (inside code blocks/args)
if (stream.match(/[A-Za-z_][\w-]*/)) {
const word = stream.current()
if (keywords.has(word)) {
return 'keyword'
}
if (atoms.has(word)) {
return 'atom'
}
if (stream.peek() === '(') {
return 'function'
}
return null
}
// Numbers, optionally with a unit
if (stream.match(/\d+(?:\.\d+)?(?:pt|mm|cm|in|em|fr|deg|rad|%)?/)) {
return 'number'
}
// Markup bold (*) and italic (_) — mapped to strong/emphasis rather than
// the generic operator tag so themes and typstHighlightStyle can style them.
if (stream.eat('*')) return 'strong'
if (stream.eat('_')) return 'emphasis'
stream.next()
return null
},
tokenTable: {
comment: t.comment,
string: t.string,
keyword: t.keyword,
number: t.number,
function: t.function(t.variableName),
heading: t.heading,
label: t.labelName,
ref: t.labelName,
operator: t.operator,
atom: t.atom,
strong: t.strong,
emphasis: t.emphasis,
},
// Labels (<name>) and references (@name)
'Label/"<" Label/">" Label/LabelName': t.labelName,
'Ref/"@" Ref/RefName': t.labelName,
// Backslash escapes
Escape: t.escape,
}),
],
}),
languageData: {
commentTokens: { line: '//', block: { open: '/*', close: '*/' } },
closeBrackets: { brackets: ['(', '[', '{', '"', '$'] },
},
}
})
// Provides formatting (bold/italic) for key structural tokens even when the
// active theme doesn't define a colour for .tok-heading / .tok-strong etc.
// Mirrors the approach used for Markdown.
// Provide bold/italic formatting for structural tokens even when the active
// theme doesn't define a colour for every highlight class.
const typstHighlightStyle = HighlightStyle.define([
{ tag: t.heading, fontWeight: 'bold' },
{ tag: t.strong, fontWeight: 'bold' },
{ tag: t.emphasis, fontStyle: 'italic' },
])
export const TypstLanguage = StreamLanguage.define(parser)
export const typst = () => {
return new LanguageSupport(TypstLanguage, [
TypstLanguage.data.of({ autocomplete: typstCompletions }),
@@ -0,0 +1,175 @@
/* Hand-written external tokenizers for the Typst Lezer grammar. */
import { ExternalTokenizer } from '@lezer/lr'
import {
HeadingMark,
RawBlockOpen,
RawBlockBody,
RawBlockClose,
RawInlineContent,
CodeBlockBody,
BlockCommentBody,
} from './typst.terms.mjs'
const BACKTICK = 96 // `
const SLASH = 47 // /
const STAR = 42 // *
const NEWLINE = 10 // \n
const EQUALS = 61 // =
const SPACE = 32 //
const TAB = 9 // \t
const OPEN_BRACE = 123 // {
const CLOSE_BRACE = 125 // }
// ── headingTokenizer ────────────────────────────────────────────────────
// Emits HeadingMark when '=+' appears at the start of a line, followed by
// at least one space. The emitted token covers the '=' chars + the space(s).
export const headingTokenizer = new ExternalTokenizer(
(input, _stack) => {
// Start-of-line check: position 0 or previous char is newline.
// (With @skip { spaces }, @skip only removes horizontal whitespace, so
// newlines remain visible and input.peek(-1) reliably finds them.)
if (input.pos > 0 && input.peek(-1) !== NEWLINE) return
if (input.next !== EQUALS) return
// Consume one or more '=' heading level markers.
while (input.next === EQUALS) input.advance()
// Must be immediately followed by whitespace.
if (input.next !== SPACE && input.next !== TAB) return
// Consume the whitespace — it's part of the marker.
while (input.next === SPACE || input.next === TAB) input.advance()
input.acceptToken(HeadingMark)
},
{ contextual: false }
)
// ── rawTokenizer ────────────────────────────────────────────────────────
// Handles all three raw-block tokens (contextual: uses stack.canShift).
//
// RawBlockOpen — ``` + optional lang tag + rest of opening line (incl. \n)
// RawBlockBody — content between the two ``` fences
// RawBlockClose — closing ```
export const rawTokenizer = new ExternalTokenizer(
(input, stack) => {
if (input.next === BACKTICK &&
input.peek(1) === BACKTICK &&
input.peek(2) === BACKTICK) {
if (stack.canShift(RawBlockOpen)) {
// Consume the opening ```.
input.advance(); input.advance(); input.advance()
// Consume an optional language tag (letters and digits).
while (
(input.next >= 65 && input.next <= 90) || // AZ
(input.next >= 97 && input.next <= 122) || // az
(input.next >= 48 && input.next <= 57) // 09
) {
input.advance()
}
// Consume the rest of the opening line (and the newline itself).
while (input.next !== -1 && input.next !== NEWLINE) input.advance()
if (input.next === NEWLINE) input.advance()
return input.acceptToken(RawBlockOpen)
}
if (stack.canShift(RawBlockClose)) {
input.advance(); input.advance(); input.advance()
return input.acceptToken(RawBlockClose)
}
}
if (stack.canShift(RawBlockBody)) {
let hasContent = false
while (input.next !== -1) {
if (
input.next === BACKTICK &&
input.peek(1) === BACKTICK &&
input.peek(2) === BACKTICK
) break
input.advance()
hasContent = true
}
if (hasContent) return input.acceptToken(RawBlockBody)
}
},
{ contextual: true }
)
// ── rawInlineTokenizer ──────────────────────────────────────────────────
// Emits RawInlineContent — everything inside backtick-delimited inline raw
// that is not a backtick or newline.
export const rawInlineTokenizer = new ExternalTokenizer(
(input, _stack) => {
let hasContent = false
while (input.next !== -1 && input.next !== BACKTICK && input.next !== NEWLINE) {
input.advance()
hasContent = true
}
if (hasContent) input.acceptToken(RawInlineContent)
},
{ contextual: false }
)
// ── codeBlockTokenizer ──────────────────────────────────────────────────
// Emits CodeBlockBody — the interior of a #{ ... } code block.
// Tracks brace nesting depth so that inner braces (e.g. #{ f({ x }) })
// are included in the body rather than closing the outer block.
export const codeBlockTokenizer = new ExternalTokenizer(
(input, _stack) => {
// The opening '{' has already been consumed by the grammar rule.
let depth = 1
let hasContent = false
while (input.next !== -1) {
const ch = input.next
if (ch === OPEN_BRACE) {
depth++
input.advance()
hasContent = true
} else if (ch === CLOSE_BRACE) {
if (depth === 1) break // leave this '}' for the grammar rule
depth--
input.advance()
hasContent = true
} else {
input.advance()
hasContent = true
}
}
if (hasContent) input.acceptToken(CodeBlockBody)
},
{ contextual: false }
)
// ── blockCommentTokenizer ───────────────────────────────────────────────
// Emits BlockCommentBody — the interior of a /* ... */ comment.
// Typst supports nested block comments (/* /* inner */ outer */), so this
// tokenizer tracks depth rather than stopping at the first */.
export const blockCommentTokenizer = new ExternalTokenizer(
(input, _stack) => {
// The opening '/*' has already been consumed by the grammar rule.
let depth = 1
let hasContent = false
while (input.next !== -1) {
if (input.next === SLASH && input.peek(1) === STAR) {
// Nested opening /*
depth++
input.advance(); input.advance()
hasContent = true
} else if (input.next === STAR && input.peek(1) === SLASH) {
depth--
if (depth === 0) break // stop before the closing */ (grammar consumes it)
// Inner closing */ — consume it as part of the body.
input.advance(); input.advance()
hasContent = true
} else {
input.advance()
hasContent = true
}
}
if (hasContent) input.acceptToken(BlockCommentBody)
},
{ contextual: false }
)
@@ -0,0 +1,204 @@
// typst.grammar — Lezer LR grammar for the Typst typesetting language.
// Covers markup mode (top-level), code mode (#expr) and math mode ($...$).
// External tokenizers handle constructs requiring context-sensitive lexing:
// headingTokenizer — start-of-line detection for heading markers
// rawTokenizer — triple-backtick raw block open/body/close
// rawInlineTokenizer — single-backtick raw inline content
// codeBlockTokenizer — brace-depth tracking inside #{ ... }
// blockCommentTokenizer — depth-tracked nested /* ... */ comments
@top Document { item* }
// ── Markup-mode items ─────────────────────────────────────────────────────
item {
Heading |
LineComment |
BlockComment |
RawBlock |
RawInline |
CodeExpr |
InlineMath |
Strong |
Emphasis |
Label |
Ref |
Escape |
Newline |
MarkupContent
}
// ── Headings ──────────────────────────────────────────────────────────────
// HeadingMark is produced by an external tokenizer that enforces the
// start-of-line constraint and captures the "=+" prefix + trailing space.
Heading { HeadingMark HeadingTitle }
HeadingTitle { headingTitleItem* }
headingTitleItem {
Strong | Emphasis | CodeExpr | InlineMath | RawInline | Label | Ref | HeadingText
}
// ── Comments ──────────────────────────────────────────────────────────────
LineComment { "//" LineCommentContent }
// BlockCommentBody is external so it can track nesting depth —
// Typst supports /* /* nested */ */ block comments.
BlockComment { "/*" BlockCommentBody? "*/" }
// ── Raw content ───────────────────────────────────────────────────────────
// Raw blocks: ```[lang]\n...\n``` — open/body/close are all external tokens.
RawBlock { RawBlockOpen RawBlockBody? RawBlockClose }
// Raw inline: `...` on one line.
RawInline { "`" RawInlineContent? "`" }
// ── Code expressions ─────────────────────────────────────────────────────
// '#' enters code mode. Forms:
// #keyword — keyword expression (let, set, show, if, …)
// #bool — boolean/null literal (true, false, none, auto)
// #ident — variable reference
// #ident(args) — function call, chainable with '.' and '()'
// #{ ... } — code block (content is depth-tracked by external token)
// #[ ... ] — content block (re-parses as markup items)
CodeExpr { "#" codeExprBody }
codeExprBody {
KeywordExpr |
AtomExpr |
CallExpr |
CodeBlock |
ContentBlock
}
KeywordExpr { CodeKeyword }
AtomExpr { CodeBool }
CallExpr { CodeIdent callSuffix* }
callSuffix {
CodeArgs |
"." CodeIdent
}
CodeArgs { "(" codeArgList? ")" }
codeArgList { codeArgItem ("," codeArgItem)* ","? }
codeArgItem {
CodeIdent ":" codeValue |
codeValue
}
codeValue {
CodeString |
CodeNumber |
CodeBool |
CallExpr |
ContentBlock |
CodeBlock |
InlineMath
}
// CodeBlockBody depth-tracks braces so #{ let x = { 1 } } parses correctly.
CodeBlock { "{" CodeBlockBody? "}" }
// ContentBlock re-enters markup mode, allowing #[*bold* text].
ContentBlock { "[" item* "]" }
// ── Math ──────────────────────────────────────────────────────────────────
// Both inline ($x^2$) and display ($ x^2 $) math use the same node type.
InlineMath { "$" MathContent? "$" }
// ── Markup formatting ─────────────────────────────────────────────────────
Strong { "*" strongItem* "*" }
strongItem { Emphasis | CodeExpr | InlineMath | RawInline | Label | Ref | StrongText }
Emphasis { "_" emphItem* "_" }
emphItem { Strong | CodeExpr | InlineMath | RawInline | Label | Ref | EmphText }
// ── Labels and references ─────────────────────────────────────────────────
Label { "<" LabelName ">" }
Ref { "@" RefName }
// ── Escapes ───────────────────────────────────────────────────────────────
Escape { "\\" EscapeChar }
// ── External tokenizer declarations ──────────────────────────────────────
@external tokens headingTokenizer from "./tokens.mjs" {
HeadingMark
}
@external tokens rawTokenizer from "./tokens.mjs" {
RawBlockOpen,
RawBlockBody,
RawBlockClose
}
@external tokens rawInlineTokenizer from "./tokens.mjs" {
RawInlineContent
}
@external tokens codeBlockTokenizer from "./tokens.mjs" {
CodeBlockBody
}
@external tokens blockCommentTokenizer from "./tokens.mjs" {
BlockCommentBody
}
// ── Regular tokens ────────────────────────────────────────────────────────
@tokens {
// Horizontal whitespace only. Newlines are kept as explicit Newline items
// so that HeadingMark (which checks start-of-line via input.peek(-1)) can
// reliably detect newlines in the raw input stream.
spaces { $[ \t]+ }
// Keywords take precedence over identifiers when they match fully
// (e.g. "let" → CodeKeyword, "letter" → CodeIdent).
CodeKeyword {
"let" | "set" | "show" | "import" | "include" |
"if" | "else" | "for" | "while" | "return" |
"break" | "continue" | "in" | "as" |
"and" | "or" | "not" | "context"
}
// Boolean / null literals — distinct from keywords for highlighting.
CodeBool { "true" | "false" | "none" | "auto" }
// General identifier: [A-Za-z_][A-Za-z0-9_-]*
CodeIdent { identHead identTail* }
identHead { @asciiLetter | "_" }
identTail { @asciiLetter | @digit | "_" | "-" }
// Double-quoted string with backslash escapes (no single-quoted strings in Typst).
CodeString { '"' (!["\\\n] | "\\" _)* '"' }
// Number literal with optional unit suffix.
CodeNumber {
@digit+ ("." @digit+)?
("pt" | "mm" | "cm" | "in" | "em" | "rem" | "fr" | "deg" | "rad" | "%")?
}
// Comment content — everything to end of line.
LineCommentContent { ![\n]* }
// Math content — everything between the $ delimiters (no crossing newlines).
MathContent { ![$\n]+ }
// Text tokens for different markup contexts; each excludes its own delimiters.
HeadingText { ![\n*_$#`<@\\]+ }
StrongText { ![\n*$#`@\\]+ }
EmphText { ![\n_$#`@\\]+ }
// Regular markup: excludes all special-character starters plus whitespace
// (whitespace is handled by @skip). The '/' is excluded so that '//' and
// '/*' are not accidentally consumed as plain text.
MarkupContent { ![\n \t=*_$#/<@`\\]+ }
// Label names: identifiers with optional dots/colons (e.g. <sec:intro>).
LabelName { (identHead | @digit) (identTail | "." | ":")* }
RefName { identHead identTail* }
// Escape: any single character after backslash.
EscapeChar { _ }
// Newline item — kept out of @skip so heading detection works.
Newline { "\n" }
// Resolve ambiguities: longer/more-specific tokens win.
@precedence { CodeKeyword CodeBool CodeIdent }
}
@skip { spaces }
@@ -33,6 +33,20 @@ const grammars = [
'../../frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs'
),
},
{
grammarPath: path.resolve(
import.meta.dirname,
'../../frontend/js/features/source-editor/lezer-typst/typst.grammar'
),
parserOutputPath: path.resolve(
import.meta.dirname,
'../../frontend/js/features/source-editor/lezer-typst/typst.mjs'
),
termsOutputPath: path.resolve(
import.meta.dirname,
'../../frontend/js/features/source-editor/lezer-typst/typst.terms.mjs'
),
},
]
function compile(grammar) {