feat(editor): native Lezer grammar for Typst syntax highlighting
Build and Deploy Verso / deploy (push) Has been cancelled
Build and Deploy Verso / deploy (push) Has been cancelled
Replace the StreamLanguage tokenizer with a full LR grammar compiled by
@lezer/generator, giving Typst the same parse-tree infrastructure that
LaTeX and BibTeX already use.
Grammar features:
- Headings (=, ==, …) via SOL-detecting external tokenizer
- Code expressions (#keyword, #func(args), #ident.method, #{…}, #[…])
- Named argument highlighting (key: value in function calls)
- Inline and display math ($…$)
- Strong (*…*) and emphasis (_…_) with bold/italic formatting
- Raw blocks (```lang…```) and inline raw (`…`)
- Nested block comments (/* /* */ */) via depth-tracking external tokenizer
- Labels (<name>) and references (@name)
- Backslash escapes
Infrastructure changes:
- lezer-typst/typst.grammar — new Lezer grammar
- lezer-typst/tokens.mjs — external tokenizers for context-sensitive lexing
- scripts/lezer-latex/generate.mjs — Typst added to grammars array so the
existing lezer-latex:generate script (and Dockerfile step) compile it
- .gitignore — generated typst.mjs / typst.terms.mjs excluded from git
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
2c0f387cef
commit
045d458875
@@ -31,6 +31,8 @@ frontend/js/features/source-editor/lezer-latex/latex.mjs
|
||||
frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
|
||||
frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
|
||||
frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
|
||||
frontend/js/features/source-editor/lezer-typst/typst.mjs
|
||||
frontend/js/features/source-editor/lezer-typst/typst.terms.mjs
|
||||
|
||||
!**/fixtures/**/*.log
|
||||
|
||||
|
||||
@@ -51,9 +51,8 @@ function computeOutline(
|
||||
return { items, status: ProjectionStatus.Complete }
|
||||
}
|
||||
|
||||
// The Typst language uses a StreamLanguage, whose syntax tree has no structural
|
||||
// heading nodes to project from, so we scan the document text directly. Typst
|
||||
// files are small, so a full rescan on each edit is cheap.
|
||||
// Scan document text for headings rather than traversing the syntax tree,
|
||||
// since a full text scan is simpler and cheap for typical Typst file sizes.
|
||||
export const typstDocumentOutline = StateField.define<
|
||||
ProjectionResult<FlatOutlineItem>
|
||||
>({
|
||||
|
||||
@@ -1,179 +1,105 @@
|
||||
import {
|
||||
StreamLanguage,
|
||||
StreamParser,
|
||||
LRLanguage,
|
||||
LanguageSupport,
|
||||
foldNodeProp,
|
||||
foldInside,
|
||||
HighlightStyle,
|
||||
syntaxHighlighting,
|
||||
} from '@codemirror/language'
|
||||
import { tags as t } from '@lezer/highlight'
|
||||
import { styleTags, tags as t } from '@lezer/highlight'
|
||||
import { parser } from '../../lezer-typst/typst.mjs'
|
||||
import { typstCompletions } from './complete'
|
||||
import { typstDocumentOutline } from './document-outline'
|
||||
|
||||
const keywords = new Set([
|
||||
'let',
|
||||
'set',
|
||||
'show',
|
||||
'import',
|
||||
'include',
|
||||
'if',
|
||||
'else',
|
||||
'for',
|
||||
'while',
|
||||
'return',
|
||||
'break',
|
||||
'continue',
|
||||
'in',
|
||||
'as',
|
||||
'and',
|
||||
'or',
|
||||
'not',
|
||||
'context',
|
||||
])
|
||||
// Note on tree structure: rules starting with a lowercase letter in the grammar
|
||||
// are inline (no tree node), so their children are promoted to the parent.
|
||||
// E.g. codeArgItem, codeValue, callSuffix, codeArgList are all inline.
|
||||
// Therefore:
|
||||
// - The named-argument key "CodeIdent" is a *direct* child of CodeArgs.
|
||||
// - Positional arguments that are identifiers are wrapped in CallExpr.
|
||||
|
||||
const atoms = new Set(['none', 'auto', 'true', 'false'])
|
||||
export const TypstLanguage = LRLanguage.define({
|
||||
name: 'typst',
|
||||
parser: parser.configure({
|
||||
props: [
|
||||
foldNodeProp.add({
|
||||
RawBlock: foldInside,
|
||||
BlockComment: foldInside,
|
||||
CodeBlock: foldInside,
|
||||
ContentBlock: foldInside,
|
||||
CodeArgs: foldInside,
|
||||
}),
|
||||
styleTags({
|
||||
// Headings
|
||||
'HeadingMark HeadingText': t.heading,
|
||||
|
||||
type TypstState = {
|
||||
inBlockComment: boolean
|
||||
inHeading: boolean
|
||||
}
|
||||
// Comments
|
||||
'LineComment LineCommentContent': t.comment,
|
||||
'BlockComment BlockCommentBody': t.comment,
|
||||
'"/*" "*/"': t.comment,
|
||||
|
||||
// A lightweight stream tokenizer for Typst. It is not a full grammar, but it
|
||||
// gives sensible highlighting for the common constructs: comments, strings,
|
||||
// headings, code (#... functions and keywords), math delimiters, labels,
|
||||
// references, numbers with units and markup bold/italic markers. Token names
|
||||
// are mapped to standard highlight tags via `tokenTable`, so the editor's
|
||||
// global class highlighter themes them automatically (light + dark).
|
||||
const parser: StreamParser<TypstState> = {
|
||||
startState() {
|
||||
return { inBlockComment: false, inHeading: false }
|
||||
},
|
||||
// Raw content
|
||||
'RawBlockOpen RawBlockBody RawBlockClose': t.monospace,
|
||||
'RawInlineContent': t.monospace,
|
||||
'RawInline/"`"': t.monospace,
|
||||
|
||||
token(stream, state) {
|
||||
if (state.inBlockComment) {
|
||||
if (stream.match(/.*?\*\//)) {
|
||||
state.inBlockComment = false
|
||||
} else {
|
||||
stream.skipToEnd()
|
||||
}
|
||||
return 'comment'
|
||||
}
|
||||
// The '#' sigil that enters code mode
|
||||
'CodeExpr/"#"': t.processingInstruction,
|
||||
|
||||
// Color the heading title text (everything after the '= ' prefix)
|
||||
if (state.inHeading) {
|
||||
stream.skipToEnd()
|
||||
state.inHeading = false
|
||||
return 'heading'
|
||||
}
|
||||
// Code keywords and boolean/null literals
|
||||
CodeKeyword: t.keyword,
|
||||
CodeBool: t.atom,
|
||||
|
||||
if (stream.eatSpace()) {
|
||||
return null
|
||||
}
|
||||
// Identifiers:
|
||||
// - direct child of CallExpr → function/method name
|
||||
// - direct child of CodeArgs → named argument key (key: value syntax)
|
||||
// - everywhere else → plain variable
|
||||
'CallExpr/CodeIdent': t.function(t.variableName),
|
||||
'CodeArgs/CodeIdent': t.attributeName,
|
||||
CodeIdent: t.variableName,
|
||||
|
||||
// Comments
|
||||
if (stream.match('/*')) {
|
||||
state.inBlockComment = true
|
||||
return 'comment'
|
||||
}
|
||||
if (stream.match('//')) {
|
||||
stream.skipToEnd()
|
||||
return 'comment'
|
||||
}
|
||||
// Literals in code mode
|
||||
CodeString: t.string,
|
||||
CodeNumber: t.number,
|
||||
|
||||
// Strings
|
||||
if (stream.match(/"(?:[^"\\]|\\.)*"/)) {
|
||||
return 'string'
|
||||
}
|
||||
// Code punctuation
|
||||
'CodeBlock/"{"': t.brace,
|
||||
'CodeBlock/"}"': t.brace,
|
||||
'ContentBlock/"["': t.squareBracket,
|
||||
'ContentBlock/"]"': t.squareBracket,
|
||||
'CodeArgs/"(" CodeArgs/")"': t.paren,
|
||||
|
||||
// Headings: one or more '=' at the start of a line followed by a space.
|
||||
// Set inHeading so the title text on the same line is also colored.
|
||||
if (stream.sol() && stream.match(/=+\s/)) {
|
||||
state.inHeading = true
|
||||
return 'heading'
|
||||
}
|
||||
// Math ($...$) — coloured like a string in most themes
|
||||
'InlineMath/"$"': t.string,
|
||||
MathContent: t.string,
|
||||
|
||||
// Labels <name> and references @name
|
||||
if (stream.match(/<[\w-]+>/)) {
|
||||
return 'label'
|
||||
}
|
||||
if (stream.match(/@[\w-]+/)) {
|
||||
return 'ref'
|
||||
}
|
||||
// Markup emphasis
|
||||
'Strong/"*" Strong/StrongText': t.strong,
|
||||
'Emphasis/"_" Emphasis/EmphText': t.emphasis,
|
||||
|
||||
// Math delimiter
|
||||
if (stream.eat('$')) {
|
||||
return 'operator'
|
||||
}
|
||||
|
||||
// Code mode: '#' introduces a function call or keyword
|
||||
if (stream.match(/#[A-Za-z_][\w-]*/)) {
|
||||
const word = stream.current().slice(1)
|
||||
return keywords.has(word) ? 'keyword' : 'function'
|
||||
}
|
||||
if (stream.eat('#')) {
|
||||
return 'operator'
|
||||
}
|
||||
|
||||
// Identifiers, keywords and atoms (inside code blocks/args)
|
||||
if (stream.match(/[A-Za-z_][\w-]*/)) {
|
||||
const word = stream.current()
|
||||
if (keywords.has(word)) {
|
||||
return 'keyword'
|
||||
}
|
||||
if (atoms.has(word)) {
|
||||
return 'atom'
|
||||
}
|
||||
if (stream.peek() === '(') {
|
||||
return 'function'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
// Numbers, optionally with a unit
|
||||
if (stream.match(/\d+(?:\.\d+)?(?:pt|mm|cm|in|em|fr|deg|rad|%)?/)) {
|
||||
return 'number'
|
||||
}
|
||||
|
||||
// Markup bold (*) and italic (_) — mapped to strong/emphasis rather than
|
||||
// the generic operator tag so themes and typstHighlightStyle can style them.
|
||||
if (stream.eat('*')) return 'strong'
|
||||
if (stream.eat('_')) return 'emphasis'
|
||||
|
||||
stream.next()
|
||||
return null
|
||||
},
|
||||
|
||||
tokenTable: {
|
||||
comment: t.comment,
|
||||
string: t.string,
|
||||
keyword: t.keyword,
|
||||
number: t.number,
|
||||
function: t.function(t.variableName),
|
||||
heading: t.heading,
|
||||
label: t.labelName,
|
||||
ref: t.labelName,
|
||||
operator: t.operator,
|
||||
atom: t.atom,
|
||||
strong: t.strong,
|
||||
emphasis: t.emphasis,
|
||||
},
|
||||
// Labels (<name>) and references (@name)
|
||||
'Label/"<" Label/">" Label/LabelName': t.labelName,
|
||||
'Ref/"@" Ref/RefName': t.labelName,
|
||||
|
||||
// Backslash escapes
|
||||
Escape: t.escape,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
languageData: {
|
||||
commentTokens: { line: '//', block: { open: '/*', close: '*/' } },
|
||||
closeBrackets: { brackets: ['(', '[', '{', '"', '$'] },
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// Provides formatting (bold/italic) for key structural tokens even when the
|
||||
// active theme doesn't define a colour for .tok-heading / .tok-strong etc.
|
||||
// Mirrors the approach used for Markdown.
|
||||
// Provide bold/italic formatting for structural tokens even when the active
|
||||
// theme doesn't define a colour for every highlight class.
|
||||
const typstHighlightStyle = HighlightStyle.define([
|
||||
{ tag: t.heading, fontWeight: 'bold' },
|
||||
{ tag: t.strong, fontWeight: 'bold' },
|
||||
{ tag: t.emphasis, fontStyle: 'italic' },
|
||||
])
|
||||
|
||||
export const TypstLanguage = StreamLanguage.define(parser)
|
||||
|
||||
export const typst = () => {
|
||||
return new LanguageSupport(TypstLanguage, [
|
||||
TypstLanguage.data.of({ autocomplete: typstCompletions }),
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
/* Hand-written external tokenizers for the Typst Lezer grammar. */
|
||||
|
||||
import { ExternalTokenizer } from '@lezer/lr'
|
||||
import {
|
||||
HeadingMark,
|
||||
RawBlockOpen,
|
||||
RawBlockBody,
|
||||
RawBlockClose,
|
||||
RawInlineContent,
|
||||
CodeBlockBody,
|
||||
BlockCommentBody,
|
||||
} from './typst.terms.mjs'
|
||||
|
||||
const BACKTICK = 96 // `
|
||||
const SLASH = 47 // /
|
||||
const STAR = 42 // *
|
||||
const NEWLINE = 10 // \n
|
||||
const EQUALS = 61 // =
|
||||
const SPACE = 32 //
|
||||
const TAB = 9 // \t
|
||||
const OPEN_BRACE = 123 // {
|
||||
const CLOSE_BRACE = 125 // }
|
||||
|
||||
// ── headingTokenizer ────────────────────────────────────────────────────
|
||||
// Emits HeadingMark when '=+' appears at the start of a line, followed by
|
||||
// at least one space. The emitted token covers the '=' chars + the space(s).
|
||||
export const headingTokenizer = new ExternalTokenizer(
|
||||
(input, _stack) => {
|
||||
// Start-of-line check: position 0 or previous char is newline.
|
||||
// (With @skip { spaces }, @skip only removes horizontal whitespace, so
|
||||
// newlines remain visible and input.peek(-1) reliably finds them.)
|
||||
if (input.pos > 0 && input.peek(-1) !== NEWLINE) return
|
||||
|
||||
if (input.next !== EQUALS) return
|
||||
|
||||
// Consume one or more '=' heading level markers.
|
||||
while (input.next === EQUALS) input.advance()
|
||||
|
||||
// Must be immediately followed by whitespace.
|
||||
if (input.next !== SPACE && input.next !== TAB) return
|
||||
|
||||
// Consume the whitespace — it's part of the marker.
|
||||
while (input.next === SPACE || input.next === TAB) input.advance()
|
||||
|
||||
input.acceptToken(HeadingMark)
|
||||
},
|
||||
{ contextual: false }
|
||||
)
|
||||
|
||||
// ── rawTokenizer ────────────────────────────────────────────────────────
|
||||
// Handles all three raw-block tokens (contextual: uses stack.canShift).
|
||||
//
|
||||
// RawBlockOpen — ``` + optional lang tag + rest of opening line (incl. \n)
|
||||
// RawBlockBody — content between the two ``` fences
|
||||
// RawBlockClose — closing ```
|
||||
export const rawTokenizer = new ExternalTokenizer(
|
||||
(input, stack) => {
|
||||
if (input.next === BACKTICK &&
|
||||
input.peek(1) === BACKTICK &&
|
||||
input.peek(2) === BACKTICK) {
|
||||
if (stack.canShift(RawBlockOpen)) {
|
||||
// Consume the opening ```.
|
||||
input.advance(); input.advance(); input.advance()
|
||||
// Consume an optional language tag (letters and digits).
|
||||
while (
|
||||
(input.next >= 65 && input.next <= 90) || // A–Z
|
||||
(input.next >= 97 && input.next <= 122) || // a–z
|
||||
(input.next >= 48 && input.next <= 57) // 0–9
|
||||
) {
|
||||
input.advance()
|
||||
}
|
||||
// Consume the rest of the opening line (and the newline itself).
|
||||
while (input.next !== -1 && input.next !== NEWLINE) input.advance()
|
||||
if (input.next === NEWLINE) input.advance()
|
||||
return input.acceptToken(RawBlockOpen)
|
||||
}
|
||||
|
||||
if (stack.canShift(RawBlockClose)) {
|
||||
input.advance(); input.advance(); input.advance()
|
||||
return input.acceptToken(RawBlockClose)
|
||||
}
|
||||
}
|
||||
|
||||
if (stack.canShift(RawBlockBody)) {
|
||||
let hasContent = false
|
||||
while (input.next !== -1) {
|
||||
if (
|
||||
input.next === BACKTICK &&
|
||||
input.peek(1) === BACKTICK &&
|
||||
input.peek(2) === BACKTICK
|
||||
) break
|
||||
input.advance()
|
||||
hasContent = true
|
||||
}
|
||||
if (hasContent) return input.acceptToken(RawBlockBody)
|
||||
}
|
||||
},
|
||||
{ contextual: true }
|
||||
)
|
||||
|
||||
// ── rawInlineTokenizer ──────────────────────────────────────────────────
|
||||
// Emits RawInlineContent — everything inside backtick-delimited inline raw
|
||||
// that is not a backtick or newline.
|
||||
export const rawInlineTokenizer = new ExternalTokenizer(
|
||||
(input, _stack) => {
|
||||
let hasContent = false
|
||||
while (input.next !== -1 && input.next !== BACKTICK && input.next !== NEWLINE) {
|
||||
input.advance()
|
||||
hasContent = true
|
||||
}
|
||||
if (hasContent) input.acceptToken(RawInlineContent)
|
||||
},
|
||||
{ contextual: false }
|
||||
)
|
||||
|
||||
// ── codeBlockTokenizer ──────────────────────────────────────────────────
|
||||
// Emits CodeBlockBody — the interior of a #{ ... } code block.
|
||||
// Tracks brace nesting depth so that inner braces (e.g. #{ f({ x }) })
|
||||
// are included in the body rather than closing the outer block.
|
||||
export const codeBlockTokenizer = new ExternalTokenizer(
|
||||
(input, _stack) => {
|
||||
// The opening '{' has already been consumed by the grammar rule.
|
||||
let depth = 1
|
||||
let hasContent = false
|
||||
while (input.next !== -1) {
|
||||
const ch = input.next
|
||||
if (ch === OPEN_BRACE) {
|
||||
depth++
|
||||
input.advance()
|
||||
hasContent = true
|
||||
} else if (ch === CLOSE_BRACE) {
|
||||
if (depth === 1) break // leave this '}' for the grammar rule
|
||||
depth--
|
||||
input.advance()
|
||||
hasContent = true
|
||||
} else {
|
||||
input.advance()
|
||||
hasContent = true
|
||||
}
|
||||
}
|
||||
if (hasContent) input.acceptToken(CodeBlockBody)
|
||||
},
|
||||
{ contextual: false }
|
||||
)
|
||||
|
||||
// ── blockCommentTokenizer ───────────────────────────────────────────────
|
||||
// Emits BlockCommentBody — the interior of a /* ... */ comment.
|
||||
// Typst supports nested block comments (/* /* inner */ outer */), so this
|
||||
// tokenizer tracks depth rather than stopping at the first */.
|
||||
export const blockCommentTokenizer = new ExternalTokenizer(
|
||||
(input, _stack) => {
|
||||
// The opening '/*' has already been consumed by the grammar rule.
|
||||
let depth = 1
|
||||
let hasContent = false
|
||||
while (input.next !== -1) {
|
||||
if (input.next === SLASH && input.peek(1) === STAR) {
|
||||
// Nested opening /*
|
||||
depth++
|
||||
input.advance(); input.advance()
|
||||
hasContent = true
|
||||
} else if (input.next === STAR && input.peek(1) === SLASH) {
|
||||
depth--
|
||||
if (depth === 0) break // stop before the closing */ (grammar consumes it)
|
||||
// Inner closing */ — consume it as part of the body.
|
||||
input.advance(); input.advance()
|
||||
hasContent = true
|
||||
} else {
|
||||
input.advance()
|
||||
hasContent = true
|
||||
}
|
||||
}
|
||||
if (hasContent) input.acceptToken(BlockCommentBody)
|
||||
},
|
||||
{ contextual: false }
|
||||
)
|
||||
@@ -0,0 +1,204 @@
|
||||
// typst.grammar — Lezer LR grammar for the Typst typesetting language.
|
||||
// Covers markup mode (top-level), code mode (#expr) and math mode ($...$).
|
||||
// External tokenizers handle constructs requiring context-sensitive lexing:
|
||||
// headingTokenizer — start-of-line detection for heading markers
|
||||
// rawTokenizer — triple-backtick raw block open/body/close
|
||||
// rawInlineTokenizer — single-backtick raw inline content
|
||||
// codeBlockTokenizer — brace-depth tracking inside #{ ... }
|
||||
// blockCommentTokenizer — depth-tracked nested /* ... */ comments
|
||||
|
||||
@top Document { item* }
|
||||
|
||||
// ── Markup-mode items ─────────────────────────────────────────────────────
|
||||
item {
|
||||
Heading |
|
||||
LineComment |
|
||||
BlockComment |
|
||||
RawBlock |
|
||||
RawInline |
|
||||
CodeExpr |
|
||||
InlineMath |
|
||||
Strong |
|
||||
Emphasis |
|
||||
Label |
|
||||
Ref |
|
||||
Escape |
|
||||
Newline |
|
||||
MarkupContent
|
||||
}
|
||||
|
||||
// ── Headings ──────────────────────────────────────────────────────────────
|
||||
// HeadingMark is produced by an external tokenizer that enforces the
|
||||
// start-of-line constraint and captures the "=+" prefix + trailing space.
|
||||
Heading { HeadingMark HeadingTitle }
|
||||
HeadingTitle { headingTitleItem* }
|
||||
headingTitleItem {
|
||||
Strong | Emphasis | CodeExpr | InlineMath | RawInline | Label | Ref | HeadingText
|
||||
}
|
||||
|
||||
// ── Comments ──────────────────────────────────────────────────────────────
|
||||
LineComment { "//" LineCommentContent }
|
||||
// BlockCommentBody is external so it can track nesting depth —
|
||||
// Typst supports /* /* nested */ */ block comments.
|
||||
BlockComment { "/*" BlockCommentBody? "*/" }
|
||||
|
||||
// ── Raw content ───────────────────────────────────────────────────────────
|
||||
// Raw blocks: ```[lang]\n...\n``` — open/body/close are all external tokens.
|
||||
RawBlock { RawBlockOpen RawBlockBody? RawBlockClose }
|
||||
// Raw inline: `...` on one line.
|
||||
RawInline { "`" RawInlineContent? "`" }
|
||||
|
||||
// ── Code expressions ─────────────────────────────────────────────────────
|
||||
// '#' enters code mode. Forms:
|
||||
// #keyword — keyword expression (let, set, show, if, …)
|
||||
// #bool — boolean/null literal (true, false, none, auto)
|
||||
// #ident — variable reference
|
||||
// #ident(args) — function call, chainable with '.' and '()'
|
||||
// #{ ... } — code block (content is depth-tracked by external token)
|
||||
// #[ ... ] — content block (re-parses as markup items)
|
||||
CodeExpr { "#" codeExprBody }
|
||||
|
||||
codeExprBody {
|
||||
KeywordExpr |
|
||||
AtomExpr |
|
||||
CallExpr |
|
||||
CodeBlock |
|
||||
ContentBlock
|
||||
}
|
||||
|
||||
KeywordExpr { CodeKeyword }
|
||||
AtomExpr { CodeBool }
|
||||
|
||||
CallExpr { CodeIdent callSuffix* }
|
||||
callSuffix {
|
||||
CodeArgs |
|
||||
"." CodeIdent
|
||||
}
|
||||
|
||||
CodeArgs { "(" codeArgList? ")" }
|
||||
codeArgList { codeArgItem ("," codeArgItem)* ","? }
|
||||
codeArgItem {
|
||||
CodeIdent ":" codeValue |
|
||||
codeValue
|
||||
}
|
||||
|
||||
codeValue {
|
||||
CodeString |
|
||||
CodeNumber |
|
||||
CodeBool |
|
||||
CallExpr |
|
||||
ContentBlock |
|
||||
CodeBlock |
|
||||
InlineMath
|
||||
}
|
||||
|
||||
// CodeBlockBody depth-tracks braces so #{ let x = { 1 } } parses correctly.
|
||||
CodeBlock { "{" CodeBlockBody? "}" }
|
||||
// ContentBlock re-enters markup mode, allowing #[*bold* text].
|
||||
ContentBlock { "[" item* "]" }
|
||||
|
||||
// ── Math ──────────────────────────────────────────────────────────────────
|
||||
// Both inline ($x^2$) and display ($ x^2 $) math use the same node type.
|
||||
InlineMath { "$" MathContent? "$" }
|
||||
|
||||
// ── Markup formatting ─────────────────────────────────────────────────────
|
||||
Strong { "*" strongItem* "*" }
|
||||
strongItem { Emphasis | CodeExpr | InlineMath | RawInline | Label | Ref | StrongText }
|
||||
|
||||
Emphasis { "_" emphItem* "_" }
|
||||
emphItem { Strong | CodeExpr | InlineMath | RawInline | Label | Ref | EmphText }
|
||||
|
||||
// ── Labels and references ─────────────────────────────────────────────────
|
||||
Label { "<" LabelName ">" }
|
||||
Ref { "@" RefName }
|
||||
|
||||
// ── Escapes ───────────────────────────────────────────────────────────────
|
||||
Escape { "\\" EscapeChar }
|
||||
|
||||
// ── External tokenizer declarations ──────────────────────────────────────
|
||||
@external tokens headingTokenizer from "./tokens.mjs" {
|
||||
HeadingMark
|
||||
}
|
||||
|
||||
@external tokens rawTokenizer from "./tokens.mjs" {
|
||||
RawBlockOpen,
|
||||
RawBlockBody,
|
||||
RawBlockClose
|
||||
}
|
||||
|
||||
@external tokens rawInlineTokenizer from "./tokens.mjs" {
|
||||
RawInlineContent
|
||||
}
|
||||
|
||||
@external tokens codeBlockTokenizer from "./tokens.mjs" {
|
||||
CodeBlockBody
|
||||
}
|
||||
|
||||
@external tokens blockCommentTokenizer from "./tokens.mjs" {
|
||||
BlockCommentBody
|
||||
}
|
||||
|
||||
// ── Regular tokens ────────────────────────────────────────────────────────
|
||||
@tokens {
|
||||
// Horizontal whitespace only. Newlines are kept as explicit Newline items
|
||||
// so that HeadingMark (which checks start-of-line via input.peek(-1)) can
|
||||
// reliably detect newlines in the raw input stream.
|
||||
spaces { $[ \t]+ }
|
||||
|
||||
// Keywords take precedence over identifiers when they match fully
|
||||
// (e.g. "let" → CodeKeyword, "letter" → CodeIdent).
|
||||
CodeKeyword {
|
||||
"let" | "set" | "show" | "import" | "include" |
|
||||
"if" | "else" | "for" | "while" | "return" |
|
||||
"break" | "continue" | "in" | "as" |
|
||||
"and" | "or" | "not" | "context"
|
||||
}
|
||||
|
||||
// Boolean / null literals — distinct from keywords for highlighting.
|
||||
CodeBool { "true" | "false" | "none" | "auto" }
|
||||
|
||||
// General identifier: [A-Za-z_][A-Za-z0-9_-]*
|
||||
CodeIdent { identHead identTail* }
|
||||
identHead { @asciiLetter | "_" }
|
||||
identTail { @asciiLetter | @digit | "_" | "-" }
|
||||
|
||||
// Double-quoted string with backslash escapes (no single-quoted strings in Typst).
|
||||
CodeString { '"' (!["\\\n] | "\\" _)* '"' }
|
||||
|
||||
// Number literal with optional unit suffix.
|
||||
CodeNumber {
|
||||
@digit+ ("." @digit+)?
|
||||
("pt" | "mm" | "cm" | "in" | "em" | "rem" | "fr" | "deg" | "rad" | "%")?
|
||||
}
|
||||
|
||||
// Comment content — everything to end of line.
|
||||
LineCommentContent { ![\n]* }
|
||||
|
||||
// Math content — everything between the $ delimiters (no crossing newlines).
|
||||
MathContent { ![$\n]+ }
|
||||
|
||||
// Text tokens for different markup contexts; each excludes its own delimiters.
|
||||
HeadingText { ![\n*_$#`<@\\]+ }
|
||||
StrongText { ![\n*$#`@\\]+ }
|
||||
EmphText { ![\n_$#`@\\]+ }
|
||||
|
||||
// Regular markup: excludes all special-character starters plus whitespace
|
||||
// (whitespace is handled by @skip). The '/' is excluded so that '//' and
|
||||
// '/*' are not accidentally consumed as plain text.
|
||||
MarkupContent { ![\n \t=*_$#/<@`\\]+ }
|
||||
|
||||
// Label names: identifiers with optional dots/colons (e.g. <sec:intro>).
|
||||
LabelName { (identHead | @digit) (identTail | "." | ":")* }
|
||||
RefName { identHead identTail* }
|
||||
|
||||
// Escape: any single character after backslash.
|
||||
EscapeChar { _ }
|
||||
|
||||
// Newline item — kept out of @skip so heading detection works.
|
||||
Newline { "\n" }
|
||||
|
||||
// Resolve ambiguities: longer/more-specific tokens win.
|
||||
@precedence { CodeKeyword CodeBool CodeIdent }
|
||||
}
|
||||
|
||||
@skip { spaces }
|
||||
@@ -33,6 +33,20 @@ const grammars = [
|
||||
'../../frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs'
|
||||
),
|
||||
},
|
||||
{
|
||||
grammarPath: path.resolve(
|
||||
import.meta.dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-typst/typst.grammar'
|
||||
),
|
||||
parserOutputPath: path.resolve(
|
||||
import.meta.dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-typst/typst.mjs'
|
||||
),
|
||||
termsOutputPath: path.resolve(
|
||||
import.meta.dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-typst/typst.terms.mjs'
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
function compile(grammar) {
|
||||
|
||||
Reference in New Issue
Block a user