claude
045d458875
Build and Deploy Verso / deploy (push) Has been cancelled
Replace the StreamLanguage tokenizer with a full LR grammar compiled by
@lezer/generator, giving Typst the same parse-tree infrastructure that
LaTeX and BibTeX already use.
Grammar features:
- Headings (=, ==, …) via SOL-detecting external tokenizer
- Code expressions (#keyword, #func(args), #ident.method, #{…}, #[…])
- Named argument highlighting (key: value in function calls)
- Inline and display math ($…$)
- Strong (*…*) and emphasis (_…_) with bold/italic formatting
- Raw blocks (```lang…```) and inline raw (`…`)
- Nested block comments (/* /* */ */) via depth-tracking external tokenizer
- Labels (<name>) and references (@name)
- Backslash escapes
Infrastructure changes:
- lezer-typst/typst.grammar — new Lezer grammar
- lezer-typst/tokens.mjs — external tokenizers for context-sensitive lexing
- scripts/lezer-latex/generate.mjs — Typst added to grammars array so the
existing lezer-latex:generate script (and Dockerfile step) compile it
- .gitignore — generated typst.mjs / typst.terms.mjs excluded from git
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
89 lines
2.5 KiB
JavaScript
89 lines
2.5 KiB
JavaScript
/* eslint-disable @overleaf/require-script-runner */
|
|
// This script doesn't work with ScriptRunner because it is run during the build process.
|
|
import { buildParserFile } from '@lezer/generator'
|
|
import { writeFileSync, readFileSync } from 'node:fs'
|
|
import path from 'node:path'
|
|
|
|
const grammars = [
|
|
{
|
|
grammarPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-latex/latex.grammar'
|
|
),
|
|
parserOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
|
|
),
|
|
termsOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs'
|
|
),
|
|
},
|
|
{
|
|
grammarPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar'
|
|
),
|
|
parserOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs'
|
|
),
|
|
termsOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs'
|
|
),
|
|
},
|
|
{
|
|
grammarPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-typst/typst.grammar'
|
|
),
|
|
parserOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-typst/typst.mjs'
|
|
),
|
|
termsOutputPath: path.resolve(
|
|
import.meta.dirname,
|
|
'../../frontend/js/features/source-editor/lezer-typst/typst.terms.mjs'
|
|
),
|
|
},
|
|
]
|
|
|
|
function compile(grammar) {
|
|
const { grammarPath, termsOutputPath, parserOutputPath } = grammar
|
|
const moduleStyle = 'es'
|
|
console.info(`Compiling ${grammarPath}`)
|
|
|
|
const grammarText = readFileSync(grammarPath, 'utf8')
|
|
console.info(`Loaded grammar from ${grammarPath}`)
|
|
|
|
const { parser, terms } = buildParserFile(grammarText, {
|
|
fileName: grammarPath,
|
|
moduleStyle,
|
|
})
|
|
console.info(`Built parser`)
|
|
|
|
writeFileSync(parserOutputPath, parser)
|
|
console.info(`Wrote parser to ${parserOutputPath}`)
|
|
|
|
writeFileSync(termsOutputPath, terms)
|
|
console.info(`Wrote terms to ${termsOutputPath}`)
|
|
|
|
console.info('Done!')
|
|
}
|
|
|
|
export default { compile, grammars }
|
|
|
|
if (
|
|
import.meta.url === process.argv[1] ||
|
|
import.meta.url === `file://${process.argv[1]}`
|
|
) {
|
|
try {
|
|
grammars.forEach(compile)
|
|
process.exit(0)
|
|
} catch (err) {
|
|
console.error(err)
|
|
process.exit(1)
|
|
}
|
|
}
|