diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index df2d7fffbf..745ee426b6 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -386,11 +386,18 @@ async function stopCompile(projectId, userId) { } async function clearProject(projectId, userId) { + // Kill any live typst watcher before deleting its files. + const compileName = getCompileName(projectId, userId) + await TypstRunner.promises.killTypst(compileName) const compileDir = getCompileDir(projectId, userId) await fsPromises.rm(compileDir, { force: true, recursive: true }) } async function clearProjectWithListing(projectId, userId, allEntries) { + // Kill any live typst watcher (e.g. timedout compile where killTypst + // was not already called) before removing files from under it. + const compileName = getCompileName(projectId, userId) + await TypstRunner.promises.killTypst(compileName) const compileDir = getCompileDir(projectId, userId) const exists = await _checkDirectory(compileDir) diff --git a/services/clsi/app/js/TypstRunner.js b/services/clsi/app/js/TypstRunner.js index 15a4b65f57..242d81c1ce 100644 --- a/services/clsi/app/js/TypstRunner.js +++ b/services/clsi/app/js/TypstRunner.js @@ -1,96 +1,373 @@ import Path from 'node:path' +import { spawn } from 'node:child_process' import { promisify } from 'node:util' import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' import CommandRunner from './CommandRunner.js' import fs from 'node:fs' -// Maps currently-running Typst jobs: compileName → PID (or docker container id) +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +// Max lines kept from the current compile cycle (prevents unbounded growth +// for documents that produce many warnings). +const MAX_LOG_LINES = 500 + +// How long to wait for the watcher process to emit its first output. +const WATCH_START_TIMEOUT_MS = 15_000 + +// Matches the three terminal lines that typst watch emits at the end of each +// compile cycle regardless of outcome: +// "[HH:MM:SS] compiled successfully in 42ms" +// "[HH:MM:SS] compiled with warnings in 42ms" +// "[HH:MM:SS] compiled with errors" +const COMPILE_DONE_RE = /compiled (successfully|with (errors|warnings))/ + +// Signals FileId exhaustion in a long-lived typst process (typst issue #7434). +const FILE_ID_EXHAUSTION_RE = /ran out of file ids/i + +// Proactively restart the watcher before FileId exhaustion. +// Typst uses ~65 IDs per compile; 1000 compiles ≈ 65 000 — safely under 65 535. +const MAX_COMPILES_BEFORE_RESTART = 1000 + +// --------------------------------------------------------------------------- +// State (module-level, never exported) +// --------------------------------------------------------------------------- + +// Active cold-start compile jobs (Docker fallback): compileName → PID const ProcessTable = {} -// Compiles a standalone Typst document (.typ) straight to output.pdf using the -// official Typst binary (installed separately — Quarto bundles a modified fork). -// This is deliberately the simplest of the three runners: Typst only ever -// produces a PDF, so there is no format detection, no HTML asset directory -// and no extension merging (cf. QuartoRunner). -function runTypst(compileName, options, callback) { - const { directory, mainFile, image, environment, compileGroup } = options - const timeout = options.timeout || 60000 +// Long-lived watcher processes: compileName → WatchEntry +// WatchEntry shape: +// proc ChildProcess +// directory compile dir (absolute path) +// mainFile root .typ filename +// environment env vars passed to the runner +// compilationCount total successful compile cycles on this watcher +// restartPending flag to restart at the next runTypst call +// accumulator incomplete trailing line from the last data chunk +// currentLines lines accumulated since the last compile-done event +// pendingResolvers Array<{resolve, reject, timeoutHandle}> +const WatchTable = {} - logger.debug( - { directory, timeout, mainFile, compileGroup }, - 'starting typst compile' - ) +// PIDs we have intentionally killed, so the close handler can distinguish +// an expected exit from an unexpected crash. +const _killedWatchPids = new Set() - const command = _buildTypstCommand(mainFile) +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- - ProcessTable[compileName] = CommandRunner.run( - compileName, - command, - directory, - image, - timeout, - environment || {}, - compileGroup, - null, - function (error, output) { - delete ProcessTable[compileName] +async function runTypstAsync(compileName, options) { + // Docker / sandboxed mode: fall back to a cold-start compile per request. + if (Settings.clsi?.dockerRunner) { + return _runColdStart(compileName, options) + } - // Propagate real process-level errors (killed, timed out) but NOT - // ordinary non-zero exit codes from Typst itself. A compile failure - // (exit code 1) is not a server error — the absence of output.pdf is - // enough for CompileController to return 'failure'. - if (error && (error.terminated || error.timedout)) { - return callback(error) - } + const timeout = options.timeout || 60_000 + const entry = WatchTable[compileName] - // On exit-code-1 errors LocalCommandRunner attaches stdout to the error - // object; merge it so _writeLogOutput can persist it. - const combined = output || (error ? { stdout: error.stdout || '' } : null) - _writeLogOutput(compileName, directory, combined, () => - callback(null, combined) - ) + const needsStart = + !entry || + entry.restartPending || + entry.proc.exitCode !== null + + if (needsStart) { + if (entry) _killWatchEntry(compileName) + // _startWatcher spawns the process, registers all handlers, then calls + // _waitForNextCompile synchronously — the resolver is in pendingResolvers + // before any I/O event can fire, eliminating the race condition. + const result = await _startWatcher(compileName, options) + await _writeLogOutputAsync(compileName, options.directory, result) + return result + } + + // Watcher is alive. _waitForNextCompile adds the resolver synchronously + // inside the Promise constructor, before this function yields — safe. + const result = await _waitForNextCompile(compileName, timeout) + await _writeLogOutputAsync(compileName, options.directory, result) + return result +} + +// --------------------------------------------------------------------------- +// Watcher lifecycle +// --------------------------------------------------------------------------- + +async function _startWatcher(compileName, options) { + const { directory, mainFile, environment } = options + const timeout = options.timeout || 60_000 + + const absInput = Path.join(directory, mainFile) + const absOutput = Path.join(directory, 'output.pdf') + const env = { ...process.env, ...(environment || {}) } + + logger.debug({ compileName, absInput }, 'starting typst watcher') + + const proc = spawn( + '/bin/sh', + ['-c', `typst watch "${absInput}" "${absOutput}" 2>&1`], + { + cwd: directory, + env, + stdio: ['ignore', 'pipe', 'ignore'], + detached: true, } ) + + const entry = { + proc, + directory, + mainFile, + environment, + compilationCount: 0, + restartPending: false, + accumulator: '', + currentLines: [], + pendingResolvers: [], + } + WatchTable[compileName] = entry + + // Register handlers synchronously — before any I/O events can fire. + proc.stdout.setEncoding('utf8') + proc.stdout.on('data', chunk => _onWatcherData(compileName, chunk)) + + proc.on('error', err => { + logger.error({ err, compileName }, 'typst watcher process error') + _rejectAllPending( + compileName, + Object.assign(err, { terminated: true }) + ) + if (WatchTable[compileName]?.proc === proc) { + delete WatchTable[compileName] + } + }) + + proc.on('close', (code, signal) => { + logger.warn({ code, signal, compileName }, 'typst watcher exited') + const wasKilled = _killedWatchPids.delete(proc.pid) + if (!wasKilled) { + _rejectAllPending( + compileName, + Object.assign(new Error('typst watcher exited unexpectedly'), { + terminated: true, + }) + ) + } + if (WatchTable[compileName]?.proc === proc) { + delete WatchTable[compileName] + } + }) + + // typst watch performs an initial compile immediately on startup. + // _waitForNextCompile adds the resolver synchronously here (inside the + // Promise constructor) before we yield, so it will catch that first event. + return _waitForNextCompile(compileName, timeout + WATCH_START_TIMEOUT_MS) } -function _buildTypstCommand(mainFile) { - // Run through a POSIX shell so stderr (where Typst writes its diagnostics) - // is merged into stdout (2>&1). LocalCommandRunner replaces $COMPILE_DIR - // before the shell sees it; the output path is relative because the shell - // CWD is already the compile directory. - const inputPath = `$COMPILE_DIR/${mainFile}` - const cmd = `typst compile ${inputPath} output.pdf 2>&1` - return ['/bin/sh', '-c', cmd] +function _killWatchEntry(compileName) { + const entry = WatchTable[compileName] + if (!entry) return + delete WatchTable[compileName] + try { + _killedWatchPids.add(entry.proc.pid) + process.kill(-entry.proc.pid) // kill entire process group + } catch (err) { + _killedWatchPids.delete(entry.proc.pid) + logger.warn({ err, compileName }, 'error killing typst watcher process group') + } } -function _writeLogOutput(compileName, directory, output, callback) { - const content = (output && output.stdout) || '' - if (!content) return callback() - // Write to output.log so the PDF-preview log panel picks it up. Typst's - // `error:`/`warning:` + `┌─ file:line:col` diagnostics are understood by the - // Quarto/Typst log parser on the web side. - const logFile = Path.join(directory, 'output.log') - fs.unlink(logFile, () => { - fs.writeFile(logFile, content, { flag: 'wx' }, err => { - if (err) { - logger.error({ err, compileName, logFile }, 'error writing typst log') +// --------------------------------------------------------------------------- +// Stdout parsing +// --------------------------------------------------------------------------- + +function _onWatcherData(compileName, chunk) { + const entry = WatchTable[compileName] + if (!entry) return + + entry.accumulator += chunk + const lines = entry.accumulator.split('\n') + entry.accumulator = lines.pop() // keep the incomplete trailing fragment + + for (const line of lines) { + entry.currentLines.push(line) + if (entry.currentLines.length > MAX_LOG_LINES) { + entry.currentLines.shift() + } + + if (FILE_ID_EXHAUSTION_RE.test(line)) { + logger.warn({ compileName }, 'typst watcher: FileId exhaustion detected') + entry.restartPending = true + } + + if (COMPILE_DONE_RE.test(line)) { + entry.compilationCount++ + const stdout = entry.currentLines.join('\n') + entry.currentLines = [] + + if (entry.compilationCount >= MAX_COMPILES_BEFORE_RESTART) { + logger.info( + { compileName, compilationCount: entry.compilationCount }, + 'typst watcher: scheduling restart (FileId threshold)' + ) + entry.restartPending = true } - callback() - }) + + _resolveAllPending(compileName, { stdout }) + } + } +} + +// --------------------------------------------------------------------------- +// Resolver helpers +// --------------------------------------------------------------------------- + +function _waitForNextCompile(compileName, timeout) { + return new Promise((resolve, reject) => { + const entry = WatchTable[compileName] + if (!entry) { + return reject(new Error('no typst watcher for ' + compileName)) + } + + // Push synchronously inside the Promise constructor — before the first + // await in the caller, so no data event can fire in the gap. + const timeoutHandle = setTimeout(() => { + entry.pendingResolvers = entry.pendingResolvers.filter(r => r !== resolver) + reject( + Object.assign(new Error('typst compile timed out'), { timedout: true }) + ) + }, timeout) + + const resolver = { resolve, reject, timeoutHandle } + entry.pendingResolvers.push(resolver) }) } +function _resolveAllPending(compileName, result) { + const entry = WatchTable[compileName] + if (!entry) return + for (const { resolve, timeoutHandle } of entry.pendingResolvers.splice(0)) { + clearTimeout(timeoutHandle) + resolve(result) + } +} + +function _rejectAllPending(compileName, err) { + const entry = WatchTable[compileName] + if (!entry) return + for (const { reject, timeoutHandle } of entry.pendingResolvers.splice(0)) { + clearTimeout(timeoutHandle) + reject(err) + } +} + +// --------------------------------------------------------------------------- +// Log output +// --------------------------------------------------------------------------- + +async function _writeLogOutputAsync(compileName, directory, output) { + const content = (output && output.stdout) || '' + if (!content) return + // Write to output.log so the PDF-preview log panel picks it up. + const logFile = Path.join(directory, 'output.log') + try { + await fs.promises.unlink(logFile) + } catch (err) { + if (err.code !== 'ENOENT') { + logger.error({ err, compileName, logFile }, 'error removing typst log') + } + } + try { + await fs.promises.writeFile(logFile, content, { flag: 'wx' }) + } catch (err) { + if (err.code !== 'EEXIST') { + logger.error({ err, compileName, logFile }, 'error writing typst log') + } + } +} + +// --------------------------------------------------------------------------- +// Cold-start fallback (Docker / sandboxed mode) +// --------------------------------------------------------------------------- + +function _runColdStart(compileName, options) { + return new Promise((resolve, reject) => { + const { directory, mainFile, image, environment, compileGroup } = options + const timeout = options.timeout || 60_000 + + logger.debug({ directory, mainFile, compileGroup }, 'typst cold-start compile') + + const inputPath = `$COMPILE_DIR/${mainFile}` + const command = ['/bin/sh', '-c', `typst compile ${inputPath} output.pdf 2>&1`] + + ProcessTable[compileName] = CommandRunner.run( + compileName, + command, + directory, + image, + timeout, + environment || {}, + compileGroup, + null, + function (error, output) { + delete ProcessTable[compileName] + if (error && (error.terminated || error.timedout)) { + return reject(error) + } + const combined = + output || (error ? { stdout: error.stdout || '' } : null) + _writeLogOutputAsync(compileName, directory, combined).then( + () => resolve(combined), + reject + ) + } + ) + }) +} + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + function isRunning(compileName) { - return ProcessTable[compileName] != null + return ( + ProcessTable[compileName] != null || + (WatchTable[compileName]?.pendingResolvers.length > 0) + ) } function killTypst(compileName, callback) { - logger.debug({ compileName }, 'killing running typst compile') - if (!isRunning(compileName)) { - logger.warn({ compileName }, 'no such compile to kill') - return callback(null) + logger.debug({ compileName }, 'killing typst (watcher + any active compile)') + + // Cold-start fallback path + if (ProcessTable[compileName] != null) { + CommandRunner.kill(ProcessTable[compileName], () => {}) + delete ProcessTable[compileName] } - CommandRunner.kill(ProcessTable[compileName], callback) + + // Reject any in-flight waiters and tear down the watcher process + _rejectAllPending( + compileName, + Object.assign(new Error('terminated'), { terminated: true }) + ) + _killWatchEntry(compileName) + + callback(null) +} + +// Kill all watcher processes when the CLSI Node process exits. +process.on('exit', () => { + for (const compileName of Object.keys(WatchTable)) { + _killWatchEntry(compileName) + } +}) + +const runTypst = (compileName, options, callback) => { + runTypstAsync(compileName, options).then( + result => callback(null, result), + err => callback(err) + ) } export default { @@ -98,7 +375,7 @@ export default { runTypst, killTypst, promises: { - runTypst: promisify(runTypst), + runTypst: runTypstAsync, killTypst: promisify(killTypst), }, }