diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index ce582cfae3..648fbfd2c2 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -47,11 +47,14 @@ COPY libraries/settings/ /overleaf/libraries/settings/ COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ COPY services/clsi/ /overleaf/services/clsi/ -FROM app AS with-texlive +FROM app AS with-quarto +ARG QUARTO_VERSION=1.6.39 RUN apt-get update \ - && apt-cache depends texlive-full | grep "Depends: " | grep -v -- "-doc" | grep -v -- "-lang-" | sed 's/Depends: //' | xargs apt-get install -y --no-install-recommends \ - && apt-get install -y --no-install-recommends fontconfig inkscape python3-pygments qpdf \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb" -o /tmp/quarto.deb \ + && dpkg -i /tmp/quarto.deb \ + && rm /tmp/quarto.deb \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p cache compiles output \ @@ -60,6 +63,15 @@ RUN mkdir -p cache compiles output \ CMD ["node", "--expose-gc", "app.js"] FROM app + +ARG QUARTO_VERSION=1.6.39 +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb" -o /tmp/quarto.deb \ + && dpkg -i /tmp/quarto.deb \ + && rm /tmp/quarto.deb \ + && rm -rf /var/lib/apt/lists/* + RUN mkdir -p cache compiles output \ && chown node:node cache compiles output diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index bd9677eb79..2802685a14 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -1,12 +1,11 @@ import fsPromises from 'node:fs/promises' -import os from 'node:os' import Path from 'node:path' import { callbackify } from 'node:util' import Settings from '@overleaf/settings' import logger from '@overleaf/logger' import OError from '@overleaf/o-error' import ResourceWriter from './ResourceWriter.js' -import LatexRunner from './LatexRunner.js' +import QuartoRunner from './QuartoRunner.js' import OutputFileFinder from './OutputFileFinder.js' import OutputCacheManager from './OutputCacheManager.js' import ClsiMetrics from './Metrics.js' @@ -18,16 +17,12 @@ import CommandRunner from './CommandRunner.js' import ContentCacheMetrics from './ContentCacheMetrics.js' import SynctexOutputParser from './SynctexOutputParser.js' import CLSICacheHandler from './CLSICacheHandler.js' -import StatsManager from './StatsManager.js' -import SafeReader from './SafeReader.js' -import LatexMetrics from './LatexMetrics.js' import { callbackifyMultiResult } from '@overleaf/promise-utils' import * as HistoryResourceWriter from './HistoryResourceWriter.js' const { downloadLatestCompileCache, downloadOutputDotSynctexFromCompileCache } = CLSICacheHandler const { emitPdfStats } = ContentCacheMetrics -const { enableLatexMkMetrics, addLatexFdbMetrics } = LatexMetrics const { shouldSkipMetrics } = ClsiMetrics const KNOWN_LATEXMK_RULES = new Set([ @@ -196,18 +191,8 @@ async function doCompile(request, stats, timings) { const compileName = getCompileName(request.project_id, request.user_id) - // Record latexmk -time stats for a subset of users - const recordPerformanceMetrics = StatsManager.sampleRequest( - request, - Settings.performanceLogSamplingPercentage - ) - - // Define a `latexmk` property on the stats object - // to collect latexmk -time stats. - enableLatexMkMetrics(stats) - try { - await LatexRunner.promises.runLatex(compileName, { + await QuartoRunner.promises.runQuarto(compileName, { directory: compileDir, mainFile: request.rootResourcePath, compiler: request.compiler, @@ -294,50 +279,13 @@ async function doCompile(request, stats, timings) { }) timings.compileE2E = Date.now() - e2eCompileStart - const status = stats['latexmk-errors'] ? 'error' : 'success' + const status = 'success' _emitMetrics(request, status, stats, timings) if (stats['pdf-size'] && !shouldSkipMetrics(request)) { emitPdfStats(stats, timings, request) } - // Record compile performance for a subset of users - if (recordPerformanceMetrics) { - // Add fdb metrics if available - try { - const fdbFileContent = await _readFdbFile(compileDir) - if (fdbFileContent) { - addLatexFdbMetrics(fdbFileContent, stats) - } - } catch (err) { - // ignore errors reading fdb file - logger.warn( - { err, projectId, userId }, - 'error reading fdb file for performance metrics' - ) - } - - const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined - - logger.info( - { - userId: request.user_id, - projectId: request.project_id, - timeTaken: timings.compile, - clsiRequest: request, - stats, - timings, - // explicitly include latexmk stats to bypass the non-enumerable property - latexmk: stats.latexmk, - loadavg1m: loadavg?.[0], - loadavg5m: loadavg?.[1], - loadavg15m: loadavg?.[2], - samplingPercentage: Settings.performanceLogSamplingPercentage, - }, - 'sampled performance log' - ) - } - return { outputFiles, buildId, baseHistoryVersion } } @@ -366,20 +314,6 @@ async function _saveOutputFiles({ return { outputFiles, allEntries, buildId } } -// Set a maximum size for reading output.fdb_latexmk files -// This limit is chosen to prevent excessive memory usage and ensure performance, -// as fdb files are typically much smaller and only metrics are extracted from them. -const MAX_FDB_FILE_SIZE = 1024 * 1024 // 1 MB - -async function _readFdbFile(compileDir) { - const fdbFile = Path.join(compileDir, 'output.fdb_latexmk') - const { result } = await SafeReader.promises.readFile( - fdbFile, - MAX_FDB_FILE_SIZE, - 'utf8' - ) - return result -} async function stopCompile(projectId, userId) { const compileName = getCompileName(projectId, userId) @@ -388,11 +322,11 @@ async function stopCompile(projectId, userId) { if (lock) { lockReleased = lock.waitForRelease() } else { - if (!LatexRunner.isRunning(compileName)) return + if (!QuartoRunner.isRunning(compileName)) return logger.warn({ projectId, userId }, 'found running compile without lock') lockReleased = Promise.resolve() } - await LatexRunner.promises.killLatex(compileName) + await QuartoRunner.promises.killQuarto(compileName) await lockReleased } diff --git a/services/clsi/app/js/QuartoRunner.js b/services/clsi/app/js/QuartoRunner.js new file mode 100644 index 0000000000..0a31b6a4c9 --- /dev/null +++ b/services/clsi/app/js/QuartoRunner.js @@ -0,0 +1,102 @@ +import Path from 'node:path' +import { promisify } from 'node:util' +import logger from '@overleaf/logger' +import CommandRunner from './CommandRunner.js' +import fs from 'node:fs' + +// Maps currently-running Quarto jobs: compileName → PID (or docker container id) +const ProcessTable = {} + +function runQuarto(compileName, options, callback) { + const { + directory, + mainFile, + image, + environment, + compileGroup, + timings, + } = options + const timeout = options.timeout || 60000 + + logger.debug({ directory, timeout, mainFile, compileGroup }, 'starting quarto compile') + + const command = _buildQuartoCommand(mainFile) + + ProcessTable[compileName] = CommandRunner.run( + compileName, + command, + directory, + image, + timeout, + environment || {}, + compileGroup, + null, + function (error, output) { + delete ProcessTable[compileName] + if (error) return callback(error) + _writeLogOutput(compileName, directory, output, () => { + callback(null, output) + }) + } + ) +} + +function _buildQuartoCommand(mainFile) { + // Use Typst as the PDF engine — it is bundled with Quarto (>= 1.4) and + // does not require a separate LaTeX installation. + return [ + 'quarto', + 'render', + Path.join('$COMPILE_DIR', mainFile), + '--to', 'typst', + '--output', 'output.pdf', + ] +} + +function _writeLogOutput(compileName, directory, output, callback) { + if (!output) return callback() + + function _writeFile(file, content, cb) { + if (content && content.length > 0) { + fs.unlink(file, () => { + fs.writeFile(file, content, { flag: 'wx' }, err => { + if (err) { + logger.error({ err, compileName, file }, 'error writing log file') + } + cb() + }) + }) + } else { + cb() + } + } + + _writeFile(Path.join(directory, 'output.stdout'), output.stdout, () => { + _writeFile(Path.join(directory, 'output.stderr'), output.stderr, () => { + callback() + }) + }) +} + +function isRunning(compileName) { + return ProcessTable[compileName] != null +} + +function killQuarto(compileName, callback) { + logger.debug({ compileName }, 'killing running quarto compile') + if (!isRunning(compileName)) { + logger.warn({ compileName }, 'no such compile to kill') + return callback(null) + } + CommandRunner.kill(ProcessTable[compileName], callback) +} + +export default { + isRunning, + runQuarto, + killQuarto, + promises: { + runQuarto: promisify(runQuarto), + killQuarto: promisify(killQuarto), + }, +} diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js index 7a81fb1eab..d3191b050f 100644 --- a/services/clsi/app/js/RequestParser.js +++ b/services/clsi/app/js/RequestParser.js @@ -2,7 +2,7 @@ import { promisify } from 'node:util' import settings from '@overleaf/settings' import OutputCacheManager from './OutputCacheManager.js' -const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex'] +const VALID_COMPILERS = ['quarto', 'pdflatex', 'latex', 'xelatex', 'lualatex'] const MAX_TIMEOUT = 600 const EDITOR_ID_REGEX = /^[a-f0-9-]{36}$/ // UUID const HISTORY_ID_REGEX = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/ // mongo id or postgres id @@ -36,7 +36,7 @@ function parse(body, callback) { } response.compiler = _parseAttribute('compiler', compile.options.compiler, { validValues: VALID_COMPILERS, - default: 'pdflatex', + default: 'quarto', type: 'string', }) response.compileFromClsiCache = _parseAttribute( @@ -180,7 +180,7 @@ function parse(body, callback) { 'rootResourcePath', compile.rootResourcePath, { - default: 'main.tex', + default: 'main.qmd', type: 'string', } ) diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 02dc00b158..b839ecdd99 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -115,7 +115,7 @@ const httpPermissionsPolicy = { }, } -const safeCompilers = ['xelatex', 'pdflatex', 'latex', 'lualatex'] +const safeCompilers = ['quarto', 'xelatex', 'pdflatex', 'latex', 'lualatex'] module.exports = { env: 'server-ce', @@ -467,7 +467,7 @@ module.exports = { process.env.DEFAULT_LATEX_COMPILER ) ? process.env.DEFAULT_LATEX_COMPILER - : 'pdflatex', + : 'quarto', enableSubscriptions: false, restrictedCountries: [], enableOnboardingEmails: process.env.ENABLE_ONBOARDING_EMAILS === 'true',