fix(typst-preview): use persistent session to avoid Rc ownership panics

Every call to renderToSvg({ artifactContent }) internally routes through
runWithSession, which calls session.free() after fn resolves. Because the JS
GC may still hold a reference to the first RenderSession wrapper, the next
render's Rc::try_unwrap() panics with 'attempted to take ownership of Rust
value while it was borrowed'.

Fix: use renderer.createModule(data) once to create a persistent RenderSession
that is never freed during the component's lifetime. Subsequent renders call
session.manipulateData({ action: 'reset', data }) (synchronous, no ownership
transfer) + session.renderToSvg({ container }) which routes through
withinOptionSession's renderSession fast-path — bypassing runWithSession and
its session.free() entirely.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/deploy-verso-prod.yml

@@ -329,6 +329,8 @@ jobs:
                       # (CE default): admin creates accounts / sends invites.
                       - name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
                         value: "true"
+                      - name: OVERLEAF_LATEX_SHELL_ESCAPE
+                        value: "true"
                       # (SMTP email vars are loaded below via envFrom.)
                     # SMTP for password-reset / invite emails. All
                     # OVERLEAF_EMAIL_* vars come from the optional 'verso-smtp'

.gitea/workflows/deploy-verso.yml

@@ -300,6 +300,8 @@ jobs:
                       # link-sharing users).
                       - name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
                         value: "true"
+                      - name: OVERLEAF_LATEX_SHELL_ESCAPE
+                        value: "true"
           ---
           apiVersion: v1
           kind: Service

.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch

@@ -1,5 +1,5 @@
 diff --git a/lib/sandboxed_module.js b/lib/sandboxed_module.js
-index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd97100e41c3e4 100644
+index 1cd6743..4718b97 100644
 --- a/lib/sandboxed_module.js
 +++ b/lib/sandboxed_module.js
 @@ -4,7 +4,7 @@ var Module = require('module');
@@ -11,3 +11,47 @@ index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd9710
  var parent = module.parent;
  var globalOptions = {};
  var registeredBuiltInSourceTransformers = ['coffee'];
+@@ -157,12 +157,20 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
+   var cache = Object.create(null);
+   var required = this._getRequires();
+   for (var key in required) {
+-    var injectedFilename = requireLike(this.filename).resolve(key);
+-    cache[injectedFilename] = required[key];
++    // Under Yarn PnP, resolution from a transitive dependency's context may fail
++    // for packages not declared in that dependency's package.json. Silently skip
++    // cache pre-population on failure; the mock will still be injected via the
++    // inject map in requireInterceptor or resolved via RecursiveRequireProxy fallback.
++    try {
++      var injectedFilename = requireLike(this.filename).resolve(key);
++      cache[injectedFilename] = required[key];
++    } catch (e) {}
+   }
+   cache[this.filename] = this.exports;
+   var globals = this.globals;
+ 
++  // Store the top-level module's filename for PnP fallback resolution
++  var topLevelFilename = this.filename;
+   var options;
+   if(!this._options.sourceTransformersSingleOnly && this._options.sourceTransformers){
+     options = {
+@@ -208,8 +216,18 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
+       if (request in cache) return cache[request];
+       return require(request);
+     }
+-    // cached modules
+-    var requestedFilename = requireLike(this.filename).resolve(request);
++    // Resolve the requested module filename.
++    // Under Yarn PnP, packages can only resolve their declared dependencies.
++    // When sandboxed-module loads a transitive dependency, the resolution context
++    // may not have access to all needed packages. Fall back to resolving from
++    // the top-level module's context (the module under test).
++    var requestedFilename;
++    try {
++      requestedFilename = requireLike(this.filename).resolve(request);
++    } catch (e) {
++      if (this.filename === topLevelFilename) throw e;
++      requestedFilename = requireLike(topLevelFilename).resolve(request);
++    }
+     if (requestedFilename in cache) return cache[requestedFilename];
+     var sandboxedModule = createInnerSandboxedModule(requestedFilename)
+     return sandboxedModule.exports;

README.md

@@ -2,73 +2,182 @@
   <img src="services/web/public/img/ol-brand/verso-logo.svg" alt="Verso" width="440">
 </p>
 
-**A collaborative, real-time editor for Quarto, LaTeX and Typst — documents and presentations.**
+**A collaborative, real-time editor for LaTeX, Quarto and Typst — self-hosted.**
 
-Verso is a fork of [Overleaf](https://github.com/overleaf/overleaf) that adds
+---
-first-class [Quarto](https://quarto.org) and [Typst](https://typst.app) support
-alongside Overleaf's LaTeX toolchain. It keeps Overleaf's real-time
-collaboration infrastructure and runs **three compilers side by side**, chosen
-automatically from the root file's extension:
 
-| Root file | Compiler | Typical output |
+## What is Verso?
-|-----------|----------|----------------|
-| `.qmd`    | Quarto   | PDF (via Typst or LaTeX), or an HTML/RevealJS deck |
-| `.tex`    | `latexmk` / TeX Live | PDF |
-| `.typ`    | Typst    | PDF |
 
-All three coexist on one server; no per-project configuration is required to
+Verso is a fork of [Overleaf](https://github.com/overleaf/overleaf) that extends its
-pick the engine.
+collaborative editing infrastructure to support [Quarto](https://quarto.org) and
+[Typst](https://typst.app) projects alongside LaTeX. Think of it as Overleaf, but
+not limited to LaTeX.
+
+### Verso vs Overleaf
+
+[Overleaf](https://www.overleaf.com) is the gold standard for collaborative LaTeX
+editing. Verso keeps everything that makes Overleaf great — real-time co-editing,
+operational-transformation history, auth, project management, file storage — and
+adds:
+
+- **Quarto and Typst compilers** running alongside TeX Live, dispatched
+  automatically from the root file's extension (`.qmd` → Quarto, `.typ` → Typst,
+  `.tex` → `latexmk`).
+- **Language-aware editor** for Quarto and Typst (syntax highlighting, completions,
+  document outline) — not just LaTeX.
+- **Publish & share compiled output** (`/p/:token` with tiered access links) — a
+  feature absent from Overleaf Community Edition.
+- **Lumière theme** — a redesigned project dashboard and editor chrome with a
+  card-based grid, thumbnails, and a teal gradient identity.
+- **Full i18n** — French, German, Italian, and Spanish UI translations on top of
+  Overleaf's English base.
+- Completely **free and self-hosted**; no Overleaf subscription required.
+
+### Verso vs Quarto
+
+[Quarto](https://quarto.org) is a command-line tool: you install it locally, write
+`.qmd` files in any text editor, and run `quarto render` in a terminal. It is
+excellent for solo authors with full control over their environment.
+
+Verso wraps Quarto in a collaborative web editor:
+
+- **No local install** — Quarto, Typst, TeX Live and Python run on the server.
+- **Real-time collaboration** — multiple people edit the same `.qmd` simultaneously
+  with live cursors and conflict-free merging.
+- **Not just Quarto** — LaTeX and Typst projects live in the same workspace, under
+  the same auth and history system.
+- **Publish in one click** — RevealJS decks and PDFs are served at a stable link
+  without leaving the browser.
+
+Verso is not a replacement for Quarto's CLI — it is a platform that makes Quarto
+accessible as a shared, always-on service.
+
+### Verso vs Typst.app
+
+[Typst.app](https://typst.app) is a cloud-hosted web editor for Typst. It is
+polished and fast, but it is a proprietary SaaS product and only supports Typst.
+
+Verso differs in that:
+
+- It is **self-hosted** and open-source (AGPL v3) — you control your data.
+- It supports **three languages** (Typst, LaTeX, Quarto) in one instance.
+- Real-time collaboration is powered by **operational transformation** (the same
+  engine as Overleaf), not CRDTs, which means it handles concurrent edits
+  gracefully for long documents.
+- It ships with a full **project history** and version-restore workflow.
+
+If you only need Typst and want a lighter, Typst-focused alternative, have a look
+at **[Collabst](https://github.com/herluf-ba/collabst)** — an open-source,
+self-hosted collaborative Typst editor that is independent of the Overleaf
+codebase and shows a lot of promise.
 
 ---
 
 ## Features
 
-- **Real-time collaboration** — multiple people editing the same file at once,
+- **Real-time collaboration** — multiple editors, live cursors, full project history and version restore.
-  powered by Overleaf's operational-transformation engine, with live cursors
+- **Three compilers, auto-dispatched** by root file extension:
-  and full project history.
+  | Root file | Compiler | Typical output |
-- **Three compilers, auto-dispatched** — Quarto, LaTeX and Typst projects live
+  |-----------|----------|----------------|
-  side by side; the runner is selected from the root file's extension.
+  | `.qmd` | Quarto | PDF (via Typst or LaTeX), HTML, or RevealJS |
-- **Language-aware editor for all three**:
+  | `.tex` | `latexmk` / TeX Live | PDF |
-  - *LaTeX* — syntax highlighting, command/environment/reference autocomplete,
+  | `.typ` | Typst | PDF |
-    linting (inherited from Overleaf).
+- **Language-aware editor for all three** — syntax highlighting, completions, and a document outline panel for LaTeX, Quarto and Typst.
-  - *Quarto (`.qmd`)* — Markdown highlighting plus Quarto-aware completions:
+- **Format badge** on the project dashboard; compiler dropdown greys out inapplicable engines.
-    code chunks (```` ```{python} ````, `{r}`, `{julia}`, `{ojs}`…), callouts
+- **Publish & share** — compile and snapshot to `/p/:token` with three independent access tiers (project members / any logged-in user / public). HTML/RevealJS decks are served live; PDFs are embedded inline. A **Present** toolbar button links directly to the published deck.
-    and fenced divs (`::: {.callout-note}`, columns, tabsets) and
+- **RevealJS thumbnails** — the first slide of a presentation is rendered as a preview card in the project list.
-    cross-references (`@fig-`, `@tbl-`, `@sec-`, `@eq-`).
+- **Quarto Python cells** — optional per-project virtual environment built from `requirements.txt`, so Python code chunks execute during render.
-  - *Typst (`.typ`)* — syntax highlighting and completions for the common
+- **Visual formatting toolbar** — bold, italic, headings and inline code shortcuts for Quarto (`.qmd`) and Typst (`.typ`) files, in addition to Overleaf's existing LaTeX toolbar.
-    functions and markup (`#import`, `#let`, `#set`, `#show`, `#figure`,
+- **Lumière theme** — card-based project dashboard with PDF/slide thumbnails, a teal gradient identity, dark editor chrome, and an XS compact list view.
-    `#table`, `#cite`, …).
+- **i18n** — French, German, Italian and Spanish UI translations.
-- **Document outline** — section headings are extracted into the sidebar
+- **Auto-compile** — preview refreshes automatically after you stop typing.
-  outline panel for LaTeX, Quarto (`#`, `##`, …) and Typst (`=`, `==`, …).
-- **Format at a glance** — the project dashboard shows a per-project format
-  badge (Quarto / Typst / LaTeX), and the compiler dropdown greys out engines
-  that don't apply to the current root file.
-- **Publish & share compiled output** — publish the compiled result as a
-  standalone page at `/p/:token`, with three independent access tiers (project
-  members / any logged-in user / public). Works for both HTML/RevealJS decks
-  (served live) and PDFs (embedded inline). HTML decks also get a one-click
-  **Present** button in the toolbar.
-- **Quarto Python cells** — optional per-project virtual environment built from
-  the project's `requirements.txt`, so Python code chunks run during render
-  (gated to the project owner and invited collaborators).
-- **Auto-compile** — the preview refreshes automatically shortly after you stop
-  typing.
 
-## Output formats
+---
 
-In the YAML frontmatter of a `.qmd` file:
+## Releases
 
-```yaml
+### Alpha 1
-format: typst     # → PDF preview, rendered via Typst (no LaTeX required)
-format: pdf       # → PDF preview, rendered via LaTeX
-format: revealjs  # → interactive HTML slideshow preview
-format: html      # → a static HTML page
-```
 
-Typst ships inside Quarto, so `format: typst` needs no separate installation.
+The initial public release. Established Verso as an Overleaf fork with first-class
+multi-language support:
 
-> **Note on display math**: keep `$$ … $$` blocks on a single line. Multi-line
+- Quarto (`.qmd`) and Typst (`.typ`) compilers running alongside TeX Live,
-> display-math blocks can trigger YAML parse errors in some Quarto versions.
+  dispatched automatically by root file extension — no per-project configuration.
+- Language-aware editor for Quarto: Markdown highlighting, code-chunk completions
+  (`{python}`, `{r}`, `{julia}`, `{ojs}`…), callout and fenced-div completions,
+  cross-reference completions (`@fig-`, `@tbl-`, `@sec-`…).
+- Language-aware editor for Typst: syntax highlighting and completions for
+  functions, imports, math and markup.
+- Document outline panel for all three languages (LaTeX `\section`, Quarto `#`,
+  Typst `=`).
+- Format badge on the project dashboard; compiler selector greys out inapplicable
+  engines for the current root file.
+- Publish & share compiled output — HTML/RevealJS decks and PDFs hosted at
+  `/p/:token` with tiered access links (project / logged-in / public), each
+  independently resettable.
+- Quarto Python code-cell execution via an optional per-project `requirements.txt`
+  virtual environment.
+- Verso branding: name, logo and Kubernetes production deploy workflow.
+
+### Alpha 2
+
+Refinements to the Typst editor and the format badge system:
+
+- **Quarto format sub-types** — the project badge now distinguishes *Quarto PDF*
+  from *Quarto Slides*, reading the frontmatter `format:` to pick the right label.
+- **Python packages for collaborators** — Quarto Python package installation
+  extended to all users who have write access to the project, not only the owner.
+- **Typst syntax highlighting overhaul** — complete grammar rewrite covering:
+  function calls and named argument keys, multi-line display math, `#{…}` code
+  blocks, content blocks, `show`-rule bodies, `let`-value bindings, and keyword
+  vs identifier disambiguation.
+- **Typst visual formatting** — bold and italic toolbar buttons and keyboard
+  shortcuts (`Ctrl+B`, `Ctrl+I`), plus underline, small-caps and hyperlink
+  buttons, matching the Quarto and LaTeX toolbar experience.
+
+### Alpha 3
+
+- **Lumière theme** — redesigned project dashboard with a card grid, PDF/slide
+  thumbnails, parallax hover effects, a teal gradient identity and a dark editor
+  chrome. Includes an XS compact list view and a tile zoom slider.
+- **Full i18n** — French, German, Italian and Spanish translations covering the
+  complete UI (login, dashboard, editor, settings, emails).
+- **Visual editors for Quarto and Typst** — bold, italic, headings and inline code
+  shortcuts in the toolbar for `.qmd` and `.typ` files.
+- **Top/bottom split view** — new editor layout that stacks the source editor
+  above the PDF preview vertically, in addition to the existing side-by-side mode.
+- **Bidirectional format export** — LaTeX projects can be converted to Typst and
+  Typst projects to LaTeX via pandoc. Available from the File menu in the editor.
+- **Mobile layout** — project dashboard, search bar, footer and editor all
+  adapted for phone screen sizes.
+
+---
+
+## Known issues
+
+- **Large file upload timeouts** — uploads of large files on slow connections
+  can time out at the proxy layer. A streaming response fix is pending.
+
+---
+
+## Security model — trusted environments only
+
+> [!CAUTION]
+> Verso is designed for **closed groups of trusted users** (a lab, a class, a
+> small team). All three compilers can execute arbitrary code on the server:
+>
+> - LaTeX with shell-escape enabled can run system commands.
+> - Quarto Python cells execute Python code directly.
+> - Typst's scripting layer is sandboxed by design, but runs server-side.
+>
+> There is **no per-project sandbox or resource isolation** beyond what the
+> operating system provides. Exposing Verso to the public internet with open
+> registration is not recommended. If you need to host a collaborative
+> LaTeX editor for untrusted users or at scale, look at
+> [Overleaf's non-Community offerings](https://www.overleaf.com/for/enterprises),
+> which include proper sandboxing and enterprise access controls.
+
+---
 
 ## Quick start
 
@@ -82,24 +191,23 @@ docker run -d \
   registry.alocoq.fr/verso:latest
 ```
 
-Open `http://localhost` in your browser, then visit `/launchpad` on first run to
+Open `http://localhost`, then visit `/launchpad` on first run to create the admin
-create the admin account.
+account.
 
 ### Build from source
 
 ```bash
-# Build the base image (system deps + Quarto + TeX Live)
 cd server-ce
-make build-base
+make build-base        # base OS image: system deps, Quarto, Typst, TeX Live
-
+make build-community   # application image: Node services + compiled frontend
-# Build the application image
-make build-community
 ```
 
 | File | Purpose |
 |------|---------|
-| `server-ce/Dockerfile-base` | Base OS image — system deps, Quarto (with Typst) and a TeX Live (`latexmk`) toolchain |
+| `server-ce/Dockerfile-base` | Base image — system deps, Quarto (with Typst) and TeX Live |
-| `server-ce/Dockerfile` | Application image — Node services and the compiled frontend |
+| `server-ce/Dockerfile` | App image — Node services and the compiled React frontend |
+
+---
 
 ## Architecture
 
@@ -108,10 +216,10 @@ single container managed by `runit`, with `nginx` as the front router.
 
 ```
 browser ──→ nginx:80
-              ├── / ──────────────────→ web:4000        (main app, React UI)
+              ├── / ──────────────────→ web:4000         (main app, React UI)
-              ├── /socket.io ──────────→ real-time:3026  (WebSocket, OT engine)
+              ├── /socket.io ──────────→ real-time:3026   (WebSocket, OT engine)
-              ├── /p/:token ───────────→ web             (published output)
+              ├── /p/:token ───────────→ web              (published output)
-              └── /project/*/output/* → clsi-nginx:8080  (compiled output files)
+              └── /project/*/output/* → clsi-nginx:8080   (compiled output files)
 
 web → document-updater → Redis pub/sub → real-time → browser
 web → CLSI (quarto render / latexmk / typst) → output files → nginx → browser
@@ -122,64 +230,12 @@ web → CLSI (quarto render / latexmk / typst) → output files → nginx → br
 | `web` | HTTP API, React frontend, auth, project & sharing management |
 | `real-time` | WebSocket layer, live cursors and edit sync |
 | `document-updater` | Operational transformation, Redis pub/sub |
-| `clsi` | Compiler — runs `quarto render` (`.qmd`), `latexmk` (`.tex`) or `typst` (`.typ`) and serves output |
+| `clsi` | Compiler — runs `quarto render`, `latexmk` or `typst` and serves output |
 | `docstore` | Document text storage (MongoDB) |
 | `filestore` | Binary file storage (S3 or local) |
 | `project-history` | Change history and version tracking |
 
-## Writing documents
-
-### Quarto (`main.qmd`)
-
-```markdown
 ---
-title: My Presentation
-author: Your Name
-date: today
-format: revealjs
----
-
-## Slide one
-
-Write **Markdown** here.
-
-## Mathematics
-
-$$\int_0^\infty e^{-x^2}\,dx = \frac{\sqrt{\pi}}{2}$$
-```
-
-Switch `format: revealjs` to `format: typst` (or `pdf`) for a PDF preview.
-
-### LaTeX (`main.tex`)
-
-LaTeX works exactly as in Overleaf: a project whose root file is a `.tex` file
-compiles with `latexmk`/TeX Live, no setting required. The **Example LaTeX
-project** in the *New project* menu is a ready-made starting point.
-
-> The bundled TeX Live is a minimal install. Documents that need extra packages
-> may not build out of the box — see `server-ce/Dockerfile-base` for how to
-> switch to a fuller TeX Live scheme.
-
-### Typst (`main.typ`)
-
-A project whose root file is a `.typ` file compiles directly to PDF with
-[Typst](https://typst.app) — fast, modern markup with a real scripting
-language. Verso drives the Typst bundled with Quarto, so no extra install is
-needed. Use the **Blank Typst project** entry in the *New project* menu to get
-started.
-
-## Publishing compiled output
-
-From **Share → Publish**, Verso compiles the project and snapshots the result to
-a standalone page at `/p/:token`:
-
-- **HTML / RevealJS** decks are served as a live page (the **Present** toolbar
-  button is a one-click shortcut to this).
-- **PDF** output is embedded inline; the raw file stays reachable at
-  `/p/:token/output.pdf`.
-
-Three stable links are issued, one per access tier — project members, any
-logged-in user, or anyone — and each can be copied or independently reset.
 
 ## Environment variables
 
@@ -195,39 +251,46 @@ The most commonly needed:
 | `OVERLEAF_SITE_URL` | — | Public URL (used in emails and published links) |
 | `OVERLEAF_SITE_LANGUAGE` | `en` | Default UI language (e.g. `fr`) |
 | `OVERLEAF_ENABLE_PROJECT_PYTHON_VENV` | `false` | Allow Quarto Python cells to use a project `requirements.txt` |
-| `OVERLEAF_ADMIN_EMAIL` | — | Email for the first admin account |
+| `OVERLEAF_ADMIN_EMAIL` | — | Email shown on the launchpad for the first admin account |
 
 See the [Overleaf Server documentation](https://github.com/overleaf/overleaf/wiki)
 for the full list.
 
+---
+
 ## Relation to Overleaf
 
 Verso is a fork of [Overleaf Community Edition](https://github.com/overleaf/overleaf).
-The main additions on top of upstream are:
+Everything that Overleaf CE provides — real-time collaboration, operational-transformation
+history, auth, project management, binary file storage — is inherited unchanged. The
+Verso-specific additions are listed in the Features section and tracked across releases above.
 
-- Quarto and Typst compilers running alongside LaTeX, dispatched by the root
+Verso is not affiliated with Overleaf Ltd.
-  file's extension.
-- Editor language support (highlighting, autocomplete, outline) for Quarto and
-  Typst.
-- A per-project format badge on the dashboard and a root-file-aware compiler
-  selector.
-- Publishing/sharing of compiled output (HTML decks and PDFs) via `/p/:token`
-  with tiered access links, and a toolbar **Present** shortcut.
-- Optional per-project Python virtual environments for Quarto code execution.
-- Verso branding (name, logo, palette, loading animation).
 
-All other infrastructure — real-time collaboration, history, auth, file
+---
-storage, project management — is unchanged from Overleaf.
 
-## Contributing
+## Supporting the ecosystem
 
-Contributions are welcome — open an issue or pull request on the
+Verso is not accepting contributions or donations at this time. If you find it
-[Verso repository](https://git.alocoq.fr/alois/verso). The upstream Overleaf
+useful and want to support the broader ecosystem it builds on:
-contribution guidelines are in [CONTRIBUTING.md](CONTRIBUTING.md).
+
+- **Support Overleaf** — Verso is built on Overleaf's infrastructure. The best
+  way to support their work is to use or subscribe to
+  [Overleaf](https://www.overleaf.com) and encourage your institution to do the
+  same.
+- **Support Typst** — [Typst GmbH](https://typst.app) is the company behind the
+  Typst compiler. Using Typst.app or sponsoring the
+  [Typst project on GitHub](https://github.com/typst/typst) helps sustain the
+  language itself.
+- **Support RevealJS** — Verso uses [Reveal.js](https://revealjs.com) for
+  HTML presentations. Consider sponsoring the
+  [RevealJS project on GitHub](https://github.com/hakimel/reveal.js).
+
+---
 
 ## License
 
 GNU Affero General Public License v3 — see [LICENSE](LICENSE).
 
-Copyright © Overleaf, 2014–2026 (original code).
+Copyright © Overleaf, 2014–2026 (original code).  
 Verso modifications © Aloïs Coquillard, 2026.

TODO.md

@@ -0,0 +1,50 @@
+# Verso — Next Alpha Roadmap
+
+Ideas and features deferred from the current alpha.
+
+---
+
+## Next alpha (post-current)
+
+### Typst editing experience (inspired by Collabst)
+
+- **typst.ts WASM preview** — Run the Typst compiler in the browser via
+  WebAssembly (typst.ts). This would give instant, sub-second preview
+  without a server round-trip, and would eliminate the entire class of
+  race conditions in the CLSI watcher (files written → typst compiles →
+  resolver missed). Could coexist with the CLSI watcher for PDF export
+  while using the WASM path for live preview.
+
+- **Tinymist LSP integration** — Wire up
+  [Tinymist](https://github.com/Myriad-Dreamin/tinymist) (the Typst
+  language server) behind a WebSocket proxy. Would give Typst files
+  first-class autocomplete, hover docs, go-to-definition, and inline
+  error diagnostics — the main editing comfort gap vs. a native editor.
+
+### Editor UX for non-LaTeX formats (.typ, .qmd, .md)
+
+- **Visual/rich-text editing mode** — A toggle between raw source and a
+  rendered-in-place view for `.typ`, `.qmd`, and `.md` files (similar to
+  Overleaf's rich-text mode for LaTeX). Users who don't know Typst or
+  Markdown syntax should be able to edit content without seeing markup.
+  CodeMirror 6 already supports this pattern via a custom `NodeView` layer
+  or a separate Prosemirror bridge.
+
+- **Toolbar / insertion shortcuts** — A formatting toolbar and keyboard
+  shortcuts for common operations, adapted per file type:
+  - **All formats**: bold, italic, underline, headings, bullet/numbered
+    lists, inline code, links.
+  - **Quarto / Markdown**: insert image, insert table, insert code block
+    with language tag.
+  - **Quarto RevealJS**: insert slide divider (`---`), insert speaker
+    notes (`::: notes`), insert columns layout, insert video embed
+    (using Quarto's `{{< video >}}` shortcode).
+
+### AI writing assistant
+
+- **In-editor AI assistant** — Inline writing help similar to what Overleaf
+  and CoCalc offer: suggest completions, rephrase selections, explain LaTeX
+  errors, and generate boilerplate (figures, tables, equations). Should work
+  across all three formats (`.tex`, `.typ`, `.qmd`). Backend would proxy
+  requests to a configurable model API (Claude, OpenAI-compatible) so
+  self-hosters can bring their own key.

libraries/access-token-encryptor/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/eslint-plugin/index.js

@@ -1,4 +1,10 @@
+const pkg = require('./package.json')
+
 module.exports = {
+  meta: {
+    name: pkg.name,
+    version: pkg.version,
+  },
   rules: {
     'no-unnecessary-trans': require('./no-unnecessary-trans'),
     'prefer-kebab-url': require('./prefer-kebab-url'),

libraries/eslint-plugin/package.json

@@ -8,10 +8,10 @@
     "lodash": "^4.18.1"
   },
   "devDependencies": {
-    "@typescript-eslint/parser": "^8.50.0"
+    "@typescript-eslint/parser": "^8.59.4"
   },
   "peerDependencies": {
-    "eslint": "^8.51.0"
+    "eslint": "^10.4.0"
   },
   "scripts": {
     "test": "node rules.test.js"

libraries/eslint-plugin/require-vi-doMock-valid-path.js

@@ -22,7 +22,7 @@ module.exports = {
     },
   },
   create(context) {
-    const currentFilePath = context.getFilename()
+    const currentFilePath = context.filename
     // ESLint can sometimes pass <text> or <input> for snippets not in a file
     if (currentFilePath === '<text>' || currentFilePath === '<input>') {
       return {}
@@ -81,9 +81,10 @@ module.exports = {
             typeof firstArg.value !== 'string'
           ) {
             if (firstArg.type === 'Identifier') {
-              const variable = context
+              const scope = context.sourceCode.getScope(node)
-                .getScope()
+              const variable = scope.variables.find(
-                .variables.find(v => v.name === firstArg.name)
+                v => v.name === firstArg.name
+              )
               if (
                 variable &&
                 variable.defs.length > 0 &&

libraries/eslint-plugin/rules.test.js

@@ -1,4 +1,5 @@
 const { RuleTester } = require('eslint')
+const tsParser = require('@typescript-eslint/parser')
 const noThrowInCallback = require('./no-throw-in-callback')
 const preferKebabUrl = require('./prefer-kebab-url')
 const noUnnecessaryTrans = require('./no-unnecessary-trans')
@@ -8,10 +9,10 @@ const viDoMockValidPath = require('./require-vi-doMock-valid-path')
 const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
 
 const ruleTester = new RuleTester({
-  parser: require.resolve('@typescript-eslint/parser'),
+  languageOptions: {
-  parserOptions: {
+    parser: tsParser,
     ecmaVersion: 'latest',
-    ecmaFeatures: { jsx: true },
+    parserOptions: { ecmaFeatures: { jsx: true } },
   },
 })
 
@@ -33,19 +34,27 @@ ruleTester.run('prefer-kebab-url', preferKebabUrl, {
   invalid: [
     {
       code: `app.get('/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `app.get('/fooBar/:id')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
   ],
 })
@@ -153,6 +162,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
         {
           message:
             'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
+          suggestions: [],
         },
       ],
     },
@@ -163,6 +173,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
         {
           message:
             'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
+          suggestions: [],
         },
       ],
     },

libraries/fetch-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/logger/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/metrics/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/metrics/package.json

@@ -10,12 +10,12 @@
   "dependencies": {
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^3.0.0",
     "@google-cloud/profiler": "^6.0.4",
-    "@opentelemetry/api": "1.9.0",
+    "@opentelemetry/api": "^1.9.1",
-    "@opentelemetry/auto-instrumentations-node": "^0.72.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.76.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.214.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
-    "@opentelemetry/resources": "^2.6.0",
+    "@opentelemetry/resources": "^2.7.1",
-    "@opentelemetry/sdk-node": "^0.214.0",
+    "@opentelemetry/sdk-node": "^0.218.0",
-    "@opentelemetry/semantic-conventions": "^1.39.0",
+    "@opentelemetry/semantic-conventions": "^1.41.1",
     "compression": "^1.7.4",
     "prom-client": "^14.1.1",
     "yn": "^3.1.1"

libraries/o-error/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/object-persistor/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/object-persistor/package.json

@@ -38,7 +38,6 @@
     "mocha": "^11.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
-    "mock-fs": "^5.2.0",
     "mongodb": "6.12.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",

libraries/object-persistor/src/FSPersistor.js

@@ -3,9 +3,12 @@ const fs = require('node:fs')
 const fsPromises = require('node:fs/promises')
 const { glob } = require('glob')
 const Path = require('node:path')
+const { promisify } = require('node:util')
 const { PassThrough } = require('node:stream')
 const { pipeline } = require('node:stream/promises')
 
+const openCb = promisify(fs.open)
+
 const AbstractPersistor = require('./AbstractPersistor')
 const { ReadError, WriteError, NotImplementedError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
@@ -85,8 +88,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
     })
     const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
 
+    let fd
     try {
-      opts.fd = await fsPromises.open(fsPath, 'r')
+      fd = await openCb(fsPath, 'r')
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -96,7 +100,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       )
     }
 
-    const stream = fs.createReadStream(null, opts)
+    const stream = fs.createReadStream(null, { ...opts, fd })
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
     pipeline(stream, observer, pass).catch(() => {})

libraries/object-persistor/test/unit/FSPersistorTests.js

@@ -1,6 +1,6 @@
 const crypto = require('node:crypto')
+const os = require('node:os')
 const { expect } = require('chai')
-const mockFs = require('mock-fs')
 const fs = require('node:fs')
 const fsPromises = require('node:fs/promises')
 const Path = require('node:path')
@@ -10,22 +10,59 @@ const Errors = require('../../src/Errors')
 
 const MODULE_PATH = '../../src/FSPersistor.js'
 
+function createTree(base, tree) {
+  fs.mkdirSync(base, { recursive: true })
+  for (const [name, content] of Object.entries(tree)) {
+    const fullPath = Path.join(base, name)
+    if (Buffer.isBuffer(content) || typeof content === 'string') {
+      fs.writeFileSync(fullPath, content)
+    } else if (content && typeof content.symlink === 'string') {
+      fs.symlinkSync(content.symlink, fullPath)
+    } else {
+      createTree(fullPath, content)
+    }
+  }
+}
+
 describe('FSPersistorTests', function () {
-  const localFiles = {
+  const fileContents = {
-    '/uploads/info.txt': Buffer.from('This information is critical', {
+    'info.txt': Buffer.from('This information is critical', {
       encoding: 'utf-8',
     }),
-    '/uploads/other.txt': Buffer.from('Some other content', {
+    'other.txt': Buffer.from('Some other content', {
       encoding: 'utf-8',
     }),
   }
-  const location = '/bucket'
+  let tmpDir
+  let location
+  let notADirPath
   const files = {
     wombat: 'animals/wombat.tex',
     giraffe: 'animals/giraffe.tex',
     potato: 'vegetables/potato.tex',
   }
 
+  beforeEach(function () {
+    tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'fs-persistor-test-'))
+    createTree(tmpDir, {
+      uploads: {
+        'info.txt': fileContents['info.txt'],
+        'other.txt': fileContents['other.txt'],
+      },
+      'not-a-dir':
+        'This regular file is meant to prevent using this path as a directory',
+      directory: {
+        subdirectory: {},
+      },
+    })
+    notADirPath = Path.join(tmpDir, 'not-a-dir')
+    location = Path.join(tmpDir, 'bucket')
+  })
+
+  afterEach(function () {
+    fs.rmSync(tmpDir, { recursive: true })
+  })
+
   const scenarios = [
     {
       description: 'default settings',
@@ -54,31 +91,26 @@ describe('FSPersistorTests', function () {
         persistor = new FSPersistor(scenario.settings)
       })
 
-      beforeEach(function () {
-        mockFs({
-          ...localFiles,
-          '/not-a-dir':
-            'This regular file is meant to prevent using this path as a directory',
-          '/directory/subdirectory': {},
-        })
-      })
-
-      afterEach(function () {
-        mockFs.restore()
-      })
-
       describe('sendFile', function () {
         it('should copy the file', async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
           const contents = await fsPromises.readFile(
             scenario.fsPath(files.wombat)
           )
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
 
         it('should return an error if the file cannot be stored', async function () {
           await expect(
-            persistor.sendFile('/not-a-dir', files.wombat, '/uploads/info.txt')
+            persistor.sendFile(
+              notADirPath,
+              files.wombat,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           ).to.be.rejectedWith(Errors.WriteError)
         })
       })
@@ -88,7 +120,9 @@ describe('FSPersistorTests', function () {
 
         describe("when the file doesn't exist", function () {
           beforeEach(function () {
-            stream = fs.createReadStream('/uploads/info.txt')
+            stream = fs.createReadStream(
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           })
 
           it('should write the stream to disk', async function () {
@@ -96,7 +130,7 @@ describe('FSPersistorTests', function () {
             const contents = await fsPromises.readFile(
               scenario.fsPath(files.wombat)
             )
-            expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+            expect(contents.equals(fileContents['info.txt'])).to.be.true
           })
 
           it('should delete the temporary file', async function () {
@@ -109,7 +143,7 @@ describe('FSPersistorTests', function () {
           describe('on error', function () {
             beforeEach(async function () {
               await expect(
-                persistor.sendStream('/not-a-dir', files.wombat, stream)
+                persistor.sendStream(notADirPath, files.wombat, stream)
               ).to.be.rejectedWith(Errors.WriteError)
             })
 
@@ -129,13 +163,12 @@ describe('FSPersistorTests', function () {
           describe('when the md5 hash matches', function () {
             it('should write the stream to disk', async function () {
               await persistor.sendStream(location, files.wombat, stream, {
-                sourceMd5: md5(localFiles['/uploads/info.txt']),
+                sourceMd5: md5(fileContents['info.txt']),
               })
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
-                .true
             })
           })
 
@@ -169,9 +202,11 @@ describe('FSPersistorTests', function () {
             await persistor.sendFile(
               location,
               files.wombat,
-              '/uploads/info.txt'
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
+            stream = fs.createReadStream(
+              Path.join(tmpDir, 'uploads', 'other.txt')
             )
-            stream = fs.createReadStream('/uploads/other.txt')
           })
 
           it('should write the stream to disk', async function () {
@@ -179,7 +214,7 @@ describe('FSPersistorTests', function () {
             const contents = await fsPromises.readFile(
               scenario.fsPath(files.wombat)
             )
-            expect(contents.equals(localFiles['/uploads/other.txt'])).to.be.true
+            expect(contents.equals(fileContents['other.txt'])).to.be.true
           })
 
           it('should delete the temporary file', async function () {
@@ -192,7 +227,7 @@ describe('FSPersistorTests', function () {
           describe('on error', function () {
             beforeEach(async function () {
               await expect(
-                persistor.sendStream('/not-a-dir', files.wombat, stream)
+                persistor.sendStream(notADirPath, files.wombat, stream)
               ).to.be.rejectedWith(Errors.WriteError)
             })
 
@@ -200,8 +235,7 @@ describe('FSPersistorTests', function () {
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
-                .true
             })
 
             it('should delete the temporary file', async function () {
@@ -215,13 +249,12 @@ describe('FSPersistorTests', function () {
           describe('when the md5 hash matches', function () {
             it('should write the stream to disk', async function () {
               await persistor.sendStream(location, files.wombat, stream, {
-                sourceMd5: md5(localFiles['/uploads/other.txt']),
+                sourceMd5: md5(fileContents['other.txt']),
               })
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/other.txt'])).to.be
+              expect(contents.equals(fileContents['other.txt'])).to.be.true
-                .true
             })
           })
 
@@ -238,8 +271,7 @@ describe('FSPersistorTests', function () {
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
-                .true
             })
 
             it('should delete the temporary file', async function () {
@@ -254,13 +286,17 @@ describe('FSPersistorTests', function () {
 
       describe('getObjectStream', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return a string with the object contents', async function () {
           const stream = await persistor.getObjectStream(location, files.wombat)
           const contents = await streamToBuffer(stream)
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
 
         it('should support ranges', async function () {
@@ -274,8 +310,8 @@ describe('FSPersistorTests', function () {
           )
           const contents = await streamToBuffer(stream)
           // end is inclusive in ranges, but exclusive in slice()
-          expect(contents.equals(localFiles['/uploads/info.txt'].slice(5, 17)))
+          expect(contents.equals(fileContents['info.txt'].slice(5, 17))).to.be
-            .to.be.true
+            .true
         })
 
         it('should give a NotFoundError if the file does not exist', async function () {
@@ -287,13 +323,17 @@ describe('FSPersistorTests', function () {
 
       describe('getObjectSize', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return the file size', async function () {
           expect(
             await persistor.getObjectSize(location, files.wombat)
-          ).to.equal(localFiles['/uploads/info.txt'].length)
+          ).to.equal(fileContents['info.txt'].length)
         })
 
         it('should throw a NotFoundError if the file does not exist', async function () {
@@ -305,7 +345,11 @@ describe('FSPersistorTests', function () {
 
       describe('copyObject', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('Should copy the file to the new location', async function () {
@@ -313,13 +357,17 @@ describe('FSPersistorTests', function () {
           const contents = await fsPromises.readFile(
             scenario.fsPath(files.potato)
           )
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
       })
 
       describe('deleteObject', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
           await fsPromises.access(scenario.fsPath(files.wombat))
         })
 
@@ -337,7 +385,11 @@ describe('FSPersistorTests', function () {
       describe('deleteDirectory', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
             await fsPromises.access(scenario.fsPath(file))
           }
         })
@@ -365,7 +417,11 @@ describe('FSPersistorTests', function () {
 
       describe('checkIfObjectExists', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return true for existing files', async function () {
@@ -384,13 +440,17 @@ describe('FSPersistorTests', function () {
       describe('directorySize', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
         it('should sum directory files size', async function () {
           expect(await persistor.directorySize(location, 'animals')).to.equal(
-            2 * localFiles['/uploads/info.txt'].length
+            2 * fileContents['info.txt'].length
           )
         })
 
@@ -404,7 +464,11 @@ describe('FSPersistorTests', function () {
       describe('listDirectoryKeys', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
@@ -427,7 +491,11 @@ describe('FSPersistorTests', function () {
       describe('listDirectoryStats', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
@@ -438,7 +506,7 @@ describe('FSPersistorTests', function () {
           expect(keys).to.include(scenario.fsPath(files.wombat))
           expect(keys).to.include(scenario.fsPath(files.giraffe))
           for (const stat of stats) {
-            expect(stat.size).to.equal(localFiles['/uploads/info.txt'].length)
+            expect(stat.size).to.equal(fileContents['info.txt'].length)
           }
         })
 

libraries/overleaf-editor-core/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/promise-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/ranges-tracker/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/redis-wrapper/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/stream-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

package.json

@@ -3,39 +3,41 @@
   "private": true,
   "packageManager": "yarn@4.14.1",
   "devDependencies": {
+    "@eslint/compat": "^2.1.0",
+    "@eslint/js": "^10.0.1",
     "@overleaf/eslint-plugin": "workspace:*",
     "@prettier/plugin-pug": "^3.4.0",
     "@types/chai": "^4.3.0",
     "@types/chai-as-promised": "^7.1.8",
     "@types/mocha": "^10.0.6",
     "@types/multer": "^2.1.0",
-    "@typescript-eslint/eslint-plugin": "8.50.0",
+    "@typescript-eslint/eslint-plugin": "^8.59.4",
-    "@typescript-eslint/parser": "^8.50.0",
+    "@typescript-eslint/parser": "^8.59.4",
     "@vitest/eslint-plugin": "^1.5.0",
-    "eslint": "^8.15.0",
+    "eslint": "^10.4.0",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^10.0.1",
-    "eslint-config-standard": "^17.0.0",
+    "eslint-formatter-unix": "^8.40.0",
-    "eslint-plugin-chai-expect": "^3.0.0",
+    "eslint-plugin-chai-expect": "^4.0.0",
-    "eslint-plugin-chai-friendly": "^0.7.2",
+    "eslint-plugin-chai-friendly": "^1.1.0",
-    "eslint-plugin-cypress": "^2.15.1",
+    "eslint-plugin-cypress": "^4.1.0",
-    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-import": "^2.32.0",
-    "eslint-plugin-mocha": "^10.1.0",
+    "eslint-plugin-mocha": "^11.0.0",
-    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-n": "^18.0.0",
-    "eslint-plugin-prettier": "^4.0.0",
+    "eslint-plugin-promise": "^7.2.1",
-    "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-unicorn": "^56.0.0",
+    "globals": "^17.6.0",
     "prettier": "3.7.4",
     "prettier-plugin-groovy": "0.2.1",
     "typescript": "^5.9.3"
   },
   "engines": {
-    "node": ">=20.0.0"
+    "node": ">=20.19.0"
   },
   "resolutions": {
     "@xmldom/xmldom": "0.8.13",
     "argparse/underscore": "1.13.8",
     "east/underscore": "1.13.8",
-    "referer-parser/js-yaml": "^4.1.0",
+    "referer-parser/js-yaml": "^4.1.1",
     "sandboxed-module": "patch:sandboxed-module@npm%3A2.0.4#~/.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch",
     "request/tough-cookie": "5.1.2",
     "request/form-data": "2.5.5",
@@ -99,7 +101,6 @@
     "knip": "5.64.1",
     "eslint-plugin-testing-library": "7.5.3",
     "chart.js": "4.0.1",
-    "mock-fs": "5.2.0",
     "@customerio/cdp-analytics-node": "0.3.9",
     "@google-cloud/bigquery": "8.1.1",
     "moment": "2.29.4",

server-ce/.eslintrc

@@ -1,29 +0,0 @@
-{
-  "extends": [
-    "eslint:recommended",
-    "standard",
-    "prettier"
-  ],
-  "plugins": [
-    "unicorn"
-  ],
-  "parserOptions": {
-    "ecmaVersion": 2020
-  },
-  "env": {
-    "node": true
-  },
-  "rules": {
-    // Do not allow importing of implicit dependencies.
-    "import/no-extraneous-dependencies": "error",
-    "unicorn/prefer-node-protocol": "error"
-  },
-  "overrides": [
-    // Extra rules for Cypress tests
-    { "files": ["**/*.spec.ts"], "extends": ["plugin:cypress/recommended"] }
-  ],
-  "ignorePatterns": [
-    "hotfix/",
-    "develop/"
-  ]
-}

server-ce/Dockerfile

@@ -54,6 +54,13 @@ RUN --mount=type=cache,target=/root/.cache \
 # Add the actual source files
 # ---------------------------
 COPY --parents libraries/ services/ tools/migrations/ /overleaf/
+
+# Syntax-check all server-side ESM modules before the expensive webpack
+# compile. node --check parses without executing, so it's fast and safe.
+# Catches things like escaped backticks from sed substitutions that webpack
+# never sees (it only bundles frontend code).
+RUN find services/web/app/src services/web/modules -name '*.mjs' | xargs node --check
+
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/root/.yarn/berry/cache,id=server-ce-yarn-cache \
     --mount=type=tmpfs,target=/usr/local/share/.cache/yarn \

server-ce/Dockerfile-base

@@ -25,6 +25,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
       unattended-upgrades \
       build-essential wget net-tools unzip time poppler-utils optipng strace nginx git python3 python-is-python3 zlib1g-dev libpcre3-dev gettext-base libwww-perl ca-certificates curl gnupg \
       qpdf \
+      pandoc \
 # upgrade base-image, batch all the upgrades together, rather than installing them on-by-one (which is slow!)
 &&  unattended-upgrade --verbose --no-minimal-upgrade-steps \
 # install Node.js https://github.com/nodesource/distributions#nodejs
@@ -48,6 +49,12 @@ RUN curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QU
 &&  mkdir -p /var/www/.cache/quarto /var/www/.local/share \
 &&  chown -R www-data:www-data /var/www/.cache /var/www/.local
 
+# Install official Typst binary (Quarto bundles a modified fork without --synctex)
+# ---------------------------------------------------------------------------------
+ARG TYPST_VERSION=0.13.1
+RUN curl -fsSL "https://github.com/typst/typst/releases/download/v${TYPST_VERSION}/typst-x86_64-unknown-linux-musl.tar.xz" \
+    | tar -xJC /usr/local/bin --strip-components=1 "typst-x86_64-unknown-linux-musl/typst"
+
 # Pre-install popular Quarto extensions
 # -----------------------------------------------------------------------
 # Extensions land in /opt/quarto-extensions/_extensions/<author>/<name>/.
@@ -98,6 +105,15 @@ RUN apt-get update \
       opencv-python-headless tqdm \
 &&  rm -rf /var/lib/apt/lists/* /root/.cache
 
+# Install Inkscape (for the LaTeX svg package via shell-escape)
+# Must come AFTER the pip installs above: inkscape pulls in python3-numpy via
+# apt, which would block pip from upgrading numpy. With pip's numpy already in
+# /usr/local/lib, apt installs its own copy into /usr/lib alongside it — no
+# conflict — and Python resolves /usr/local/lib first at import time.
+RUN apt-get update \
+&&  apt-get install -y inkscape \
+&&  rm -rf /var/lib/apt/lists/*
+
 # Install decktape + headless Chromium (for exporting RevealJS decks to PDF)
 # -----------------------------------------------------------------------
 # decktape drives a headless Chromium (via Puppeteer) to print the rendered
@@ -142,15 +158,11 @@ RUN mkdir /install-tl-unx \
 &&  echo "tlpdbopt_autobackup 0"       >> /install-tl-unx/texlive.profile \
 &&  echo "tlpdbopt_install_docfiles 0" >> /install-tl-unx/texlive.profile \
 &&  echo "tlpdbopt_install_srcfiles 0" >> /install-tl-unx/texlive.profile \
-&&  echo "selected_scheme scheme-basic" >> /install-tl-unx/texlive.profile \
+&&  echo "selected_scheme scheme-full" >> /install-tl-unx/texlive.profile \
 &&  echo "TEXDIR /usr/local/texlive"   >> /install-tl-unx/texlive.profile \
 &&  /install-tl-unx/install-tl \
       -profile /install-tl-unx/texlive.profile \
       -repository ${TEXLIVE_MIRROR} \
-&&  /usr/local/texlive/bin/x86_64-linux/tlmgr install \
-      --repository ${TEXLIVE_MIRROR} \
-      latexmk \
-      texcount \
 &&  rm -rf /install-tl-unx
 
 

server-ce/config/env.sh

@@ -1,3 +1,4 @@
+export ENABLE_PANDOC_CONVERSIONS=true
 export CHAT_HOST=127.0.0.1
 export CLSI_HOST=127.0.0.1
 export DOCSTORE_HOST=127.0.0.1

server-ce/config/settings.js

@@ -50,6 +50,7 @@ const TMP_DIR = '/var/lib/overleaf/tmp'
 const settings = {
   clsi: {
     optimiseInDocker: process.env.OPTIMISE_PDF === 'true',
+    latexShellEscape: process.env.OVERLEAF_LATEX_SHELL_ESCAPE === 'true',
   },
 
   brandPrefix: '',

server-ce/eslint.config.mjs

@@ -0,0 +1,33 @@
+import { defineConfig, globalIgnores } from 'eslint/config'
+import cypress from 'eslint-plugin-cypress/flat'
+import path from 'node:path'
+import baseConfig from '../eslint.config.mjs'
+
+const ROOT_DIR = path.resolve(import.meta.dirname, '..')
+
+export default defineConfig([
+  globalIgnores(['**/hotfix/', '**/develop/']),
+  {
+    basePath: ROOT_DIR,
+    extends: baseConfig,
+    languageOptions: {
+      ecmaVersion: 2020,
+    },
+  },
+  {
+    // The cypress block in baseConfig has patterns rooted at the
+    // monorepo root (`server-ce/test/helpers/*.ts`). When ESLint loads
+    // this file (server-ce/eslint.config.mjs) as the closest config --
+    // which happens when running `yarn run lint` from
+    // /overleaf/server-ce/test/ -- patterns from baseConfig are
+    // resolved relative to /overleaf/server-ce/, so those cross-dir
+    // patterns don't match. Re-declare with paths relative to this
+    // config file.
+    files: [
+      'test/helpers/*.ts',
+      'test/cypress/support/*.{js,jsx,mjs,cjs,ts,tsx}',
+      '**/*.spec.ts',
+    ],
+    ...cypress.configs.recommended,
+  },
+])

server-ce/nginx/clsi-nginx.conf

@@ -48,9 +48,9 @@ server {
     rewrite ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$4 break;
     root /var/lib/overleaf/data/output/$1-$2/generated-files/$3/;
   }
-  # handle output files for anonymous users
+  # handle output files for anonymous users (project ID may be a UUID with hyphens for conversions)
-  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
+  location ~ ^/project/([0-9a-f-]+)/build/([0-9a-f-]+)/output/(.+)$ {
-    rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
+    rewrite ^/project/([0-9a-f-]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
     root /var/lib/overleaf/data/output/$1/generated-files/$2/;
   }
 

server-ce/nginx/nginx.conf.template

@@ -47,6 +47,7 @@ http {
 	gzip_proxied any; # allow upstream server to compress.
 
 	client_max_body_size 500m;
+	client_body_timeout 15m;
 
 	# gzip_vary on;
 	# gzip_proxied any;

server-ce/nginx/overleaf.conf

@@ -9,6 +9,26 @@ server {
 		internal;
 	}
 
+	# File upload endpoints: extended timeouts for large files on slow connections.
+	# proxy_request_buffering off: forward the request body to Node.js immediately
+	# rather than buffering first, so Node.js can send a keepalive response byte
+	# before the full body arrives (preventing upstream proxy "first-byte" timeouts).
+	# proxy_buffering off: forward that keepalive byte to Traefik/LB without delay.
+	location ~ ^/project/[^/]+/upload$ {
+		proxy_pass http://127.0.0.1:4000;
+		proxy_http_version 1.1;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-Host $host;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_request_buffering off;
+		proxy_buffering off;
+		proxy_read_timeout 15m;
+		proxy_send_timeout 15m;
+		send_timeout 15m;
+		client_body_timeout 15m;
+		client_max_body_size 550m;
+	}
+
 	location / {
 		proxy_pass http://127.0.0.1:4000;
 		proxy_http_version 1.1;

server-ce/test/Makefile

@@ -11,6 +11,7 @@ export TEX_LIVE_DOCKER_IMAGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/t
 export ALL_TEX_LIVE_DOCKER_IMAGES ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1,us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2022.1
 export IMAGE_TAG_CE ?= sharelatex/sharelatex:main
 export IMAGE_TAG_PRO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:main
+export IMAGE_TAG_GIT_BRIDGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/git-bridge:main
 export CYPRESS_SHARD ?=
 export COMPOSE_PROJECT_NAME ?= test
 export USER_UID=$(shell id -u)

server-ce/test/admin.spec.ts

@@ -245,6 +245,7 @@ describe('admin panel', function () {
           'Deleted Projects',
           'Audit Log',
           'Sessions',
+          'Personal Access Tokens',
         ]
         cy.findAllByRole('tab').should('have.length', tabs.length)
         tabs.forEach(tabName => {

server-ce/test/docker-compose.yml

@@ -78,12 +78,12 @@ services:
     working_dir: $PWD
     volumes:
       - $PWD:$PWD
-      - $MONOREPO/libraries:$MONOREPO/libraries:ro
+      - $MONOREPO/libraries:$MONOREPO/libraries
-      - $MONOREPO/node_modules:$MONOREPO/node_modules:ro
+      - $MONOREPO/node_modules:$MONOREPO/node_modules
-      - $MONOREPO/.yarn:$MONOREPO/.yarn:ro
+      - $MONOREPO/.yarn:$MONOREPO/.yarn
-      - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml:ro
+      - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml
-      - $MONOREPO/package.json:$MONOREPO/package.json:ro
+      - $MONOREPO/package.json:$MONOREPO/package.json
-      - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock:ro
+      - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock
     environment:
       MONOREPO:
       CYPRESS_SHARD:
@@ -130,6 +130,7 @@ services:
       ALL_TEX_LIVE_DOCKER_IMAGES:
       IMAGE_TAG_CE:
       IMAGE_TAG_PRO:
+      IMAGE_TAG_GIT_BRIDGE:
     healthcheck:
       test: curl --fail http://localhost/status
       interval: 3s

server-ce/test/editor.spec.ts

@@ -44,6 +44,7 @@ describe('editor', function () {
       cy.log(`change project language to '${lng}'`)
       cy.findByRole('button', { name: 'Settings' }).click()
       cy.findByRole('dialog').within(() => {
+        cy.findByRole('tab', { name: 'Spelling and language' }).click()
         cy.findByLabelText('Spellcheck language').select(lng)
       })
       cy.get('body').type('{esc}')
@@ -76,6 +77,7 @@ describe('editor', function () {
       cy.log('remove word from dictionary')
       cy.findByRole('button', { name: 'Settings' }).click()
       cy.findByRole('dialog').within(() => {
+        cy.findByRole('tab', { name: 'Spelling and language' }).click()
         cy.findByLabelText('Dictionary').click()
       })
       cy.findByTestId('dictionary-modal').within(() => {

server-ce/test/host-admin.js

@@ -32,11 +32,12 @@ const PATHS = {
 const IMAGES = {
   CE: process.env.IMAGE_TAG_CE.replace(/:.+/, ''),
   PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
+  GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/:.+/, ''),
 }
 const LATEST = {
   CE: process.env.IMAGE_TAG_CE.replace(/.+:/, '') || 'latest',
   PRO: process.env.IMAGE_TAG_PRO.replace(/.+:/, '') || 'latest',
-  GIT_BRIDGE: 'latest', // TODO, build in CI?
+  GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/.+:/, '') || 'latest',
 }
 
 function defaultDockerComposeOverride() {
@@ -242,7 +243,7 @@ function setVarsDockerCompose({
 
   cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version === 'latest' ? (pro ? LATEST.PRO : LATEST.CE) : version}`
   cfg.services['git-bridge'].image =
-    `quay.io/sharelatex/git-bridge:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
+    `${IMAGES.GIT_BRIDGE}:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
 
   cfg.services.sharelatex.environment = vars
 

server-ce/test/sandboxed-compiles.spec.ts

@@ -333,7 +333,7 @@ describe('SandboxedCompiles', function () {
   })
 
   // https://github.com/overleaf/internal/issues/20216
-  // eslint-disable-next-line mocha/no-skipped-tests
+  // eslint-disable-next-line mocha/no-pending-tests
   describe.skip('unavailable in CE', function () {
     if (isExcludedBySharding('CE_CUSTOM_1')) return
     startWith({ pro: false, vars: enabledVars, resetData: true })

services/chat/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/chat/Makefile

@@ -54,6 +54,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -66,8 +70,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/chat/buildscript.txt

@@ -1,5 +1,6 @@
 chat
 --dependencies=mongo
+--deploy-pipeline=chat
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/clsi/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/clsi/Dockerfile

@@ -32,6 +32,7 @@ COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-ed
 COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
+COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/clsi/package.json /overleaf/services/clsi/package.json
 COPY .yarn/patches/ /overleaf/.yarn/patches/
 
@@ -45,6 +46,7 @@ COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
 COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
+COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/clsi/ /overleaf/services/clsi/
 
 FROM app AS with-quarto

services/clsi/Makefile

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/promise-utils/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
+	$(MONOREPO)/libraries/validation-tools/package.json \
 	$(MONOREPO)/services/clsi/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
@@ -54,6 +55,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -67,8 +72,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \
@@ -165,8 +170,9 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
+	-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
-	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
+	-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
+	-cd ../../ && docker build -t us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pdftocairo:24.02 dockerfiles/pdftocairo
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/clsi/app.js

@@ -145,6 +145,15 @@ app.post(
   bodyParser.json({ limit: Settings.compileSizeLimit }),
   ConversionController.convertProjectToDocument
 )
+app.post(
+  '/convert/pdf-to-jpeg',
+  FileUploadMiddleware.multerMiddleware,
+  ConversionController.convertPDFToJPEG
+)
+app.get(
+  '/project/:project_id/user/:user_id/build/:build_id/thumbnail',
+  ConversionController.thumbnailFromBuild
+)
 
 if (process.env.NODE_ENV === 'development' && global.__coverage__) {
   app.get('/coverage', (req, res) => {

services/clsi/app/js/CompileController.js

@@ -80,6 +80,10 @@ function compile(req, res, next) {
                 { err: error, projectId: request.project_id },
                 'timeout running compile'
               )
+            } else if (error?.typstCompileFailure) {
+              // Typst compiled but with errors — treat as a compile failure so
+              // the frontend shows the error log rather than the old PDF.
+              status = 'failure'
             } else if (error) {
               status = 'error'
               code = 500

services/clsi/app/js/CompileManager.js

@@ -386,11 +386,18 @@ async function stopCompile(projectId, userId) {
 }
 
 async function clearProject(projectId, userId) {
+  // Kill any live typst watcher before deleting its files.
+  const compileName = getCompileName(projectId, userId)
+  await TypstRunner.promises.killTypst(compileName)
   const compileDir = getCompileDir(projectId, userId)
   await fsPromises.rm(compileDir, { force: true, recursive: true })
 }
 
 async function clearProjectWithListing(projectId, userId, allEntries) {
+  // Kill any live typst watcher (e.g. timedout compile where killTypst
+  // was not already called) before removing files from under it.
+  const compileName = getCompileName(projectId, userId)
+  await TypstRunner.promises.killTypst(compileName)
   const compileDir = getCompileDir(projectId, userId)
 
   const exists = await _checkDirectory(compileDir)

services/clsi/app/js/ConversionController.js

@@ -1,4 +1,7 @@
 import crypto from 'node:crypto'
+import { execFile } from 'node:child_process'
+import os from 'node:os'
+import { promisify } from 'node:util'
 import logger from '@overleaf/logger'
 import { expressify } from '@overleaf/promise-utils'
 import fs from 'node:fs/promises'
@@ -14,10 +17,16 @@ import RequestParser from './RequestParser.js'
 import { pipeline } from 'node:stream/promises'
 import Settings from '@overleaf/settings'
 import Path from 'node:path'
+import { z } from '@overleaf/validation-tools'
+
+const execFileAsync = promisify(execFile)
 
 const CONVERSION_CONFIGS = {
   docx: { extension: 'docx' },
   markdown: { extension: 'zip' },
+  html: { extension: 'zip' },
+  typst: { extension: 'typ' },
+  latex: { extension: 'tex' },
 }
 
 async function convertDocumentToLaTeX(req, res) {
@@ -27,7 +36,7 @@ async function convertDocumentToLaTeX(req, res) {
     await fs.unlink(path).catch(() => {})
     return res.sendStatus(404)
   }
-  if (!conversionType || !['docx', 'markdown'].includes(conversionType)) {
+  if (!conversionType || !['docx', 'markdown', 'typst'].includes(conversionType)) {
     await fs.unlink(path).catch(() => {})
     return res.sendStatus(400)
   }
@@ -77,6 +86,51 @@ async function convertDocumentToLaTeX(req, res) {
   }
 }
 
+const PDFToJPEGQuerySchema = z.object({
+  mode: z.enum(['preview', 'thumbnail']),
+})
+
+async function convertPDFToJPEG(req, res) {
+  const { path } = req.file
+  if (!Settings.enablePdfConversions) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(404)
+  }
+  const parsed = PDFToJPEGQuerySchema.safeParse(req.query)
+  if (!parsed.success) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(400)
+  }
+  const { mode } = parsed.data
+  logger.debug({ path, mode }, 'received pdf for conversion to jpeg')
+  const conversionId = crypto.randomUUID()
+  let jpegPath
+  try {
+    jpegPath = await ConversionManager.promises.convertPDFToJPEGWithLock(
+      conversionId,
+      path,
+      mode
+    )
+  } finally {
+    await fs.unlink(path).catch(() => {})
+  }
+
+  try {
+    const jpegStat = await fs.stat(jpegPath)
+
+    res.setHeader('Content-Length', jpegStat.size)
+    res.attachment('output.jpg')
+    res.setHeader('X-Content-Type-Options', 'nosniff')
+
+    const readStream = fsSync.createReadStream(jpegPath)
+    await pipeline(readStream, res)
+  } finally {
+    await fs
+      .rm(Path.dirname(jpegPath), { recursive: true, force: true })
+      .catch(() => {})
+  }
+}
+
 async function convertProjectToDocument(req, res) {
   if (!Settings.enablePandocConversions) {
     return res.sendStatus(404)
@@ -204,7 +258,117 @@ async function convertProjectToDocument(req, res) {
   }
 }
 
+// Generates a JPEG thumbnail of page 1 of the compiled output using
+// pdftocairo (poppler-utils). Tries output.pdf first (LaTeX / Quarto-PDF),
+// then output-slides.pdf (Quarto RevealJS after a PDF-export compile), then
+// falls back to rendering slide 1 of output.html via decktape (normal
+// RevealJS preview compile). All temp dirs are cleaned up in finally.
+async function thumbnailFromBuild(req, res) {
+  const { project_id: projectId, user_id: userId, build_id: buildId } = req.params
+  if (!buildId?.match(OutputCacheManager.BUILD_REGEX)) return res.sendStatus(400)
+
+  const compileName = userId ? `${projectId}-${userId}` : projectId
+  const buildDir = Path.join(
+    Settings.path.outputDir,
+    compileName,
+    OutputCacheManager.CACHE_SUBDIR,
+    buildId
+  )
+
+  let pdfPath = null
+  let deckTapeDir = null
+
+  for (const name of ['output.pdf', 'output-slides.pdf']) {
+    try {
+      const p = Path.join(buildDir, name)
+      await fs.access(p)
+      pdfPath = p
+      break
+    } catch {}
+  }
+
+  if (!pdfPath) {
+    const htmlPath = Path.join(buildDir, 'output.html')
+    try {
+      await fs.access(htmlPath)
+      deckTapeDir = await fs.mkdtemp(Path.join(os.tmpdir(), 'clsi-deck-'))
+      const chromeHome = Path.join(deckTapeDir, 'chrome')
+      await fs.mkdir(chromeHome, { recursive: true })
+      const slidePdf = Path.join(deckTapeDir, 'slide1.pdf')
+      await execFileAsync(
+        'decktape',
+        [
+          '--slides', '1',
+          '--chrome-arg=--no-sandbox',
+          '--chrome-arg=--disable-dev-shm-usage',
+          '--chrome-arg=--disable-gpu',
+          `--chrome-arg=--user-data-dir=${chromeHome}/data`,
+          htmlPath,
+          slidePdf,
+        ],
+        {
+          timeout: 60000,
+          env: {
+            ...process.env,
+            HOME: chromeHome,
+            XDG_CONFIG_HOME: chromeHome,
+            XDG_CACHE_HOME: chromeHome,
+          },
+        }
+      )
+      pdfPath = slidePdf
+    } catch (err) {
+      logger.warn({ err, projectId, buildId }, 'decktape slide1 thumbnail failed')
+      if (deckTapeDir) {
+        await fs.rm(deckTapeDir, { recursive: true, force: true }).catch(() => {})
+        deckTapeDir = null
+      }
+      return res.sendStatus(404)
+    }
+  }
+
+  const tmpDir = await fs.mkdtemp(Path.join(os.tmpdir(), 'clsi-thumb-'))
+  const outputBase = Path.join(tmpDir, 'thumb')
+  const jpegPath = outputBase + '.jpg'
+
+  try {
+    await execFileAsync(
+      'pdftocairo',
+      [
+        '-jpeg',
+        '-jpegopt', 'quality=90',
+        '-singlefile',
+        '-scale-to-x', '794',
+        '-scale-to-y', '-1',
+        '-f', '1',
+        '-l', '1',
+        pdfPath,
+        outputBase,
+      ],
+      { timeout: 30000 }
+    )
+
+    const jpegStat = await fs.stat(jpegPath)
+    res.setHeader('Content-Type', 'image/jpeg')
+    res.setHeader('Content-Length', jpegStat.size)
+    res.setHeader('Cache-Control', 'public, max-age=86400')
+    res.setHeader('X-Content-Type-Options', 'nosniff')
+    const readStream = fsSync.createReadStream(jpegPath)
+    await pipeline(readStream, res)
+  } catch (err) {
+    logger.warn({ err, projectId, buildId }, 'thumbnail generation failed')
+    if (!res.headersSent) res.sendStatus(500)
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {})
+    if (deckTapeDir) {
+      await fs.rm(deckTapeDir, { recursive: true, force: true }).catch(() => {})
+    }
+  }
+}
+
 export default {
   convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
   convertProjectToDocument: expressify(convertProjectToDocument),
+  convertPDFToJPEG: expressify(convertPDFToJPEG),
+  thumbnailFromBuild: expressify(thumbnailFromBuild),
 }

services/clsi/app/js/ConversionManager.js

@@ -16,8 +16,24 @@ const CONVERSION_CONFIGS = {
     inputFilename: 'input.md',
     pandocArgs: ['--from', 'markdown'],
   },
+  typst: {
+    inputFilename: 'input.typ',
+    pandocArgs: ['--from', 'typst'],
+  },
 }
 
+const PDF_TO_JPEG_CONFIGS = {
+  preview: { width: 794, quality: 90 },
+  thumbnail: { width: 794, quality: 90 },
+}
+
+const PDF_TO_JPEG_INPUT_FILENAME = 'input.pdf'
+const PDF_TO_JPEG_OUTPUT_FILENAME = 'output.jpg'
+const PDF_TO_JPEG_OUTPUT_BASENAME = Path.basename(
+  PDF_TO_JPEG_OUTPUT_FILENAME,
+  '.jpg'
+)
+
 async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
   const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
   const lock = LockManager.acquire(conversionDir)
@@ -150,6 +166,44 @@ const LATEX_EXPORT_CONFIGS = {
       'markdown',
     ],
   },
+  html: {
+    fileExtension: 'html',
+    compressOutput: true,
+    getPandocArgs: ({ outputPath }) => [
+      '--output',
+      outputPath,
+      '--from',
+      'latex',
+      '--to',
+      'html',
+      '--standalone',
+    ],
+  },
+  typst: {
+    fileExtension: 'typ',
+    compressOutput: false,
+    getPandocArgs: ({ outputPath }) => [
+      '--output',
+      outputPath,
+      '--from',
+      'latex',
+      '--to',
+      'typst',
+    ],
+  },
+  latex: {
+    fileExtension: 'tex',
+    compressOutput: false,
+    getPandocArgs: ({ outputPath }) => [
+      '--output',
+      outputPath,
+      '--from',
+      'typst',
+      '--to',
+      'latex',
+      '--standalone',
+    ],
+  },
 }
 
 async function convertLaTeXToDocumentInDirWithLock(
@@ -298,9 +352,76 @@ async function convertLaTeXToDocumentInDir(
   return Path.join(compileDir, finalOutputName)
 }
 
+async function convertPDFToJPEGWithLock(conversionId, inputPath, mode) {
+  const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
+  const lock = LockManager.acquire(conversionDir)
+  try {
+    return await convertPDFToJPEG(conversionId, conversionDir, inputPath, mode)
+  } finally {
+    lock.release()
+  }
+}
+
+async function convertPDFToJPEG(conversionId, conversionDir, inputPath, mode) {
+  const config = PDF_TO_JPEG_CONFIGS[mode]
+  await fs.mkdir(conversionDir, { recursive: true })
+  const newSourcePath = Path.join(conversionDir, PDF_TO_JPEG_INPUT_FILENAME)
+  await fs.copyFile(inputPath, newSourcePath)
+  const dstPath = Path.join(conversionDir, PDF_TO_JPEG_OUTPUT_FILENAME)
+
+  try {
+    const { stdout, stderr, exitCode } = await CommandRunner.promises.run(
+      conversionId,
+      [
+        'pdftocairo',
+        '-jpeg',
+        '-jpegopt',
+        `quality=${config.quality}`,
+        '-singlefile',
+        '-scale-to-x',
+        config.width.toString(),
+        '-scale-to-y',
+        '-1', // maintain aspect ratio
+        PDF_TO_JPEG_INPUT_FILENAME,
+        PDF_TO_JPEG_OUTPUT_BASENAME,
+      ],
+      conversionDir,
+      Settings.pdftocairoImage,
+      Settings.conversionTimeoutSeconds * 1000,
+      {},
+      'conversions',
+      null
+    )
+    if (exitCode !== 0) {
+      throw new OError('Non-zero exit code from pdftocairo', {
+        exitCode,
+        stderr,
+      })
+    }
+    logger.debug(
+      { stdout, stderr, exitCode },
+      'pdf-to-jpeg conversion completed'
+    )
+
+    const stat = await fs.lstat(dstPath)
+    if (!stat.isFile()) {
+      throw new OError('output.jpg is not a regular file', { stat })
+    }
+
+    // Clean up the source PDF to leave only the conversion result
+    await fs.unlink(newSourcePath).catch(() => {})
+  } catch (error) {
+    await fs.rm(conversionDir, { force: true, recursive: true }).catch(() => {})
+    throw new OError('pdf-to-jpeg conversion failed').withCause(error)
+  }
+
+  return dstPath
+}
+
 export default {
   promises: {
     convertToLaTeXWithLock,
     convertLaTeXToDocumentInDirWithLock,
+    convertPDFToJPEGWithLock,
   },
 }

services/clsi/app/js/LatexRunner.js

@@ -197,6 +197,10 @@ function _buildLatexCommand(mainFile, opts = {}) {
     command.push(...opts.flags)
   }
 
+  if (Settings.clsi?.latexShellEscape) {
+    command.push('-shell-escape')
+  }
+
   // TeX Engine selection. A .tex project may carry a non-LaTeX compiler value
   // (e.g. 'quarto', the fork-wide default for Project.compiler) because the
   // runner is chosen by file extension, not by this setting. In that case fall

services/clsi/app/js/ResourceWriter.js

@@ -209,8 +209,16 @@ export default ResourceWriter = {
           return callback(error)
         }
 
+        // Project input resources are in outputFiles only to be served from
+        // the output cache (HTML media exception in OutputFileFinder).  They
+        // must never be deleted here — incremental compiles don't re-sync
+        // unchanged binary files, so deleting them would leave them missing
+        // for Quarto and for _appendMissingResourceWarnings.
+        const incomingPaths = new Set(resources.map(r => r.path))
+
         const jobs = []
         for (const { path } of outputFiles || []) {
+          if (incomingPaths.has(path)) continue
           const shouldDelete = ResourceWriter.isExtraneousFile(path)
           if (shouldDelete) {
             jobs.push(callback =>

services/clsi/app/js/TypstRunner.js

@@ -1,96 +1,485 @@
 import Path from 'node:path'
+import { spawn } from 'node:child_process'
 import { promisify } from 'node:util'
 import logger from '@overleaf/logger'
+import Settings from '@overleaf/settings'
 import CommandRunner from './CommandRunner.js'
 import fs from 'node:fs'
 
-// Maps currently-running Typst jobs: compileName → PID (or docker container id)
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+// Max lines kept from the current compile cycle (prevents unbounded growth
+// for documents that produce many warnings).
+const MAX_LOG_LINES = 500
+
+// How long to wait for the watcher process to emit its first output.
+const WATCH_START_TIMEOUT_MS = 15_000
+
+// Matches the start-of-compile marker typst watch emits before each cycle:
+//   "[HH:MM:SS] compiling ..."
+// Used to reset the line buffer so stale output from a failed compile that
+// didn't emit a "compiled with errors" footer cannot bleed into the next log.
+const COMPILE_START_RE = /^\[\d{2}:\d{2}:\d{2}\] compiling/
+
+// Matches the three terminal lines that typst watch emits at the end of each
+// compile cycle regardless of outcome:
+//   "[HH:MM:SS] compiled successfully in 42ms"
+//   "[HH:MM:SS] compiled with warnings in 42ms"
+//   "[HH:MM:SS] compiled with errors"
+const COMPILE_DONE_RE = /compiled (successfully|with (errors|warnings))/
+
+// Signals FileId exhaustion in a long-lived typst process (typst issue #7434).
+const FILE_ID_EXHAUSTION_RE = /ran out of file ids/i
+
+// Proactively restart the watcher before FileId exhaustion.
+// Typst uses ~65 IDs per compile; 1000 compiles ≈ 65 000 — safely under 65 535.
+const MAX_COMPILES_BEFORE_RESTART = 1000
+
+// typst watch emits the "[HH:MM:SS] compiled with errors" status line FIRST,
+// then the full diagnostic output (file:line:col, code snippets) AFTERWARDS.
+// We buffer post-done lines and resolve after this delay if no new compile
+// cycle starts sooner. 150 ms is well above the ~1 chunk latency for typst's
+// diagnostic flush and imperceptible on top of a typical compile time.
+const FLUSH_DELAY_MS = 150
+
+// ---------------------------------------------------------------------------
+// State (module-level, never exported)
+// ---------------------------------------------------------------------------
+
+// Active cold-start compile jobs (Docker fallback): compileName → PID
 const ProcessTable = {}
 
-// Compiles a standalone Typst document (.typ) straight to output.pdf. We reuse
+// Long-lived watcher processes: compileName → WatchEntry
-// the Typst that ships inside Quarto via `quarto typst compile`, so there is no
+// WatchEntry shape:
-// extra binary to install. This is deliberately the simplest of the three
+//   proc              ChildProcess
-// runners: Typst only ever produces a PDF, so there is no format detection,
+//   directory         compile dir (absolute path)
-// no HTML asset directory and no extension merging (cf. QuartoRunner).
+//   mainFile          root .typ filename
-function runTypst(compileName, options, callback) {
+//   environment       env vars passed to the runner
-  const { directory, mainFile, image, environment, compileGroup } = options
+//   compilationCount  total successful compile cycles on this watcher
-  const timeout = options.timeout || 60000
+//   restartPending    flag to restart at the next runTypst call
+//   accumulator       incomplete trailing line from the last data chunk
+//   currentLines      lines accumulated in the current phase (pre-done or
+//                     post-done, see _onWatcherData for the two-phase logic)
+//   doneResult        { preLines, compiledWithErrors } held between the
+//                     COMPILE_DONE_RE line and the post-done diagnostic flush;
+//                     null when not in post-done phase
+//   flushTimeout      timeout handle that finalises doneResult when no next
+//                     compile cycle starts within FLUSH_DELAY_MS
+//   pendingResolvers  Array<{resolve, reject, timeoutHandle}>
+//   pendingResult     compile result cached when typst finished before a
+//                     resolver was registered (race-condition safety net)
+const WatchTable = {}
 
-  logger.debug(
+// PIDs we have intentionally killed, so the close handler can distinguish
-    { directory, timeout, mainFile, compileGroup },
+// an expected exit from an unexpected crash.
-    'starting typst compile'
+const _killedWatchPids = new Set()
-  )
 
-  const command = _buildTypstCommand(mainFile)
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
 
-  ProcessTable[compileName] = CommandRunner.run(
+async function runTypstAsync(compileName, options) {
-    compileName,
+  // Docker / sandboxed mode: fall back to a cold-start compile per request.
-    command,
+  if (Settings.clsi?.dockerRunner) {
-    directory,
+    return _runColdStart(compileName, options)
-    image,
+  }
-    timeout,
-    environment || {},
-    compileGroup,
-    null,
-    function (error, output) {
-      delete ProcessTable[compileName]
 
-      // Propagate real process-level errors (killed, timed out) but NOT
+  const timeout = options.timeout || 60_000
-      // ordinary non-zero exit codes from Typst itself. A compile failure
+  const entry = WatchTable[compileName]
-      // (exit code 1) is not a server error — the absence of output.pdf is
-      // enough for CompileController to return 'failure'.
-      if (error && (error.terminated || error.timedout)) {
-        return callback(error)
-      }
 
-      // On exit-code-1 errors LocalCommandRunner attaches stdout to the error
+  const needsStart =
-      // object; merge it so _writeLogOutput can persist it.
+    !entry ||
-      const combined = output || (error ? { stdout: error.stdout || '' } : null)
+    entry.restartPending ||
-      _writeLogOutput(compileName, directory, combined, () =>
+    entry.proc.exitCode !== null
-        callback(null, combined)
+
-      )
+  if (needsStart) {
+    if (entry) _killWatchEntry(compileName)
+    // _startWatcher spawns the process, registers all handlers, then calls
+    // _waitForNextCompile synchronously — the resolver is in pendingResolvers
+    // before any I/O event can fire, eliminating the race condition.
+    const result = await _startWatcher(compileName, options)
+    await _writeLogOutputAsync(compileName, options.directory, result)
+    if (result.compiledWithErrors) {
+      throw Object.assign(new Error('typst-compile-failure'), {
+        typstCompileFailure: true,
+      })
+    }
+    return result
+  }
+
+  // Watcher is alive. _waitForNextCompile adds the resolver synchronously
+  // inside the Promise constructor, before this function yields — safe.
+  const result = await _waitForNextCompile(compileName, timeout)
+  await _writeLogOutputAsync(compileName, options.directory, result)
+  if (result.compiledWithErrors) {
+    throw Object.assign(new Error('typst-compile-failure'), {
+      typstCompileFailure: true,
+    })
+  }
+  return result
+}
+
+// ---------------------------------------------------------------------------
+// Watcher lifecycle
+// ---------------------------------------------------------------------------
+
+async function _startWatcher(compileName, options) {
+  const { directory, mainFile, environment } = options
+  const timeout = options.timeout || 60_000
+
+  const absInput = Path.join(directory, mainFile)
+  const absOutput = Path.join(directory, 'output.pdf')
+  const env = { ...process.env, ...(environment || {}) }
+
+  logger.debug({ compileName, absInput }, 'starting typst watcher')
+
+  const proc = spawn(
+    '/bin/sh',
+    ['-c', `typst watch "${absInput}" "${absOutput}" 2>&1`],
+    {
+      cwd: directory,
+      env,
+      stdio: ['ignore', 'pipe', 'ignore'],
+      detached: true,
     }
   )
+
+  const entry = {
+    proc,
+    directory,
+    mainFile,
+    environment,
+    compilationCount: 0,
+    restartPending: false,
+    accumulator: '',
+    currentLines: [],
+    doneResult: null,
+    flushTimeout: null,
+    pendingResolvers: [],
+    pendingResult: null,
+  }
+  WatchTable[compileName] = entry
+
+  // Register handlers synchronously — before any I/O events can fire.
+  proc.stdout.setEncoding('utf8')
+  proc.stdout.on('data', chunk => _onWatcherData(compileName, chunk))
+
+  proc.on('error', err => {
+    logger.error({ err, compileName }, 'typst watcher process error')
+    _rejectAllPending(
+      compileName,
+      Object.assign(err, { terminated: true })
+    )
+    if (WatchTable[compileName]?.proc === proc) {
+      delete WatchTable[compileName]
+    }
+  })
+
+  proc.on('close', (code, signal) => {
+    logger.warn({ code, signal, compileName }, 'typst watcher exited')
+    const wasKilled = _killedWatchPids.delete(proc.pid)
+    if (!wasKilled) {
+      _rejectAllPending(
+        compileName,
+        Object.assign(new Error('typst watcher exited unexpectedly'), {
+          terminated: true,
+        })
+      )
+    }
+    if (WatchTable[compileName]?.proc === proc) {
+      delete WatchTable[compileName]
+    }
+  })
+
+  // typst watch performs an initial compile immediately on startup.
+  // _waitForNextCompile adds the resolver synchronously here (inside the
+  // Promise constructor) before we yield, so it will catch that first event.
+  return _waitForNextCompile(compileName, timeout + WATCH_START_TIMEOUT_MS)
 }
 
-function _buildTypstCommand(mainFile) {
+function _killWatchEntry(compileName) {
-  // Run through a POSIX shell so stderr (where Typst writes its diagnostics)
+  const entry = WatchTable[compileName]
-  // is merged into stdout (2>&1). LocalCommandRunner replaces $COMPILE_DIR
+  if (!entry) return
-  // before the shell sees it; the output path is relative because the shell
+  clearTimeout(entry.flushTimeout)
-  // CWD is already the compile directory.
+  delete WatchTable[compileName]
-  const inputPath = `$COMPILE_DIR/${mainFile}`
+  try {
-  const cmd = `quarto typst compile ${inputPath} output.pdf 2>&1`
+    _killedWatchPids.add(entry.proc.pid)
-  return ['/bin/sh', '-c', cmd]
+    process.kill(-entry.proc.pid) // kill entire process group
+  } catch (err) {
+    _killedWatchPids.delete(entry.proc.pid)
+    logger.warn({ err, compileName }, 'error killing typst watcher process group')
+  }
 }
 
-function _writeLogOutput(compileName, directory, output, callback) {
+// ---------------------------------------------------------------------------
-  const content = (output && output.stdout) || ''
+// Stdout parsing
-  if (!content) return callback()
+// ---------------------------------------------------------------------------
-  // Write to output.log so the PDF-preview log panel picks it up. Typst's
+
-  // `error:`/`warning:` + `┌─ file:line:col` diagnostics are understood by the
+function _onWatcherData(compileName, chunk) {
-  // Quarto/Typst log parser on the web side.
+  const entry = WatchTable[compileName]
-  const logFile = Path.join(directory, 'output.log')
+  if (!entry) return
-  fs.unlink(logFile, () => {
+
-    fs.writeFile(logFile, content, { flag: 'wx' }, err => {
+  entry.accumulator += chunk
-      if (err) {
+  const lines = entry.accumulator.split('\n')
-        logger.error({ err, compileName, logFile }, 'error writing typst log')
+  entry.accumulator = lines.pop() // keep the incomplete trailing fragment
+
+  for (const line of lines) {
+    if (COMPILE_START_RE.test(line)) {
+      // A new compile cycle is starting.  If we were in the post-done phase
+      // (collecting diagnostic lines that typst emits AFTER the status line),
+      // finalise the previous result now — all diagnostics have arrived.
+      if (entry.doneResult) {
+        _finalizeCompile(compileName)
       }
-      callback()
+      // Start fresh for the new cycle.
-    })
+      entry.currentLines = [line]
+      continue
+    }
+
+    entry.currentLines.push(line)
+    if (entry.currentLines.length > MAX_LOG_LINES) {
+      entry.currentLines.shift()
+    }
+
+    if (FILE_ID_EXHAUSTION_RE.test(line)) {
+      logger.warn({ compileName }, 'typst watcher: FileId exhaustion detected')
+      entry.restartPending = true
+    }
+
+    if (COMPILE_DONE_RE.test(line)) {
+      entry.compilationCount++
+
+      if (entry.compilationCount >= MAX_COMPILES_BEFORE_RESTART) {
+        logger.info(
+          { compileName, compilationCount: entry.compilationCount },
+          'typst watcher: scheduling restart (FileId threshold)'
+        )
+        entry.restartPending = true
+      }
+
+      // typst watch outputs the "[HH:MM:SS] compiled with errors" status
+      // line FIRST, then the full diagnostics (file:line:col, code snippets)
+      // AFTERWARDS. Enter post-done phase: keep accumulating into currentLines
+      // and flush after FLUSH_DELAY_MS (or immediately when the next compile
+      // cycle's COMPILE_START_RE arrives, whichever comes first).
+      entry.doneResult = {
+        preLines: entry.currentLines,
+        compiledWithErrors: /compiled with errors/.test(line),
+      }
+      entry.currentLines = []
+
+      clearTimeout(entry.flushTimeout)
+      entry.flushTimeout = setTimeout(
+        () => _finalizeCompile(compileName),
+        FLUSH_DELAY_MS
+      )
+    }
+  }
+}
+
+// Combines the pre-done lines (up to/including the status line) with any
+// post-done diagnostic lines and resolves all pending waiters.
+function _finalizeCompile(compileName) {
+  const entry = WatchTable[compileName]
+  if (!entry || !entry.doneResult) return
+
+  clearTimeout(entry.flushTimeout)
+  entry.flushTimeout = null
+
+  const { preLines, compiledWithErrors } = entry.doneResult
+  entry.doneResult = null
+
+  // Merge: status line(s) first, then the post-done diagnostics.
+  const allLines = preLines.concat(entry.currentLines)
+  entry.currentLines = []
+
+  _resolveAllPending(compileName, {
+    stdout: allLines.join('\n'),
+    compiledWithErrors,
   })
 }
 
+// ---------------------------------------------------------------------------
+// Resolver helpers
+// ---------------------------------------------------------------------------
+
+function _waitForNextCompile(compileName, timeout) {
+  return new Promise((resolve, reject) => {
+    const entry = WatchTable[compileName]
+    if (!entry) {
+      return reject(new Error('no typst watcher for ' + compileName))
+    }
+
+    // If typst finished a compile cycle before this resolver was registered
+    // (race: ResourceWriter wrote files → typst compiled → runTypst called),
+    // consume the cached result immediately instead of waiting for a timeout.
+    if (entry.pendingResult) {
+      const result = entry.pendingResult
+      entry.pendingResult = null
+      return resolve(result)
+    }
+
+    // Push synchronously inside the Promise constructor — before the first
+    // await in the caller, so no data event can fire in the gap.
+    const timeoutHandle = setTimeout(() => {
+      entry.pendingResolvers = entry.pendingResolvers.filter(r => r !== resolver)
+      reject(
+        Object.assign(new Error('typst compile timed out'), { timedout: true })
+      )
+    }, timeout)
+
+    const resolver = { resolve, reject, timeoutHandle }
+    entry.pendingResolvers.push(resolver)
+  })
+}
+
+function _resolveAllPending(compileName, result) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+  const resolvers = entry.pendingResolvers.splice(0)
+  if (resolvers.length === 0) {
+    // typst compiled before a resolver was registered — cache the result so
+    // the next _waitForNextCompile call can consume it immediately.
+    entry.pendingResult = result
+    return
+  }
+  for (const { resolve, timeoutHandle } of resolvers) {
+    clearTimeout(timeoutHandle)
+    resolve(result)
+  }
+}
+
+function _rejectAllPending(compileName, err) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+  for (const { reject, timeoutHandle } of entry.pendingResolvers.splice(0)) {
+    clearTimeout(timeoutHandle)
+    reject(err)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Log output
+// ---------------------------------------------------------------------------
+
+async function _writeLogOutputAsync(compileName, directory, output) {
+  const content = (output && output.stdout) || ''
+  if (!content) return
+  // Write to output.log so the PDF-preview log panel picks it up.
+  const logFile = Path.join(directory, 'output.log')
+  try {
+    await fs.promises.unlink(logFile)
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      logger.error({ err, compileName, logFile }, 'error removing typst log')
+    }
+  }
+  try {
+    await fs.promises.writeFile(logFile, content, { flag: 'wx' })
+  } catch (err) {
+    if (err.code !== 'EEXIST') {
+      logger.error({ err, compileName, logFile }, 'error writing typst log')
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Cold-start fallback (Docker / sandboxed mode)
+// ---------------------------------------------------------------------------
+
+function _runColdStart(compileName, options) {
+  return new Promise((resolve, reject) => {
+    const { directory, mainFile, image, environment, compileGroup } = options
+    const timeout = options.timeout || 60_000
+
+    logger.debug({ directory, mainFile, compileGroup }, 'typst cold-start compile')
+
+    const inputPath = `$COMPILE_DIR/${mainFile}`
+    const command = ['/bin/sh', '-c', `typst compile ${inputPath} output.pdf 2>&1`]
+
+    ProcessTable[compileName] = CommandRunner.run(
+      compileName,
+      command,
+      directory,
+      image,
+      timeout,
+      environment || {},
+      compileGroup,
+      null,
+      function (error, output) {
+        delete ProcessTable[compileName]
+        if (error && (error.terminated || error.timedout)) {
+          return reject(error)
+        }
+        const combined =
+          output || (error ? { stdout: error.stdout || '' } : null)
+        _writeLogOutputAsync(compileName, directory, combined).then(
+          () => {
+            if (error && combined?.stdout) {
+              // Non-zero exit with output = typst compile error (not a
+              // system/infra error). Signal failure so the log panel opens.
+              reject(
+                Object.assign(new Error('typst-compile-failure'), {
+                  typstCompileFailure: true,
+                })
+              )
+            } else if (error) {
+              reject(error)
+            } else {
+              resolve(combined)
+            }
+          },
+          reject
+        )
+      }
+    )
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Public interface
+// ---------------------------------------------------------------------------
+
 function isRunning(compileName) {
-  return ProcessTable[compileName] != null
+  return (
+    ProcessTable[compileName] != null ||
+    (WatchTable[compileName]?.pendingResolvers.length > 0)
+  )
 }
 
 function killTypst(compileName, callback) {
-  logger.debug({ compileName }, 'killing running typst compile')
+  logger.debug({ compileName }, 'killing typst (watcher + any active compile)')
-  if (!isRunning(compileName)) {
+
-    logger.warn({ compileName }, 'no such compile to kill')
+  // Cold-start fallback path
-    return callback(null)
+  if (ProcessTable[compileName] != null) {
+    CommandRunner.kill(ProcessTable[compileName], () => {})
+    delete ProcessTable[compileName]
   }
-  CommandRunner.kill(ProcessTable[compileName], callback)
+
+  // Reject any in-flight waiters and tear down the watcher process
+  _rejectAllPending(
+    compileName,
+    Object.assign(new Error('terminated'), { terminated: true })
+  )
+  _killWatchEntry(compileName)
+
+  callback(null)
+}
+
+// Kill all watcher processes when the CLSI Node process exits.
+process.on('exit', () => {
+  for (const compileName of Object.keys(WatchTable)) {
+    _killWatchEntry(compileName)
+  }
+})
+
+const runTypst = (compileName, options, callback) => {
+  runTypstAsync(compileName, options).then(
+    result => callback(null, result),
+    err => callback(err)
+  )
 }
 
 export default {
@@ -98,7 +487,7 @@ export default {
   runTypst,
   killTypst,
   promises: {
-    runTypst: promisify(runTypst),
+    runTypst: runTypstAsync,
     killTypst: promisify(killTypst),
   },
 }

services/clsi/buildscript.txt

@@ -1,7 +1,7 @@
 clsi
 --data-dirs=cache,compiles,output
 --dependencies=
---env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true
+--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true,ENABLE_PDF_CONVERSIONS=true
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.14.1

services/clsi/config/settings.defaults.cjs

@@ -31,6 +31,9 @@ module.exports = {
     parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
   pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
   enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
+  pdftocairoImage:
+    process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
+  enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
   maxUploadSize: 50 * 1024 * 1024,
 
   internal: {

services/clsi/docker-compose.ci.yml

@@ -30,7 +30,7 @@ services:
       ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
       ENABLE_PDF_CACHING: true
       PDF_CACHING_ENABLE_WORKER_POOL: true
-      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
+      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
@@ -38,6 +38,7 @@ services:
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
       ENABLE_PANDOC_CONVERSIONS: true
+      ENABLE_PDF_CONVERSIONS: true
     volumes:
       - ./reports:/overleaf/services/clsi/reports
       - ./compiles:/overleaf/services/clsi/compiles

services/clsi/docker-compose.yml

@@ -53,7 +53,7 @@ services:
       ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
       ENABLE_PDF_CACHING: true
       PDF_CACHING_ENABLE_WORKER_POOL: true
-      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
+      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
@@ -61,6 +61,7 @@ services:
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
       ENABLE_PANDOC_CONVERSIONS: true
+      ENABLE_PDF_CONVERSIONS: true
     depends_on:
       clsi-nginx:
         condition: service_started

services/clsi/package.json

@@ -23,6 +23,7 @@
     "@overleaf/promise-utils": "workspace:*",
     "@overleaf/settings": "workspace:*",
     "@overleaf/stream-utils": "workspace:*",
+    "@overleaf/validation-tools": "workspace:*",
     "archiver": "5.3.2",
     "async": "^3.2.5",
     "body-parser": "1.20.4",
@@ -45,7 +46,6 @@
     "mocha": "^11.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
-    "mock-fs": "^5.1.2",
     "node-fetch": "^2.7.0",
     "nyc": "^17.1.0",
     "sinon": "~9.0.1",

services/clsi/test/acceptance/js/PdfToJpegConversionTests.js

@@ -0,0 +1,83 @@
+import Client from './helpers/Client.js'
+import ClsiApp from './helpers/ClsiApp.js'
+import Path from 'node:path'
+import fs from 'node:fs/promises'
+import { promisify } from 'node:util'
+import { execFile as execFileCb } from 'node:child_process'
+import { expect } from 'chai'
+
+const execFile = promisify(execFileCb)
+
+const FIXTURE_PDF = Path.join(import.meta.dirname, '../fixtures/minimal.pdf')
+
+const MODE_EXPECTATIONS = {
+  preview: { width: 794 },
+  thumbnail: { width: 190 },
+}
+
+async function writeResponseToTempfile(response) {
+  const buffer = Buffer.from(await response.arrayBuffer())
+  const tmpPath = `/tmp/clsi-acceptance-pdf-to-jpeg-${crypto.randomUUID()}.jpg`
+  await fs.writeFile(tmpPath, buffer)
+  return { tmpPath, buffer }
+}
+
+describe('pdf-to-jpeg conversion', function () {
+  before(async function () {
+    await ClsiApp.ensureRunning()
+  })
+
+  for (const [mode, { width: expectedWidth }] of Object.entries(
+    MODE_EXPECTATIONS
+  )) {
+    describe(`with mode=${mode}`, function () {
+      let response
+      let tmpPath
+      let buffer
+
+      before(async function () {
+        response = await Client.convertPdfToJpeg(FIXTURE_PDF, mode)
+        expect(response.status).to.equal(200)
+        ;({ tmpPath, buffer } = await writeResponseToTempfile(response))
+      })
+
+      after(async function () {
+        if (tmpPath) {
+          await fs.unlink(tmpPath).catch(() => {})
+        }
+      })
+
+      it('returns a JPEG (per `file`)', async function () {
+        const { stdout } = await execFile('file', ['--brief', tmpPath])
+        expect(stdout).to.match(/JPEG image data/)
+      })
+
+      it(`has the expected width of ${expectedWidth}px`, async function () {
+        const { stdout } = await execFile('identify', [
+          '-format',
+          '%w %h',
+          tmpPath,
+        ])
+        const [width, height] = stdout.trim().split(' ').map(Number)
+        expect(width).to.equal(expectedWidth)
+        // A4 portrait is taller than wide; height must be positive and
+        // larger than the width (so the aspect ratio was preserved).
+        expect(height).to.be.greaterThan(width)
+      })
+
+      it('returns a non-empty body matching Content-Length', function () {
+        expect(buffer.length).to.be.greaterThan(0)
+        expect(buffer.length).to.equal(
+          Number(response.headers.get('content-length'))
+        )
+      })
+    })
+  }
+
+  describe('with an unsupported mode', function () {
+    it('returns 400', async function () {
+      const response = await Client.convertPdfToJpeg(FIXTURE_PDF, 'not-a-mode')
+      expect(response.status).to.equal(400)
+    })
+  })
+})

services/clsi/test/acceptance/js/helpers/Client.js

@@ -53,6 +53,16 @@ async function convertDocument(path, type) {
   }
 }
 
+async function convertPdfToJpeg(path, mode) {
+  const formData = new FormData()
+  formData.append('qqfile', await fsPromises.readFile(path), 'input.pdf')
+  return await fetch(`${host}/convert/pdf-to-jpeg?mode=${mode}`, {
+    method: 'POST',
+    headers: formData.getHeaders(),
+    body: formData.getBuffer(),
+  })
+}
+
 async function convertProjectToDocument(
   projectId,
   userId,
@@ -239,6 +249,7 @@ export default {
   compile,
   convertProjectToDocument,
   convertDocument,
+  convertPdfToJpeg,
   stopCompile,
   clearCache,
   getOutputFile,

services/clsi/test/unit/js/ConversionController.test.js

@@ -18,6 +18,7 @@ describe('ConversionController', function () {
     ctx.documentStat = { size: 5678 }
     ctx.Settings = {
       enablePandocConversions: true,
+      enablePdfConversions: true,
       path: {
         compilesDir: '/compiles',
         outputDir: '/output',
@@ -591,6 +592,37 @@ describe('ConversionController', function () {
       })
     })
 
+    describe('with conversionType=html', function () {
+      beforeEach(async function (ctx) {
+        ctx.req.query = { type: 'html' }
+        ctx.fs.stat.resolves(ctx.documentStat)
+
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          sinon.stub()
+        )
+      })
+
+      it('should call convertLaTeXToDocumentInDirWithLock with type=html', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
+          sinon.match(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          ),
+          sinon.match(
+            /^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          ),
+          'main.tex',
+          'html'
+        )
+      })
+
+      it('should set the attachment filename with .zip extension', function (ctx) {
+        sinon.assert.calledWith(ctx.res.attachment, 'output.zip')
+      })
+    })
+
     describe('when conversion fails', function () {
       beforeEach(async function (ctx) {
         ctx.next = sinon.stub()

services/clsi/test/unit/js/ConversionManager.test.js

@@ -77,6 +77,24 @@ const LATEX_TO_DOCUMENT_CASES = [
       '--extract-media=.',
     ],
   },
+  {
+    type: 'html',
+    extension: 'html',
+    compressOutput: true,
+    pandocArgs: outputId => [
+      'pandoc',
+      Path.join('..', 'main.tex'),
+      '--output',
+      'main.html',
+      '--from',
+      'latex',
+      '--to',
+      'html',
+      '--standalone',
+      '--resource-path=..',
+      '--extract-media=.',
+    ],
+  },
 ]
 
 describe('ConversionManager', function () {

services/clsi/test/unit/js/DraftModeManager.test.js

@@ -1,7 +1,8 @@
 import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest'
 import Path from 'node:path'
+import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
-import mockFs from 'mock-fs'
+import os from 'node:os'
 
 const MODULE_PATH = Path.join(
   import.meta.dirname,
@@ -15,20 +16,19 @@ describe('DraftModeManager', () => {
     }))
 
     ctx.DraftModeManager = (await import(MODULE_PATH)).default
-    ctx.filename = '/mock/filename.tex'
+    ctx.tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'draft-mode-test-'))
+    ctx.filename = Path.join(ctx.tmpDir, 'filename.tex')
     ctx.contents = `\
 \\documentclass{article}
 \\begin{document}
 Hello world
 \\end{document}\
 `
-    mockFs({
+    fs.writeFileSync(ctx.filename, ctx.contents)
-      [ctx.filename]: ctx.contents,
-    })
   })
 
-  afterEach(() => {
+  afterEach(ctx => {
-    mockFs.restore()
+    fs.rmSync(ctx.tmpDir, { recursive: true })
   })
 
   describe('injectDraftMode', () => {

services/clsi/test/unit/js/OutputFileFinder.test.js

@@ -1,6 +1,6 @@
-import sinon from 'sinon'
 import { expect, describe, beforeEach, afterEach, it } from 'vitest'
-import mockFs from 'mock-fs'
+import fs from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 
 const modulePath = path.join(
@@ -8,30 +8,40 @@ const modulePath = path.join(
   '../../../app/js/OutputFileFinder'
 )
 
+function createTree(base, tree) {
+  fs.mkdirSync(base, { recursive: true })
+  for (const [name, content] of Object.entries(tree)) {
+    const fullPath = path.join(base, name)
+    if (Buffer.isBuffer(content) || typeof content === 'string') {
+      fs.writeFileSync(fullPath, content)
+    } else if (content && content.symlink) {
+      fs.symlinkSync(content.symlink, fullPath)
+    } else {
+      createTree(fullPath, content)
+    }
+  }
+}
+
 describe('OutputFileFinder', function () {
   beforeEach(async function (ctx) {
     ctx.OutputFileFinder = (await import(modulePath)).default
-    ctx.directory = '/test/dir'
+    ctx.directory = fs.mkdtempSync(
-    ctx.callback = sinon.stub()
+      path.join(os.tmpdir(), 'output-finder-test-')
-
+    )
-    mockFs({
+    createTree(ctx.directory, {
-      [ctx.directory]: {
+      resource: {
-        resource: {
+        'path.tex': 'a source file',
-          'path.tex': 'a source file',
-        },
-        'output.pdf': 'a generated pdf file',
-        extra: {
-          'file.tex': 'a generated tex file',
-        },
-        'sneaky-file': mockFs.symlink({
-          path: '../foo',
-        }),
       },
+      'output.pdf': 'a generated pdf file',
+      extra: {
+        'file.tex': 'a generated tex file',
+      },
+      'sneaky-file': { symlink: '../foo' },
     })
   })
 
-  afterEach(function () {
+  afterEach(function (ctx) {
-    mockFs.restore()
+    fs.rmSync(ctx.directory, { recursive: true })
   })
 
   describe('findOutputFiles', function () {

services/docstore/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/docstore/Makefile

@@ -56,6 +56,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -68,8 +72,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/docstore/buildscript.txt

@@ -1,5 +1,6 @@
 docstore
 --dependencies=mongo,gcs
+--deploy-pipeline=docstore
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/document-updater/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/document-updater/Makefile

@@ -57,6 +57,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -69,8 +73,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/document-updater/buildscript.txt

@@ -1,5 +1,6 @@
 document-updater
 --dependencies=mongo,redis
+--deploy-pipeline=document-updater
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/document-updater/types/mongodb-legacy.d.ts

@@ -0,0 +1,3 @@
+declare module 'mongodb-legacy' {
+  export * from 'mongodb'
+}

services/filestore/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/filestore/Makefile

@@ -53,6 +53,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -66,8 +70,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/filestore/buildscript.txt

@@ -1,6 +1,7 @@
 filestore
 --data-dirs=uploads,template_files
 --dependencies=s3,gcs
+--deploy-pipeline=filestore-readonly
 --env-add=ENABLE_CONVERSIONS=true,USE_PROM_METRICS=true,AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
 --env-pass-through=
 --esmock-loader=False

services/git-bridge/Makefile

@@ -15,6 +15,10 @@ IMAGE_REPO_BRANCH ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)
 IMAGE_REPO_MAIN ?= $(IMAGE_REPO):main
 IMAGE_REPO_FINAL ?= $(IMAGE_REPO_BRANCH)-$(BUILD_NUMBER)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 runtime-conf:
 	/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java

@@ -5,6 +5,10 @@ import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpResponse;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSyntaxException;
 import jakarta.servlet.*;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
@@ -107,15 +111,18 @@ public class Oauth2Filter implements Filter {
       // fail later (for example, in the unlikely event that the token
       // expired between the two requests). In that case, JGit will
       // return a 401 without a custom error message.
-      int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
+      AccessTokenCheck check = checkAccessToken(this.oauth2Server, password, getClientIp(request));
-      if (statusCode == 429) {
+      if (check.statusCode == 429) {
         handleRateLimit(projectId, username, request, response);
         return;
-      } else if (statusCode == 401) {
+      } else if (check.statusCode == 401 && "token_expired".equals(check.errorCode)) {
+        handleExpiredAccessToken(projectId, request, response);
+        return;
+      } else if (check.statusCode == 401) {
         handleBadAccessToken(projectId, request, response);
         return;
-      } else if (statusCode >= 400) {
+      } else if (check.statusCode >= 400) {
-        handleUnknownOauthServerError(projectId, statusCode, request, response);
+        handleUnknownOauthServerError(projectId, check.statusCode, request, response);
         return;
       }
       cred.setAccessToken(password);
@@ -229,8 +236,52 @@ public class Oauth2Filter implements Filter {
             "https://www.overleaf.com/learn/how-to/Git_integration"));
   }
 
-  private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
+  private void handleExpiredAccessToken(
+      String projectId, HttpServletRequest request, HttpServletResponse response)
       throws IOException {
+    Log.debug("[{}] Expired access token, ip={}", projectId, getClientIp(request));
+    sendResponse(
+        response,
+        401,
+        Arrays.asList(
+            "Your Overleaf Git authentication token has expired.",
+            "",
+            "Generate a new authentication token in your Overleaf Account Settings,",
+            "then run the git command again."));
+  }
+
+  static class AccessTokenCheck {
+    final int statusCode;
+    final String errorCode;
+
+    AccessTokenCheck(int statusCode, String errorCode) {
+      this.statusCode = statusCode;
+      this.errorCode = errorCode;
+    }
+  }
+
+  static String parseErrorCode(String body) {
+    if (body == null || body.isEmpty()) {
+      return null;
+    }
+    try {
+      JsonElement element = new Gson().fromJson(body, JsonElement.class);
+      if (element == null || !element.isJsonObject()) {
+        return null;
+      }
+      JsonObject obj = element.getAsJsonObject();
+      JsonElement codeElement = obj.get("error_code");
+      if (codeElement == null || codeElement.isJsonNull()) {
+        return null;
+      }
+      return codeElement.getAsString();
+    } catch (JsonSyntaxException | UnsupportedOperationException e) {
+      return null;
+    }
+  }
+
+  private AccessTokenCheck checkAccessToken(
+      String oauth2Server, String accessToken, String clientIp) throws IOException {
     GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
     HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
     HttpHeaders headers = new HttpHeaders();
@@ -239,8 +290,12 @@ public class Oauth2Filter implements Filter {
     request.setThrowExceptionOnExecuteError(false);
     HttpResponse response = request.execute();
     int statusCode = response.getStatusCode();
+    String errorCode = null;
+    if (statusCode >= 400 && statusCode < 500) {
+      errorCode = parseErrorCode(response.parseAsString());
+    }
     response.disconnect();
-    return statusCode;
+    return new AccessTokenCheck(statusCode, errorCode);
   }
 
   private void handleUnknownOauthServerError(

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/server/Oauth2FilterTest.java

@@ -0,0 +1,51 @@
+package uk.ac.ic.wlgitbridge.server;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class Oauth2FilterTest {
+
+  @Test
+  public void parseErrorCode_returnsTokenExpired_whenBodyContainsIt() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_expired\"}";
+    Assert.assertEquals("token_expired", Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsTokenInvalid_whenBodyContainsIt() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_invalid\"}";
+    Assert.assertEquals("token_invalid", Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenErrorCodeFieldIsMissing() {
+    String body = "{\"error\":\"invalid_token\"}";
+    Assert.assertNull(Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsNull() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode(null));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsEmpty() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode(""));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsNotJson() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode("not json at all"));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsJsonArray() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode("[1, 2, 3]"));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenErrorCodeFieldIsJsonNull() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":null}";
+    Assert.assertNull(Oauth2Filter.parseErrorCode(body));
+  }
+}

services/history-v1/.eslintignore

@@ -1,2 +0,0 @@
-archive/
-

services/history-v1/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/history-v1/Makefile

@@ -59,6 +59,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -71,8 +75,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/history-v1/buildscript.txt

@@ -1,5 +1,6 @@
 history-v1
 --dependencies=postgres,gcs,mongo,redis,s3
+--deploy-pipeline=history-v1
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/history-v1/docker-compose.ci.yml

@@ -109,7 +109,7 @@ services:
       - mongo:127.0.0.1
 
   postgres:
-    image: postgres:10
+    image: postgres:14
     environment:
       POSTGRES_USER: overleaf
       POSTGRES_PASSWORD: overleaf

services/history-v1/docker-compose.yml

@@ -124,7 +124,7 @@ services:
       - mongo:127.0.0.1
 
   postgres:
-    image: postgres:10
+    image: postgres:14
     environment:
       POSTGRES_USER: overleaf
       POSTGRES_PASSWORD: overleaf

services/history-v1/package.json

@@ -44,7 +44,8 @@
     "temp": "^0.8.3",
     "throng": "^4.0.0",
     "tsscmp": "^1.0.6",
-    "utf-8-validate": "^5.0.4"
+    "utf-8-validate": "^5.0.4",
+    "zip-stream": "^7.0.2"
   },
   "devDependencies": {
     "@overleaf/migrations": "workspace:*",

services/history-v1/storage/lib/backupArchiver.mjs

@@ -185,13 +185,30 @@ class BackupBlobStore {
  */
 
 /**
- * @typedef {(import('archiver').Archiver)} Archiver
+ * @typedef {(import('zip-stream').default)} ZipStream
  */
 
 /**
  * @typedef {(import('overleaf-editor-core').FileMap)} FileMap
  */
 
+/**
+ * Promisified wrapper for ZipStream's entry method.
+ *
+ * @param {ZipStream} archive
+ * @param {Buffer|NodeJS.ReadableStream|string} source
+ * @param {{ name: string }} data
+ * @return {Promise<void>}
+ */
+function addEntry(archive, source, data) {
+  return new Promise((resolve, reject) => {
+    archive.entry(source, data, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 /**
  *
  * @param historyId
@@ -254,14 +271,15 @@ async function fetchBlob(historyId, hash, persistor) {
 
 /**
  * @typedef {object} AddChunkOptions
- * @property {string} [prefix] Should include trailing slash (if length > 0)
+ * @property {string} [prefix]
  * @property {boolean} [useBackupGlobalBlobs]
+ * @property {boolean} [verbose]
  */
 
 /**
  *
  * @param {History} history
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {CachedPerProjectEncryptedS3Persistor} projectCache
  * @param {string} historyId
  * @param {AddChunkOptions} [options]
@@ -272,7 +290,7 @@ async function addChunkToArchive(
   archive,
   projectCache,
   historyId,
-  { prefix = '', useBackupGlobalBlobs = false } = {}
+  { prefix = '', useBackupGlobalBlobs = false, verbose = false } = {}
 ) {
   const chunkBlobs = new Set()
   history.findBlobHashes(chunkBlobs)
@@ -334,9 +352,16 @@ async function addChunkToArchive(
         }
         content = await blobStore.getStream(hash)
       }
-      archive.append(content, {
+      if (content == null) {
+        logger.error({ filePath }, 'File content is empty')
+        continue
+      }
+      await addEntry(archive, content, {
         name: `${prefix}${filePath}`,
       })
+      if (verbose) {
+        logger.info({ filePath: `${prefix}${filePath}` }, 'added to archive')
+      }
     }
   })
 }
@@ -358,17 +383,20 @@ async function findStartVersionOfLatestChunk(historyId) {
 /**
  * Restore a project from the latest snapshot
  *
- * There is an assumption that the database backup has been restored.
+ * There is an assumption that the database backup
+ * has been restored.
  *
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {string} historyId
  * @param {boolean} [useBackupGlobalBlobs]
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
 export async function archiveLatestChunk(
   archive,
   historyId,
-  useBackupGlobalBlobs = false
+  useBackupGlobalBlobs = false,
+  verbose = false
 ) {
   logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
 
@@ -386,20 +414,28 @@ export async function archiveLatestChunk(
 
   await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
     useBackupGlobalBlobs,
+    verbose,
   })
 
   return archive
 }
 
 /**
- * Fetches all raw blobs from the project and adds them to the archive.
+ * Fetches all raw blobs from the project and adds
+ * them to the archive.
  *
  * @param {string} historyId
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {CachedPerProjectEncryptedS3Persistor} projectCache
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
-async function addRawBlobsToArchive(historyId, archive, projectCache) {
+async function addRawBlobsToArchive(
+  historyId,
+  archive,
+  projectCache,
+  verbose = false
+) {
   const blobKeys = await projectCache.listDirectoryKeys(
     projectBlobsBucket,
     projectKey.format(historyId)
@@ -411,9 +447,13 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
         key,
         { autoGunzip: true }
       )
-      archive.append(stream, {
+      const entryName = path.join(historyId, 'blobs', key)
-        name: path.join(historyId, 'blobs', key),
+      await addEntry(archive, stream, {
+        name: entryName,
       })
+      if (verbose) {
+        logger.info({ entryName }, 'added to archive')
+      }
     } catch (err) {
       logger.warn({ err, path: key }, 'Failed to append blob to archive')
     }
@@ -425,17 +465,20 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
  *
  * This can work without the database being backed up.
  *
- * It will split the project into chunks per directory and download the blobs alongside the chunk.
+ * It will split the project into chunks per directory
+ * and download the blobs alongside the chunk.
  *
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {string} historyId
  * @param {boolean} [useBackupGlobalBlobs]
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
 export async function archiveRawProject(
   archive,
   historyId,
-  useBackupGlobalBlobs = false
+  useBackupGlobalBlobs = false,
+  verbose = false
 ) {
   const projectCache = await getProjectPersistor(historyId)
 
@@ -454,11 +497,15 @@ export async function archiveRawProject(
 
     const { buffer } = await loadChunkByKey(projectCache, key)
 
-    archive.append(buffer, {
+    const entryName = `${historyId}/chunks/${chunkId}/chunk.json`
-      name: `${historyId}/chunks/${chunkId}/chunk.json`,
+    await addEntry(archive, buffer, {
+      name: entryName,
     })
+    if (verbose) {
+      logger.info({ entryName }, 'added to archive')
+    }
   }
-  await addRawBlobsToArchive(historyId, archive, projectCache)
+  await addRawBlobsToArchive(historyId, archive, projectCache, verbose)
 }
 
 export class BackupPersistorError extends OError {}

services/history-v1/storage/lib/streams.js

@@ -6,19 +6,19 @@
  */
 'use strict'
 
-const Stream = require('node:stream')
+const { pipeline } = require('node:stream/promises')
 const zlib = require('node:zlib')
 const { WritableBuffer } = require('@overleaf/stream-utils')
 
 /**
  * Create a promise for the result of reading a stream to a buffer.
  *
- * @param {Stream.Readable} readStream
+ * @param {import('node:stream').Readable} readStream
  * @return {Promise<Buffer>}
  */
 async function readStreamToBuffer(readStream) {
   const bufferStream = new WritableBuffer()
-  await Stream.promises.pipeline(readStream, bufferStream)
+  await pipeline(readStream, bufferStream)
   return bufferStream.contents()
 }
 
@@ -33,7 +33,7 @@ exports.readStreamToBuffer = readStreamToBuffer
 async function gunzipStreamToBuffer(readStream) {
   const gunzip = zlib.createGunzip()
   const bufferStream = new WritableBuffer()
-  await Stream.promises.pipeline(readStream, gunzip, bufferStream)
+  await pipeline(readStream, gunzip, bufferStream)
   return bufferStream.contents()
 }
 

services/history-v1/storage/scripts/backup_worker.mjs

@@ -19,7 +19,7 @@ const LAG_TIME_BUCKETS_HRS = [
 ] // hours
 
 // Configure backup settings to match worker concurrency
-configureBackup({ concurrency: UPLOAD_CONCURRENCY, useSecondary: true })
+configureBackup({ concurrency: UPLOAD_CONCURRENCY })
 
 let gracefulShutdownInitiated = false
 

services/history-v1/storage/scripts/finalise_chunk.mjs

@@ -0,0 +1,129 @@
+// Finalise the current chunk for a project and start a new empty chunk
+// whose starting snapshot is the end snapshot of the (now-closed) current
+// chunk.
+//
+// This is intended as a recovery tool for projects whose current chunk has
+// become corrupted in such a way that further changes can no longer be
+// persisted, but where the end snapshot of the current chunk can still be
+// computed.
+
+import logger from '@overleaf/logger'
+import commandLineArgs from 'command-line-args'
+import { Change, Chunk, History, NoOperation } from 'overleaf-editor-core'
+import * as redis from '../lib/redis.js'
+import knex from '../lib/knex.js'
+import knexReadOnly from '../lib/knex_read_only.js'
+import { client as mongoClient } from '../lib/mongodb.js'
+import chunkStore from '../lib/chunk_store/index.js'
+import redisBackend from '../lib/chunk_store/redis.js'
+import { loadGlobalBlobs } from '../lib/blob_store/index.js'
+import { fileURLToPath } from 'node:url'
+import { EventEmitter } from 'node:events'
+
+EventEmitter.defaultMaxListeners = 20
+
+logger.initialize('finalise-chunk')
+
+const optionDefinitions = [
+  { name: 'historyId', type: String },
+  { name: 'dry-run', alias: 'd', type: Boolean },
+]
+const options = commandLineArgs(optionDefinitions)
+const HISTORY_ID = options.historyId
+const DRY_RUN = options['dry-run'] || false
+
+if (!HISTORY_ID) {
+  console.error('Usage: finalise_chunk.mjs --historyId <id> [--dry-run]')
+  process.exit(2)
+}
+
+async function finaliseCurrentChunk(historyId) {
+  // Validates the history id and selects the backend (postgres or mongo).
+  chunkStore.getBackend(historyId)
+
+  await loadGlobalBlobs()
+
+  const currentChunk = await chunkStore.loadLatest(historyId, {
+    persistedOnly: true,
+  })
+  const startVersion = currentChunk.getStartVersion()
+  const endVersion = currentChunk.getEndVersion()
+  const numChanges = currentChunk.getChanges().length
+
+  logger.info(
+    { historyId, startVersion, endVersion, numChanges },
+    'loaded current chunk'
+  )
+
+  if (endVersion === startVersion) {
+    throw new Error(
+      `current chunk for history ${historyId} is already empty (no changes); refusing to create another empty chunk`
+    )
+  }
+
+  let nonPersistedChanges
+  try {
+    nonPersistedChanges = await redisBackend.getNonPersistedChanges(
+      historyId,
+      endVersion
+    )
+  } catch (err) {
+    throw new Error(
+      `unable to read non-persisted changes from redis for history ${historyId}: ${err.message}`
+    )
+  }
+  if (nonPersistedChanges.length > 0) {
+    throw new Error(
+      `history ${historyId} has ${nonPersistedChanges.length} non-persisted change(s) in redis; persist or expire them before running this script`
+    )
+  }
+
+  const endSnapshot = currentChunk.getSnapshot().clone()
+  endSnapshot.applyAll(currentChunk.getChanges())
+
+  // The chunks table has a unique constraint on (doc_id, end_version), so the
+  // new chunk cannot share an end_version with the chunk we are closing. Add a
+  // single NoOperation change to bump end_version by 1 without mutating the
+  // snapshot.
+  const recoveryChange = new Change([new NoOperation()], new Date(), [])
+  const newChunk = new Chunk(
+    new History(endSnapshot, [recoveryChange]),
+    endVersion
+  )
+
+  if (DRY_RUN) {
+    logger.info(
+      { historyId, endVersion },
+      'dry run: would close current chunk and create new empty chunk'
+    )
+    return
+  }
+
+  await chunkStore.create(historyId, newChunk)
+
+  logger.info(
+    { historyId, endVersion },
+    'closed current chunk and created new empty chunk'
+  )
+}
+
+async function main() {
+  try {
+    await finaliseCurrentChunk(HISTORY_ID)
+  } catch (err) {
+    logger.fatal({ err, historyId: HISTORY_ID }, 'failed to finalise chunk')
+    process.exitCode = 1
+  } finally {
+    await redis.disconnect()
+    await mongoClient.close()
+    await knex.destroy()
+    await knexReadOnly.destroy()
+  }
+}
+
+const currentScriptPath = fileURLToPath(import.meta.url)
+if (process.argv[1] === currentScriptPath) {
+  main()
+}
+
+export { finaliseCurrentChunk }

services/history-v1/storage/scripts/recover_zip.js

@@ -17,9 +17,10 @@ const fs = require('node:fs')
 const os = require('node:os')
 const path = require('node:path')
 const util = require('node:util')
+const { pipeline } = require('node:stream/promises')
 
-// Something is registering 11 listeners, over the limit of 10, which generates
+// Something is registering 11 listeners, over the limit
-// a lot of warning noise.
+// of 10, which generates a lot of warning noise.
 require('node:events').EventEmitter.defaultMaxListeners = 11
 
 const config = require('config')
@@ -27,11 +28,23 @@ const config = require('config')
 // eslint-disable-next-line import/no-extraneous-dependencies
 const { Storage } = require('@google-cloud/storage')
 const isValidUtf8 = require('utf-8-validate')
+// zip-stream@7 uses ESM default export
+const ZipStream = require('zip-stream').default
+
+function createStorage() {
+  const opts = {}
+  if (config.has('persistor.gcs.endpoint.apiEndpoint')) {
+    opts.apiEndpoint = config.get('persistor.gcs.endpoint.apiEndpoint')
+  }
+  if (config.has('persistor.gcs.endpoint.projectId')) {
+    opts.projectId = config.get('persistor.gcs.endpoint.projectId')
+  }
+  return new Storage(opts)
+}
 
 const core = require('overleaf-editor-core')
 const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js')
 const streams = require('../lib/streams')
-const ProjectArchive = require('../lib/project_archive')
 
 const {
   values: { verbose: VERBOSE },
@@ -53,7 +66,7 @@ if (HISTORY_IDS.length === 0) {
 
 async function listDeletedChunks(historyId) {
   const bucketName = config.get('chunkStore.bucket')
-  const storage = new Storage()
+  const storage = createStorage()
   const [files] = await storage.bucket(bucketName).getFiles({
     prefix: projectKey.format(historyId),
     versions: true,
@@ -137,7 +150,7 @@ class RecoveryBlobStore {
     if (VERBOSE) console.log('fetching blob', hash)
 
     const bucketName = config.get('blobStore.projectBucket')
-    const storage = new Storage()
+    const storage = createStorage()
     const [files] = await storage.bucket(bucketName).getFiles({
       prefix: this.makeProjectBlobKey(hash),
       versions: true,
@@ -158,7 +171,7 @@ class RecoveryBlobStore {
 
   async fetchGlobalBlob(hash, destination) {
     const bucketName = config.get('blobStore.globalBucket')
-    const storage = new Storage()
+    const storage = createStorage()
     const file = storage.bucket(bucketName).file(this.makeGlobalBlobKey(hash))
     await file.download({ destination })
   }
@@ -203,9 +216,18 @@ class RecoveryBlobStore {
 async function uploadZip(historyId, zipPathname) {
   const bucketName = config.get('zipStore.bucket')
   const deadline = 24 * 3600 * 1000 // lifecycle limit on the zips bucket
-  const storage = new Storage()
+  const storage = createStorage()
   const destination = `${historyId}-recovered.zip`
-  await storage.bucket(bucketName).upload(zipPathname, { destination })
+  await storage.bucket(bucketName).upload(zipPathname, {
+    destination,
+    resumable: false,
+  })
+
+  if (config.has('persistor.gcs.endpoint.apiEndpoint')) {
+    // In emulator mode, signed URLs aren't available
+    const apiEndpoint = config.get('persistor.gcs.endpoint.apiEndpoint')
+    return `${apiEndpoint}/storage/v1/b/${bucketName}/o/${encodeURIComponent(destination)}?alt=media`
+  }
 
   const signedUrls = await storage
     .bucket(bucketName)
@@ -219,6 +241,23 @@ async function uploadZip(historyId, zipPathname) {
   return signedUrls[0]
 }
 
+/**
+ * Promisified wrapper for ZipStream's entry method.
+ *
+ * @param {ZipStream} archive
+ * @param {Buffer|NodeJS.ReadableStream|string} source
+ * @param {{ name: string }} data
+ * @return {Promise<void>}
+ */
+function addEntry(archive, source, data) {
+  return new Promise((resolve, reject) => {
+    archive.entry(source, data, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 async function restoreProject(historyId) {
   const tmp = await fs.promises.mkdtemp(
     path.join(os.tmpdir(), historyId.toString())
@@ -237,9 +276,40 @@ async function restoreProject(historyId) {
   if (VERBOSE) console.log('zipping', historyId)
 
   const zipPathname = path.join(tmp, `${historyId}.zip`)
-  const zipTimeoutMs = 60 * 1000
+  const outputFile = fs.createWriteStream(zipPathname)
-  const archive = new ProjectArchive(snapshot, zipTimeoutMs)
+  const archive = new ZipStream()
-  await archive.writeZip(blobStore, zipPathname)
+
+  const pipelinePromise = pipeline(archive, outputFile)
+
+  for (const pathname of snapshot.getFilePathnames()) {
+    const file = snapshot.getFile(pathname)
+    if (!file) continue
+
+    await file.load('eager', blobStore)
+    let content = file.getContent({
+      filterTrackedDeletes: true,
+    })
+
+    if (content === null) {
+      const hash = file.getHash()
+      content = await blobStore.getStream(hash)
+    }
+
+    if (content == null) continue
+
+    if (typeof content === 'string') {
+      content = Buffer.from(content)
+    }
+    await addEntry(archive, content, { name: pathname })
+    if (VERBOSE) console.log(`${pathname} added`)
+  }
+
+  archive.finalize()
+  await pipelinePromise
+
+  if (VERBOSE) {
+    console.log(`Wrote ${archive.getBytesWritten()} bytes`)
+  }
 
   if (VERBOSE) console.log('uploading', historyId)
 
@@ -252,4 +322,7 @@ async function main() {
     console.log(signedUrl)
   }
 }
-main().catch(console.error)
+main().catch(err => {
+  console.error(err)
+  process.exit(1)
+})

services/history-v1/storage/scripts/recover_zip_from_backup.mjs

@@ -4,6 +4,7 @@ import commandLineArgs from 'command-line-args'
 import assert from '../lib/assert.js'
 import fs from 'node:fs'
 import { setTimeout } from 'node:timers/promises'
+import { pipeline } from 'node:stream/promises'
 import {
   archiveLatestChunk,
   archiveRawProject,
@@ -11,17 +12,13 @@ import {
 } from '../lib/backupArchiver.mjs'
 import knex from '../lib/knex.js'
 import { client } from '../lib/mongodb.js'
-import archiver from 'archiver'
+import ZipStream from 'zip-stream'
-import Events from 'node:events'
 import { Chunk } from 'overleaf-editor-core'
 import _ from 'lodash'
 
-// Silence warning.
-Events.setMaxListeners(20)
-
 const SUPPORTED_MODES = ['raw', 'latest']
 
-// Pads the mode name to a fixed length for better alignment in output.
+// Pads the mode name to a fixed length for alignment.
 const padModeName = _.partialRight(
   _.padEnd,
   Math.max(...SUPPORTED_MODES.map(mode => mode.length))
@@ -65,25 +62,6 @@ function usage() {
   })
 }
 
-/**
- * @typedef {import('archiver').ZipArchive} ZipArchive
- */
-
-/**
- * @typedef {import('archiver').ProgressData} ProgressData
- */
-
-/**
- * @typedef {import('archiver').EntryData} EntryData
- */
-
-/**
- * @typedef {Object} ArchiverError
- * @property {string} message
- * @property {string} code
- * @property {Object} data
- */
-
 let historyId, help, mode, output, useBackupGlobalBlobs, verbose
 
 try {
@@ -136,80 +114,37 @@ await loadGlobalBlobs()
 
 outputFile = fs.createWriteStream(output)
 
-const archive = archiver.create('zip', {})
+const archive = new ZipStream()
 
-archive.on('close', function () {
+archive.on('error', function (e) {
-  console.log(archive.pointer() + ' total bytes')
+  console.error(`Error writing archive: ${e.message}`)
-  console.log(`Wrote ${output}`)
-  shutdown().catch(e => console.error('Error shutting down', e))
 })
 
-archive.on(
-  'error',
-  /**
-   *
-   * @param {ArchiverError} e
-   */
-  function (e) {
-    console.error(`Error writing archive: ${e.message}`)
-  }
-)
-
-archive.on('end', function () {
-  console.log(`Wrote ${archive.pointer()} total bytes to ${output}`)
-  shutdown().catch(e => console.error('Error shutting down', e))
-})
-
-archive.on(
-  'progress',
-  /**
-   *
-   * @param {ProgressData} progress
-   */
-  function (progress) {
-    if (verbose) {
-      console.log(
-        `${progress.entries.processed} processed out of ${progress.entries.total}`
-      )
-    }
-  }
-)
-
-archive.on(
-  'entry',
-  /**
-   *
-   * @param {EntryData} entry
-   */
-  function (entry) {
-    if (verbose) {
-      console.log(`${entry.name} added`)
-    }
-  }
-)
-
-archive.on(
-  'warning',
-  /**
-   *
-   * @param {ArchiverError} warning
-   */
-  function (warning) {
-    console.warn(`Warning encountered when writing archive: ${warning.message}`)
-  }
-)
-
 try {
+  // Pipe archive to the output file before adding entries.
+  // pipeline handles backpressure and will resolve when
+  // the archive stream ends.
+  const pipelinePromise = pipeline(archive, outputFile)
+
   switch (mode) {
     case 'latest':
-      await archiveLatestChunk(archive, historyId, useBackupGlobalBlobs)
+      await archiveLatestChunk(
+        archive,
+        historyId,
+        useBackupGlobalBlobs,
+        verbose
+      )
       break
     case 'raw':
     default:
-      await archiveRawProject(archive, historyId, useBackupGlobalBlobs)
+      await archiveRawProject(archive, historyId, useBackupGlobalBlobs, verbose)
       break
   }
-  archive.pipe(outputFile)
+
+  archive.finalize()
+  await pipelinePromise
+
+  console.log(`Wrote ${archive.getBytesWritten()} total bytes to ${output}`)
 } catch (error) {
   if (error instanceof BackupPersistorError) {
     console.error(error.message)
@@ -222,12 +157,7 @@ try {
   } else {
     console.error('Error encountered when writing archive')
   }
-} finally {
+  await shutdown(1)
-  await Promise.race([
-    await archive.finalize(),
-    setTimeout(10000).then(() => {
-      console.error('Archive did not finalize in time')
-      return shutdown(1)
-    }),
-  ])
 }
+
+await shutdown(0)

services/history-v1/test/acceptance/js/api/projects.test.js

@@ -507,7 +507,7 @@ describe('project controller', function () {
     })
   })
 
-  // eslint-disable-next-line mocha/no-skipped-tests
+  // eslint-disable-next-line mocha/no-pending-tests
   describe.skip('getLatestContent', function () {
     // TODO: remove this endpoint entirely, see
     // https://github.com/overleaf/write_latex/pull/5120#discussion_r244291862

services/history-v1/test/acceptance/js/storage/backup.test.mjs

@@ -647,6 +647,68 @@ describe('backup script', function () {
       expect(newBackupStatus.backupStatus.lastBackedUpVersion).to.equal(50) // backup fails on final chunk
       expect(newBackupStatus.currentEndVersion).to.equal(54) // backup is incomplete due to missing blob
     })
+
+    it('can recover zip file from backup in raw mode', async function () {
+      // First, run backup so data is available
+      await runBackupScript(['--projectId', projectId])
+
+      const zipPath = `/tmp/test-recover-raw-${historyId}.zip`
+      try {
+        await runRecoverZipFromBackupScript([
+          '--historyId',
+          historyId,
+          '--output',
+          zipPath,
+          '--mode=raw',
+        ])
+
+        // Verify the zip file is valid
+        const { stdout } = await promisify(execFile)('unzip', ['-l', zipPath], {
+          encoding: 'utf-8',
+        })
+
+        // Raw mode includes chunk and blob keys
+        // Verify chunks are present
+        expect(stdout).to.include('chunk.json')
+
+        // Verify blob hashes are present (hashes are stored as {hash[0:2]}/{hash[2:]})
+        expect(stdout).to.include(testFiles.GRAPH_PNG_HASH.slice(2))
+        expect(stdout).to.include(testFiles.NON_BMP_TXT_HASH.slice(2))
+      } finally {
+        await fs.promises.unlink(zipPath).catch(() => {})
+      }
+    })
+
+    it('can recover zip file from backup in latest mode', async function () {
+      // First, run backup so data is available
+      await runBackupScript(['--projectId', projectId])
+
+      const zipPath = `/tmp/test-recover-latest-${historyId}.zip`
+      try {
+        await runRecoverZipFromBackupScript([
+          '--historyId',
+          historyId,
+          '--output',
+          zipPath,
+          '--mode=latest',
+        ])
+
+        // Verify the zip file is valid
+        const { stdout } = await promisify(execFile)('unzip', ['-l', zipPath], {
+          encoding: 'utf-8',
+        })
+
+        // Latest mode includes the project files
+        expect(stdout).to.include('main.tex')
+        expect(stdout).to.include('chapter1.tex')
+        expect(stdout).to.include('chapter2.tex')
+        expect(stdout).to.include('bibliography.bib')
+        expect(stdout).to.include('graph.png')
+        expect(stdout).to.include('unicodeFile.tex')
+      } finally {
+        await fs.promises.unlink(zipPath).catch(() => {})
+      }
+    })
   })
 })
 
@@ -683,3 +745,37 @@ async function runBackupScript(args) {
   }
   return result
 }
+
+/**
+ * Run the recover_zip_from_backup script with given arguments
+ * @param {string[]} args
+ */
+async function runRecoverZipFromBackupScript(args) {
+  const TIMEOUT = 30 * 1000
+  let result
+  try {
+    result = await promisify(execFile)(
+      'node',
+      ['storage/scripts/recover_zip_from_backup.mjs', ...args],
+      {
+        encoding: 'utf-8',
+        timeout: TIMEOUT,
+        env: {
+          ...process.env,
+          LOG_LEVEL: 'debug',
+        },
+      }
+    )
+    result.status = 0
+  } catch (err) {
+    const { stdout, stderr, code } = err
+    if (typeof code !== 'number') {
+      console.log(err)
+    }
+    result = { stdout, stderr, status: code }
+  }
+  if (result.status !== 0) {
+    throw new Error(`recover_zip_from_backup failed: ${result.stderr}`)
+  }
+  return result
+}

services/history-v1/test/acceptance/js/storage/finalise_chunk.test.js

@@ -0,0 +1,205 @@
+'use strict'
+
+const { promisify } = require('node:util')
+const { execFile } = require('node:child_process')
+const { expect } = require('chai')
+const {
+  Change,
+  AddFileOperation,
+  EditFileOperation,
+  TextOperation,
+  File,
+} = require('overleaf-editor-core')
+const cleanup = require('./support/cleanup')
+const fixtures = require('./support/fixtures')
+const { setupProjectState } = require('./support/redis')
+const chunkStore = require('../../../../storage/lib/chunk_store')
+const persistChanges = require('../../../../storage/lib/persist_changes')
+const knex = require('../../../../storage/lib/knex')
+
+const SCRIPT_PATH = 'storage/scripts/finalise_chunk.mjs'
+
+async function runScript(args) {
+  try {
+    const result = await promisify(execFile)('node', [SCRIPT_PATH, ...args], {
+      encoding: 'utf-8',
+      timeout: 10000,
+      env: { ...process.env, LOG_LEVEL: 'debug' },
+    })
+    return { ...result, status: 0 }
+  } catch (err) {
+    if (typeof err.code !== 'number') {
+      throw err
+    }
+    return { stdout: err.stdout, stderr: err.stderr, status: err.code }
+  }
+}
+
+async function getChunkRows(projectId) {
+  return await knex('chunks')
+    .select('id', 'start_version', 'end_version', 'closed')
+    .where('doc_id', parseInt(projectId, 10))
+    .orderBy('end_version')
+}
+
+describe('finalise_chunk script', function () {
+  before(cleanup.everything)
+
+  let limitsToPersistImmediately
+  before(function () {
+    const farFuture = new Date()
+    farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+    limitsToPersistImmediately = {
+      minChangeTimestamp: farFuture,
+      maxChangeTimestamp: farFuture,
+      maxChunkChanges: 100,
+    }
+  })
+
+  beforeEach(async function () {
+    await cleanup.everything()
+    await fixtures.create()
+  })
+
+  describe('with a populated current chunk', function () {
+    let projectId
+    let initialContent
+    let initialEndVersion
+    let initialChunkId
+
+    beforeEach(async function () {
+      projectId = await chunkStore.initializeProject()
+      initialContent = 'Hello world.'
+      const change1 = new Change(
+        [new AddFileOperation('main.tex', File.fromString(initialContent))],
+        new Date(Date.now() - 30000),
+        []
+      )
+      const change2 = new Change(
+        [
+          new EditFileOperation(
+            'main.tex',
+            new TextOperation().retain(initialContent.length).insert(' More.')
+          ),
+        ],
+        new Date(Date.now() - 20000),
+        []
+      )
+      await persistChanges(
+        projectId,
+        [change1, change2],
+        limitsToPersistImmediately,
+        0
+      )
+      const metadata = await chunkStore.getLatestChunkMetadata(projectId)
+      initialEndVersion = metadata.endVersion
+      initialChunkId = metadata.id
+      expect(initialEndVersion).to.equal(2)
+    })
+
+    it('closes the current chunk and creates a new empty chunk', async function () {
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status, result.stderr).to.equal(0)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(2)
+
+      const oldRow = rows.find(r => r.id.toString() === initialChunkId)
+      expect(oldRow, 'old chunk still in DB').to.exist
+      expect(oldRow.closed).to.equal(true)
+      expect(oldRow.end_version).to.equal(initialEndVersion)
+
+      const newRow = rows.find(r => r.id.toString() !== initialChunkId)
+      expect(newRow, 'new chunk inserted').to.exist
+      expect(newRow.start_version).to.equal(initialEndVersion)
+      expect(newRow.end_version).to.equal(initialEndVersion + 1)
+      expect(newRow.closed).to.equal(false)
+
+      const newChunk = await chunkStore.loadLatest(projectId, {
+        persistedOnly: true,
+      })
+      expect(newChunk.getStartVersion()).to.equal(initialEndVersion)
+      expect(newChunk.getEndVersion()).to.equal(initialEndVersion + 1)
+      // The new chunk holds a single NoOperation change as a placeholder so
+      // its end_version is distinct from the closed chunk's end_version.
+      expect(newChunk.getChanges()).to.have.length(1)
+
+      const newSnapshot = newChunk.getSnapshot()
+      const file = newSnapshot.getFile('main.tex')
+      expect(file, 'main.tex carried over to new chunk snapshot').to.exist
+    })
+
+    it('does not modify state on a dry-run', async function () {
+      const result = await runScript(['--historyId', projectId, '--dry-run'])
+      expect(result.status, result.stderr).to.equal(0)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].id.toString()).to.equal(initialChunkId)
+      expect(rows[0].closed).to.equal(false)
+    })
+  })
+
+  describe('safety checks', function () {
+    it('refuses to act on an already-empty current chunk', async function () {
+      const projectId = await chunkStore.initializeProject()
+
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status).to.not.equal(0)
+      expect(result.stdout + result.stderr).to.match(/already empty/)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].start_version).to.equal(0)
+      expect(rows[0].end_version).to.equal(0)
+      expect(rows[0].closed).to.equal(false)
+    })
+
+    it('refuses when there are non-persisted changes in redis', async function () {
+      const projectId = await chunkStore.initializeProject()
+      const initialContent = 'Initial.'
+      const initialChange = new Change(
+        [new AddFileOperation('main.tex', File.fromString(initialContent))],
+        new Date(Date.now() - 30000),
+        []
+      )
+      await persistChanges(
+        projectId,
+        [initialChange],
+        limitsToPersistImmediately,
+        0
+      )
+      const metadata = await chunkStore.getLatestChunkMetadata(projectId)
+      const initialChunkId = metadata.id
+
+      const bufferedChange = new Change(
+        [
+          new EditFileOperation(
+            'main.tex',
+            new TextOperation()
+              .retain(initialContent.length)
+              .insert(' Buffered.')
+          ),
+        ],
+        new Date(Date.now() - 10000),
+        []
+      )
+      await setupProjectState(projectId, {
+        persistTime: Date.now() - 1000,
+        headVersion: 2,
+        persistedVersion: 1,
+        changes: [bufferedChange],
+        expireTimeFuture: true,
+      })
+
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status).to.not.equal(0)
+      expect(result.stdout + result.stderr).to.match(/non-persisted/)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].id.toString()).to.equal(initialChunkId)
+      expect(rows[0].closed).to.equal(false)
+    })
+  })
+})

services/history-v1/test/acceptance/js/storage/recover_zip.test.mjs

@@ -0,0 +1,169 @@
+import { expect } from 'chai'
+import config from 'config'
+import { execFile } from 'node:child_process'
+import fs from 'node:fs'
+import { promisify } from 'node:util'
+import { Change, Operation, File, TextOperation } from 'overleaf-editor-core'
+// We depend on this via object-persistor.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Storage } from '@google-cloud/storage'
+import {
+  loadGlobalBlobs,
+  BlobStore,
+} from '../../../../storage/lib/blob_store/index.js'
+import ChunkStore from '../../../../storage/lib/chunk_store/index.js'
+import persistChanges from '../../../../storage/lib/persist_changes.js'
+import testFiles from '../storage/support/test_files.js'
+import cleanup from './support/cleanup.js'
+import { getZipEntries } from './support/unzip.js'
+
+describe('recover_zip script', function () {
+  let projectId
+  let limitsToPersistImmediately
+
+  before(async function () {
+    const farFuture = new Date()
+    farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+    limitsToPersistImmediately = {
+      minChangeTimestamp: farFuture,
+      maxChangeTimestamp: farFuture,
+      maxChanges: 10,
+      maxChunkChanges: 10,
+    }
+
+    const gcsEndpoint = config.get('persistor.gcs.endpoint')
+    const storage = new Storage({
+      apiEndpoint: gcsEndpoint.apiEndpoint,
+      projectId: gcsEndpoint.projectId,
+    })
+    const bucketName = config.get('zipStore.bucket')
+    try {
+      const [exists] = await storage.bucket(bucketName).exists()
+      if (!exists) {
+        await storage.createBucket(bucketName)
+      }
+    } catch (err) {
+      if (err.code !== 409) throw err
+    }
+  })
+
+  beforeEach(cleanup.everything)
+
+  beforeEach(async function () {
+    await loadGlobalBlobs()
+    projectId = '123'
+
+    // Initialize the project in the chunk store
+    await ChunkStore.initializeProject(projectId)
+
+    const blobStore = new BlobStore(projectId)
+
+    // Upload binary file blob
+    await blobStore.putFile(testFiles.path('graph.png'))
+
+    // Create initial snapshot with text and binary files
+    const addMainTex = Operation.addFile(
+      'main.tex',
+      File.fromString('hello world')
+    )
+    const addGraphPng = Operation.addFile(
+      'graph.png',
+      File.fromHash(testFiles.GRAPH_PNG_HASH)
+    )
+    const change1 = new Change([addMainTex, addGraphPng], new Date(), [])
+    await persistChanges(projectId, [change1], limitsToPersistImmediately, 0)
+
+    // Add a text edit
+    const textOp = TextOperation.fromJSON({
+      textOperation: ['hello world'.length, ' more'],
+    })
+    const editOp = Operation.editFile('main.tex', textOp)
+    const change2 = new Change([editOp], new Date(), [])
+    await persistChanges(projectId, [change2], limitsToPersistImmediately, 1)
+  })
+
+  it('creates a valid zip from GCS data', async function () {
+    this.timeout(30 * 1000)
+
+    const zipPath = `/tmp/test-recover-zip-${projectId}.zip`
+    try {
+      const { stdout } = await runRecoverZipScript([projectId])
+
+      // The script logs the signed URL to stdout
+      const urlMatch = stdout.match(/(https?:\/\/[^\s]+)/)
+      expect(urlMatch).to.not.be.null
+      const signedUrl = urlMatch[1]
+
+      // Download the zip via fetch
+      const res = await fetch(signedUrl)
+      expect(res.ok).to.be.true
+      const buffer = await res.arrayBuffer()
+      await fs.promises.writeFile(zipPath, Buffer.from(buffer))
+
+      const zipEntries = await getZipEntries(zipPath)
+      const fileNames = zipEntries.map(e => e.fileName).sort()
+
+      expect(fileNames).to.deep.equal(['graph.png', 'main.tex'])
+
+      // Verify text content size (after edit)
+      const mainTexEntry = zipEntries.find(e => e.fileName === 'main.tex')
+      expect(mainTexEntry.uncompressedSize).to.equal('hello world more'.length)
+
+      // Verify binary content size
+      const graphEntry = zipEntries.find(e => e.fileName === 'graph.png')
+      expect(graphEntry.uncompressedSize).to.equal(
+        testFiles.GRAPH_PNG_BYTE_LENGTH
+      )
+    } finally {
+      await fs.promises.unlink(zipPath).catch(() => {})
+    }
+  })
+
+  it('supports the --verbose flag', async function () {
+    this.timeout(30 * 1000)
+
+    const { stdout } = await runRecoverZipScript(['--verbose', projectId])
+
+    // Verbose mode logs each file as it's added
+    expect(stdout).to.include('main.tex added')
+    expect(stdout).to.include('graph.png added')
+  })
+})
+
+/**
+ * Run the recover_zip.js script with given arguments
+ * @param {string[]} args
+ */
+async function runRecoverZipScript(args) {
+  const TIMEOUT = 30 * 1000
+  let result
+  try {
+    result = await promisify(execFile)(
+      'node',
+      ['storage/scripts/recover_zip.js', ...args],
+      {
+        encoding: 'utf-8',
+        timeout: TIMEOUT,
+        env: {
+          ...process.env,
+          LOG_LEVEL: 'debug',
+        },
+      }
+    )
+    result.status = 0
+  } catch (err) {
+    const { stdout, stderr, code } = err
+    if (typeof code !== 'number') {
+      console.log(err)
+    }
+    result = { stdout, stderr, status: code }
+  }
+  if (result.status !== 0 || result.stderr) {
+    throw new Error(
+      `recover_zip failed (exit ${result.status}):\n` +
+        `stdout: ${result.stdout}\n` +
+        `stderr: ${result.stderr}`
+    )
+  }
+  return result
+}

services/notifications/Makefile

@@ -55,6 +55,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -67,8 +71,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/notifications/app.ts

@@ -9,7 +9,6 @@ import express, {
   type ErrorRequestHandler,
   type NextFunction,
 } from 'express'
-import methodOverride from 'method-override'
 import { mongoClient } from './app/js/mongodb.js'
 import NotificationsController from './app/js/NotificationsController.ts'
 import HealthCheckController from './app/js/HealthCheckController.ts'
@@ -25,7 +24,6 @@ logger.initialize('notifications')
 metrics.memory.monitor(logger)
 metrics.open_sockets.monitor()
 
-app.use(methodOverride())
 app.use(express.json())
 app.use(metrics.http.monitor(logger))
 

services/notifications/buildscript.txt

@@ -1,5 +1,6 @@
 notifications
 --dependencies=mongo
+--deploy-pipeline=notifications
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/notifications/package.json

@@ -30,14 +30,12 @@
     "body-parser": "1.20.4",
     "bunyan": "^1.8.15",
     "express": "4.22.1",
-    "method-override": "^3.0.0",
     "mongodb-legacy": "6.1.3",
     "zod": "^4.1.7",
     "zod-validation-error": "^4.0.1"
   },
   "devDependencies": {
     "@overleaf/migrations": "workspace:*",
-    "@types/method-override": "^3.0.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",

services/project-history/.eslintignore

@@ -1 +0,0 @@
-app/lib/*.js