fix: pass session token so anonymous users can install python packages

userCanInstallPython passed null as the token, so anonymous users
accessing via a share link got privilege level NONE from the WithoutUser
path and allowPythonInstall was always false for them.

Read the token from req.session.anonTokenAccess via
TokenAccessHandler.getRequestToken and forward it through
userCanInstallPython to getPrivilegeLevelForProject.  For TOKEN_BASED
projects this resolves the anonymous user's access level via
getPrivilegeLevelForProjectWithToken, enabling package installation.

Also update Quarto Slides badge color to #e4637c.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch

@@ -1,5 +1,5 @@
 diff --git a/lib/sandboxed_module.js b/lib/sandboxed_module.js
-index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd97100e41c3e4 100644
+index 1cd6743..4718b97 100644
 --- a/lib/sandboxed_module.js
 +++ b/lib/sandboxed_module.js
 @@ -4,7 +4,7 @@ var Module = require('module');
@@ -11,3 +11,47 @@ index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd9710
  var parent = module.parent;
  var globalOptions = {};
  var registeredBuiltInSourceTransformers = ['coffee'];
+@@ -157,12 +157,20 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
+   var cache = Object.create(null);
+   var required = this._getRequires();
+   for (var key in required) {
+-    var injectedFilename = requireLike(this.filename).resolve(key);
+-    cache[injectedFilename] = required[key];
++    // Under Yarn PnP, resolution from a transitive dependency's context may fail
++    // for packages not declared in that dependency's package.json. Silently skip
++    // cache pre-population on failure; the mock will still be injected via the
++    // inject map in requireInterceptor or resolved via RecursiveRequireProxy fallback.
++    try {
++      var injectedFilename = requireLike(this.filename).resolve(key);
++      cache[injectedFilename] = required[key];
++    } catch (e) {}
+   }
+   cache[this.filename] = this.exports;
+   var globals = this.globals;
+ 
++  // Store the top-level module's filename for PnP fallback resolution
++  var topLevelFilename = this.filename;
+   var options;
+   if(!this._options.sourceTransformersSingleOnly && this._options.sourceTransformers){
+     options = {
+@@ -208,8 +216,18 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
+       if (request in cache) return cache[request];
+       return require(request);
+     }
+-    // cached modules
+-    var requestedFilename = requireLike(this.filename).resolve(request);
++    // Resolve the requested module filename.
++    // Under Yarn PnP, packages can only resolve their declared dependencies.
++    // When sandboxed-module loads a transitive dependency, the resolution context
++    // may not have access to all needed packages. Fall back to resolving from
++    // the top-level module's context (the module under test).
++    var requestedFilename;
++    try {
++      requestedFilename = requireLike(this.filename).resolve(request);
++    } catch (e) {
++      if (this.filename === topLevelFilename) throw e;
++      requestedFilename = requireLike(topLevelFilename).resolve(request);
++    }
+     if (requestedFilename in cache) return cache[requestedFilename];
+     var sandboxedModule = createInnerSandboxedModule(requestedFilename)
+     return sandboxedModule.exports;

TODO.md

@@ -0,0 +1,41 @@
+# Verso — Next Alpha Roadmap
+
+Ideas and features deferred from the current alpha.
+
+---
+
+## Next alpha (post-current)
+
+### Typst editing experience (inspired by Collabst)
+
+- **typst.ts WASM preview** — Run the Typst compiler in the browser via
+  WebAssembly (typst.ts). This would give instant, sub-second preview
+  without a server round-trip, and would eliminate the entire class of
+  race conditions in the CLSI watcher (files written → typst compiles →
+  resolver missed). Could coexist with the CLSI watcher for PDF export
+  while using the WASM path for live preview.
+
+- **Tinymist LSP integration** — Wire up
+  [Tinymist](https://github.com/Myriad-Dreamin/tinymist) (the Typst
+  language server) behind a WebSocket proxy. Would give Typst files
+  first-class autocomplete, hover docs, go-to-definition, and inline
+  error diagnostics — the main editing comfort gap vs. a native editor.
+
+### Editor UX for non-LaTeX formats (.typ, .qmd, .md)
+
+- **Visual/rich-text editing mode** — A toggle between raw source and a
+  rendered-in-place view for `.typ`, `.qmd`, and `.md` files (similar to
+  Overleaf's rich-text mode for LaTeX). Users who don't know Typst or
+  Markdown syntax should be able to edit content without seeing markup.
+  CodeMirror 6 already supports this pattern via a custom `NodeView` layer
+  or a separate Prosemirror bridge.
+
+- **Toolbar / insertion shortcuts** — A formatting toolbar and keyboard
+  shortcuts for common operations, adapted per file type:
+  - **All formats**: bold, italic, underline, headings, bullet/numbered
+    lists, inline code, links.
+  - **Quarto / Markdown**: insert image, insert table, insert code block
+    with language tag.
+  - **Quarto RevealJS**: insert slide divider (`---`), insert speaker
+    notes (`::: notes`), insert columns layout, insert video embed
+    (using Quarto's `{{< video >}}` shortcode).

libraries/access-token-encryptor/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/eslint-plugin/index.js

@@ -1,4 +1,10 @@
+const pkg = require('./package.json')
+
 module.exports = {
+  meta: {
+    name: pkg.name,
+    version: pkg.version,
+  },
   rules: {
     'no-unnecessary-trans': require('./no-unnecessary-trans'),
     'prefer-kebab-url': require('./prefer-kebab-url'),

libraries/eslint-plugin/package.json

@@ -8,10 +8,10 @@
     "lodash": "^4.18.1"
   },
   "devDependencies": {
-    "@typescript-eslint/parser": "^8.50.0"
+    "@typescript-eslint/parser": "^8.59.4"
   },
   "peerDependencies": {
-    "eslint": "^8.51.0"
+    "eslint": "^10.4.0"
   },
   "scripts": {
     "test": "node rules.test.js"

libraries/eslint-plugin/require-vi-doMock-valid-path.js

@@ -22,7 +22,7 @@ module.exports = {
     },
   },
   create(context) {
-    const currentFilePath = context.getFilename()
+    const currentFilePath = context.filename
     // ESLint can sometimes pass <text> or <input> for snippets not in a file
     if (currentFilePath === '<text>' || currentFilePath === '<input>') {
       return {}
@@ -81,9 +81,10 @@ module.exports = {
             typeof firstArg.value !== 'string'
           ) {
             if (firstArg.type === 'Identifier') {
-              const variable = context
-                .getScope()
-                .variables.find(v => v.name === firstArg.name)
+              const scope = context.sourceCode.getScope(node)
+              const variable = scope.variables.find(
+                v => v.name === firstArg.name
+              )
               if (
                 variable &&
                 variable.defs.length > 0 &&

libraries/eslint-plugin/rules.test.js

@@ -1,4 +1,5 @@
 const { RuleTester } = require('eslint')
+const tsParser = require('@typescript-eslint/parser')
 const noThrowInCallback = require('./no-throw-in-callback')
 const preferKebabUrl = require('./prefer-kebab-url')
 const noUnnecessaryTrans = require('./no-unnecessary-trans')
@@ -8,10 +9,10 @@ const viDoMockValidPath = require('./require-vi-doMock-valid-path')
 const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
 
 const ruleTester = new RuleTester({
-  parser: require.resolve('@typescript-eslint/parser'),
-  parserOptions: {
+  languageOptions: {
+    parser: tsParser,
     ecmaVersion: 'latest',
-    ecmaFeatures: { jsx: true },
+    parserOptions: { ecmaFeatures: { jsx: true } },
   },
 })
 
@@ -33,19 +34,27 @@ ruleTester.run('prefer-kebab-url', preferKebabUrl, {
   invalid: [
     {
       code: `app.get('/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `app.get('/fooBar/:id')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
     {
       code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
+      errors: [
+        { message: 'Route path should be in kebab-case.', suggestions: 1 },
+      ],
     },
   ],
 })
@@ -153,6 +162,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
         {
           message:
             'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
+          suggestions: [],
         },
       ],
     },
@@ -163,6 +173,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
         {
           message:
             'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
+          suggestions: [],
         },
       ],
     },

libraries/fetch-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/logger/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/metrics/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/metrics/package.json

@@ -10,12 +10,12 @@
   "dependencies": {
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^3.0.0",
     "@google-cloud/profiler": "^6.0.4",
-    "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.72.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.214.0",
-    "@opentelemetry/resources": "^2.6.0",
-    "@opentelemetry/sdk-node": "^0.214.0",
-    "@opentelemetry/semantic-conventions": "^1.39.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.76.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-node": "^0.218.0",
+    "@opentelemetry/semantic-conventions": "^1.41.1",
     "compression": "^1.7.4",
     "prom-client": "^14.1.1",
     "yn": "^3.1.1"

libraries/o-error/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/object-persistor/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/object-persistor/package.json

@@ -38,7 +38,6 @@
     "mocha": "^11.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
-    "mock-fs": "^5.2.0",
     "mongodb": "6.12.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",

libraries/object-persistor/src/FSPersistor.js

@@ -3,9 +3,12 @@ const fs = require('node:fs')
 const fsPromises = require('node:fs/promises')
 const { glob } = require('glob')
 const Path = require('node:path')
+const { promisify } = require('node:util')
 const { PassThrough } = require('node:stream')
 const { pipeline } = require('node:stream/promises')
 
+const openCb = promisify(fs.open)
+
 const AbstractPersistor = require('./AbstractPersistor')
 const { ReadError, WriteError, NotImplementedError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
@@ -85,8 +88,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
     })
     const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
 
+    let fd
     try {
-      opts.fd = await fsPromises.open(fsPath, 'r')
+      fd = await openCb(fsPath, 'r')
     } catch (err) {
       throw PersistorHelper.wrapError(
         err,
@@ -96,7 +100,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       )
     }
 
-    const stream = fs.createReadStream(null, opts)
+    const stream = fs.createReadStream(null, { ...opts, fd })
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
     pipeline(stream, observer, pass).catch(() => {})

libraries/object-persistor/test/unit/FSPersistorTests.js

@@ -1,6 +1,6 @@
 const crypto = require('node:crypto')
+const os = require('node:os')
 const { expect } = require('chai')
-const mockFs = require('mock-fs')
 const fs = require('node:fs')
 const fsPromises = require('node:fs/promises')
 const Path = require('node:path')
@@ -10,22 +10,59 @@ const Errors = require('../../src/Errors')
 
 const MODULE_PATH = '../../src/FSPersistor.js'
 
+function createTree(base, tree) {
+  fs.mkdirSync(base, { recursive: true })
+  for (const [name, content] of Object.entries(tree)) {
+    const fullPath = Path.join(base, name)
+    if (Buffer.isBuffer(content) || typeof content === 'string') {
+      fs.writeFileSync(fullPath, content)
+    } else if (content && typeof content.symlink === 'string') {
+      fs.symlinkSync(content.symlink, fullPath)
+    } else {
+      createTree(fullPath, content)
+    }
+  }
+}
+
 describe('FSPersistorTests', function () {
-  const localFiles = {
-    '/uploads/info.txt': Buffer.from('This information is critical', {
+  const fileContents = {
+    'info.txt': Buffer.from('This information is critical', {
       encoding: 'utf-8',
     }),
-    '/uploads/other.txt': Buffer.from('Some other content', {
+    'other.txt': Buffer.from('Some other content', {
       encoding: 'utf-8',
     }),
   }
-  const location = '/bucket'
+  let tmpDir
+  let location
+  let notADirPath
   const files = {
     wombat: 'animals/wombat.tex',
     giraffe: 'animals/giraffe.tex',
     potato: 'vegetables/potato.tex',
   }
 
+  beforeEach(function () {
+    tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'fs-persistor-test-'))
+    createTree(tmpDir, {
+      uploads: {
+        'info.txt': fileContents['info.txt'],
+        'other.txt': fileContents['other.txt'],
+      },
+      'not-a-dir':
+        'This regular file is meant to prevent using this path as a directory',
+      directory: {
+        subdirectory: {},
+      },
+    })
+    notADirPath = Path.join(tmpDir, 'not-a-dir')
+    location = Path.join(tmpDir, 'bucket')
+  })
+
+  afterEach(function () {
+    fs.rmSync(tmpDir, { recursive: true })
+  })
+
   const scenarios = [
     {
       description: 'default settings',
@@ -54,31 +91,26 @@ describe('FSPersistorTests', function () {
         persistor = new FSPersistor(scenario.settings)
       })
 
-      beforeEach(function () {
-        mockFs({
-          ...localFiles,
-          '/not-a-dir':
-            'This regular file is meant to prevent using this path as a directory',
-          '/directory/subdirectory': {},
-        })
-      })
-
-      afterEach(function () {
-        mockFs.restore()
-      })
-
       describe('sendFile', function () {
         it('should copy the file', async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
           const contents = await fsPromises.readFile(
             scenario.fsPath(files.wombat)
           )
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
 
         it('should return an error if the file cannot be stored', async function () {
           await expect(
-            persistor.sendFile('/not-a-dir', files.wombat, '/uploads/info.txt')
+            persistor.sendFile(
+              notADirPath,
+              files.wombat,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           ).to.be.rejectedWith(Errors.WriteError)
         })
       })
@@ -88,7 +120,9 @@ describe('FSPersistorTests', function () {
 
         describe("when the file doesn't exist", function () {
           beforeEach(function () {
-            stream = fs.createReadStream('/uploads/info.txt')
+            stream = fs.createReadStream(
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           })
 
           it('should write the stream to disk', async function () {
@@ -96,7 +130,7 @@ describe('FSPersistorTests', function () {
             const contents = await fsPromises.readFile(
               scenario.fsPath(files.wombat)
             )
-            expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+            expect(contents.equals(fileContents['info.txt'])).to.be.true
           })
 
           it('should delete the temporary file', async function () {
@@ -109,7 +143,7 @@ describe('FSPersistorTests', function () {
           describe('on error', function () {
             beforeEach(async function () {
               await expect(
-                persistor.sendStream('/not-a-dir', files.wombat, stream)
+                persistor.sendStream(notADirPath, files.wombat, stream)
               ).to.be.rejectedWith(Errors.WriteError)
             })
 
@@ -129,13 +163,12 @@ describe('FSPersistorTests', function () {
           describe('when the md5 hash matches', function () {
             it('should write the stream to disk', async function () {
               await persistor.sendStream(location, files.wombat, stream, {
-                sourceMd5: md5(localFiles['/uploads/info.txt']),
+                sourceMd5: md5(fileContents['info.txt']),
               })
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
-                .true
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
             })
           })
 
@@ -169,9 +202,11 @@ describe('FSPersistorTests', function () {
             await persistor.sendFile(
               location,
               files.wombat,
-              '/uploads/info.txt'
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
+            stream = fs.createReadStream(
+              Path.join(tmpDir, 'uploads', 'other.txt')
             )
-            stream = fs.createReadStream('/uploads/other.txt')
           })
 
           it('should write the stream to disk', async function () {
@@ -179,7 +214,7 @@ describe('FSPersistorTests', function () {
             const contents = await fsPromises.readFile(
               scenario.fsPath(files.wombat)
             )
-            expect(contents.equals(localFiles['/uploads/other.txt'])).to.be.true
+            expect(contents.equals(fileContents['other.txt'])).to.be.true
           })
 
           it('should delete the temporary file', async function () {
@@ -192,7 +227,7 @@ describe('FSPersistorTests', function () {
           describe('on error', function () {
             beforeEach(async function () {
               await expect(
-                persistor.sendStream('/not-a-dir', files.wombat, stream)
+                persistor.sendStream(notADirPath, files.wombat, stream)
               ).to.be.rejectedWith(Errors.WriteError)
             })
 
@@ -200,8 +235,7 @@ describe('FSPersistorTests', function () {
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
-                .true
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
             })
 
             it('should delete the temporary file', async function () {
@@ -215,13 +249,12 @@ describe('FSPersistorTests', function () {
           describe('when the md5 hash matches', function () {
             it('should write the stream to disk', async function () {
               await persistor.sendStream(location, files.wombat, stream, {
-                sourceMd5: md5(localFiles['/uploads/other.txt']),
+                sourceMd5: md5(fileContents['other.txt']),
               })
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/other.txt'])).to.be
-                .true
+              expect(contents.equals(fileContents['other.txt'])).to.be.true
             })
           })
 
@@ -238,8 +271,7 @@ describe('FSPersistorTests', function () {
               const contents = await fsPromises.readFile(
                 scenario.fsPath(files.wombat)
               )
-              expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
-                .true
+              expect(contents.equals(fileContents['info.txt'])).to.be.true
             })
 
             it('should delete the temporary file', async function () {
@@ -254,13 +286,17 @@ describe('FSPersistorTests', function () {
 
       describe('getObjectStream', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return a string with the object contents', async function () {
           const stream = await persistor.getObjectStream(location, files.wombat)
           const contents = await streamToBuffer(stream)
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
 
         it('should support ranges', async function () {
@@ -274,8 +310,8 @@ describe('FSPersistorTests', function () {
           )
           const contents = await streamToBuffer(stream)
           // end is inclusive in ranges, but exclusive in slice()
-          expect(contents.equals(localFiles['/uploads/info.txt'].slice(5, 17)))
-            .to.be.true
+          expect(contents.equals(fileContents['info.txt'].slice(5, 17))).to.be
+            .true
         })
 
         it('should give a NotFoundError if the file does not exist', async function () {
@@ -287,13 +323,17 @@ describe('FSPersistorTests', function () {
 
       describe('getObjectSize', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return the file size', async function () {
           expect(
             await persistor.getObjectSize(location, files.wombat)
-          ).to.equal(localFiles['/uploads/info.txt'].length)
+          ).to.equal(fileContents['info.txt'].length)
         })
 
         it('should throw a NotFoundError if the file does not exist', async function () {
@@ -305,7 +345,11 @@ describe('FSPersistorTests', function () {
 
       describe('copyObject', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('Should copy the file to the new location', async function () {
@@ -313,13 +357,17 @@ describe('FSPersistorTests', function () {
           const contents = await fsPromises.readFile(
             scenario.fsPath(files.potato)
           )
-          expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
+          expect(contents.equals(fileContents['info.txt'])).to.be.true
         })
       })
 
       describe('deleteObject', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
           await fsPromises.access(scenario.fsPath(files.wombat))
         })
 
@@ -337,7 +385,11 @@ describe('FSPersistorTests', function () {
       describe('deleteDirectory', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
             await fsPromises.access(scenario.fsPath(file))
           }
         })
@@ -365,7 +417,11 @@ describe('FSPersistorTests', function () {
 
       describe('checkIfObjectExists', function () {
         beforeEach(async function () {
-          await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
+          await persistor.sendFile(
+            location,
+            files.wombat,
+            Path.join(tmpDir, 'uploads', 'info.txt')
+          )
         })
 
         it('should return true for existing files', async function () {
@@ -384,13 +440,17 @@ describe('FSPersistorTests', function () {
       describe('directorySize', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
         it('should sum directory files size', async function () {
           expect(await persistor.directorySize(location, 'animals')).to.equal(
-            2 * localFiles['/uploads/info.txt'].length
+            2 * fileContents['info.txt'].length
           )
         })
 
@@ -404,7 +464,11 @@ describe('FSPersistorTests', function () {
       describe('listDirectoryKeys', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
@@ -427,7 +491,11 @@ describe('FSPersistorTests', function () {
       describe('listDirectoryStats', function () {
         beforeEach(async function () {
           for (const file of Object.values(files)) {
-            await persistor.sendFile(location, file, '/uploads/info.txt')
+            await persistor.sendFile(
+              location,
+              file,
+              Path.join(tmpDir, 'uploads', 'info.txt')
+            )
           }
         })
 
@@ -438,7 +506,7 @@ describe('FSPersistorTests', function () {
           expect(keys).to.include(scenario.fsPath(files.wombat))
           expect(keys).to.include(scenario.fsPath(files.giraffe))
           for (const stat of stats) {
-            expect(stat.size).to.equal(localFiles['/uploads/info.txt'].length)
+            expect(stat.size).to.equal(fileContents['info.txt'].length)
           }
         })
 

libraries/overleaf-editor-core/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/promise-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/ranges-tracker/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/redis-wrapper/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

libraries/stream-utils/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

package.json

@@ -3,39 +3,41 @@
   "private": true,
   "packageManager": "yarn@4.14.1",
   "devDependencies": {
+    "@eslint/compat": "^2.1.0",
+    "@eslint/js": "^10.0.1",
     "@overleaf/eslint-plugin": "workspace:*",
     "@prettier/plugin-pug": "^3.4.0",
     "@types/chai": "^4.3.0",
     "@types/chai-as-promised": "^7.1.8",
     "@types/mocha": "^10.0.6",
     "@types/multer": "^2.1.0",
-    "@typescript-eslint/eslint-plugin": "8.50.0",
-    "@typescript-eslint/parser": "^8.50.0",
+    "@typescript-eslint/eslint-plugin": "^8.59.4",
+    "@typescript-eslint/parser": "^8.59.4",
     "@vitest/eslint-plugin": "^1.5.0",
-    "eslint": "^8.15.0",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-config-standard": "^17.0.0",
-    "eslint-plugin-chai-expect": "^3.0.0",
-    "eslint-plugin-chai-friendly": "^0.7.2",
-    "eslint-plugin-cypress": "^2.15.1",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-mocha": "^10.1.0",
-    "eslint-plugin-n": "^15.7.0",
-    "eslint-plugin-prettier": "^4.0.0",
-    "eslint-plugin-promise": "^6.0.0",
+    "eslint": "^10.4.0",
+    "eslint-config-prettier": "^10.0.1",
+    "eslint-formatter-unix": "^8.40.0",
+    "eslint-plugin-chai-expect": "^4.0.0",
+    "eslint-plugin-chai-friendly": "^1.1.0",
+    "eslint-plugin-cypress": "^4.1.0",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-mocha": "^11.0.0",
+    "eslint-plugin-n": "^18.0.0",
+    "eslint-plugin-promise": "^7.2.1",
     "eslint-plugin-unicorn": "^56.0.0",
+    "globals": "^17.6.0",
     "prettier": "3.7.4",
     "prettier-plugin-groovy": "0.2.1",
     "typescript": "^5.9.3"
   },
   "engines": {
-    "node": ">=20.0.0"
+    "node": ">=20.19.0"
   },
   "resolutions": {
     "@xmldom/xmldom": "0.8.13",
     "argparse/underscore": "1.13.8",
     "east/underscore": "1.13.8",
-    "referer-parser/js-yaml": "^4.1.0",
+    "referer-parser/js-yaml": "^4.1.1",
     "sandboxed-module": "patch:sandboxed-module@npm%3A2.0.4#~/.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch",
     "request/tough-cookie": "5.1.2",
     "request/form-data": "2.5.5",
@@ -99,7 +101,6 @@
     "knip": "5.64.1",
     "eslint-plugin-testing-library": "7.5.3",
     "chart.js": "4.0.1",
-    "mock-fs": "5.2.0",
     "@customerio/cdp-analytics-node": "0.3.9",
     "@google-cloud/bigquery": "8.1.1",
     "moment": "2.29.4",

server-ce/.eslintrc

@@ -1,29 +0,0 @@
-{
-  "extends": [
-    "eslint:recommended",
-    "standard",
-    "prettier"
-  ],
-  "plugins": [
-    "unicorn"
-  ],
-  "parserOptions": {
-    "ecmaVersion": 2020
-  },
-  "env": {
-    "node": true
-  },
-  "rules": {
-    // Do not allow importing of implicit dependencies.
-    "import/no-extraneous-dependencies": "error",
-    "unicorn/prefer-node-protocol": "error"
-  },
-  "overrides": [
-    // Extra rules for Cypress tests
-    { "files": ["**/*.spec.ts"], "extends": ["plugin:cypress/recommended"] }
-  ],
-  "ignorePatterns": [
-    "hotfix/",
-    "develop/"
-  ]
-}

server-ce/Dockerfile-base

@@ -48,6 +48,12 @@ RUN curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QU
 &&  mkdir -p /var/www/.cache/quarto /var/www/.local/share \
 &&  chown -R www-data:www-data /var/www/.cache /var/www/.local
 
+# Install official Typst binary (Quarto bundles a modified fork without --synctex)
+# ---------------------------------------------------------------------------------
+ARG TYPST_VERSION=0.13.1
+RUN curl -fsSL "https://github.com/typst/typst/releases/download/v${TYPST_VERSION}/typst-x86_64-unknown-linux-musl.tar.xz" \
+    | tar -xJC /usr/local/bin --strip-components=1 "typst-x86_64-unknown-linux-musl/typst"
+
 # Pre-install popular Quarto extensions
 # -----------------------------------------------------------------------
 # Extensions land in /opt/quarto-extensions/_extensions/<author>/<name>/.
@@ -142,15 +148,11 @@ RUN mkdir /install-tl-unx \
 &&  echo "tlpdbopt_autobackup 0"       >> /install-tl-unx/texlive.profile \
 &&  echo "tlpdbopt_install_docfiles 0" >> /install-tl-unx/texlive.profile \
 &&  echo "tlpdbopt_install_srcfiles 0" >> /install-tl-unx/texlive.profile \
-&&  echo "selected_scheme scheme-basic" >> /install-tl-unx/texlive.profile \
+&&  echo "selected_scheme scheme-full" >> /install-tl-unx/texlive.profile \
 &&  echo "TEXDIR /usr/local/texlive"   >> /install-tl-unx/texlive.profile \
 &&  /install-tl-unx/install-tl \
       -profile /install-tl-unx/texlive.profile \
       -repository ${TEXLIVE_MIRROR} \
-&&  /usr/local/texlive/bin/x86_64-linux/tlmgr install \
-      --repository ${TEXLIVE_MIRROR} \
-      latexmk \
-      texcount \
 &&  rm -rf /install-tl-unx
 
 

server-ce/eslint.config.mjs

@@ -0,0 +1,33 @@
+import { defineConfig, globalIgnores } from 'eslint/config'
+import cypress from 'eslint-plugin-cypress/flat'
+import path from 'node:path'
+import baseConfig from '../eslint.config.mjs'
+
+const ROOT_DIR = path.resolve(import.meta.dirname, '..')
+
+export default defineConfig([
+  globalIgnores(['**/hotfix/', '**/develop/']),
+  {
+    basePath: ROOT_DIR,
+    extends: baseConfig,
+    languageOptions: {
+      ecmaVersion: 2020,
+    },
+  },
+  {
+    // The cypress block in baseConfig has patterns rooted at the
+    // monorepo root (`server-ce/test/helpers/*.ts`). When ESLint loads
+    // this file (server-ce/eslint.config.mjs) as the closest config --
+    // which happens when running `yarn run lint` from
+    // /overleaf/server-ce/test/ -- patterns from baseConfig are
+    // resolved relative to /overleaf/server-ce/, so those cross-dir
+    // patterns don't match. Re-declare with paths relative to this
+    // config file.
+    files: [
+      'test/helpers/*.ts',
+      'test/cypress/support/*.{js,jsx,mjs,cjs,ts,tsx}',
+      '**/*.spec.ts',
+    ],
+    ...cypress.configs.recommended,
+  },
+])

server-ce/test/Makefile

@@ -11,6 +11,7 @@ export TEX_LIVE_DOCKER_IMAGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/t
 export ALL_TEX_LIVE_DOCKER_IMAGES ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1,us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2022.1
 export IMAGE_TAG_CE ?= sharelatex/sharelatex:main
 export IMAGE_TAG_PRO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:main
+export IMAGE_TAG_GIT_BRIDGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/git-bridge:main
 export CYPRESS_SHARD ?=
 export COMPOSE_PROJECT_NAME ?= test
 export USER_UID=$(shell id -u)

server-ce/test/admin.spec.ts

@@ -245,6 +245,7 @@ describe('admin panel', function () {
           'Deleted Projects',
           'Audit Log',
           'Sessions',
+          'Personal Access Tokens',
         ]
         cy.findAllByRole('tab').should('have.length', tabs.length)
         tabs.forEach(tabName => {

server-ce/test/docker-compose.yml

@@ -78,12 +78,12 @@ services:
     working_dir: $PWD
     volumes:
       - $PWD:$PWD
-      - $MONOREPO/libraries:$MONOREPO/libraries:ro
-      - $MONOREPO/node_modules:$MONOREPO/node_modules:ro
-      - $MONOREPO/.yarn:$MONOREPO/.yarn:ro
-      - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml:ro
-      - $MONOREPO/package.json:$MONOREPO/package.json:ro
-      - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock:ro
+      - $MONOREPO/libraries:$MONOREPO/libraries
+      - $MONOREPO/node_modules:$MONOREPO/node_modules
+      - $MONOREPO/.yarn:$MONOREPO/.yarn
+      - $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml
+      - $MONOREPO/package.json:$MONOREPO/package.json
+      - $MONOREPO/yarn.lock:$MONOREPO/yarn.lock
     environment:
       MONOREPO:
       CYPRESS_SHARD:
@@ -130,6 +130,7 @@ services:
       ALL_TEX_LIVE_DOCKER_IMAGES:
       IMAGE_TAG_CE:
       IMAGE_TAG_PRO:
+      IMAGE_TAG_GIT_BRIDGE:
     healthcheck:
       test: curl --fail http://localhost/status
       interval: 3s

server-ce/test/editor.spec.ts

@@ -44,6 +44,7 @@ describe('editor', function () {
       cy.log(`change project language to '${lng}'`)
       cy.findByRole('button', { name: 'Settings' }).click()
       cy.findByRole('dialog').within(() => {
+        cy.findByRole('tab', { name: 'Spelling and language' }).click()
         cy.findByLabelText('Spellcheck language').select(lng)
       })
       cy.get('body').type('{esc}')
@@ -76,6 +77,7 @@ describe('editor', function () {
       cy.log('remove word from dictionary')
       cy.findByRole('button', { name: 'Settings' }).click()
       cy.findByRole('dialog').within(() => {
+        cy.findByRole('tab', { name: 'Spelling and language' }).click()
         cy.findByLabelText('Dictionary').click()
       })
       cy.findByTestId('dictionary-modal').within(() => {

server-ce/test/host-admin.js

@@ -32,11 +32,12 @@ const PATHS = {
 const IMAGES = {
   CE: process.env.IMAGE_TAG_CE.replace(/:.+/, ''),
   PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
+  GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/:.+/, ''),
 }
 const LATEST = {
   CE: process.env.IMAGE_TAG_CE.replace(/.+:/, '') || 'latest',
   PRO: process.env.IMAGE_TAG_PRO.replace(/.+:/, '') || 'latest',
-  GIT_BRIDGE: 'latest', // TODO, build in CI?
+  GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/.+:/, '') || 'latest',
 }
 
 function defaultDockerComposeOverride() {
@@ -242,7 +243,7 @@ function setVarsDockerCompose({
 
   cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version === 'latest' ? (pro ? LATEST.PRO : LATEST.CE) : version}`
   cfg.services['git-bridge'].image =
-    `quay.io/sharelatex/git-bridge:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
+    `${IMAGES.GIT_BRIDGE}:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
 
   cfg.services.sharelatex.environment = vars
 

server-ce/test/sandboxed-compiles.spec.ts

@@ -333,7 +333,7 @@ describe('SandboxedCompiles', function () {
   })
 
   // https://github.com/overleaf/internal/issues/20216
-  // eslint-disable-next-line mocha/no-skipped-tests
+  // eslint-disable-next-line mocha/no-pending-tests
   describe.skip('unavailable in CE', function () {
     if (isExcludedBySharding('CE_CUSTOM_1')) return
     startWith({ pro: false, vars: enabledVars, resetData: true })

services/chat/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/chat/Makefile

@@ -54,6 +54,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -66,8 +70,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/chat/buildscript.txt

@@ -1,5 +1,6 @@
 chat
 --dependencies=mongo
+--deploy-pipeline=chat
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/clsi/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/clsi/Dockerfile

@@ -32,6 +32,7 @@ COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-ed
 COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
+COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/clsi/package.json /overleaf/services/clsi/package.json
 COPY .yarn/patches/ /overleaf/.yarn/patches/
 
@@ -45,6 +46,7 @@ COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
 COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
+COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/clsi/ /overleaf/services/clsi/
 
 FROM app AS with-quarto

services/clsi/Makefile

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/promise-utils/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
+	$(MONOREPO)/libraries/validation-tools/package.json \
 	$(MONOREPO)/services/clsi/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
@@ -54,6 +55,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -67,8 +72,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \
@@ -165,8 +170,9 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
-	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
-	docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
+	-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
+	-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
+	-cd ../../ && docker build -t us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pdftocairo:24.02 dockerfiles/pdftocairo
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/clsi/app.js

@@ -145,6 +145,11 @@ app.post(
   bodyParser.json({ limit: Settings.compileSizeLimit }),
   ConversionController.convertProjectToDocument
 )
+app.post(
+  '/convert/pdf-to-jpeg',
+  FileUploadMiddleware.multerMiddleware,
+  ConversionController.convertPDFToJPEG
+)
 
 if (process.env.NODE_ENV === 'development' && global.__coverage__) {
   app.get('/coverage', (req, res) => {

services/clsi/app/js/CompileController.js

@@ -80,6 +80,10 @@ function compile(req, res, next) {
                 { err: error, projectId: request.project_id },
                 'timeout running compile'
               )
+            } else if (error?.typstCompileFailure) {
+              // Typst compiled but with errors — treat as a compile failure so
+              // the frontend shows the error log rather than the old PDF.
+              status = 'failure'
             } else if (error) {
               status = 'error'
               code = 500

services/clsi/app/js/CompileManager.js

@@ -386,11 +386,18 @@ async function stopCompile(projectId, userId) {
 }
 
 async function clearProject(projectId, userId) {
+  // Kill any live typst watcher before deleting its files.
+  const compileName = getCompileName(projectId, userId)
+  await TypstRunner.promises.killTypst(compileName)
   const compileDir = getCompileDir(projectId, userId)
   await fsPromises.rm(compileDir, { force: true, recursive: true })
 }
 
 async function clearProjectWithListing(projectId, userId, allEntries) {
+  // Kill any live typst watcher (e.g. timedout compile where killTypst
+  // was not already called) before removing files from under it.
+  const compileName = getCompileName(projectId, userId)
+  await TypstRunner.promises.killTypst(compileName)
   const compileDir = getCompileDir(projectId, userId)
 
   const exists = await _checkDirectory(compileDir)

services/clsi/app/js/ConversionController.js

@@ -14,10 +14,12 @@ import RequestParser from './RequestParser.js'
 import { pipeline } from 'node:stream/promises'
 import Settings from '@overleaf/settings'
 import Path from 'node:path'
+import { z } from '@overleaf/validation-tools'
 
 const CONVERSION_CONFIGS = {
   docx: { extension: 'docx' },
   markdown: { extension: 'zip' },
+  html: { extension: 'zip' },
 }
 
 async function convertDocumentToLaTeX(req, res) {
@@ -77,6 +79,51 @@ async function convertDocumentToLaTeX(req, res) {
   }
 }
 
+const PDFToJPEGQuerySchema = z.object({
+  mode: z.enum(['preview', 'thumbnail']),
+})
+
+async function convertPDFToJPEG(req, res) {
+  const { path } = req.file
+  if (!Settings.enablePdfConversions) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(404)
+  }
+  const parsed = PDFToJPEGQuerySchema.safeParse(req.query)
+  if (!parsed.success) {
+    await fs.unlink(path).catch(() => {})
+    return res.sendStatus(400)
+  }
+  const { mode } = parsed.data
+  logger.debug({ path, mode }, 'received pdf for conversion to jpeg')
+  const conversionId = crypto.randomUUID()
+  let jpegPath
+  try {
+    jpegPath = await ConversionManager.promises.convertPDFToJPEGWithLock(
+      conversionId,
+      path,
+      mode
+    )
+  } finally {
+    await fs.unlink(path).catch(() => {})
+  }
+
+  try {
+    const jpegStat = await fs.stat(jpegPath)
+
+    res.setHeader('Content-Length', jpegStat.size)
+    res.attachment('output.jpg')
+    res.setHeader('X-Content-Type-Options', 'nosniff')
+
+    const readStream = fsSync.createReadStream(jpegPath)
+    await pipeline(readStream, res)
+  } finally {
+    await fs
+      .rm(Path.dirname(jpegPath), { recursive: true, force: true })
+      .catch(() => {})
+  }
+}
+
 async function convertProjectToDocument(req, res) {
   if (!Settings.enablePandocConversions) {
     return res.sendStatus(404)
@@ -207,4 +254,5 @@ async function convertProjectToDocument(req, res) {
 export default {
   convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
   convertProjectToDocument: expressify(convertProjectToDocument),
+  convertPDFToJPEG: expressify(convertPDFToJPEG),
 }

services/clsi/app/js/ConversionManager.js

@@ -18,6 +18,18 @@ const CONVERSION_CONFIGS = {
   },
 }
 
+const PDF_TO_JPEG_CONFIGS = {
+  preview: { width: 794, quality: 90 },
+  thumbnail: { width: 190, quality: 50 },
+}
+
+const PDF_TO_JPEG_INPUT_FILENAME = 'input.pdf'
+const PDF_TO_JPEG_OUTPUT_FILENAME = 'output.jpg'
+const PDF_TO_JPEG_OUTPUT_BASENAME = Path.basename(
+  PDF_TO_JPEG_OUTPUT_FILENAME,
+  '.jpg'
+)
+
 async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
   const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
   const lock = LockManager.acquire(conversionDir)
@@ -150,6 +162,19 @@ const LATEX_EXPORT_CONFIGS = {
       'markdown',
     ],
   },
+  html: {
+    fileExtension: 'html',
+    compressOutput: true,
+    getPandocArgs: ({ outputPath }) => [
+      '--output',
+      outputPath,
+      '--from',
+      'latex',
+      '--to',
+      'html',
+      '--standalone',
+    ],
+  },
 }
 
 async function convertLaTeXToDocumentInDirWithLock(
@@ -298,9 +323,76 @@ async function convertLaTeXToDocumentInDir(
   return Path.join(compileDir, finalOutputName)
 }
 
+async function convertPDFToJPEGWithLock(conversionId, inputPath, mode) {
+  const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
+  const lock = LockManager.acquire(conversionDir)
+  try {
+    return await convertPDFToJPEG(conversionId, conversionDir, inputPath, mode)
+  } finally {
+    lock.release()
+  }
+}
+
+async function convertPDFToJPEG(conversionId, conversionDir, inputPath, mode) {
+  const config = PDF_TO_JPEG_CONFIGS[mode]
+  await fs.mkdir(conversionDir, { recursive: true })
+  const newSourcePath = Path.join(conversionDir, PDF_TO_JPEG_INPUT_FILENAME)
+  await fs.copyFile(inputPath, newSourcePath)
+  const dstPath = Path.join(conversionDir, PDF_TO_JPEG_OUTPUT_FILENAME)
+
+  try {
+    const { stdout, stderr, exitCode } = await CommandRunner.promises.run(
+      conversionId,
+      [
+        'pdftocairo',
+        '-jpeg',
+        '-jpegopt',
+        `quality=${config.quality}`,
+        '-singlefile',
+        '-scale-to-x',
+        config.width.toString(),
+        '-scale-to-y',
+        '-1', // maintain aspect ratio
+        PDF_TO_JPEG_INPUT_FILENAME,
+        PDF_TO_JPEG_OUTPUT_BASENAME,
+      ],
+      conversionDir,
+      Settings.pdftocairoImage,
+      Settings.conversionTimeoutSeconds * 1000,
+      {},
+      'conversions',
+      null
+    )
+    if (exitCode !== 0) {
+      throw new OError('Non-zero exit code from pdftocairo', {
+        exitCode,
+        stderr,
+      })
+    }
+    logger.debug(
+      { stdout, stderr, exitCode },
+      'pdf-to-jpeg conversion completed'
+    )
+
+    const stat = await fs.lstat(dstPath)
+    if (!stat.isFile()) {
+      throw new OError('output.jpg is not a regular file', { stat })
+    }
+
+    // Clean up the source PDF to leave only the conversion result
+    await fs.unlink(newSourcePath).catch(() => {})
+  } catch (error) {
+    await fs.rm(conversionDir, { force: true, recursive: true }).catch(() => {})
+    throw new OError('pdf-to-jpeg conversion failed').withCause(error)
+  }
+
+  return dstPath
+}
+
 export default {
   promises: {
     convertToLaTeXWithLock,
     convertLaTeXToDocumentInDirWithLock,
+    convertPDFToJPEGWithLock,
   },
 }

services/clsi/app/js/ResourceWriter.js

@@ -209,8 +209,16 @@ export default ResourceWriter = {
           return callback(error)
         }
 
+        // Project input resources are in outputFiles only to be served from
+        // the output cache (HTML media exception in OutputFileFinder).  They
+        // must never be deleted here — incremental compiles don't re-sync
+        // unchanged binary files, so deleting them would leave them missing
+        // for Quarto and for _appendMissingResourceWarnings.
+        const incomingPaths = new Set(resources.map(r => r.path))
+
         const jobs = []
         for (const { path } of outputFiles || []) {
+          if (incomingPaths.has(path)) continue
           const shouldDelete = ResourceWriter.isExtraneousFile(path)
           if (shouldDelete) {
             jobs.push(callback =>

services/clsi/app/js/TypstRunner.js

@@ -1,96 +1,485 @@
 import Path from 'node:path'
+import { spawn } from 'node:child_process'
 import { promisify } from 'node:util'
 import logger from '@overleaf/logger'
+import Settings from '@overleaf/settings'
 import CommandRunner from './CommandRunner.js'
 import fs from 'node:fs'
 
-// Maps currently-running Typst jobs: compileName → PID (or docker container id)
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+// Max lines kept from the current compile cycle (prevents unbounded growth
+// for documents that produce many warnings).
+const MAX_LOG_LINES = 500
+
+// How long to wait for the watcher process to emit its first output.
+const WATCH_START_TIMEOUT_MS = 15_000
+
+// Matches the start-of-compile marker typst watch emits before each cycle:
+//   "[HH:MM:SS] compiling ..."
+// Used to reset the line buffer so stale output from a failed compile that
+// didn't emit a "compiled with errors" footer cannot bleed into the next log.
+const COMPILE_START_RE = /^\[\d{2}:\d{2}:\d{2}\] compiling/
+
+// Matches the three terminal lines that typst watch emits at the end of each
+// compile cycle regardless of outcome:
+//   "[HH:MM:SS] compiled successfully in 42ms"
+//   "[HH:MM:SS] compiled with warnings in 42ms"
+//   "[HH:MM:SS] compiled with errors"
+const COMPILE_DONE_RE = /compiled (successfully|with (errors|warnings))/
+
+// Signals FileId exhaustion in a long-lived typst process (typst issue #7434).
+const FILE_ID_EXHAUSTION_RE = /ran out of file ids/i
+
+// Proactively restart the watcher before FileId exhaustion.
+// Typst uses ~65 IDs per compile; 1000 compiles ≈ 65 000 — safely under 65 535.
+const MAX_COMPILES_BEFORE_RESTART = 1000
+
+// typst watch emits the "[HH:MM:SS] compiled with errors" status line FIRST,
+// then the full diagnostic output (file:line:col, code snippets) AFTERWARDS.
+// We buffer post-done lines and resolve after this delay if no new compile
+// cycle starts sooner. 150 ms is well above the ~1 chunk latency for typst's
+// diagnostic flush and imperceptible on top of a typical compile time.
+const FLUSH_DELAY_MS = 150
+
+// ---------------------------------------------------------------------------
+// State (module-level, never exported)
+// ---------------------------------------------------------------------------
+
+// Active cold-start compile jobs (Docker fallback): compileName → PID
 const ProcessTable = {}
 
-// Compiles a standalone Typst document (.typ) straight to output.pdf. We reuse
-// the Typst that ships inside Quarto via `quarto typst compile`, so there is no
-// extra binary to install. This is deliberately the simplest of the three
-// runners: Typst only ever produces a PDF, so there is no format detection,
-// no HTML asset directory and no extension merging (cf. QuartoRunner).
-function runTypst(compileName, options, callback) {
-  const { directory, mainFile, image, environment, compileGroup } = options
-  const timeout = options.timeout || 60000
+// Long-lived watcher processes: compileName → WatchEntry
+// WatchEntry shape:
+//   proc              ChildProcess
+//   directory         compile dir (absolute path)
+//   mainFile          root .typ filename
+//   environment       env vars passed to the runner
+//   compilationCount  total successful compile cycles on this watcher
+//   restartPending    flag to restart at the next runTypst call
+//   accumulator       incomplete trailing line from the last data chunk
+//   currentLines      lines accumulated in the current phase (pre-done or
+//                     post-done, see _onWatcherData for the two-phase logic)
+//   doneResult        { preLines, compiledWithErrors } held between the
+//                     COMPILE_DONE_RE line and the post-done diagnostic flush;
+//                     null when not in post-done phase
+//   flushTimeout      timeout handle that finalises doneResult when no next
+//                     compile cycle starts within FLUSH_DELAY_MS
+//   pendingResolvers  Array<{resolve, reject, timeoutHandle}>
+//   pendingResult     compile result cached when typst finished before a
+//                     resolver was registered (race-condition safety net)
+const WatchTable = {}
 
-  logger.debug(
-    { directory, timeout, mainFile, compileGroup },
-    'starting typst compile'
-  )
+// PIDs we have intentionally killed, so the close handler can distinguish
+// an expected exit from an unexpected crash.
+const _killedWatchPids = new Set()
 
-  const command = _buildTypstCommand(mainFile)
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
 
-  ProcessTable[compileName] = CommandRunner.run(
-    compileName,
-    command,
-    directory,
-    image,
-    timeout,
-    environment || {},
-    compileGroup,
-    null,
-    function (error, output) {
-      delete ProcessTable[compileName]
+async function runTypstAsync(compileName, options) {
+  // Docker / sandboxed mode: fall back to a cold-start compile per request.
+  if (Settings.clsi?.dockerRunner) {
+    return _runColdStart(compileName, options)
+  }
 
-      // Propagate real process-level errors (killed, timed out) but NOT
-      // ordinary non-zero exit codes from Typst itself. A compile failure
-      // (exit code 1) is not a server error — the absence of output.pdf is
-      // enough for CompileController to return 'failure'.
-      if (error && (error.terminated || error.timedout)) {
-        return callback(error)
-      }
+  const timeout = options.timeout || 60_000
+  const entry = WatchTable[compileName]
 
-      // On exit-code-1 errors LocalCommandRunner attaches stdout to the error
-      // object; merge it so _writeLogOutput can persist it.
-      const combined = output || (error ? { stdout: error.stdout || '' } : null)
-      _writeLogOutput(compileName, directory, combined, () =>
-        callback(null, combined)
-      )
+  const needsStart =
+    !entry ||
+    entry.restartPending ||
+    entry.proc.exitCode !== null
+
+  if (needsStart) {
+    if (entry) _killWatchEntry(compileName)
+    // _startWatcher spawns the process, registers all handlers, then calls
+    // _waitForNextCompile synchronously — the resolver is in pendingResolvers
+    // before any I/O event can fire, eliminating the race condition.
+    const result = await _startWatcher(compileName, options)
+    await _writeLogOutputAsync(compileName, options.directory, result)
+    if (result.compiledWithErrors) {
+      throw Object.assign(new Error('typst-compile-failure'), {
+        typstCompileFailure: true,
+      })
+    }
+    return result
+  }
+
+  // Watcher is alive. _waitForNextCompile adds the resolver synchronously
+  // inside the Promise constructor, before this function yields — safe.
+  const result = await _waitForNextCompile(compileName, timeout)
+  await _writeLogOutputAsync(compileName, options.directory, result)
+  if (result.compiledWithErrors) {
+    throw Object.assign(new Error('typst-compile-failure'), {
+      typstCompileFailure: true,
+    })
+  }
+  return result
+}
+
+// ---------------------------------------------------------------------------
+// Watcher lifecycle
+// ---------------------------------------------------------------------------
+
+async function _startWatcher(compileName, options) {
+  const { directory, mainFile, environment } = options
+  const timeout = options.timeout || 60_000
+
+  const absInput = Path.join(directory, mainFile)
+  const absOutput = Path.join(directory, 'output.pdf')
+  const env = { ...process.env, ...(environment || {}) }
+
+  logger.debug({ compileName, absInput }, 'starting typst watcher')
+
+  const proc = spawn(
+    '/bin/sh',
+    ['-c', `typst watch "${absInput}" "${absOutput}" 2>&1`],
+    {
+      cwd: directory,
+      env,
+      stdio: ['ignore', 'pipe', 'ignore'],
+      detached: true,
     }
   )
+
+  const entry = {
+    proc,
+    directory,
+    mainFile,
+    environment,
+    compilationCount: 0,
+    restartPending: false,
+    accumulator: '',
+    currentLines: [],
+    doneResult: null,
+    flushTimeout: null,
+    pendingResolvers: [],
+    pendingResult: null,
+  }
+  WatchTable[compileName] = entry
+
+  // Register handlers synchronously — before any I/O events can fire.
+  proc.stdout.setEncoding('utf8')
+  proc.stdout.on('data', chunk => _onWatcherData(compileName, chunk))
+
+  proc.on('error', err => {
+    logger.error({ err, compileName }, 'typst watcher process error')
+    _rejectAllPending(
+      compileName,
+      Object.assign(err, { terminated: true })
+    )
+    if (WatchTable[compileName]?.proc === proc) {
+      delete WatchTable[compileName]
+    }
+  })
+
+  proc.on('close', (code, signal) => {
+    logger.warn({ code, signal, compileName }, 'typst watcher exited')
+    const wasKilled = _killedWatchPids.delete(proc.pid)
+    if (!wasKilled) {
+      _rejectAllPending(
+        compileName,
+        Object.assign(new Error('typst watcher exited unexpectedly'), {
+          terminated: true,
+        })
+      )
+    }
+    if (WatchTable[compileName]?.proc === proc) {
+      delete WatchTable[compileName]
+    }
+  })
+
+  // typst watch performs an initial compile immediately on startup.
+  // _waitForNextCompile adds the resolver synchronously here (inside the
+  // Promise constructor) before we yield, so it will catch that first event.
+  return _waitForNextCompile(compileName, timeout + WATCH_START_TIMEOUT_MS)
 }
 
-function _buildTypstCommand(mainFile) {
-  // Run through a POSIX shell so stderr (where Typst writes its diagnostics)
-  // is merged into stdout (2>&1). LocalCommandRunner replaces $COMPILE_DIR
-  // before the shell sees it; the output path is relative because the shell
-  // CWD is already the compile directory.
-  const inputPath = `$COMPILE_DIR/${mainFile}`
-  const cmd = `quarto typst compile ${inputPath} output.pdf 2>&1`
-  return ['/bin/sh', '-c', cmd]
+function _killWatchEntry(compileName) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+  clearTimeout(entry.flushTimeout)
+  delete WatchTable[compileName]
+  try {
+    _killedWatchPids.add(entry.proc.pid)
+    process.kill(-entry.proc.pid) // kill entire process group
+  } catch (err) {
+    _killedWatchPids.delete(entry.proc.pid)
+    logger.warn({ err, compileName }, 'error killing typst watcher process group')
+  }
 }
 
-function _writeLogOutput(compileName, directory, output, callback) {
-  const content = (output && output.stdout) || ''
-  if (!content) return callback()
-  // Write to output.log so the PDF-preview log panel picks it up. Typst's
-  // `error:`/`warning:` + `┌─ file:line:col` diagnostics are understood by the
-  // Quarto/Typst log parser on the web side.
-  const logFile = Path.join(directory, 'output.log')
-  fs.unlink(logFile, () => {
-    fs.writeFile(logFile, content, { flag: 'wx' }, err => {
-      if (err) {
-        logger.error({ err, compileName, logFile }, 'error writing typst log')
+// ---------------------------------------------------------------------------
+// Stdout parsing
+// ---------------------------------------------------------------------------
+
+function _onWatcherData(compileName, chunk) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+
+  entry.accumulator += chunk
+  const lines = entry.accumulator.split('\n')
+  entry.accumulator = lines.pop() // keep the incomplete trailing fragment
+
+  for (const line of lines) {
+    if (COMPILE_START_RE.test(line)) {
+      // A new compile cycle is starting.  If we were in the post-done phase
+      // (collecting diagnostic lines that typst emits AFTER the status line),
+      // finalise the previous result now — all diagnostics have arrived.
+      if (entry.doneResult) {
+        _finalizeCompile(compileName)
       }
-      callback()
-    })
+      // Start fresh for the new cycle.
+      entry.currentLines = [line]
+      continue
+    }
+
+    entry.currentLines.push(line)
+    if (entry.currentLines.length > MAX_LOG_LINES) {
+      entry.currentLines.shift()
+    }
+
+    if (FILE_ID_EXHAUSTION_RE.test(line)) {
+      logger.warn({ compileName }, 'typst watcher: FileId exhaustion detected')
+      entry.restartPending = true
+    }
+
+    if (COMPILE_DONE_RE.test(line)) {
+      entry.compilationCount++
+
+      if (entry.compilationCount >= MAX_COMPILES_BEFORE_RESTART) {
+        logger.info(
+          { compileName, compilationCount: entry.compilationCount },
+          'typst watcher: scheduling restart (FileId threshold)'
+        )
+        entry.restartPending = true
+      }
+
+      // typst watch outputs the "[HH:MM:SS] compiled with errors" status
+      // line FIRST, then the full diagnostics (file:line:col, code snippets)
+      // AFTERWARDS. Enter post-done phase: keep accumulating into currentLines
+      // and flush after FLUSH_DELAY_MS (or immediately when the next compile
+      // cycle's COMPILE_START_RE arrives, whichever comes first).
+      entry.doneResult = {
+        preLines: entry.currentLines,
+        compiledWithErrors: /compiled with errors/.test(line),
+      }
+      entry.currentLines = []
+
+      clearTimeout(entry.flushTimeout)
+      entry.flushTimeout = setTimeout(
+        () => _finalizeCompile(compileName),
+        FLUSH_DELAY_MS
+      )
+    }
+  }
+}
+
+// Combines the pre-done lines (up to/including the status line) with any
+// post-done diagnostic lines and resolves all pending waiters.
+function _finalizeCompile(compileName) {
+  const entry = WatchTable[compileName]
+  if (!entry || !entry.doneResult) return
+
+  clearTimeout(entry.flushTimeout)
+  entry.flushTimeout = null
+
+  const { preLines, compiledWithErrors } = entry.doneResult
+  entry.doneResult = null
+
+  // Merge: status line(s) first, then the post-done diagnostics.
+  const allLines = preLines.concat(entry.currentLines)
+  entry.currentLines = []
+
+  _resolveAllPending(compileName, {
+    stdout: allLines.join('\n'),
+    compiledWithErrors,
   })
 }
 
+// ---------------------------------------------------------------------------
+// Resolver helpers
+// ---------------------------------------------------------------------------
+
+function _waitForNextCompile(compileName, timeout) {
+  return new Promise((resolve, reject) => {
+    const entry = WatchTable[compileName]
+    if (!entry) {
+      return reject(new Error('no typst watcher for ' + compileName))
+    }
+
+    // If typst finished a compile cycle before this resolver was registered
+    // (race: ResourceWriter wrote files → typst compiled → runTypst called),
+    // consume the cached result immediately instead of waiting for a timeout.
+    if (entry.pendingResult) {
+      const result = entry.pendingResult
+      entry.pendingResult = null
+      return resolve(result)
+    }
+
+    // Push synchronously inside the Promise constructor — before the first
+    // await in the caller, so no data event can fire in the gap.
+    const timeoutHandle = setTimeout(() => {
+      entry.pendingResolvers = entry.pendingResolvers.filter(r => r !== resolver)
+      reject(
+        Object.assign(new Error('typst compile timed out'), { timedout: true })
+      )
+    }, timeout)
+
+    const resolver = { resolve, reject, timeoutHandle }
+    entry.pendingResolvers.push(resolver)
+  })
+}
+
+function _resolveAllPending(compileName, result) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+  const resolvers = entry.pendingResolvers.splice(0)
+  if (resolvers.length === 0) {
+    // typst compiled before a resolver was registered — cache the result so
+    // the next _waitForNextCompile call can consume it immediately.
+    entry.pendingResult = result
+    return
+  }
+  for (const { resolve, timeoutHandle } of resolvers) {
+    clearTimeout(timeoutHandle)
+    resolve(result)
+  }
+}
+
+function _rejectAllPending(compileName, err) {
+  const entry = WatchTable[compileName]
+  if (!entry) return
+  for (const { reject, timeoutHandle } of entry.pendingResolvers.splice(0)) {
+    clearTimeout(timeoutHandle)
+    reject(err)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Log output
+// ---------------------------------------------------------------------------
+
+async function _writeLogOutputAsync(compileName, directory, output) {
+  const content = (output && output.stdout) || ''
+  if (!content) return
+  // Write to output.log so the PDF-preview log panel picks it up.
+  const logFile = Path.join(directory, 'output.log')
+  try {
+    await fs.promises.unlink(logFile)
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      logger.error({ err, compileName, logFile }, 'error removing typst log')
+    }
+  }
+  try {
+    await fs.promises.writeFile(logFile, content, { flag: 'wx' })
+  } catch (err) {
+    if (err.code !== 'EEXIST') {
+      logger.error({ err, compileName, logFile }, 'error writing typst log')
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Cold-start fallback (Docker / sandboxed mode)
+// ---------------------------------------------------------------------------
+
+function _runColdStart(compileName, options) {
+  return new Promise((resolve, reject) => {
+    const { directory, mainFile, image, environment, compileGroup } = options
+    const timeout = options.timeout || 60_000
+
+    logger.debug({ directory, mainFile, compileGroup }, 'typst cold-start compile')
+
+    const inputPath = `$COMPILE_DIR/${mainFile}`
+    const command = ['/bin/sh', '-c', `typst compile ${inputPath} output.pdf 2>&1`]
+
+    ProcessTable[compileName] = CommandRunner.run(
+      compileName,
+      command,
+      directory,
+      image,
+      timeout,
+      environment || {},
+      compileGroup,
+      null,
+      function (error, output) {
+        delete ProcessTable[compileName]
+        if (error && (error.terminated || error.timedout)) {
+          return reject(error)
+        }
+        const combined =
+          output || (error ? { stdout: error.stdout || '' } : null)
+        _writeLogOutputAsync(compileName, directory, combined).then(
+          () => {
+            if (error && combined?.stdout) {
+              // Non-zero exit with output = typst compile error (not a
+              // system/infra error). Signal failure so the log panel opens.
+              reject(
+                Object.assign(new Error('typst-compile-failure'), {
+                  typstCompileFailure: true,
+                })
+              )
+            } else if (error) {
+              reject(error)
+            } else {
+              resolve(combined)
+            }
+          },
+          reject
+        )
+      }
+    )
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Public interface
+// ---------------------------------------------------------------------------
+
 function isRunning(compileName) {
-  return ProcessTable[compileName] != null
+  return (
+    ProcessTable[compileName] != null ||
+    (WatchTable[compileName]?.pendingResolvers.length > 0)
+  )
 }
 
 function killTypst(compileName, callback) {
-  logger.debug({ compileName }, 'killing running typst compile')
-  if (!isRunning(compileName)) {
-    logger.warn({ compileName }, 'no such compile to kill')
-    return callback(null)
+  logger.debug({ compileName }, 'killing typst (watcher + any active compile)')
+
+  // Cold-start fallback path
+  if (ProcessTable[compileName] != null) {
+    CommandRunner.kill(ProcessTable[compileName], () => {})
+    delete ProcessTable[compileName]
   }
-  CommandRunner.kill(ProcessTable[compileName], callback)
+
+  // Reject any in-flight waiters and tear down the watcher process
+  _rejectAllPending(
+    compileName,
+    Object.assign(new Error('terminated'), { terminated: true })
+  )
+  _killWatchEntry(compileName)
+
+  callback(null)
+}
+
+// Kill all watcher processes when the CLSI Node process exits.
+process.on('exit', () => {
+  for (const compileName of Object.keys(WatchTable)) {
+    _killWatchEntry(compileName)
+  }
+})
+
+const runTypst = (compileName, options, callback) => {
+  runTypstAsync(compileName, options).then(
+    result => callback(null, result),
+    err => callback(err)
+  )
 }
 
 export default {
@@ -98,7 +487,7 @@ export default {
   runTypst,
   killTypst,
   promises: {
-    runTypst: promisify(runTypst),
+    runTypst: runTypstAsync,
     killTypst: promisify(killTypst),
   },
 }

services/clsi/buildscript.txt

@@ -1,7 +1,7 @@
 clsi
 --data-dirs=cache,compiles,output
 --dependencies=
---env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true
+--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true,ENABLE_PDF_CONVERSIONS=true
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.14.1

services/clsi/config/settings.defaults.cjs

@@ -31,6 +31,9 @@ module.exports = {
     parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
   pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
   enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
+  pdftocairoImage:
+    process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
+  enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
   maxUploadSize: 50 * 1024 * 1024,
 
   internal: {

services/clsi/docker-compose.ci.yml

@@ -30,7 +30,7 @@ services:
       ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
       ENABLE_PDF_CACHING: true
       PDF_CACHING_ENABLE_WORKER_POOL: true
-      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
+      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
@@ -38,6 +38,7 @@ services:
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
       ENABLE_PANDOC_CONVERSIONS: true
+      ENABLE_PDF_CONVERSIONS: true
     volumes:
       - ./reports:/overleaf/services/clsi/reports
       - ./compiles:/overleaf/services/clsi/compiles

services/clsi/docker-compose.yml

@@ -53,7 +53,7 @@ services:
       ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
       ENABLE_PDF_CACHING: true
       PDF_CACHING_ENABLE_WORKER_POOL: true
-      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
+      ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
       TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
@@ -61,6 +61,7 @@ services:
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
       SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
       ENABLE_PANDOC_CONVERSIONS: true
+      ENABLE_PDF_CONVERSIONS: true
     depends_on:
       clsi-nginx:
         condition: service_started

services/clsi/package.json

@@ -23,6 +23,7 @@
     "@overleaf/promise-utils": "workspace:*",
     "@overleaf/settings": "workspace:*",
     "@overleaf/stream-utils": "workspace:*",
+    "@overleaf/validation-tools": "workspace:*",
     "archiver": "5.3.2",
     "async": "^3.2.5",
     "body-parser": "1.20.4",
@@ -45,7 +46,6 @@
     "mocha": "^11.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",
-    "mock-fs": "^5.1.2",
     "node-fetch": "^2.7.0",
     "nyc": "^17.1.0",
     "sinon": "~9.0.1",

services/clsi/test/acceptance/js/PdfToJpegConversionTests.js

@@ -0,0 +1,83 @@
+import Client from './helpers/Client.js'
+import ClsiApp from './helpers/ClsiApp.js'
+import Path from 'node:path'
+import fs from 'node:fs/promises'
+import { promisify } from 'node:util'
+import { execFile as execFileCb } from 'node:child_process'
+import { expect } from 'chai'
+
+const execFile = promisify(execFileCb)
+
+const FIXTURE_PDF = Path.join(import.meta.dirname, '../fixtures/minimal.pdf')
+
+const MODE_EXPECTATIONS = {
+  preview: { width: 794 },
+  thumbnail: { width: 190 },
+}
+
+async function writeResponseToTempfile(response) {
+  const buffer = Buffer.from(await response.arrayBuffer())
+  const tmpPath = `/tmp/clsi-acceptance-pdf-to-jpeg-${crypto.randomUUID()}.jpg`
+  await fs.writeFile(tmpPath, buffer)
+  return { tmpPath, buffer }
+}
+
+describe('pdf-to-jpeg conversion', function () {
+  before(async function () {
+    await ClsiApp.ensureRunning()
+  })
+
+  for (const [mode, { width: expectedWidth }] of Object.entries(
+    MODE_EXPECTATIONS
+  )) {
+    describe(`with mode=${mode}`, function () {
+      let response
+      let tmpPath
+      let buffer
+
+      before(async function () {
+        response = await Client.convertPdfToJpeg(FIXTURE_PDF, mode)
+        expect(response.status).to.equal(200)
+        ;({ tmpPath, buffer } = await writeResponseToTempfile(response))
+      })
+
+      after(async function () {
+        if (tmpPath) {
+          await fs.unlink(tmpPath).catch(() => {})
+        }
+      })
+
+      it('returns a JPEG (per `file`)', async function () {
+        const { stdout } = await execFile('file', ['--brief', tmpPath])
+        expect(stdout).to.match(/JPEG image data/)
+      })
+
+      it(`has the expected width of ${expectedWidth}px`, async function () {
+        const { stdout } = await execFile('identify', [
+          '-format',
+          '%w %h',
+          tmpPath,
+        ])
+        const [width, height] = stdout.trim().split(' ').map(Number)
+        expect(width).to.equal(expectedWidth)
+        // A4 portrait is taller than wide; height must be positive and
+        // larger than the width (so the aspect ratio was preserved).
+        expect(height).to.be.greaterThan(width)
+      })
+
+      it('returns a non-empty body matching Content-Length', function () {
+        expect(buffer.length).to.be.greaterThan(0)
+        expect(buffer.length).to.equal(
+          Number(response.headers.get('content-length'))
+        )
+      })
+    })
+  }
+
+  describe('with an unsupported mode', function () {
+    it('returns 400', async function () {
+      const response = await Client.convertPdfToJpeg(FIXTURE_PDF, 'not-a-mode')
+      expect(response.status).to.equal(400)
+    })
+  })
+})

services/clsi/test/acceptance/js/helpers/Client.js

@@ -53,6 +53,16 @@ async function convertDocument(path, type) {
   }
 }
 
+async function convertPdfToJpeg(path, mode) {
+  const formData = new FormData()
+  formData.append('qqfile', await fsPromises.readFile(path), 'input.pdf')
+  return await fetch(`${host}/convert/pdf-to-jpeg?mode=${mode}`, {
+    method: 'POST',
+    headers: formData.getHeaders(),
+    body: formData.getBuffer(),
+  })
+}
+
 async function convertProjectToDocument(
   projectId,
   userId,
@@ -239,6 +249,7 @@ export default {
   compile,
   convertProjectToDocument,
   convertDocument,
+  convertPdfToJpeg,
   stopCompile,
   clearCache,
   getOutputFile,

services/clsi/test/unit/js/ConversionController.test.js

@@ -18,6 +18,7 @@ describe('ConversionController', function () {
     ctx.documentStat = { size: 5678 }
     ctx.Settings = {
       enablePandocConversions: true,
+      enablePdfConversions: true,
       path: {
         compilesDir: '/compiles',
         outputDir: '/output',
@@ -591,6 +592,37 @@ describe('ConversionController', function () {
       })
     })
 
+    describe('with conversionType=html', function () {
+      beforeEach(async function (ctx) {
+        ctx.req.query = { type: 'html' }
+        ctx.fs.stat.resolves(ctx.documentStat)
+
+        await ctx.ConversionController.convertProjectToDocument(
+          ctx.req,
+          ctx.res,
+          sinon.stub()
+        )
+      })
+
+      it('should call convertLaTeXToDocumentInDirWithLock with type=html', function (ctx) {
+        sinon.assert.calledWith(
+          ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
+          sinon.match(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          ),
+          sinon.match(
+            /^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          ),
+          'main.tex',
+          'html'
+        )
+      })
+
+      it('should set the attachment filename with .zip extension', function (ctx) {
+        sinon.assert.calledWith(ctx.res.attachment, 'output.zip')
+      })
+    })
+
     describe('when conversion fails', function () {
       beforeEach(async function (ctx) {
         ctx.next = sinon.stub()

services/clsi/test/unit/js/ConversionManager.test.js

@@ -77,6 +77,24 @@ const LATEX_TO_DOCUMENT_CASES = [
       '--extract-media=.',
     ],
   },
+  {
+    type: 'html',
+    extension: 'html',
+    compressOutput: true,
+    pandocArgs: outputId => [
+      'pandoc',
+      Path.join('..', 'main.tex'),
+      '--output',
+      'main.html',
+      '--from',
+      'latex',
+      '--to',
+      'html',
+      '--standalone',
+      '--resource-path=..',
+      '--extract-media=.',
+    ],
+  },
 ]
 
 describe('ConversionManager', function () {

services/clsi/test/unit/js/DraftModeManager.test.js

@@ -1,7 +1,8 @@
 import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest'
 import Path from 'node:path'
+import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
-import mockFs from 'mock-fs'
+import os from 'node:os'
 
 const MODULE_PATH = Path.join(
   import.meta.dirname,
@@ -15,20 +16,19 @@ describe('DraftModeManager', () => {
     }))
 
     ctx.DraftModeManager = (await import(MODULE_PATH)).default
-    ctx.filename = '/mock/filename.tex'
+    ctx.tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'draft-mode-test-'))
+    ctx.filename = Path.join(ctx.tmpDir, 'filename.tex')
     ctx.contents = `\
 \\documentclass{article}
 \\begin{document}
 Hello world
 \\end{document}\
 `
-    mockFs({
-      [ctx.filename]: ctx.contents,
-    })
+    fs.writeFileSync(ctx.filename, ctx.contents)
   })
 
-  afterEach(() => {
-    mockFs.restore()
+  afterEach(ctx => {
+    fs.rmSync(ctx.tmpDir, { recursive: true })
   })
 
   describe('injectDraftMode', () => {

services/clsi/test/unit/js/OutputFileFinder.test.js

@@ -1,6 +1,6 @@
-import sinon from 'sinon'
 import { expect, describe, beforeEach, afterEach, it } from 'vitest'
-import mockFs from 'mock-fs'
+import fs from 'node:fs'
+import os from 'node:os'
 import path from 'node:path'
 
 const modulePath = path.join(
@@ -8,30 +8,40 @@ const modulePath = path.join(
   '../../../app/js/OutputFileFinder'
 )
 
+function createTree(base, tree) {
+  fs.mkdirSync(base, { recursive: true })
+  for (const [name, content] of Object.entries(tree)) {
+    const fullPath = path.join(base, name)
+    if (Buffer.isBuffer(content) || typeof content === 'string') {
+      fs.writeFileSync(fullPath, content)
+    } else if (content && content.symlink) {
+      fs.symlinkSync(content.symlink, fullPath)
+    } else {
+      createTree(fullPath, content)
+    }
+  }
+}
+
 describe('OutputFileFinder', function () {
   beforeEach(async function (ctx) {
     ctx.OutputFileFinder = (await import(modulePath)).default
-    ctx.directory = '/test/dir'
-    ctx.callback = sinon.stub()
-
-    mockFs({
-      [ctx.directory]: {
-        resource: {
-          'path.tex': 'a source file',
-        },
-        'output.pdf': 'a generated pdf file',
-        extra: {
-          'file.tex': 'a generated tex file',
-        },
-        'sneaky-file': mockFs.symlink({
-          path: '../foo',
-        }),
+    ctx.directory = fs.mkdtempSync(
+      path.join(os.tmpdir(), 'output-finder-test-')
+    )
+    createTree(ctx.directory, {
+      resource: {
+        'path.tex': 'a source file',
       },
+      'output.pdf': 'a generated pdf file',
+      extra: {
+        'file.tex': 'a generated tex file',
+      },
+      'sneaky-file': { symlink: '../foo' },
     })
   })
 
-  afterEach(function () {
-    mockFs.restore()
+  afterEach(function (ctx) {
+    fs.rmSync(ctx.directory, { recursive: true })
   })
 
   describe('findOutputFiles', function () {

services/docstore/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/docstore/Makefile

@@ -56,6 +56,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -68,8 +72,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/docstore/buildscript.txt

@@ -1,5 +1,6 @@
 docstore
 --dependencies=mongo,gcs
+--deploy-pipeline=docstore
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/document-updater/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/document-updater/Makefile

@@ -57,6 +57,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -69,8 +73,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/document-updater/buildscript.txt

@@ -1,5 +1,6 @@
 document-updater
 --dependencies=mongo,redis
+--deploy-pipeline=document-updater
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/document-updater/types/mongodb-legacy.d.ts

@@ -0,0 +1,3 @@
+declare module 'mongodb-legacy' {
+  export * from 'mongodb'
+}

services/filestore/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/filestore/Makefile

@@ -53,6 +53,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -66,8 +70,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/filestore/buildscript.txt

@@ -1,6 +1,7 @@
 filestore
 --data-dirs=uploads,template_files
 --dependencies=s3,gcs
+--deploy-pipeline=filestore-readonly
 --env-add=ENABLE_CONVERSIONS=true,USE_PROM_METRICS=true,AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
 --env-pass-through=
 --esmock-loader=False

services/git-bridge/Makefile

@@ -15,6 +15,10 @@ IMAGE_REPO_BRANCH ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)
 IMAGE_REPO_MAIN ?= $(IMAGE_REPO):main
 IMAGE_REPO_FINAL ?= $(IMAGE_REPO_BRANCH)-$(BUILD_NUMBER)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 runtime-conf:
 	/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
 

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java

@@ -5,6 +5,10 @@ import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpResponse;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSyntaxException;
 import jakarta.servlet.*;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
@@ -107,15 +111,18 @@ public class Oauth2Filter implements Filter {
       // fail later (for example, in the unlikely event that the token
       // expired between the two requests). In that case, JGit will
       // return a 401 without a custom error message.
-      int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
-      if (statusCode == 429) {
+      AccessTokenCheck check = checkAccessToken(this.oauth2Server, password, getClientIp(request));
+      if (check.statusCode == 429) {
         handleRateLimit(projectId, username, request, response);
         return;
-      } else if (statusCode == 401) {
+      } else if (check.statusCode == 401 && "token_expired".equals(check.errorCode)) {
+        handleExpiredAccessToken(projectId, request, response);
+        return;
+      } else if (check.statusCode == 401) {
         handleBadAccessToken(projectId, request, response);
         return;
-      } else if (statusCode >= 400) {
-        handleUnknownOauthServerError(projectId, statusCode, request, response);
+      } else if (check.statusCode >= 400) {
+        handleUnknownOauthServerError(projectId, check.statusCode, request, response);
         return;
       }
       cred.setAccessToken(password);
@@ -229,8 +236,52 @@ public class Oauth2Filter implements Filter {
             "https://www.overleaf.com/learn/how-to/Git_integration"));
   }
 
-  private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
+  private void handleExpiredAccessToken(
+      String projectId, HttpServletRequest request, HttpServletResponse response)
       throws IOException {
+    Log.debug("[{}] Expired access token, ip={}", projectId, getClientIp(request));
+    sendResponse(
+        response,
+        401,
+        Arrays.asList(
+            "Your Overleaf Git authentication token has expired.",
+            "",
+            "Generate a new authentication token in your Overleaf Account Settings,",
+            "then run the git command again."));
+  }
+
+  static class AccessTokenCheck {
+    final int statusCode;
+    final String errorCode;
+
+    AccessTokenCheck(int statusCode, String errorCode) {
+      this.statusCode = statusCode;
+      this.errorCode = errorCode;
+    }
+  }
+
+  static String parseErrorCode(String body) {
+    if (body == null || body.isEmpty()) {
+      return null;
+    }
+    try {
+      JsonElement element = new Gson().fromJson(body, JsonElement.class);
+      if (element == null || !element.isJsonObject()) {
+        return null;
+      }
+      JsonObject obj = element.getAsJsonObject();
+      JsonElement codeElement = obj.get("error_code");
+      if (codeElement == null || codeElement.isJsonNull()) {
+        return null;
+      }
+      return codeElement.getAsString();
+    } catch (JsonSyntaxException | UnsupportedOperationException e) {
+      return null;
+    }
+  }
+
+  private AccessTokenCheck checkAccessToken(
+      String oauth2Server, String accessToken, String clientIp) throws IOException {
     GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
     HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
     HttpHeaders headers = new HttpHeaders();
@@ -239,8 +290,12 @@ public class Oauth2Filter implements Filter {
     request.setThrowExceptionOnExecuteError(false);
     HttpResponse response = request.execute();
     int statusCode = response.getStatusCode();
+    String errorCode = null;
+    if (statusCode >= 400 && statusCode < 500) {
+      errorCode = parseErrorCode(response.parseAsString());
+    }
     response.disconnect();
-    return statusCode;
+    return new AccessTokenCheck(statusCode, errorCode);
   }
 
   private void handleUnknownOauthServerError(

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/server/Oauth2FilterTest.java

@@ -0,0 +1,51 @@
+package uk.ac.ic.wlgitbridge.server;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class Oauth2FilterTest {
+
+  @Test
+  public void parseErrorCode_returnsTokenExpired_whenBodyContainsIt() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_expired\"}";
+    Assert.assertEquals("token_expired", Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsTokenInvalid_whenBodyContainsIt() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_invalid\"}";
+    Assert.assertEquals("token_invalid", Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenErrorCodeFieldIsMissing() {
+    String body = "{\"error\":\"invalid_token\"}";
+    Assert.assertNull(Oauth2Filter.parseErrorCode(body));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsNull() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode(null));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsEmpty() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode(""));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsNotJson() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode("not json at all"));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenBodyIsJsonArray() {
+    Assert.assertNull(Oauth2Filter.parseErrorCode("[1, 2, 3]"));
+  }
+
+  @Test
+  public void parseErrorCode_returnsNull_whenErrorCodeFieldIsJsonNull() {
+    String body = "{\"error\":\"invalid_token\",\"error_code\":null}";
+    Assert.assertNull(Oauth2Filter.parseErrorCode(body));
+  }
+}

services/history-v1/.eslintignore

@@ -1,2 +0,0 @@
-archive/
-

services/history-v1/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/history-v1/Makefile

@@ -59,6 +59,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -71,8 +75,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/history-v1/buildscript.txt

@@ -1,5 +1,6 @@
 history-v1
 --dependencies=postgres,gcs,mongo,redis,s3
+--deploy-pipeline=history-v1
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/history-v1/docker-compose.ci.yml

@@ -109,7 +109,7 @@ services:
       - mongo:127.0.0.1
 
   postgres:
-    image: postgres:10
+    image: postgres:14
     environment:
       POSTGRES_USER: overleaf
       POSTGRES_PASSWORD: overleaf

services/history-v1/docker-compose.yml

@@ -124,7 +124,7 @@ services:
       - mongo:127.0.0.1
 
   postgres:
-    image: postgres:10
+    image: postgres:14
     environment:
       POSTGRES_USER: overleaf
       POSTGRES_PASSWORD: overleaf

services/history-v1/package.json

@@ -44,7 +44,8 @@
     "temp": "^0.8.3",
     "throng": "^4.0.0",
     "tsscmp": "^1.0.6",
-    "utf-8-validate": "^5.0.4"
+    "utf-8-validate": "^5.0.4",
+    "zip-stream": "^7.0.2"
   },
   "devDependencies": {
     "@overleaf/migrations": "workspace:*",

services/history-v1/storage/lib/backupArchiver.mjs

@@ -185,13 +185,30 @@ class BackupBlobStore {
  */
 
 /**
- * @typedef {(import('archiver').Archiver)} Archiver
+ * @typedef {(import('zip-stream').default)} ZipStream
  */
 
 /**
  * @typedef {(import('overleaf-editor-core').FileMap)} FileMap
  */
 
+/**
+ * Promisified wrapper for ZipStream's entry method.
+ *
+ * @param {ZipStream} archive
+ * @param {Buffer|NodeJS.ReadableStream|string} source
+ * @param {{ name: string }} data
+ * @return {Promise<void>}
+ */
+function addEntry(archive, source, data) {
+  return new Promise((resolve, reject) => {
+    archive.entry(source, data, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 /**
  *
  * @param historyId
@@ -254,14 +271,15 @@ async function fetchBlob(historyId, hash, persistor) {
 
 /**
  * @typedef {object} AddChunkOptions
- * @property {string} [prefix] Should include trailing slash (if length > 0)
+ * @property {string} [prefix]
  * @property {boolean} [useBackupGlobalBlobs]
+ * @property {boolean} [verbose]
  */
 
 /**
  *
  * @param {History} history
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {CachedPerProjectEncryptedS3Persistor} projectCache
  * @param {string} historyId
  * @param {AddChunkOptions} [options]
@@ -272,7 +290,7 @@ async function addChunkToArchive(
   archive,
   projectCache,
   historyId,
-  { prefix = '', useBackupGlobalBlobs = false } = {}
+  { prefix = '', useBackupGlobalBlobs = false, verbose = false } = {}
 ) {
   const chunkBlobs = new Set()
   history.findBlobHashes(chunkBlobs)
@@ -334,9 +352,16 @@ async function addChunkToArchive(
         }
         content = await blobStore.getStream(hash)
       }
-      archive.append(content, {
+      if (content == null) {
+        logger.error({ filePath }, 'File content is empty')
+        continue
+      }
+      await addEntry(archive, content, {
         name: `${prefix}${filePath}`,
       })
+      if (verbose) {
+        logger.info({ filePath: `${prefix}${filePath}` }, 'added to archive')
+      }
     }
   })
 }
@@ -358,17 +383,20 @@ async function findStartVersionOfLatestChunk(historyId) {
 /**
  * Restore a project from the latest snapshot
  *
- * There is an assumption that the database backup has been restored.
+ * There is an assumption that the database backup
+ * has been restored.
  *
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {string} historyId
  * @param {boolean} [useBackupGlobalBlobs]
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
 export async function archiveLatestChunk(
   archive,
   historyId,
-  useBackupGlobalBlobs = false
+  useBackupGlobalBlobs = false,
+  verbose = false
 ) {
   logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
 
@@ -386,20 +414,28 @@ export async function archiveLatestChunk(
 
   await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
     useBackupGlobalBlobs,
+    verbose,
   })
 
   return archive
 }
 
 /**
- * Fetches all raw blobs from the project and adds them to the archive.
+ * Fetches all raw blobs from the project and adds
+ * them to the archive.
  *
  * @param {string} historyId
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {CachedPerProjectEncryptedS3Persistor} projectCache
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
-async function addRawBlobsToArchive(historyId, archive, projectCache) {
+async function addRawBlobsToArchive(
+  historyId,
+  archive,
+  projectCache,
+  verbose = false
+) {
   const blobKeys = await projectCache.listDirectoryKeys(
     projectBlobsBucket,
     projectKey.format(historyId)
@@ -411,9 +447,13 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
         key,
         { autoGunzip: true }
       )
-      archive.append(stream, {
-        name: path.join(historyId, 'blobs', key),
+      const entryName = path.join(historyId, 'blobs', key)
+      await addEntry(archive, stream, {
+        name: entryName,
       })
+      if (verbose) {
+        logger.info({ entryName }, 'added to archive')
+      }
     } catch (err) {
       logger.warn({ err, path: key }, 'Failed to append blob to archive')
     }
@@ -425,17 +465,20 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
  *
  * This can work without the database being backed up.
  *
- * It will split the project into chunks per directory and download the blobs alongside the chunk.
+ * It will split the project into chunks per directory
+ * and download the blobs alongside the chunk.
  *
- * @param {Archiver} archive
+ * @param {ZipStream} archive
  * @param {string} historyId
  * @param {boolean} [useBackupGlobalBlobs]
+ * @param {boolean} [verbose]
  * @return {Promise<void>}
  */
 export async function archiveRawProject(
   archive,
   historyId,
-  useBackupGlobalBlobs = false
+  useBackupGlobalBlobs = false,
+  verbose = false
 ) {
   const projectCache = await getProjectPersistor(historyId)
 
@@ -454,11 +497,15 @@ export async function archiveRawProject(
 
     const { buffer } = await loadChunkByKey(projectCache, key)
 
-    archive.append(buffer, {
-      name: `${historyId}/chunks/${chunkId}/chunk.json`,
+    const entryName = `${historyId}/chunks/${chunkId}/chunk.json`
+    await addEntry(archive, buffer, {
+      name: entryName,
     })
+    if (verbose) {
+      logger.info({ entryName }, 'added to archive')
+    }
   }
-  await addRawBlobsToArchive(historyId, archive, projectCache)
+  await addRawBlobsToArchive(historyId, archive, projectCache, verbose)
 }
 
 export class BackupPersistorError extends OError {}

services/history-v1/storage/lib/streams.js

@@ -6,19 +6,19 @@
  */
 'use strict'
 
-const Stream = require('node:stream')
+const { pipeline } = require('node:stream/promises')
 const zlib = require('node:zlib')
 const { WritableBuffer } = require('@overleaf/stream-utils')
 
 /**
  * Create a promise for the result of reading a stream to a buffer.
  *
- * @param {Stream.Readable} readStream
+ * @param {import('node:stream').Readable} readStream
  * @return {Promise<Buffer>}
  */
 async function readStreamToBuffer(readStream) {
   const bufferStream = new WritableBuffer()
-  await Stream.promises.pipeline(readStream, bufferStream)
+  await pipeline(readStream, bufferStream)
   return bufferStream.contents()
 }
 
@@ -33,7 +33,7 @@ exports.readStreamToBuffer = readStreamToBuffer
 async function gunzipStreamToBuffer(readStream) {
   const gunzip = zlib.createGunzip()
   const bufferStream = new WritableBuffer()
-  await Stream.promises.pipeline(readStream, gunzip, bufferStream)
+  await pipeline(readStream, gunzip, bufferStream)
   return bufferStream.contents()
 }
 

services/history-v1/storage/scripts/backup_worker.mjs

@@ -19,7 +19,7 @@ const LAG_TIME_BUCKETS_HRS = [
 ] // hours
 
 // Configure backup settings to match worker concurrency
-configureBackup({ concurrency: UPLOAD_CONCURRENCY, useSecondary: true })
+configureBackup({ concurrency: UPLOAD_CONCURRENCY })
 
 let gracefulShutdownInitiated = false
 

services/history-v1/storage/scripts/finalise_chunk.mjs

@@ -0,0 +1,129 @@
+// Finalise the current chunk for a project and start a new empty chunk
+// whose starting snapshot is the end snapshot of the (now-closed) current
+// chunk.
+//
+// This is intended as a recovery tool for projects whose current chunk has
+// become corrupted in such a way that further changes can no longer be
+// persisted, but where the end snapshot of the current chunk can still be
+// computed.
+
+import logger from '@overleaf/logger'
+import commandLineArgs from 'command-line-args'
+import { Change, Chunk, History, NoOperation } from 'overleaf-editor-core'
+import * as redis from '../lib/redis.js'
+import knex from '../lib/knex.js'
+import knexReadOnly from '../lib/knex_read_only.js'
+import { client as mongoClient } from '../lib/mongodb.js'
+import chunkStore from '../lib/chunk_store/index.js'
+import redisBackend from '../lib/chunk_store/redis.js'
+import { loadGlobalBlobs } from '../lib/blob_store/index.js'
+import { fileURLToPath } from 'node:url'
+import { EventEmitter } from 'node:events'
+
+EventEmitter.defaultMaxListeners = 20
+
+logger.initialize('finalise-chunk')
+
+const optionDefinitions = [
+  { name: 'historyId', type: String },
+  { name: 'dry-run', alias: 'd', type: Boolean },
+]
+const options = commandLineArgs(optionDefinitions)
+const HISTORY_ID = options.historyId
+const DRY_RUN = options['dry-run'] || false
+
+if (!HISTORY_ID) {
+  console.error('Usage: finalise_chunk.mjs --historyId <id> [--dry-run]')
+  process.exit(2)
+}
+
+async function finaliseCurrentChunk(historyId) {
+  // Validates the history id and selects the backend (postgres or mongo).
+  chunkStore.getBackend(historyId)
+
+  await loadGlobalBlobs()
+
+  const currentChunk = await chunkStore.loadLatest(historyId, {
+    persistedOnly: true,
+  })
+  const startVersion = currentChunk.getStartVersion()
+  const endVersion = currentChunk.getEndVersion()
+  const numChanges = currentChunk.getChanges().length
+
+  logger.info(
+    { historyId, startVersion, endVersion, numChanges },
+    'loaded current chunk'
+  )
+
+  if (endVersion === startVersion) {
+    throw new Error(
+      `current chunk for history ${historyId} is already empty (no changes); refusing to create another empty chunk`
+    )
+  }
+
+  let nonPersistedChanges
+  try {
+    nonPersistedChanges = await redisBackend.getNonPersistedChanges(
+      historyId,
+      endVersion
+    )
+  } catch (err) {
+    throw new Error(
+      `unable to read non-persisted changes from redis for history ${historyId}: ${err.message}`
+    )
+  }
+  if (nonPersistedChanges.length > 0) {
+    throw new Error(
+      `history ${historyId} has ${nonPersistedChanges.length} non-persisted change(s) in redis; persist or expire them before running this script`
+    )
+  }
+
+  const endSnapshot = currentChunk.getSnapshot().clone()
+  endSnapshot.applyAll(currentChunk.getChanges())
+
+  // The chunks table has a unique constraint on (doc_id, end_version), so the
+  // new chunk cannot share an end_version with the chunk we are closing. Add a
+  // single NoOperation change to bump end_version by 1 without mutating the
+  // snapshot.
+  const recoveryChange = new Change([new NoOperation()], new Date(), [])
+  const newChunk = new Chunk(
+    new History(endSnapshot, [recoveryChange]),
+    endVersion
+  )
+
+  if (DRY_RUN) {
+    logger.info(
+      { historyId, endVersion },
+      'dry run: would close current chunk and create new empty chunk'
+    )
+    return
+  }
+
+  await chunkStore.create(historyId, newChunk)
+
+  logger.info(
+    { historyId, endVersion },
+    'closed current chunk and created new empty chunk'
+  )
+}
+
+async function main() {
+  try {
+    await finaliseCurrentChunk(HISTORY_ID)
+  } catch (err) {
+    logger.fatal({ err, historyId: HISTORY_ID }, 'failed to finalise chunk')
+    process.exitCode = 1
+  } finally {
+    await redis.disconnect()
+    await mongoClient.close()
+    await knex.destroy()
+    await knexReadOnly.destroy()
+  }
+}
+
+const currentScriptPath = fileURLToPath(import.meta.url)
+if (process.argv[1] === currentScriptPath) {
+  main()
+}
+
+export { finaliseCurrentChunk }

services/history-v1/storage/scripts/recover_zip.js

@@ -17,9 +17,10 @@ const fs = require('node:fs')
 const os = require('node:os')
 const path = require('node:path')
 const util = require('node:util')
+const { pipeline } = require('node:stream/promises')
 
-// Something is registering 11 listeners, over the limit of 10, which generates
-// a lot of warning noise.
+// Something is registering 11 listeners, over the limit
+// of 10, which generates a lot of warning noise.
 require('node:events').EventEmitter.defaultMaxListeners = 11
 
 const config = require('config')
@@ -27,11 +28,23 @@ const config = require('config')
 // eslint-disable-next-line import/no-extraneous-dependencies
 const { Storage } = require('@google-cloud/storage')
 const isValidUtf8 = require('utf-8-validate')
+// zip-stream@7 uses ESM default export
+const ZipStream = require('zip-stream').default
+
+function createStorage() {
+  const opts = {}
+  if (config.has('persistor.gcs.endpoint.apiEndpoint')) {
+    opts.apiEndpoint = config.get('persistor.gcs.endpoint.apiEndpoint')
+  }
+  if (config.has('persistor.gcs.endpoint.projectId')) {
+    opts.projectId = config.get('persistor.gcs.endpoint.projectId')
+  }
+  return new Storage(opts)
+}
 
 const core = require('overleaf-editor-core')
 const projectKey = require('@overleaf/object-persistor/src/ProjectKey.js')
 const streams = require('../lib/streams')
-const ProjectArchive = require('../lib/project_archive')
 
 const {
   values: { verbose: VERBOSE },
@@ -53,7 +66,7 @@ if (HISTORY_IDS.length === 0) {
 
 async function listDeletedChunks(historyId) {
   const bucketName = config.get('chunkStore.bucket')
-  const storage = new Storage()
+  const storage = createStorage()
   const [files] = await storage.bucket(bucketName).getFiles({
     prefix: projectKey.format(historyId),
     versions: true,
@@ -137,7 +150,7 @@ class RecoveryBlobStore {
     if (VERBOSE) console.log('fetching blob', hash)
 
     const bucketName = config.get('blobStore.projectBucket')
-    const storage = new Storage()
+    const storage = createStorage()
     const [files] = await storage.bucket(bucketName).getFiles({
       prefix: this.makeProjectBlobKey(hash),
       versions: true,
@@ -158,7 +171,7 @@ class RecoveryBlobStore {
 
   async fetchGlobalBlob(hash, destination) {
     const bucketName = config.get('blobStore.globalBucket')
-    const storage = new Storage()
+    const storage = createStorage()
     const file = storage.bucket(bucketName).file(this.makeGlobalBlobKey(hash))
     await file.download({ destination })
   }
@@ -203,9 +216,18 @@ class RecoveryBlobStore {
 async function uploadZip(historyId, zipPathname) {
   const bucketName = config.get('zipStore.bucket')
   const deadline = 24 * 3600 * 1000 // lifecycle limit on the zips bucket
-  const storage = new Storage()
+  const storage = createStorage()
   const destination = `${historyId}-recovered.zip`
-  await storage.bucket(bucketName).upload(zipPathname, { destination })
+  await storage.bucket(bucketName).upload(zipPathname, {
+    destination,
+    resumable: false,
+  })
+
+  if (config.has('persistor.gcs.endpoint.apiEndpoint')) {
+    // In emulator mode, signed URLs aren't available
+    const apiEndpoint = config.get('persistor.gcs.endpoint.apiEndpoint')
+    return `${apiEndpoint}/storage/v1/b/${bucketName}/o/${encodeURIComponent(destination)}?alt=media`
+  }
 
   const signedUrls = await storage
     .bucket(bucketName)
@@ -219,6 +241,23 @@ async function uploadZip(historyId, zipPathname) {
   return signedUrls[0]
 }
 
+/**
+ * Promisified wrapper for ZipStream's entry method.
+ *
+ * @param {ZipStream} archive
+ * @param {Buffer|NodeJS.ReadableStream|string} source
+ * @param {{ name: string }} data
+ * @return {Promise<void>}
+ */
+function addEntry(archive, source, data) {
+  return new Promise((resolve, reject) => {
+    archive.entry(source, data, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 async function restoreProject(historyId) {
   const tmp = await fs.promises.mkdtemp(
     path.join(os.tmpdir(), historyId.toString())
@@ -237,9 +276,40 @@ async function restoreProject(historyId) {
   if (VERBOSE) console.log('zipping', historyId)
 
   const zipPathname = path.join(tmp, `${historyId}.zip`)
-  const zipTimeoutMs = 60 * 1000
-  const archive = new ProjectArchive(snapshot, zipTimeoutMs)
-  await archive.writeZip(blobStore, zipPathname)
+  const outputFile = fs.createWriteStream(zipPathname)
+  const archive = new ZipStream()
+
+  const pipelinePromise = pipeline(archive, outputFile)
+
+  for (const pathname of snapshot.getFilePathnames()) {
+    const file = snapshot.getFile(pathname)
+    if (!file) continue
+
+    await file.load('eager', blobStore)
+    let content = file.getContent({
+      filterTrackedDeletes: true,
+    })
+
+    if (content === null) {
+      const hash = file.getHash()
+      content = await blobStore.getStream(hash)
+    }
+
+    if (content == null) continue
+
+    if (typeof content === 'string') {
+      content = Buffer.from(content)
+    }
+    await addEntry(archive, content, { name: pathname })
+    if (VERBOSE) console.log(`${pathname} added`)
+  }
+
+  archive.finalize()
+  await pipelinePromise
+
+  if (VERBOSE) {
+    console.log(`Wrote ${archive.getBytesWritten()} bytes`)
+  }
 
   if (VERBOSE) console.log('uploading', historyId)
 
@@ -252,4 +322,7 @@ async function main() {
     console.log(signedUrl)
   }
 }
-main().catch(console.error)
+main().catch(err => {
+  console.error(err)
+  process.exit(1)
+})

services/history-v1/storage/scripts/recover_zip_from_backup.mjs

@@ -4,6 +4,7 @@ import commandLineArgs from 'command-line-args'
 import assert from '../lib/assert.js'
 import fs from 'node:fs'
 import { setTimeout } from 'node:timers/promises'
+import { pipeline } from 'node:stream/promises'
 import {
   archiveLatestChunk,
   archiveRawProject,
@@ -11,17 +12,13 @@ import {
 } from '../lib/backupArchiver.mjs'
 import knex from '../lib/knex.js'
 import { client } from '../lib/mongodb.js'
-import archiver from 'archiver'
-import Events from 'node:events'
+import ZipStream from 'zip-stream'
 import { Chunk } from 'overleaf-editor-core'
 import _ from 'lodash'
 
-// Silence warning.
-Events.setMaxListeners(20)
-
 const SUPPORTED_MODES = ['raw', 'latest']
 
-// Pads the mode name to a fixed length for better alignment in output.
+// Pads the mode name to a fixed length for alignment.
 const padModeName = _.partialRight(
   _.padEnd,
   Math.max(...SUPPORTED_MODES.map(mode => mode.length))
@@ -65,25 +62,6 @@ function usage() {
   })
 }
 
-/**
- * @typedef {import('archiver').ZipArchive} ZipArchive
- */
-
-/**
- * @typedef {import('archiver').ProgressData} ProgressData
- */
-
-/**
- * @typedef {import('archiver').EntryData} EntryData
- */
-
-/**
- * @typedef {Object} ArchiverError
- * @property {string} message
- * @property {string} code
- * @property {Object} data
- */
-
 let historyId, help, mode, output, useBackupGlobalBlobs, verbose
 
 try {
@@ -136,80 +114,37 @@ await loadGlobalBlobs()
 
 outputFile = fs.createWriteStream(output)
 
-const archive = archiver.create('zip', {})
+const archive = new ZipStream()
 
-archive.on('close', function () {
-  console.log(archive.pointer() + ' total bytes')
-  console.log(`Wrote ${output}`)
-  shutdown().catch(e => console.error('Error shutting down', e))
+archive.on('error', function (e) {
+  console.error(`Error writing archive: ${e.message}`)
 })
 
-archive.on(
-  'error',
-  /**
-   *
-   * @param {ArchiverError} e
-   */
-  function (e) {
-    console.error(`Error writing archive: ${e.message}`)
-  }
-)
-
-archive.on('end', function () {
-  console.log(`Wrote ${archive.pointer()} total bytes to ${output}`)
-  shutdown().catch(e => console.error('Error shutting down', e))
-})
-
-archive.on(
-  'progress',
-  /**
-   *
-   * @param {ProgressData} progress
-   */
-  function (progress) {
-    if (verbose) {
-      console.log(
-        `${progress.entries.processed} processed out of ${progress.entries.total}`
-      )
-    }
-  }
-)
-
-archive.on(
-  'entry',
-  /**
-   *
-   * @param {EntryData} entry
-   */
-  function (entry) {
-    if (verbose) {
-      console.log(`${entry.name} added`)
-    }
-  }
-)
-
-archive.on(
-  'warning',
-  /**
-   *
-   * @param {ArchiverError} warning
-   */
-  function (warning) {
-    console.warn(`Warning encountered when writing archive: ${warning.message}`)
-  }
-)
-
 try {
+  // Pipe archive to the output file before adding entries.
+  // pipeline handles backpressure and will resolve when
+  // the archive stream ends.
+  const pipelinePromise = pipeline(archive, outputFile)
+
   switch (mode) {
     case 'latest':
-      await archiveLatestChunk(archive, historyId, useBackupGlobalBlobs)
+      await archiveLatestChunk(
+        archive,
+        historyId,
+        useBackupGlobalBlobs,
+        verbose
+      )
       break
     case 'raw':
     default:
-      await archiveRawProject(archive, historyId, useBackupGlobalBlobs)
+      await archiveRawProject(archive, historyId, useBackupGlobalBlobs, verbose)
       break
   }
-  archive.pipe(outputFile)
+
+  archive.finalize()
+  await pipelinePromise
+
+  console.log(`Wrote ${archive.getBytesWritten()} total bytes to ${output}`)
 } catch (error) {
   if (error instanceof BackupPersistorError) {
     console.error(error.message)
@@ -222,12 +157,7 @@ try {
   } else {
     console.error('Error encountered when writing archive')
   }
-} finally {
-  await Promise.race([
-    await archive.finalize(),
-    setTimeout(10000).then(() => {
-      console.error('Archive did not finalize in time')
-      return shutdown(1)
-    }),
-  ])
+  await shutdown(1)
 }
+
+await shutdown(0)

services/history-v1/test/acceptance/js/api/projects.test.js

@@ -507,7 +507,7 @@ describe('project controller', function () {
     })
   })
 
-  // eslint-disable-next-line mocha/no-skipped-tests
+  // eslint-disable-next-line mocha/no-pending-tests
   describe.skip('getLatestContent', function () {
     // TODO: remove this endpoint entirely, see
     // https://github.com/overleaf/write_latex/pull/5120#discussion_r244291862

services/history-v1/test/acceptance/js/storage/backup.test.mjs

@@ -647,6 +647,68 @@ describe('backup script', function () {
       expect(newBackupStatus.backupStatus.lastBackedUpVersion).to.equal(50) // backup fails on final chunk
       expect(newBackupStatus.currentEndVersion).to.equal(54) // backup is incomplete due to missing blob
     })
+
+    it('can recover zip file from backup in raw mode', async function () {
+      // First, run backup so data is available
+      await runBackupScript(['--projectId', projectId])
+
+      const zipPath = `/tmp/test-recover-raw-${historyId}.zip`
+      try {
+        await runRecoverZipFromBackupScript([
+          '--historyId',
+          historyId,
+          '--output',
+          zipPath,
+          '--mode=raw',
+        ])
+
+        // Verify the zip file is valid
+        const { stdout } = await promisify(execFile)('unzip', ['-l', zipPath], {
+          encoding: 'utf-8',
+        })
+
+        // Raw mode includes chunk and blob keys
+        // Verify chunks are present
+        expect(stdout).to.include('chunk.json')
+
+        // Verify blob hashes are present (hashes are stored as {hash[0:2]}/{hash[2:]})
+        expect(stdout).to.include(testFiles.GRAPH_PNG_HASH.slice(2))
+        expect(stdout).to.include(testFiles.NON_BMP_TXT_HASH.slice(2))
+      } finally {
+        await fs.promises.unlink(zipPath).catch(() => {})
+      }
+    })
+
+    it('can recover zip file from backup in latest mode', async function () {
+      // First, run backup so data is available
+      await runBackupScript(['--projectId', projectId])
+
+      const zipPath = `/tmp/test-recover-latest-${historyId}.zip`
+      try {
+        await runRecoverZipFromBackupScript([
+          '--historyId',
+          historyId,
+          '--output',
+          zipPath,
+          '--mode=latest',
+        ])
+
+        // Verify the zip file is valid
+        const { stdout } = await promisify(execFile)('unzip', ['-l', zipPath], {
+          encoding: 'utf-8',
+        })
+
+        // Latest mode includes the project files
+        expect(stdout).to.include('main.tex')
+        expect(stdout).to.include('chapter1.tex')
+        expect(stdout).to.include('chapter2.tex')
+        expect(stdout).to.include('bibliography.bib')
+        expect(stdout).to.include('graph.png')
+        expect(stdout).to.include('unicodeFile.tex')
+      } finally {
+        await fs.promises.unlink(zipPath).catch(() => {})
+      }
+    })
   })
 })
 
@@ -683,3 +745,37 @@ async function runBackupScript(args) {
   }
   return result
 }
+
+/**
+ * Run the recover_zip_from_backup script with given arguments
+ * @param {string[]} args
+ */
+async function runRecoverZipFromBackupScript(args) {
+  const TIMEOUT = 30 * 1000
+  let result
+  try {
+    result = await promisify(execFile)(
+      'node',
+      ['storage/scripts/recover_zip_from_backup.mjs', ...args],
+      {
+        encoding: 'utf-8',
+        timeout: TIMEOUT,
+        env: {
+          ...process.env,
+          LOG_LEVEL: 'debug',
+        },
+      }
+    )
+    result.status = 0
+  } catch (err) {
+    const { stdout, stderr, code } = err
+    if (typeof code !== 'number') {
+      console.log(err)
+    }
+    result = { stdout, stderr, status: code }
+  }
+  if (result.status !== 0) {
+    throw new Error(`recover_zip_from_backup failed: ${result.stderr}`)
+  }
+  return result
+}

services/history-v1/test/acceptance/js/storage/finalise_chunk.test.js

@@ -0,0 +1,205 @@
+'use strict'
+
+const { promisify } = require('node:util')
+const { execFile } = require('node:child_process')
+const { expect } = require('chai')
+const {
+  Change,
+  AddFileOperation,
+  EditFileOperation,
+  TextOperation,
+  File,
+} = require('overleaf-editor-core')
+const cleanup = require('./support/cleanup')
+const fixtures = require('./support/fixtures')
+const { setupProjectState } = require('./support/redis')
+const chunkStore = require('../../../../storage/lib/chunk_store')
+const persistChanges = require('../../../../storage/lib/persist_changes')
+const knex = require('../../../../storage/lib/knex')
+
+const SCRIPT_PATH = 'storage/scripts/finalise_chunk.mjs'
+
+async function runScript(args) {
+  try {
+    const result = await promisify(execFile)('node', [SCRIPT_PATH, ...args], {
+      encoding: 'utf-8',
+      timeout: 10000,
+      env: { ...process.env, LOG_LEVEL: 'debug' },
+    })
+    return { ...result, status: 0 }
+  } catch (err) {
+    if (typeof err.code !== 'number') {
+      throw err
+    }
+    return { stdout: err.stdout, stderr: err.stderr, status: err.code }
+  }
+}
+
+async function getChunkRows(projectId) {
+  return await knex('chunks')
+    .select('id', 'start_version', 'end_version', 'closed')
+    .where('doc_id', parseInt(projectId, 10))
+    .orderBy('end_version')
+}
+
+describe('finalise_chunk script', function () {
+  before(cleanup.everything)
+
+  let limitsToPersistImmediately
+  before(function () {
+    const farFuture = new Date()
+    farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+    limitsToPersistImmediately = {
+      minChangeTimestamp: farFuture,
+      maxChangeTimestamp: farFuture,
+      maxChunkChanges: 100,
+    }
+  })
+
+  beforeEach(async function () {
+    await cleanup.everything()
+    await fixtures.create()
+  })
+
+  describe('with a populated current chunk', function () {
+    let projectId
+    let initialContent
+    let initialEndVersion
+    let initialChunkId
+
+    beforeEach(async function () {
+      projectId = await chunkStore.initializeProject()
+      initialContent = 'Hello world.'
+      const change1 = new Change(
+        [new AddFileOperation('main.tex', File.fromString(initialContent))],
+        new Date(Date.now() - 30000),
+        []
+      )
+      const change2 = new Change(
+        [
+          new EditFileOperation(
+            'main.tex',
+            new TextOperation().retain(initialContent.length).insert(' More.')
+          ),
+        ],
+        new Date(Date.now() - 20000),
+        []
+      )
+      await persistChanges(
+        projectId,
+        [change1, change2],
+        limitsToPersistImmediately,
+        0
+      )
+      const metadata = await chunkStore.getLatestChunkMetadata(projectId)
+      initialEndVersion = metadata.endVersion
+      initialChunkId = metadata.id
+      expect(initialEndVersion).to.equal(2)
+    })
+
+    it('closes the current chunk and creates a new empty chunk', async function () {
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status, result.stderr).to.equal(0)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(2)
+
+      const oldRow = rows.find(r => r.id.toString() === initialChunkId)
+      expect(oldRow, 'old chunk still in DB').to.exist
+      expect(oldRow.closed).to.equal(true)
+      expect(oldRow.end_version).to.equal(initialEndVersion)
+
+      const newRow = rows.find(r => r.id.toString() !== initialChunkId)
+      expect(newRow, 'new chunk inserted').to.exist
+      expect(newRow.start_version).to.equal(initialEndVersion)
+      expect(newRow.end_version).to.equal(initialEndVersion + 1)
+      expect(newRow.closed).to.equal(false)
+
+      const newChunk = await chunkStore.loadLatest(projectId, {
+        persistedOnly: true,
+      })
+      expect(newChunk.getStartVersion()).to.equal(initialEndVersion)
+      expect(newChunk.getEndVersion()).to.equal(initialEndVersion + 1)
+      // The new chunk holds a single NoOperation change as a placeholder so
+      // its end_version is distinct from the closed chunk's end_version.
+      expect(newChunk.getChanges()).to.have.length(1)
+
+      const newSnapshot = newChunk.getSnapshot()
+      const file = newSnapshot.getFile('main.tex')
+      expect(file, 'main.tex carried over to new chunk snapshot').to.exist
+    })
+
+    it('does not modify state on a dry-run', async function () {
+      const result = await runScript(['--historyId', projectId, '--dry-run'])
+      expect(result.status, result.stderr).to.equal(0)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].id.toString()).to.equal(initialChunkId)
+      expect(rows[0].closed).to.equal(false)
+    })
+  })
+
+  describe('safety checks', function () {
+    it('refuses to act on an already-empty current chunk', async function () {
+      const projectId = await chunkStore.initializeProject()
+
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status).to.not.equal(0)
+      expect(result.stdout + result.stderr).to.match(/already empty/)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].start_version).to.equal(0)
+      expect(rows[0].end_version).to.equal(0)
+      expect(rows[0].closed).to.equal(false)
+    })
+
+    it('refuses when there are non-persisted changes in redis', async function () {
+      const projectId = await chunkStore.initializeProject()
+      const initialContent = 'Initial.'
+      const initialChange = new Change(
+        [new AddFileOperation('main.tex', File.fromString(initialContent))],
+        new Date(Date.now() - 30000),
+        []
+      )
+      await persistChanges(
+        projectId,
+        [initialChange],
+        limitsToPersistImmediately,
+        0
+      )
+      const metadata = await chunkStore.getLatestChunkMetadata(projectId)
+      const initialChunkId = metadata.id
+
+      const bufferedChange = new Change(
+        [
+          new EditFileOperation(
+            'main.tex',
+            new TextOperation()
+              .retain(initialContent.length)
+              .insert(' Buffered.')
+          ),
+        ],
+        new Date(Date.now() - 10000),
+        []
+      )
+      await setupProjectState(projectId, {
+        persistTime: Date.now() - 1000,
+        headVersion: 2,
+        persistedVersion: 1,
+        changes: [bufferedChange],
+        expireTimeFuture: true,
+      })
+
+      const result = await runScript(['--historyId', projectId])
+      expect(result.status).to.not.equal(0)
+      expect(result.stdout + result.stderr).to.match(/non-persisted/)
+
+      const rows = await getChunkRows(projectId)
+      expect(rows).to.have.length(1)
+      expect(rows[0].id.toString()).to.equal(initialChunkId)
+      expect(rows[0].closed).to.equal(false)
+    })
+  })
+})

services/history-v1/test/acceptance/js/storage/recover_zip.test.mjs

@@ -0,0 +1,169 @@
+import { expect } from 'chai'
+import config from 'config'
+import { execFile } from 'node:child_process'
+import fs from 'node:fs'
+import { promisify } from 'node:util'
+import { Change, Operation, File, TextOperation } from 'overleaf-editor-core'
+// We depend on this via object-persistor.
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Storage } from '@google-cloud/storage'
+import {
+  loadGlobalBlobs,
+  BlobStore,
+} from '../../../../storage/lib/blob_store/index.js'
+import ChunkStore from '../../../../storage/lib/chunk_store/index.js'
+import persistChanges from '../../../../storage/lib/persist_changes.js'
+import testFiles from '../storage/support/test_files.js'
+import cleanup from './support/cleanup.js'
+import { getZipEntries } from './support/unzip.js'
+
+describe('recover_zip script', function () {
+  let projectId
+  let limitsToPersistImmediately
+
+  before(async function () {
+    const farFuture = new Date()
+    farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
+    limitsToPersistImmediately = {
+      minChangeTimestamp: farFuture,
+      maxChangeTimestamp: farFuture,
+      maxChanges: 10,
+      maxChunkChanges: 10,
+    }
+
+    const gcsEndpoint = config.get('persistor.gcs.endpoint')
+    const storage = new Storage({
+      apiEndpoint: gcsEndpoint.apiEndpoint,
+      projectId: gcsEndpoint.projectId,
+    })
+    const bucketName = config.get('zipStore.bucket')
+    try {
+      const [exists] = await storage.bucket(bucketName).exists()
+      if (!exists) {
+        await storage.createBucket(bucketName)
+      }
+    } catch (err) {
+      if (err.code !== 409) throw err
+    }
+  })
+
+  beforeEach(cleanup.everything)
+
+  beforeEach(async function () {
+    await loadGlobalBlobs()
+    projectId = '123'
+
+    // Initialize the project in the chunk store
+    await ChunkStore.initializeProject(projectId)
+
+    const blobStore = new BlobStore(projectId)
+
+    // Upload binary file blob
+    await blobStore.putFile(testFiles.path('graph.png'))
+
+    // Create initial snapshot with text and binary files
+    const addMainTex = Operation.addFile(
+      'main.tex',
+      File.fromString('hello world')
+    )
+    const addGraphPng = Operation.addFile(
+      'graph.png',
+      File.fromHash(testFiles.GRAPH_PNG_HASH)
+    )
+    const change1 = new Change([addMainTex, addGraphPng], new Date(), [])
+    await persistChanges(projectId, [change1], limitsToPersistImmediately, 0)
+
+    // Add a text edit
+    const textOp = TextOperation.fromJSON({
+      textOperation: ['hello world'.length, ' more'],
+    })
+    const editOp = Operation.editFile('main.tex', textOp)
+    const change2 = new Change([editOp], new Date(), [])
+    await persistChanges(projectId, [change2], limitsToPersistImmediately, 1)
+  })
+
+  it('creates a valid zip from GCS data', async function () {
+    this.timeout(30 * 1000)
+
+    const zipPath = `/tmp/test-recover-zip-${projectId}.zip`
+    try {
+      const { stdout } = await runRecoverZipScript([projectId])
+
+      // The script logs the signed URL to stdout
+      const urlMatch = stdout.match(/(https?:\/\/[^\s]+)/)
+      expect(urlMatch).to.not.be.null
+      const signedUrl = urlMatch[1]
+
+      // Download the zip via fetch
+      const res = await fetch(signedUrl)
+      expect(res.ok).to.be.true
+      const buffer = await res.arrayBuffer()
+      await fs.promises.writeFile(zipPath, Buffer.from(buffer))
+
+      const zipEntries = await getZipEntries(zipPath)
+      const fileNames = zipEntries.map(e => e.fileName).sort()
+
+      expect(fileNames).to.deep.equal(['graph.png', 'main.tex'])
+
+      // Verify text content size (after edit)
+      const mainTexEntry = zipEntries.find(e => e.fileName === 'main.tex')
+      expect(mainTexEntry.uncompressedSize).to.equal('hello world more'.length)
+
+      // Verify binary content size
+      const graphEntry = zipEntries.find(e => e.fileName === 'graph.png')
+      expect(graphEntry.uncompressedSize).to.equal(
+        testFiles.GRAPH_PNG_BYTE_LENGTH
+      )
+    } finally {
+      await fs.promises.unlink(zipPath).catch(() => {})
+    }
+  })
+
+  it('supports the --verbose flag', async function () {
+    this.timeout(30 * 1000)
+
+    const { stdout } = await runRecoverZipScript(['--verbose', projectId])
+
+    // Verbose mode logs each file as it's added
+    expect(stdout).to.include('main.tex added')
+    expect(stdout).to.include('graph.png added')
+  })
+})
+
+/**
+ * Run the recover_zip.js script with given arguments
+ * @param {string[]} args
+ */
+async function runRecoverZipScript(args) {
+  const TIMEOUT = 30 * 1000
+  let result
+  try {
+    result = await promisify(execFile)(
+      'node',
+      ['storage/scripts/recover_zip.js', ...args],
+      {
+        encoding: 'utf-8',
+        timeout: TIMEOUT,
+        env: {
+          ...process.env,
+          LOG_LEVEL: 'debug',
+        },
+      }
+    )
+    result.status = 0
+  } catch (err) {
+    const { stdout, stderr, code } = err
+    if (typeof code !== 'number') {
+      console.log(err)
+    }
+    result = { stdout, stderr, status: code }
+  }
+  if (result.status !== 0 || result.stderr) {
+    throw new Error(
+      `recover_zip failed (exit ${result.status}):\n` +
+        `stdout: ${result.stdout}\n` +
+        `stderr: ${result.stderr}`
+    )
+  }
+  return result
+}

services/notifications/Makefile

@@ -55,6 +55,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -67,8 +71,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/notifications/reports:/overleaf/services/notifications/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/notifications/app.ts

@@ -9,7 +9,6 @@ import express, {
   type ErrorRequestHandler,
   type NextFunction,
 } from 'express'
-import methodOverride from 'method-override'
 import { mongoClient } from './app/js/mongodb.js'
 import NotificationsController from './app/js/NotificationsController.ts'
 import HealthCheckController from './app/js/HealthCheckController.ts'
@@ -25,7 +24,6 @@ logger.initialize('notifications')
 metrics.memory.monitor(logger)
 metrics.open_sockets.monitor()
 
-app.use(methodOverride())
 app.use(express.json())
 app.use(metrics.http.monitor(logger))
 

services/notifications/buildscript.txt

@@ -1,5 +1,6 @@
 notifications
 --dependencies=mongo
+--deploy-pipeline=notifications
 --env-add=
 --env-pass-through=
 --esmock-loader=False

services/notifications/package.json

@@ -30,14 +30,12 @@
     "body-parser": "1.20.4",
     "bunyan": "^1.8.15",
     "express": "4.22.1",
-    "method-override": "^3.0.0",
     "mongodb-legacy": "6.1.3",
     "zod": "^4.1.7",
     "zod-validation-error": "^4.0.1"
   },
   "devDependencies": {
     "@overleaf/migrations": "workspace:*",
-    "@types/method-override": "^3.0.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",

services/project-history/.eslintignore

@@ -1 +0,0 @@
-app/lib/*.js

services/project-history/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/project-history/Makefile

@@ -58,6 +58,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -70,8 +74,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/app/js/types.ts:/overleaf/services/document-updater/app/js/types.ts --volume $(MONOREPO)/services/project-history/reports:/overleaf/services/project-history/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/project-history/buildscript.txt

@@ -1,5 +1,6 @@
 project-history
 --dependencies=mongo,redis
+--deploy-pipeline=project-history
 --env-add=
 --env-pass-through=
 --esmock-loader=True

services/project-history/scripts/add_index_for_sync_state.js

@@ -1,5 +1,3 @@
-/* eslint-env mongo */
-
 // add a TTL index to expire entries for completed resyncs in the
 // projectHistorySyncState collection.  The entries should only be expired if
 // resyncProjectStructure is false and resyncDocContents is a zero-length array.

services/real-time/.mocharc.cjs

@@ -1,7 +1,7 @@
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
+    reporter: require.resolve('mocha-multi-reporters'),
     'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
   }
 }

services/real-time/Makefile

@@ -54,6 +54,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
 	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
 
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 clean:
 	-docker rmi $(IMAGE_CI)
 	-docker rmi $(IMAGE_REPO_FINAL)
@@ -66,8 +70,8 @@ clean:
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
-RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/real-time/reports:/overleaf/services/real-time/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
 
 SHELLCHECK_OPTS = \
 	--shell=bash \

services/web/.eslintignore

@@ -1,12 +0,0 @@
-# NOTE: changing paths may require updating them in the Makefile too.
-data/
-scripts/translations/.cache/
-node_modules
-frontend/js/vendor
-modules/**/frontend/js/vendor
-/public/
-frontend/js/features/source-editor/lezer-latex/latex.mjs
-frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
-frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
-frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
-frontend/js/features/source-editor/hunspell/wasm/hunspell.mjs

services/web/.eslintrc.js

@@ -1,665 +0,0 @@
-const _ = require('lodash')
-const confusingBrowserGlobals = require('confusing-browser-globals')
-const globals = require('globals')
-
-module.exports = {
-  root: true,
-  parser: '@typescript-eslint/parser',
-  extends: [
-    'eslint:recommended',
-    'plugin:@typescript-eslint/recommended',
-    'standard',
-    'prettier',
-    'plugin:storybook/recommended',
-  ],
-  plugins: ['@overleaf'],
-  env: {
-    es2020: true,
-  },
-  settings: {
-    // Tell eslint-plugin-react to detect which version of React we are using
-    react: {
-      version: 'detect',
-    },
-  },
-  rules: {
-    'no-constant-binary-expression': 'error',
-    'no-restricted-globals': ['error', ...confusingBrowserGlobals],
-
-    // do not allow importing of implicit dependencies.
-    'import/no-extraneous-dependencies': 'error',
-
-    '@overleaf/prefer-kebab-url': 'error',
-    '@overleaf/require-cio-snake-case-properties': 'error',
-
-    // disable some TypeScript rules
-    '@typescript-eslint/no-var-requires': 'off',
-    '@typescript-eslint/no-unused-vars': 'off',
-    '@typescript-eslint/no-empty-function': 'off',
-    '@typescript-eslint/no-explicit-any': 'off',
-    '@typescript-eslint/no-this-alias': 'off',
-    '@typescript-eslint/no-non-null-assertion': 'off',
-    '@typescript-eslint/ban-ts-comment': 'off',
-
-    'no-use-before-define': 'off',
-    '@typescript-eslint/no-use-before-define': [
-      'error',
-      { functions: false, classes: false, variables: false },
-    ],
-    'react-hooks/exhaustive-deps': [
-      'warn',
-      {
-        additionalHooks: '(useCommandProvider)',
-      },
-    ],
-  },
-  overrides: [
-    // NOTE: changing paths may require updating them in the Makefile too.
-    {
-      // Node
-      files: [
-        '**/app/src/**/*.{js,mjs}',
-        'app.{js,mjs}',
-        'i18next-scanner.config.js',
-        'scripts/**/*.{js,mjs}',
-        'webpack.config*.js',
-      ],
-      env: {
-        node: true,
-      },
-    },
-    {
-      // Test specific rules
-      files: ['**/test/**/*.*'],
-      excludedFiles: [
-        '**/test/unit/src/**/*.test.mjs',
-        'test/unit/bootstrap.mjs',
-      ], // exclude vitest files
-      plugins: ['mocha', 'chai-expect', 'chai-friendly'],
-      env: {
-        mocha: true,
-      },
-      rules: {
-        // mocha-specific rules
-        'mocha/handle-done-callback': 'error',
-        'mocha/no-exclusive-tests': 'error',
-        'mocha/no-global-tests': 'error',
-        'mocha/no-identical-title': 'error',
-        'mocha/no-nested-tests': 'error',
-        'mocha/no-pending-tests': 'error',
-        'mocha/no-skipped-tests': 'error',
-        'mocha/no-mocha-arrows': 'error',
-
-        // Swap the no-unused-expressions rule with a more chai-friendly one
-        'no-unused-expressions': 'off',
-        'chai-friendly/no-unused-expressions': 'error',
-
-        // chai-specific rules
-        'chai-expect/missing-assertion': 'error',
-        'chai-expect/terminating-properties': 'error',
-
-        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
-        // we don't enforce this at the top-level - just in tests to manage `this` scope
-        // based on mocha's context mechanism
-        'mocha/prefer-arrow-callback': 'error',
-
-        '@typescript-eslint/no-unused-expressions': 'off',
-      },
-    },
-    {
-      files: ['**/test/unit/src/**/*.test.mjs', 'test/unit/bootstrap.mjs'],
-      env: {
-        jest: true, // best match for vitest API etc.
-      },
-      plugins: ['@vitest', 'chai-expect', 'chai-friendly'], // still using chai for now
-      rules: {
-        // vitest-specific rules
-        '@vitest/no-focused-tests': 'error',
-        '@vitest/no-disabled-tests': 'error',
-
-        // Swap the no-unused-expressions rule with a more chai-friendly one
-        'no-unused-expressions': 'off',
-        'chai-friendly/no-unused-expressions': 'error',
-
-        // chai-specific rules
-        'chai-expect/missing-assertion': 'error',
-        'chai-expect/terminating-properties': 'error',
-        '@typescript-eslint/no-unused-expressions': 'off',
-        '@overleaf/require-vi-doMock-valid-path': 'error',
-      },
-    },
-    {
-      // ES specific rules
-      files: [
-        '**/app/src/**/*.mjs',
-        'modules/*/index.mjs',
-        'app.mjs',
-        'scripts/**/*.mjs',
-        'migrations/**/*.mjs',
-        '**/test/acceptance/src/**/*.mjs',
-        '**/test/unit/src/**/*.mjs',
-      ],
-      excludedFiles: [
-        // migration template file
-        'migrations/lib/template.mjs',
-      ],
-      parserOptions: {
-        sourceType: 'module',
-      },
-      plugins: ['unicorn'],
-      rules: {
-        'import/no-unresolved': [
-          'error',
-          {
-            // eslint-plugin-import does not support exports directive in package.json
-            // https://github.com/import-js/eslint-plugin-import/issues/1810
-            ignore: ['^p-queue$'],
-          },
-        ],
-        'import/named': 'error',
-        'import/default': 'error',
-        'import/extensions': [
-          'error',
-          'ignorePackages',
-          {
-            js: 'always',
-            mjs: 'always',
-          },
-        ],
-        'unicorn/prefer-module': 'error',
-        'unicorn/prefer-node-protocol': 'error',
-      },
-    },
-    {
-      // Backend specific rules
-      files: ['**/app/src/**/*.{js,mjs}', 'app.{js,mjs}'],
-      parserOptions: {
-        tsconfigRootDir: __dirname,
-        project: './tsconfig.backend.json',
-      },
-      rules: {
-        // do not allow importing of implicit dependencies.
-        'import/no-extraneous-dependencies': [
-          'error',
-          {
-            // do not allow importing of devDependencies.
-            devDependencies: false,
-          },
-        ],
-        'no-restricted-syntax': [
-          'error',
-          // do not allow node-fetch in backend code
-          {
-            selector:
-              "CallExpression[callee.name='require'] > .arguments[value='node-fetch']",
-            message:
-              'Requiring node-fetch is not allowed in production services, please use fetch-utils.',
-          },
-          // mongoose populate must set fields to populate
-          {
-            selector:
-              "CallExpression[callee.property.name='populate'][arguments.length<2]",
-            message:
-              "Populate without a second argument returns the whole document. Use populate('field',['prop1','prop2']) instead",
-          },
-          // Require `new` when constructing ObjectId (For mongo + mongoose upgrade)
-          {
-            selector:
-              "CallExpression[callee.name='ObjectId'], CallExpression[callee.property.name='ObjectId']",
-            message:
-              'Construct ObjectId with `new ObjectId()` instead of `ObjectId()`',
-          },
-          // Require `new` when mapping a list of ids to a list of ObjectId (For mongo + mongoose upgrade)
-          {
-            selector:
-              "CallExpression[callee.property.name='map'] Identifier[name='ObjectId']:first-child, CallExpression[callee.property.name='map'] MemberExpression[property.name='ObjectId']:first-child",
-            message:
-              "Don't map ObjectId directly. Use `id => new ObjectId(id)` instead",
-          },
-          // Catch incorrect usage of `await db.collection.find()`
-          {
-            selector:
-              "AwaitExpression > CallExpression > MemberExpression[property.name='find'][object.object.name='db']",
-            message:
-              'Mongo find returns a cursor not a promise, use `for await (const result of cursor)` or `.toArray()` instead.',
-          },
-        ],
-        '@typescript-eslint/no-floating-promises': [
-          'error',
-          { checkThenables: true },
-        ],
-      },
-    },
-    {
-      // Backend scripts specific rules
-      files: ['**/scripts/**/*.{js,mjs}'],
-      rules: {
-        'no-restricted-syntax': [
-          'error',
-          // Require `new` when constructing ObjectId (For mongo + mongoose upgrade)
-          {
-            selector:
-              "CallExpression[callee.name='ObjectId'], CallExpression[callee.property.name='ObjectId']",
-            message:
-              'Construct ObjectId with `new ObjectId()` instead of `ObjectId()`',
-          },
-          // Require `new` when mapping a list of ids to a list of ObjectId (For mongo + mongoose upgrade)
-          {
-            selector:
-              "CallExpression[callee.property.name='map'] Identifier[name='ObjectId']:first-child, CallExpression[callee.property.name='map'] MemberExpression[property.name='ObjectId']:first-child",
-            message:
-              "Don't map ObjectId directly. Use `id => new ObjectId(id)` instead",
-          },
-          // Catch incorrect usage of `await db.collection.find()`
-          {
-            selector:
-              "AwaitExpression > CallExpression > MemberExpression[property.name='find'][object.object.name='db']",
-            message:
-              'Mongo find returns a cursor not a promise, use `for await (const result of cursor)` or `.toArray()` instead.',
-          },
-        ],
-      },
-    },
-    {
-      // Insist on using Script Runner for new scripts. Old scripts should be
-      // converted to use Script Runner in future, but are excluded for now
-      rules: {
-        '@overleaf/require-script-runner': 'error',
-      },
-      files: ['**/scripts/**/*.mjs'], // ESM only
-      excludedFiles: [
-        'modules/admin-roles/scripts/import_admin_role_assignments.mjs',
-        'modules/admin-roles/scripts/remove_admin_role_from_user.mjs',
-        'modules/admin-roles/scripts/remove_admin_roles_from_non_admins.mjs',
-        'modules/admin-roles/scripts/utils.mjs',
-        'modules/institutions/scripts/apply_policy_to_institution.mjs',
-        'modules/server-ce-scripts/scripts/change-compile-timeout.mjs',
-        'modules/server-ce-scripts/scripts/check-mongodb.mjs',
-        'modules/server-ce-scripts/scripts/check-redis.mjs',
-        'modules/server-ce-scripts/scripts/check-texlive-images.mjs',
-        'modules/server-ce-scripts/scripts/create-user.mjs',
-        'modules/server-ce-scripts/scripts/delete-user.mjs',
-        'modules/server-ce-scripts/scripts/export-user-projects.mjs',
-        'modules/server-ce-scripts/scripts/migrate-user-emails.mjs',
-        'modules/server-ce-scripts/scripts/rename-tag.mjs',
-        'modules/server-ce-scripts/scripts/transfer-all-projects-to-user.mjs',
-        'modules/server-ce-scripts/scripts/upgrade-user-features.mjs',
-        'modules/subscriptions/scripts/backfill_user_last_trial.mjs',
-        'scripts/add_feature_override.mjs',
-        'scripts/add_subscription_members_csv.mjs',
-        'scripts/analytics/helpers/GoogleBigQueryHelper.mjs',
-        'scripts/attach_dangling_comments_to_doc.mjs',
-        'scripts/backfill_mixpanel_user_properties.mjs',
-        'scripts/backfill_project_image_name.mjs',
-        'scripts/backfill_user_properties.mjs',
-        'scripts/backfill_users_sso_attribute.mjs',
-        'scripts/bench_bcrypt.mjs',
-        'scripts/check_institution_users.mjs',
-        'scripts/check_overleafModuleImports.mjs',
-        'scripts/check_saml_emails.mjs',
-        'scripts/clear_feedback_collection.mjs',
-        'scripts/clear_sessions_set_must_reconfirm.mjs',
-        'scripts/count_files_in_projects.mjs',
-        'scripts/count_project_size.mjs',
-        'scripts/create_oauth_personal_access_token.mjs',
-        'scripts/create_project.mjs',
-        'scripts/deactivate_projects.mjs',
-        'scripts/delete-duplicate-splittest-versions/delete_test_dupes.mjs',
-        'scripts/delete-orphaned-docs/delete-orphaned-docs.mjs',
-        'scripts/delete_dangling_comments.mjs',
-        'scripts/delete_orphaned_chat_threads.mjs',
-        'scripts/delete_orphaned_data_helper.mjs',
-        'scripts/delete_subscriptions.mjs',
-        'scripts/devcontainer_setup.mjs',
-        'scripts/e2e_test_setup.mjs',
-        'scripts/ensure_affiliations.mjs',
-        'scripts/esm-check-migration.mjs',
-        'scripts/example/script_for_migration.mjs',
-        'scripts/fix_collaborator_refs_null.mjs',
-        'scripts/fix_comment_id.mjs',
-        'scripts/helpers/chunkArray.mjs',
-        'scripts/helpers/env_variable_helper.mjs',
-        'scripts/inst_table.mjs',
-        'scripts/invalidate_tokens.mjs',
-        'scripts/ip_matcher_ranges.mjs',
-        'scripts/learn/checkSanitize/checkSanitizeOptions.mjs',
-        'scripts/learn/checkSanitize/scrape.mjs',
-        'scripts/lezer-latex/benchmark.mjs',
-        'scripts/lezer-latex/print-tree.mjs',
-        'scripts/lezer-latex/random.mjs',
-        'scripts/lezer-latex/run.mjs',
-        'scripts/lezer-latex/test-incremental-parser.mjs',
-        'scripts/mark_migration.mjs',
-        'scripts/marketing-exports/error-assistant-export.mjs',
-        'scripts/marketing-exports/export.mjs',
-        'scripts/marketing-exports/linked-papers-users.mjs',
-        'scripts/marketing-exports/papers-export.mjs',
-        'scripts/marketing-exports/writefull-export.mjs',
-        'scripts/oauth/upgrade_token_scopes.mjs',
-        'scripts/plan-prices/plans.mjs',
-        'scripts/process_lapsed_reconfirmations.mjs',
-        'scripts/purge_non_logged_in_sessions.mjs',
-        'scripts/recurly/generate_recurly_prices.mjs',
-        'scripts/recurly/get_paypal_accounts_csv.mjs',
-        'scripts/recurly/recurly_prices.mjs',
-        'scripts/recurly/resync_recurly_state_single_subscription.mjs',
-        'scripts/recurly/resync_subscriptions.mjs',
-        'scripts/recurly/set_manually_collected_subscriptions.mjs',
-        'scripts/refresh_features.mjs',
-        'scripts/regenerate_duplicate_referral_ids.mjs',
-        'scripts/remove_deleted_users_from_token_access_refs.mjs',
-        'scripts/remove_email.mjs',
-        'scripts/remove_user_enrollment.mjs',
-        'scripts/sso_id_migration_check.mjs',
-        'scripts/stress_test.mjs',
-        'scripts/suspend_users.mjs',
-        'scripts/sync-user-entitlements/sync-user-entitlements.mjs',
-        'scripts/update_project_image_name.mjs',
-        'scripts/user-export/analytics.mjs',
-        'scripts/user-export/fs.mjs',
-        'scripts/user-export/http.mjs',
-        'scripts/user-export/observer.mjs',
-        'scripts/user-export/options.mjs',
-        'scripts/user-export/project.mjs',
-        'scripts/user-export/scrubber.mjs',
-        'scripts/user-export/stream.mjs',
-        'scripts/user-export/user.mjs',
-        'scripts/validate-data-of-model.mjs',
-      ],
-    },
-    {
-      // Cypress specific rules
-      files: [
-        'cypress/**/*.{js,jsx,ts,tsx}',
-        '**/test/frontend/**/*.spec.{js,jsx,ts,tsx}',
-      ],
-      extends: ['plugin:cypress/recommended'],
-    },
-    {
-      // Frontend test specific rules
-      files: ['**/frontend/**/*.test.{js,jsx,ts,tsx}'],
-      plugins: ['testing-library'],
-      extends: ['plugin:testing-library/react'],
-      rules: {
-        'testing-library/no-await-sync-events': 'off',
-        'testing-library/no-await-sync-queries': 'off',
-        'testing-library/no-container': 'off',
-        'testing-library/no-node-access': 'off',
-        'testing-library/no-render-in-lifecycle': 'off',
-        'testing-library/no-wait-for-multiple-assertions': 'off',
-        'testing-library/no-wait-for-side-effects': 'off',
-        'testing-library/prefer-query-by-disappearance': 'off',
-        'testing-library/prefer-screen-queries': 'off',
-        'testing-library/render-result-naming-convention': 'off',
-      },
-    },
-    {
-      // Frontend specific rules
-      files: [
-        '**/frontend/js/**/*.{js,jsx,ts,tsx}',
-        '**/frontend/stories/**/*.{js,jsx,ts,tsx}',
-        '**/*.stories.{js,jsx,ts,tsx}',
-        '**/test/frontend/**/*.{js,jsx,ts,tsx}',
-        '**/test/frontend/components/**/*.spec.{js,jsx,ts,tsx}',
-      ],
-      env: {
-        browser: true,
-      },
-      parserOptions: {
-        sourceType: 'module',
-      },
-      plugins: ['jsx-a11y'],
-      extends: [
-        'plugin:react/recommended',
-        'plugin:react-hooks/recommended',
-        'plugin:jsx-a11y/recommended',
-        'standard-jsx',
-        'prettier',
-      ],
-      globals: {
-        __webpack_public_path__: true,
-        $: true,
-        ga: true,
-      },
-      rules: {
-        // TODO: remove once https://github.com/standard/eslint-config-standard-react/issues/68 (support eslint@8) is fixed.
-        // START: inline standard-react rules
-        // "react/jsx-no-bind": ["error", {
-        //   "allowArrowFunctions": true,
-        //   "allowBind": false,
-        //   "ignoreRefs": true
-        // },],
-        'react/no-did-update-set-state': 'error',
-        'react/no-unused-prop-types': 'error',
-        'react/prop-types': 'error',
-        '@overleaf/no-generated-editor-themes': 'error',
-        // "react/react-in-jsx-scope": "error",
-        // END: inline standard-react rules
-
-        'react/no-unknown-property': [
-          'error',
-          {
-            ignore: ['dnd-container', 'dropdown-toggle'],
-          },
-        ],
-
-        'react/jsx-no-target-blank': [
-          'error',
-          {
-            allowReferrer: true,
-          },
-        ],
-        // Prevent usage of legacy string refs
-        'react/no-string-refs': 'error',
-
-        // Prevent curly braces around strings (as they're unnecessary)
-        'react/jsx-curly-brace-presence': [
-          'error',
-          {
-            props: 'never',
-            children: 'never',
-          },
-        ],
-
-        // Don't import React for JSX; the JSX runtime is added by a Babel plugin
-        'react/react-in-jsx-scope': 'off',
-        'react/jsx-uses-react': 'off',
-
-        // Allow functions as JSX props
-        'react/jsx-no-bind': 'off', // TODO: fix occurrences and re-enable this
-
-        // Fix conflict between prettier & standard by overriding to prefer
-        // double quotes
-        'jsx-quotes': ['error', 'prefer-double'],
-
-        // Override weird behaviour of jsx-a11y label-has-for (says labels must be
-        // nested *and* have for/id attributes)
-        'jsx-a11y/label-has-for': [
-          'error',
-          {
-            required: {
-              some: ['nesting', 'id'],
-            },
-          },
-        ],
-
-        // Require .jsx or .tsx file extension when using JSX
-        'react/jsx-filename-extension': [
-          'error',
-          {
-            extensions: ['.jsx', '.tsx'],
-          },
-        ],
-        'no-restricted-syntax': [
-          'error',
-          // prohibit direct calls to methods of window.localStorage
-          {
-            selector:
-              "CallExpression[callee.object.object.name='window'][callee.object.property.name='localStorage']",
-            message:
-              'Modify location via customLocalStorage instead of calling window.localStorage methods directly',
-          },
-        ],
-        'no-unused-vars': 'off',
-        '@typescript-eslint/no-unused-vars': [
-          'error',
-          {
-            args: 'after-used',
-            argsIgnorePattern: '^_',
-            ignoreRestSiblings: false,
-            caughtErrors: 'none',
-            vars: 'all',
-            varsIgnorePattern: '^_',
-          },
-        ],
-      },
-    },
-    {
-      // Sorting for Meta
-      files: ['frontend/js/utils/meta.ts'],
-      rules: {
-        '@typescript-eslint/member-ordering': [
-          'error',
-          { interfaces: { order: 'alphabetically' } },
-        ],
-      },
-    },
-    {
-      // React component specific rules
-      //
-      files: [
-        '**/frontend/js/**/components/**/*.{js,jsx,ts,tsx}',
-        '**/frontend/js/**/hooks/**/*.{js,jsx,ts,tsx}',
-      ],
-      rules: {
-        '@overleaf/no-unnecessary-trans': 'error',
-        '@overleaf/should-unescape-trans': 'error',
-        '@overleaf/require-loading-label': 'error',
-
-        // https://astexplorer.net/
-        'no-restricted-syntax': [
-          'error',
-          // prohibit direct calls to methods of window.location
-          {
-            selector:
-              "CallExpression[callee.object.object.name='window'][callee.object.property.name='location']",
-            message:
-              'Modify location via useLocation instead of calling window.location methods directly',
-          },
-          // prohibit assignment to window.location
-          {
-            selector:
-              "AssignmentExpression[left.object.name='window'][left.property.name='location']",
-            message:
-              'Modify location via useLocation instead of calling window.location methods directly',
-          },
-          // prohibit assignment to window.location.href
-          {
-            selector:
-              "AssignmentExpression[left.object.object.name='window'][left.object.property.name='location'][left.property.name='href']",
-            message:
-              'Modify location via useLocation instead of calling window.location methods directly',
-          },
-          // prohibit using lookbehinds due to incidents with Safari simply crashing when the script is parsed
-          {
-            selector: 'Literal[regex.pattern=/\\(\\?<[!=]/]',
-            message: 'Lookbehind is not supported in older Safari versions.',
-          },
-          // prohibit direct calls to methods of window.localStorage
-          // NOTE: this rule is also defined for all frontend files, but those rules are overriden by the React component-specific config
-          {
-            selector:
-              "CallExpression[callee.object.object.name='window'][callee.object.property.name='localStorage']",
-            message:
-              'Modify location via customLocalStorage instead of calling window.localStorage methods directly',
-          },
-        ],
-      },
-    },
-    // React + TypeScript-specific rules
-    {
-      files: ['**/*.tsx'],
-      rules: {
-        'react/prop-types': 'off',
-        'no-undef': 'off',
-      },
-    },
-    // TypeScript-specific rules
-    {
-      files: ['**/*.ts'],
-      rules: {
-        'no-undef': 'off',
-      },
-    },
-    // JavaScript-specific rules
-    {
-      files: ['**/*.js'],
-      rules: {
-        '@typescript-eslint/no-require-imports': 'off',
-      },
-    },
-    {
-      files: ['scripts/ukamf/*.js'],
-      rules: {
-        // Do not allow importing of any dependencies unless specified in either
-        //  - web/package.json
-        //  - web/scripts/ukamf/package.json
-        'import/no-extraneous-dependencies': [
-          'error',
-          { packageDir: ['.', 'scripts/ukamf'] },
-        ],
-      },
-    },
-    {
-      files: ['scripts/learn/checkSanitize/*.js'],
-      rules: {
-        // The checkSanitize script is used in the dev-env only.
-        'import/no-extraneous-dependencies': [
-          'error',
-          {
-            devDependencies: true,
-            packageDir: ['.', '../../'],
-          },
-        ],
-      },
-    },
-    {
-      files: [
-        // Backend: Use @overleaf/logger
-        //          Docs: https://manual.dev-overleaf.com/development/code/logging/#structured-logging
-        '**/app/**/*.{js,cjs,mjs}',
-        'app.{js,mjs}',
-        'modules/*/*.{js,mjs}',
-        // Frontend: Prefer debugConsole over bare console
-        //           Docs: https://manual.dev-overleaf.com/development/code/logging/#frontend
-        '**/frontend/**/*.{js,jsx,ts,tsx}',
-        // Tests
-        '**/test/**/*.{js,cjs,mjs,jsx,ts,tsx}',
-      ],
-      excludedFiles: [
-        // Allow console logs in scripts
-        '**/scripts/**/*.js',
-        // Allow console logs in stories
-        '**/stories/**/*.{js,jsx,ts,tsx}',
-        // Workers do not have access to the search params for enabling ?debug=true.
-        // self.location.url is the URL of the worker script.
-        '*.worker.{js,ts}',
-      ],
-      rules: {
-        'no-console': 'error',
-      },
-    },
-    {
-      files: ['**/*.worker.{js,ts}'],
-      rules: {
-        'no-restricted-globals': [
-          'error',
-          ..._.difference(
-            Object.keys({ ...globals.browser, ...globals.node }),
-            Object.keys(globals.worker)
-          ),
-        ],
-      },
-    },
-  ],
-}

services/web/.gitignore

@@ -26,11 +26,12 @@ cypress/results/
 # Ace themes for conversion
 frontend/js/features/source-editor/themes/ace/
 
-# Compiled parser files
+# Compiled parser files (latex/bibtex are generated by webpack plugin at build time)
 frontend/js/features/source-editor/lezer-latex/latex.mjs
 frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
 frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
 frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
+# typst compiled files are committed (generated via node scripts/lezer-latex/generate.mjs)
 
 !**/fixtures/**/*.log
 

services/web/Makefile

@@ -6,6 +6,11 @@ ifeq ($(BRANCH_NAME),)
   export BRANCH_NAME := $(shell git rev-parse --abbrev-ref HEAD)
 endif
 BRANCH_NAME_TAG_SAFE := $(shell echo "$(BRANCH_NAME)" | sed 's/\//\-\-/g' | cut -c1-96)
+
+.PHONY: print-branch-tag-safe
+print-branch-tag-safe:
+	@echo $(BRANCH_NAME_TAG_SAFE)
+
 ifeq ($(COMMIT_SHA),)
   export COMMIT_SHA := $(shell git rev-parse HEAD)
 endif
@@ -488,13 +493,13 @@ ci:
 
 # Run the linting commands in the scope of the monorepo.
 # Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/web/data/reports:/overleaf/services/web/data/reports --volume $(MONOREPO)/node_modules/.cache/:/overleaf/node_modules/.cache/ -w /overleaf $(IMAGE_CI) yarn run --silent
+RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/web/data/reports:/overleaf/services/web/data/reports --volume $(MONOREPO)/node_modules/.cache/:/overleaf/node_modules/.cache/ -w /overleaf $(IMAGE_CI) yarn run --silent
 RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
 RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
 
 ORG_PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 RUN_LINT_FORMAT ?= \
-	docker run --rm --env BRANCH_NAME --env CI --env COMMIT_SHA --env MONOREPO --env PATH=/overleaf/node_modules/.bin:$(ORG_PATH) --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume ${PWD}/data/reports:/overleaf/services/web/data/reports --volume $(MONOREPO)/node_modules/.cache/:/overleaf/node_modules/.cache/ ${IMAGE_CI}
+	docker run --rm --env BRANCH_NAME --env CI --env COMMIT_SHA --env MONOREPO --env PATH=/overleaf/node_modules/.bin:$(ORG_PATH) --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume ${PWD}/data/reports:/overleaf/services/web/data/reports --volume $(MONOREPO)/node_modules/.cache/:/overleaf/node_modules/.cache/ ${IMAGE_CI}
 
 NODE_MODULES_PATH := ${PATH}:${PWD}/node_modules/.bin:/overleaf/services/web/node_modules/.bin
 WITH_NODE_MODULES_PATH = \