fix(git-sync): remove promisor config to prevent lazy-fetch during commit

remote.origin.promisor=true causes git to fetch missing blobs from the
remote during the commit connectivity check, failing if the server is
slow or unavailable. --filter=blob:none on fetch works independently of
promisor config so trees-only fetching is preserved. Existing cached
repos have the config unset on next use.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/deploy-verso-prod.yml

@@ -331,6 +331,8 @@ jobs:
                         value: "true"
                       - name: OVERLEAF_LATEX_SHELL_ESCAPE
                         value: "true"
+                      - name: OVERLEAF_ENABLE_GIT_SYNC
+                        value: "true"
                       # (SMTP email vars are loaded below via envFrom.)
                     # SMTP for password-reset / invite emails. All
                     # OVERLEAF_EMAIL_* vars come from the optional 'verso-smtp'

services/web/app/src/Features/GitSync/GitSyncController.mjs

@@ -81,12 +81,12 @@ async function pullFromGit(req, res) {
   const projectId = req.params.project_id
   try {
     const userId = SessionManager.getLoggedInUserId(req.session)
-    const { remoteUrl, subPath } = await GitSyncHandler.getConfig(projectId)
+    const { remoteUrl, subPath, branch } = await GitSyncHandler.getConfig(projectId)
     if (!remoteUrl) {
       return res.status(400).json({ error: 'No git remote configured for this project' })
     }
     logger.debug({ projectId }, 'git sync: starting pull')
-    await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, userId)
+    await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, branch, userId)
     res.sendStatus(204)
   } catch (err) {
     logger.warn({ err, projectId }, 'git sync: pull failed')

services/web/app/src/Features/GitSync/GitSyncHandler.mjs

@@ -1,13 +1,14 @@
-import { mkdtemp, mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
+import { mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
+import pLimit from 'p-limit'
 import { createWriteStream } from 'node:fs'
 import { join, dirname, extname, basename } from 'node:path'
-import { tmpdir } from 'node:os'
 import { spawn } from 'node:child_process'
 import { pipeline } from 'node:stream/promises'
 import logger from '@overleaf/logger'
 import Settings from '@overleaf/settings'
 import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
 import ProjectEntityUpdateHandler from '../Project/ProjectEntityUpdateHandler.mjs'
+import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
 import HistoryManager from '../History/HistoryManager.mjs'
 import ClsiManager from '../Compile/ClsiManager.mjs'
 import { Project } from '../../models/Project.mjs'
@@ -20,6 +21,13 @@ const EDITABLE_FILENAMES = new Set(
   (Settings.editableFilenames || []).map(n => n.toLowerCase())
 )
 
+// Data root is one level above the dumpFolder (e.g. /var/lib/overleaf/data)
+const GIT_CACHE_ROOT = join(dirname(Settings.path.dumpFolder), 'git-sync-cache')
+
+function repoCachePath(projectId) {
+  return join(GIT_CACHE_ROOT, projectId.toString())
+}
+
 function isTextFile(filename) {
   const ext = extname(filename).toLowerCase()
   return TEXT_EXTENSIONS.has(ext) || EDITABLE_FILENAMES.has(filename.toLowerCase())
@@ -59,6 +67,24 @@ async function spawnGit(args, cwd) {
   })
 }
 
+async function spawnGitOutput(args, cwd) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn('git', args, {
+      cwd,
+      env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
+    })
+    let stdout = ''
+    let stderr = ''
+    proc.stdout?.on('data', d => (stdout += d.toString()))
+    proc.stderr?.on('data', d => (stderr += d.toString()))
+    proc.on('close', code => {
+      if (code !== 0) reject(new Error(`git ${args[0]} exited ${code}: ${stderr.trim()}`))
+      else resolve(stdout.trim())
+    })
+    proc.on('error', reject)
+  })
+}
+
 async function setConfig(projectId, { remoteUrl, subPath, pdfPath, pushFiles, pushPdf, branch }) {
   await Project.updateOne(
     { _id: new ObjectId(projectId) },
@@ -94,150 +120,222 @@ async function getConfig(projectId) {
   }
 }
 
+// Ensure a persistent cached repo exists and is connected to the right remote.
+// Returns the repo directory path.
+async function ensureRepo(projectId, remoteUrl, branch) {
+  const repoDir = repoCachePath(projectId)
+  await mkdir(repoDir, { recursive: true })
+
+  let needsInit = false
+  try {
+    const currentRemote = await spawnGitOutput(['remote', 'get-url', 'origin'], repoDir)
+    if (currentRemote !== remoteUrl) {
+      // Remote URL changed (e.g. new token) — update it in place
+      await spawnGit(['remote', 'set-url', 'origin', remoteUrl], repoDir)
+    }
+  } catch {
+    needsInit = true
+  }
+
+  if (needsInit) {
+    await rm(repoDir, { recursive: true, force: true })
+    await mkdir(repoDir, { recursive: true })
+    await spawnGit(['init'], repoDir)
+    await spawnGit(['symbolic-ref', 'HEAD', `refs/heads/${branch}`], repoDir)
+    await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
+    await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
+    await spawnGit(['remote', 'add', 'origin', remoteUrl], repoDir)
+  } else {
+    await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
+    await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
+    // Remove promisor config if present from a prior version — it causes git to lazy-fetch
+    // missing blobs during commit (connectivity check), which we don't want.
+    // --filter=blob:none on fetch works independently of this config.
+    for (const key of ['remote.origin.partialclonefilter', 'remote.origin.promisor']) {
+      try { await spawnGit(['config', '--unset', key], repoDir) } catch {}
+    }
+  }
+
+  return repoDir
+}
+
 async function pushToRemote(
   projectId,
   remoteUrl,
   subPath,
   { pdfPath, pdfBuildId, pdfClsiServerId, userId, pushFiles = true, pushPdf = true, branch = 'main' } = {}
 ) {
-  const tmpDir = await mkdtemp(join(tmpdir(), 'verso-git-'))
-  logger.debug({ projectId, tmpDir, subPath, pushFiles, pushPdf, branch }, 'git sync: starting push')
-  try {
-    const repoDir = join(tmpDir, 'repo')
-    await mkdir(repoDir)
+  logger.debug({ projectId, subPath, pushFiles, pushPdf, branch }, 'git sync: starting push')
 
-    // Init, connect to remote, fetch existing state so files outside our
-    // managed area are preserved in the push.
-    await spawnGit(['init'], repoDir)
-    await spawnGit(['symbolic-ref', 'HEAD', `refs/heads/${branch}`], repoDir)
-    await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
-    await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
-    await spawnGit(['remote', 'add', 'origin', remoteUrl], repoDir)
-    try {
-      await spawnGit(['fetch', '--depth', '1', 'origin', branch], repoDir)
-      await spawnGit(['reset', '--hard', 'FETCH_HEAD'], repoDir)
-    } catch {
-      // Remote branch doesn't exist yet (first push to empty repo) — proceed
-    }
-
-    const fileRoot = subPath ? join(repoDir, subPath) : repoDir
-
-    if (pushFiles) {
-      // Clear the managed area so files deleted in Verso are removed from git.
-      // With a subPath this leaves everything outside the subPath untouched.
-      // Without a subPath the entire repo is considered Verso-owned.
-      if (subPath) {
-        await rm(fileRoot, { recursive: true, force: true })
-        await mkdir(fileRoot, { recursive: true })
-      } else {
-        const entries = await readdir(repoDir, { withFileTypes: true })
-        for (const entry of entries) {
-          if (entry.name === '.git') continue
-          await rm(join(repoDir, entry.name), { recursive: true, force: true })
-        }
-      }
-
-      const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
-      for (const [path, doc] of Object.entries(docs)) {
-        const rel = path.startsWith('/') ? path.slice(1) : path
-        const dest = join(fileRoot, rel)
-        await mkdir(dirname(dest), { recursive: true })
-        await writeFile(dest, doc.lines.join('\n'))
-      }
-
-      const files = await ProjectEntityHandler.promises.getAllFiles(projectId)
-      for (const [path, file] of Object.entries(files)) {
-        const rel = path.startsWith('/') ? path.slice(1) : path
-        const dest = join(fileRoot, rel)
-        await mkdir(dirname(dest), { recursive: true })
-        const { stream } =
-          await HistoryManager.promises.requestBlobWithProjectId(
-            projectId,
-            file.hash
-          )
-        await pipeline(stream, createWriteStream(dest))
-      }
-    }
-
-    // PDF path is always relative to repo root, independent of subPath
-    if (pushPdf && pdfPath && pdfBuildId) {
-      const pdfStream = await ClsiManager.promises.getOutputFileStream(
-        projectId,
-        userId,
-        pdfClsiServerId ?? undefined,
-        pdfBuildId,
-        'output.pdf'
-      )
-      const pdfDest = join(repoDir, pdfPath)
-      await mkdir(dirname(pdfDest), { recursive: true })
-      await pipeline(pdfStream, createWriteStream(pdfDest))
-      logger.debug({ projectId }, 'git sync: PDF included')
-    }
-
-    await spawnGit(['add', '-A'], repoDir)
-    await spawnGit(['commit', '--allow-empty', '-m', 'Verso sync'], repoDir)
-    await spawnGit(['push', '--force', 'origin', `HEAD:${branch}`], repoDir)
-
-    logger.debug({ projectId }, 'git sync: push complete')
-  } finally {
-    await rm(tmpDir, { recursive: true, force: true })
+  // Flush in-memory document edits to MongoDB before reading, so getAllDocs returns current content.
+  // Without this, edits made in the editor may not yet be persisted and would be silently skipped.
+  if (pushFiles) {
+    await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
   }
+
+  // Use a persistent repo so git's delta mechanism works across pushes.
+  // On first push all objects are new; on subsequent pushes only changed
+  // objects are sent — git handles the diff automatically.
+  const repoDir = await ensureRepo(projectId, remoteUrl, branch)
+
+  // Fetch remote state (trees only, latest commit only) to preserve files outside the managed area.
+  // Skipped when remote HEAD already matches local HEAD (common case: re-push without external changes).
+  try {
+    let needsFetch = true
+    try {
+      const remoteRef = await spawnGitOutput(['ls-remote', 'origin', branch], repoDir)
+      const remoteHead = remoteRef.split(/\s/)[0].trim()
+      const localHead = await spawnGitOutput(['rev-parse', 'HEAD'], repoDir)
+      needsFetch = remoteHead !== localHead
+    } catch {
+      // Can't compare — proceed with fetch
+    }
+    if (needsFetch) {
+      // --depth=1 --filter=blob:none: only the latest commit's tree objects, no blobs
+      await spawnGit(['fetch', '--filter=blob:none', '--depth=1', 'origin', branch], repoDir)
+      // --mixed: moves HEAD to FETCH_HEAD + updates index, no working tree change.
+      // This makes FETCH_HEAD the parent of our next commit so the diff only shows subPath changes.
+      await spawnGit(['reset', '--mixed', 'FETCH_HEAD'], repoDir)
+    }
+  } catch {
+    // Empty remote or first push — proceed with empty index
+  }
+
+  const fileRoot = subPath ? join(repoDir, subPath) : repoDir
+
+  if (pushFiles) {
+    // Clear the managed area so files deleted in Verso are removed from git.
+    // With a subPath this leaves everything outside the subPath untouched.
+    if (subPath) {
+      await rm(fileRoot, { recursive: true, force: true })
+      await mkdir(fileRoot, { recursive: true })
+    } else {
+      const entries = await readdir(repoDir, { withFileTypes: true })
+      for (const entry of entries) {
+        if (entry.name === '.git') continue
+        await rm(join(repoDir, entry.name), { recursive: true, force: true })
+      }
+    }
+
+    const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
+    for (const [path, doc] of Object.entries(docs)) {
+      const rel = path.startsWith('/') ? path.slice(1) : path
+      const dest = join(fileRoot, rel)
+      await mkdir(dirname(dest), { recursive: true })
+      await writeFile(dest, doc.lines.join('\n'))
+    }
+
+    const files = await ProjectEntityHandler.promises.getAllFiles(projectId)
+    for (const [path, file] of Object.entries(files)) {
+      const rel = path.startsWith('/') ? path.slice(1) : path
+      const dest = join(fileRoot, rel)
+      await mkdir(dirname(dest), { recursive: true })
+      const { stream } =
+        await HistoryManager.promises.requestBlobWithProjectId(
+          projectId,
+          file.hash
+        )
+      await pipeline(stream, createWriteStream(dest))
+    }
+  }
+
+  // PDF path is always relative to repo root, independent of subPath
+  if (pushPdf && pdfPath && pdfBuildId) {
+    const pdfStream = await ClsiManager.promises.getOutputFileStream(
+      projectId,
+      userId,
+      pdfClsiServerId ?? undefined,
+      pdfBuildId,
+      'output.pdf'
+    )
+    const pdfDest = join(repoDir, pdfPath)
+    await mkdir(dirname(pdfDest), { recursive: true })
+    await pipeline(pdfStream, createWriteStream(pdfDest))
+    logger.debug({ projectId }, 'git sync: PDF included')
+  }
+
+  // Stage changes. For subPath, explicitly clear the index entries first then re-add from disk.
+  // This bypasses git's cached-tree extension which can cause modified files to be silently
+  // skipped by git add -A when the directory mtime hasn't changed since the last checkout.
+  if (pushFiles) {
+    if (subPath) {
+      await spawnGit(['rm', '-r', '--cached', '--ignore-unmatch', '--', subPath], repoDir)
+      await spawnGit(['add', '--', subPath], repoDir)
+    } else {
+      await spawnGit(['add', '-A'], repoDir)
+    }
+  }
+  if (pushPdf && pdfPath && pdfBuildId) {
+    await spawnGit(['add', '--', pdfPath], repoDir)
+  }
+  await spawnGit(['commit', '--allow-empty', '-m', 'Verso sync'], repoDir)
+  await spawnGit(['push', '--force', 'origin', `HEAD:${branch}`], repoDir)
+
+  logger.debug({ projectId }, 'git sync: push complete')
+  // Note: repoDir is intentionally not deleted — it persists as the cache
+  // for the next push so git can compute a proper delta.
 }
 
-async function pullFromRemote(projectId, remoteUrl, subPath, userId) {
-  const tmpDir = await mkdtemp(join(tmpdir(), 'verso-pull-'))
-  logger.debug({ projectId, subPath }, 'git sync: cloning remote for pull')
-  try {
-    await spawnGit(['clone', '--depth', '1', '--', remoteUrl, 'repo'], tmpDir)
+async function pullFromRemote(projectId, remoteUrl, subPath, branch, userId) {
+  logger.debug({ projectId, subPath }, 'git sync: fetching remote for pull')
 
-    const repoDir = join(tmpDir, 'repo')
-    const fileRoot = subPath ? join(repoDir, subPath) : repoDir
+  const repoDir = await ensureRepo(projectId, remoteUrl, branch)
+  // depth=1 + blob filter: only the latest commit's tree objects, no blobs upfront
+  await spawnGit(['fetch', '--filter=blob:none', '--depth=1', 'origin', branch], repoDir)
 
-    // Verify the subdirectory exists in the repo
-    try {
-      await access(fileRoot)
-    } catch {
-      throw new Error(
-        subPath
-          ? `Subdirectory "${subPath}" not found in the repository`
-          : 'Repository is empty'
-      )
-    }
-
-    const allFiles = await walkDir(fileRoot)
-    logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
-
-    for (const absPath of allFiles) {
-      // Path relative to fileRoot → Verso project path (leading slash required)
-      const rel = absPath.slice(fileRoot.length)
-      const projectPath = rel.startsWith('/') ? rel : `/${rel}`
-
-      if (isTextFile(basename(absPath))) {
-        const content = await readFile(absPath, 'utf8')
-        const lines = content.split('\n')
-        await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
-          projectId,
-          projectPath,
-          lines,
-          'git-sync',
-          userId
-        )
-      } else {
-        await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
-          projectId,
-          projectPath,
-          absPath,
-          null,
-          userId,
-          'git-sync'
-        )
-      }
-    }
-
-    logger.debug({ projectId }, 'git sync: pull complete')
-  } finally {
-    await rm(tmpDir, { recursive: true, force: true })
+  if (subPath) {
+    // Checkout only the subPath directory — triggers lazy blob fetch for those files only
+    await spawnGit(['checkout', 'FETCH_HEAD', '--', subPath], repoDir)
+  } else {
+    // Full reset — all blobs downloaded (unavoidable when no subPath)
+    await spawnGit(['reset', '--hard', 'FETCH_HEAD'], repoDir)
   }
+
+  const fileRoot = subPath ? join(repoDir, subPath) : repoDir
+
+  try {
+    await access(fileRoot)
+  } catch {
+    throw new Error(
+      subPath
+        ? `Subdirectory "${subPath}" not found in the repository`
+        : 'Repository is empty'
+    )
+  }
+
+  const allFiles = await walkDir(fileRoot)
+  logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
+
+  const limit = pLimit(5)
+  await Promise.all(
+    allFiles.map(absPath =>
+      limit(async () => {
+        const rel = absPath.slice(fileRoot.length)
+        const projectPath = rel.startsWith('/') ? rel : `/${rel}`
+        if (isTextFile(basename(absPath))) {
+          const content = await readFile(absPath, 'utf8')
+          await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
+            projectId,
+            projectPath,
+            content.split('\n'),
+            'git-sync',
+            userId
+          )
+        } else {
+          await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
+            projectId,
+            projectPath,
+            absPath,
+            null,
+            userId,
+            'git-sync'
+          )
+        }
+      })
+    )
+  )
+
+  logger.debug({ projectId }, 'git sync: pull complete')
 }
 
 export default { setConfig, getConfig, pushToRemote, pullFromRemote }

services/web/locales/en.json

@@ -1044,7 +1044,7 @@
   "git_bridge_modal_learn_more_about_authentication_tokens": "Learn more about Git integration authentication tokens.",
   "git_bridge_modal_read_only": "<strong>You have read-only access to this project.</strong> This means you can pull from __appName__ but you can’t push any changes you make back to this project.",
   "git_sync": "Git Sync",
-  "git_sync_description": "Force-push your project files to an external git repository. Include your auth token in the URL (e.g. https://user:token@github.com/org/repo.git).",
+  "git_sync_description": "Sync your project with an external git repository. Include your auth token in the URL (e.g. https://user:token@gitea.example.com/org/repo.git).",
   "git_sync_pull_now": "Pull from git",
   "git_sync_pull_success": "Project updated from git remote successfully.",
   "git_sync_pulling": "Pulling…",

services/web/locales/fr.json

@@ -1047,7 +1047,7 @@
   "git_bridge_modal_learn_more_about_authentication_tokens": "Apprenez-en davantage sur les jetons d’authentification de l’intégration Git.",
   "git_bridge_modal_read_only": "<strong>Vous disposez d'un accès en lecture seule à ce projet.</strong> Cela signifie que vous pouvez extraire des données de __appName__, mais que vous ne pouvez pas transférer les modifications que vous apportez à ce projet.",
   "git_sync": "Synchronisation Git",
-  "git_sync_description": "Pousse de force les fichiers du projet vers un dépôt git externe. Incluez votre jeton d'authentification dans l'URL (ex. https://user:token@github.com/org/repo.git).",
+  "git_sync_description": "Synchronisez votre projet avec un dépôt git externe. Incluez votre jeton d'authentification dans l'URL (ex. https://user:token@gitea.example.com/org/repo.git).",
   "git_sync_pull_now": "Tirer depuis git",
   "git_sync_pull_success": "Projet mis à jour depuis le dépôt git avec succès.",
   "git_sync_pulling": "Récupération…",