perf(git-sync): sparse partial-clone fetch for pull and push

Pull now reuses the persistent repo cache with --filter=blob:none and
checks out only the configured subPath, so only relevant blobs are
downloaded. Push likewise fetches trees-only and uses git read-tree to
update the index without touching the working tree, preserving files
outside the managed area without downloading their content.
Upserts on pull are now parallelised (p-limit 5) to speed up import.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

services/web/app/src/Features/GitSync/GitSyncController.mjs

@@ -81,12 +81,12 @@ async function pullFromGit(req, res) {
   const projectId = req.params.project_id
   try {
     const userId = SessionManager.getLoggedInUserId(req.session)
-    const { remoteUrl, subPath } = await GitSyncHandler.getConfig(projectId)
+    const { remoteUrl, subPath, branch } = await GitSyncHandler.getConfig(projectId)
     if (!remoteUrl) {
       return res.status(400).json({ error: 'No git remote configured for this project' })
     }
     logger.debug({ projectId }, 'git sync: starting pull')
-    await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, userId)
+    await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, branch, userId)
     res.sendStatus(204)
   } catch (err) {
     logger.warn({ err, projectId }, 'git sync: pull failed')

services/web/app/src/Features/GitSync/GitSyncHandler.mjs

@@ -1,4 +1,5 @@
 import { mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
+import pLimit from 'p-limit'
 import { createWriteStream } from 'node:fs'
 import { join, dirname, extname, basename } from 'node:path'
 import { spawn } from 'node:child_process'
@@ -143,9 +144,13 @@ async function ensureRepo(projectId, remoteUrl, branch) {
     await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
     await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
     await spawnGit(['remote', 'add', 'origin', remoteUrl], repoDir)
+    await spawnGit(['config', 'remote.origin.partialclonefilter', 'blob:none'], repoDir)
+    await spawnGit(['config', 'remote.origin.promisor', 'true'], repoDir)
   } else {
     await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
     await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
+    await spawnGit(['config', 'remote.origin.partialclonefilter', 'blob:none'], repoDir)
+    await spawnGit(['config', 'remote.origin.promisor', 'true'], repoDir)
   }
 
   return repoDir
@@ -164,13 +169,13 @@ async function pushToRemote(
   // objects are sent — git handles the diff automatically.
   const repoDir = await ensureRepo(projectId, remoteUrl, branch)
 
-  // Fetch current remote state. This is a delta fetch — fast after the
-  // first push because git only downloads new commits/objects.
+  // Fetch remote state (trees only, no blobs) to preserve files outside the managed area.
+  // read-tree updates the index without checking out any files, so no blobs are downloaded.
   try {
-    await spawnGit(['fetch', 'origin', branch], repoDir)
-    await spawnGit(['reset', '--hard', `origin/${branch}`], repoDir)
+    await spawnGit(['fetch', '--filter=blob:none', 'origin', branch], repoDir)
+    await spawnGit(['read-tree', 'FETCH_HEAD'], repoDir)
   } catch {
-    // Remote branch doesn't exist yet (first push to empty repo) — proceed
+    // Empty remote or first push — proceed with empty index
   }
 
   const fileRoot = subPath ? join(repoDir, subPath) : repoDir
@@ -226,7 +231,17 @@ async function pushToRemote(
     logger.debug({ projectId }, 'git sync: PDF included')
   }
 
-  await spawnGit(['add', '-A'], repoDir)
+  // Stage only what we changed; files outside the managed area are preserved in the index from read-tree
+  if (pushFiles) {
+    if (subPath) {
+      await spawnGit(['add', '-A', '--', subPath], repoDir)
+    } else {
+      await spawnGit(['add', '-A'], repoDir)
+    }
+  }
+  if (pushPdf && pdfPath && pdfBuildId) {
+    await spawnGit(['add', '--', pdfPath], repoDir)
+  }
   await spawnGit(['commit', '--allow-empty', '-m', 'Verso sync'], repoDir)
   await spawnGit(['push', '--force', 'origin', `HEAD:${branch}`], repoDir)
 
@@ -235,59 +250,66 @@ async function pushToRemote(
   // for the next push so git can compute a proper delta.
 }
 
-async function pullFromRemote(projectId, remoteUrl, subPath, userId) {
-  const tmpDir = join(GIT_CACHE_ROOT, `pull-${projectId}`)
-  await mkdir(tmpDir, { recursive: true })
-  logger.debug({ projectId, subPath }, 'git sync: cloning remote for pull')
-  try {
-    await rm(join(tmpDir, 'repo'), { recursive: true, force: true })
-    await spawnGit(['clone', '--depth', '1', '--', remoteUrl, 'repo'], tmpDir)
+async function pullFromRemote(projectId, remoteUrl, subPath, branch, userId) {
+  logger.debug({ projectId, subPath }, 'git sync: fetching remote for pull')
 
-    const repoDir = join(tmpDir, 'repo')
-    const fileRoot = subPath ? join(repoDir, subPath) : repoDir
+  const repoDir = await ensureRepo(projectId, remoteUrl, branch)
+  // Fetch with blob filter — only trees/commits downloaded upfront
+  await spawnGit(['fetch', '--filter=blob:none', 'origin', branch], repoDir)
 
-    try {
-      await access(fileRoot)
-    } catch {
-      throw new Error(
-        subPath
-          ? `Subdirectory "${subPath}" not found in the repository`
-          : 'Repository is empty'
-      )
-    }
-
-    const allFiles = await walkDir(fileRoot)
-    logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
-
-    for (const absPath of allFiles) {
-      const rel = absPath.slice(fileRoot.length)
-      const projectPath = rel.startsWith('/') ? rel : `/${rel}`
-
-      if (isTextFile(basename(absPath))) {
-        const content = await readFile(absPath, 'utf8')
-        await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
-          projectId,
-          projectPath,
-          content.split('\n'),
-          'git-sync',
-          userId
-        )
-      } else {
-        await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
-          projectId,
-          projectPath,
-          absPath,
-          null,
-          userId,
-          'git-sync'
-        )
-      }
-    }
-
-    logger.debug({ projectId }, 'git sync: pull complete')
-  } finally {
-    await rm(tmpDir, { recursive: true, force: true })
+  if (subPath) {
+    // Checkout only the subPath directory — triggers lazy blob fetch for those files only
+    await spawnGit(['checkout', 'FETCH_HEAD', '--', subPath], repoDir)
+  } else {
+    // Full reset — all blobs downloaded (unavoidable when no subPath)
+    await spawnGit(['reset', '--hard', 'FETCH_HEAD'], repoDir)
   }
+
+  const fileRoot = subPath ? join(repoDir, subPath) : repoDir
+
+  try {
+    await access(fileRoot)
+  } catch {
+    throw new Error(
+      subPath
+        ? `Subdirectory "${subPath}" not found in the repository`
+        : 'Repository is empty'
+    )
+  }
+
+  const allFiles = await walkDir(fileRoot)
+  logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
+
+  const limit = pLimit(5)
+  await Promise.all(
+    allFiles.map(absPath =>
+      limit(async () => {
+        const rel = absPath.slice(fileRoot.length)
+        const projectPath = rel.startsWith('/') ? rel : `/${rel}`
+        if (isTextFile(basename(absPath))) {
+          const content = await readFile(absPath, 'utf8')
+          await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
+            projectId,
+            projectPath,
+            content.split('\n'),
+            'git-sync',
+            userId
+          )
+        } else {
+          await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
+            projectId,
+            projectPath,
+            absPath,
+            null,
+            userId,
+            'git-sync'
+          )
+        }
+      })
+    )
+  )
+
+  logger.debug({ projectId }, 'git sync: pull complete')
 }
 
 export default { setConfig, getConfig, pushToRemote, pullFromRemote }