[history-v1] add endpoint for downloading latest zip (#33181)

* [history-v1] add endpoint for downloading latest zip * [web] address review feedback * [web] tests: do not overwrite db.projects.overleaf, extend it * [web] set includeReferer flag from downloading zip GitOrigin-RevId: e63e549f004230086f82eccf03b43fd62bde6071
2026-05-13 08:06:53 +00:00
parent b1931d0b3b
commit 7c50dc9990
13 changed files with 319 additions and 63 deletions
@@ -10,6 +10,7 @@ import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs'
 import Validation from '../../infrastructure/Validation.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import { pipeline } from 'node:stream/promises'
+import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'

 const { z, zz, parseReq } = Validation

@@ -142,7 +143,7 @@ export default {
      }
      ProjectGetter.getProject(
        projectId,
-        { name: true },
+        { name: true, 'overleaf.history.id': true },
        function (error, project) {
          if (error) {
            return next(error)
@@ -153,14 +154,33 @@ export default {
            userId,
            req.ip
          )
-          ProjectZipStreamManager.createZipStreamForProject(
-            projectId,
-            function (error, stream) {
+          SplitTestHandler.featureFlagEnabled(
+            req,
+            res,
+            'zip-from-history',
+            { includeReferer: true },
+            function (error, enabled) {
              if (error) {
                return next(error)
              }
-              prepareZipAttachment(res, `${getSafeProjectName(project)}.zip`)
-              stream.pipe(res)
+              ProjectZipStreamManager.createZipStreamForProject(
+                projectId,
+                enabled,
+                project.overleaf.history.id,
+                function (error, stream, historyVersion) {
+                  if (error) {
+                    return next(error)
+                  }
+                  prepareZipAttachment(
+                    res,
+                    `${getSafeProjectName(project)}.zip`
+                  )
+                  if (historyVersion != null) {
+                    res.setHeader('X-History-Version', historyVersion)
+                  }
+                  stream.pipe(res)
+                }
+              )
            }
          )
        }
@@ -187,17 +207,29 @@ export default {
            req.ip
          )
        }
-        ProjectZipStreamManager.createZipStreamForMultipleProjects(
-          projectIds,
-          function (error, stream) {
+        SplitTestHandler.featureFlagEnabled(
+          req,
+          res,
+          'zip-from-history',
+          { includeReferer: true },
+          function (error, enabled) {
            if (error) {
              return next(error)
            }
-            prepareZipAttachment(
-              res,
-              `Overleaf Projects (${projectIds.length} items).zip`
+            ProjectZipStreamManager.createZipStreamForMultipleProjects(
+              projectIds,
+              enabled,
+              function (error, stream) {
+                if (error) {
+                  return next(error)
+                }
+                prepareZipAttachment(
+                  res,
+                  `Overleaf Projects (${projectIds.length} items).zip`
+                )
+                stream.pipe(res)
+              }
            )
-            stream.pipe(res)
          }
        )
      }
@@ -4,10 +4,11 @@ import logger from '@overleaf/logger'
 import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import HistoryManager from '../History/HistoryManager.mjs'
+import Metrics from '@overleaf/metrics'
 let ProjectZipStreamManager

 export default ProjectZipStreamManager = {
-  createZipStreamForMultipleProjects(projectIds, callback) {
+  createZipStreamForMultipleProjects(projectIds, zipFromHistory, callback) {
    // We'll build up a zip file that contains multiple zip files
    const archive = archiver('zip')
    archive.on('error', err =>
@@ -19,38 +20,44 @@ export default ProjectZipStreamManager = {
    callback(null, archive)

    const jobs = projectIds.map(projectId => cb => {
-      ProjectGetter.getProject(projectId, { name: true }, (error, project) => {
-        if (error) {
-          return cb(error)
-        }
-        if (!project) {
-          logger.debug(
-            { projectId },
-            'cannot append project to zip stream: project not found'
-          )
-          return cb()
-        }
-        logger.debug(
-          { projectId, name: project.name },
-          'appending project to zip stream'
-        )
-        ProjectZipStreamManager.createZipStreamForProject(
-          projectId,
-          (error, stream) => {
-            if (error) {
-              return cb(error)
-            }
-            archive.append(stream, { name: `${project.name}.zip` })
-            stream.on('end', () => {
-              logger.debug(
-                { projectId, name: project.name },
-                'zip stream ended'
-              )
-              cb()
-            })
+      ProjectGetter.getProject(
+        projectId,
+        { name: true, 'overleaf.history.id': true },
+        (error, project) => {
+          if (error) {
+            return cb(error)
          }
-        )
-      })
+          if (!project) {
+            logger.debug(
+              { projectId },
+              'cannot append project to zip stream: project not found'
+            )
+            return cb()
+          }
+          logger.debug(
+            { projectId, name: project.name },
+            'appending project to zip stream'
+          )
+          ProjectZipStreamManager.createZipStreamForProject(
+            projectId,
+            zipFromHistory,
+            project.overleaf.history.id,
+            (error, stream) => {
+              if (error) {
+                return cb(error)
+              }
+              archive.append(stream, { name: `${project.name}.zip` })
+              stream.on('end', () => {
+                logger.debug(
+                  { projectId, name: project.name },
+                  'zip stream ended'
+                )
+                cb()
+              })
+            }
+          )
+        }
+      )
    })

    async.series(jobs, () => {
@@ -62,7 +69,16 @@ export default ProjectZipStreamManager = {
    })
  },

-  createZipStreamForProject(projectId, callback) {
+  createZipStreamForProject(projectId, zipFromHistory, historyId, callback) {
+    Metrics.inc('project_zip_download', 1, {
+      method: zipFromHistory ? 'history-v1' : 'web',
+    })
+    if (zipFromHistory) {
+      return HistoryManager.flushProject(projectId, error => {
+        if (error) return callback(error)
+        HistoryManager.getLatestZipWithHistoryId(historyId, callback)
+      })
+    }
    const archive = archiver('zip')
    // return stream immediately before we start adding things to it
    archive.on('error', err =>
@@ -1,4 +1,4 @@
-import { callbackify } from 'node:util'
+import { callbackify, callbackifyMultiResult } from '@overleaf/promise-utils'
 import {
  fetchJson,
  fetchNothing,
@@ -303,6 +303,22 @@ async function getLatestHistoryWithHistoryId(historyId) {
  )
 }

+/**
+ * Get the latest chunk from history using already resolved historyId
+ *
+ * @param {string} historyId
+ */
+async function getLatestZipWithHistoryId(historyId) {
+  const { response, stream } = await fetchStreamWithResponse(
+    `${HISTORY_V1_URL}/projects/${historyId}/latest/zip`,
+    {
+      basicAuth: HISTORY_V1_BASIC_AUTH,
+      signal: AbortSignal.timeout(10 * 60 * 1000),
+    }
+  )
+  return { stream, historyVersion: response.headers.get('X-History-Version') }
+}
+
 async function ensureNoResyncPending(projectId) {
  const { resyncPending } = await fetchJson(
    `${settings.apis.project_history.url}/project/${projectId}/resync-pending`
@@ -475,6 +491,10 @@ export default {
  requestBlob: callbackify(requestBlob),
  requestBlobWithProjectId: callbackify(requestBlobWithProjectId),
  getLatestHistory: callbackify(getLatestHistory),
+  getLatestZipWithHistoryId: callbackifyMultiResult(getLatestZipWithHistoryId, [
+    'stream',
+    'historyVersion',
+  ]),
  getChanges: callbackify(getChanges),
  promises: {
    initializeProject,
@@ -491,6 +511,7 @@ export default {
    requestBlob,
    requestBlobWithProjectId,
    getLatestHistory,
+    getLatestZipWithHistoryId,
    getChanges,
    getChangesWithHistoryId,
    getProjectBlobStats,
@@ -333,6 +333,30 @@ async function getOneTimeAssignment(splitTestName) {
  }
 }

+/**
+ * Checks if a feature flag is enabled for a specific user
+ *
+ * Retrieves the feature flag assignment for a user and determines if the assigned variant is 'enabled'
+ *
+ * @param req the request
+ * @param res the Express response object
+ * @param {string} splitTestName - The unique name of the feature flag
+ * @param {Object} options
+ * @param {boolean} options.includeReferer For ajax requests and downloads include the split test overrides of the page
+ * @returns {Promise<boolean>} True if the user's assigned variant is 'enabled', false otherwise
+ */
+async function featureFlagEnabled(
+  req,
+  res,
+  splitTestName,
+  { includeReferer = false } = { includeReferer: false }
+) {
+  const { variant } = await getAssignment(req, res, splitTestName, {
+    includeReferer,
+  })
+  return variant === 'enabled'
+}
+
 /**
 * Checks if a feature flag is enabled for a specific user
 *
@@ -997,6 +1021,7 @@ export default {
  getPercentile,
  getAssignment: callbackify(getAssignment),
  getAssignmentForUser: callbackify(getAssignmentForUser),
+  featureFlagEnabled: callbackify(featureFlagEnabled),
  featureFlagEnabledForUser: callbackify(featureFlagEnabledForUser),
  getOneTimeAssignment: callbackify(getOneTimeAssignment),
  getActiveAssignmentsForUser: callbackify(getActiveAssignmentsForUser),
@@ -1006,6 +1031,7 @@ export default {
  promises: {
    getAssignment,
    getAssignmentForUser,
+    featureFlagEnabled,
    featureFlagEnabledForUser,
    getOneTimeAssignment,
    getActiveAssignmentsForUser,