Merge pull request #22202 from overleaf/jpa-tpds

[misc] tpds: read files from history-v1 with fallback to filestore

GitOrigin-RevId: afdde47e453b0cb07985392b5e2b03abf348debf

services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js

@@ -596,7 +596,9 @@ const upsertFile = wrapWithLock({
       const projectHistoryId = project.overleaf?.history?.id
       await TpdsUpdateSender.promises.addFile({
         projectId: project._id,
+        historyId: projectHistoryId,
         fileId: fileRef._id,
+        hash: fileRef.hash,
         path: path.fileSystem,
         rev: fileRef.rev,
         projectName: project.name,
@@ -1298,9 +1300,15 @@ const ProjectEntityUpdateHandler = {
       })
     }
 
+    const historyId = project?.overleaf?.history?.id
+    if (!historyId) {
+      throw new OError('project does not have a history id', { projectId })
+    }
     await TpdsUpdateSender.promises.addFile({
       projectId,
+      historyId,
       fileId: fileRef._id,
+      hash: fileRef.hash,
       path: result?.path?.fileSystem,
       projectName: project.name,
       rev: fileRef.rev,
@@ -1346,7 +1354,9 @@ const ProjectEntityUpdateHandler = {
     const projectHistoryId = project.overleaf?.history?.id
     await TpdsUpdateSender.promises.addFile({
       projectId: project._id,
+      historyId: projectHistoryId,
       fileId: updatedFileRef._id,
+      hash: updatedFileRef.hash,
       path: path.fileSystem,
       rev: updatedFileRef.rev,
       projectName: project.name,

services/web/app/src/Features/ThirdPartyDataStore/TpdsProjectFlusher.js

@@ -5,6 +5,7 @@ const ProjectGetter = require('../Project/ProjectGetter')
 const ProjectEntityHandler = require('../Project/ProjectEntityHandler')
 const { Project } = require('../../models/Project')
 const TpdsUpdateSender = require('./TpdsUpdateSender')
+const OError = require('@overleaf/o-error')
 
 module.exports = {
   flushProjectToTpds: callbackify(flushProjectToTpds),
@@ -24,6 +25,7 @@ async function flushProjectToTpds(projectId) {
   const project = await ProjectGetter.promises.getProject(projectId, {
     name: true,
     deferredTpdsFlushCounter: true,
+    'overleaf.history.id': 1,
   })
   await _flushProjectToTpds(project)
 }
@@ -37,6 +39,7 @@ async function flushProjectToTpdsIfNeeded(projectId) {
   const project = await ProjectGetter.promises.getProject(projectId, {
     name: true,
     deferredTpdsFlushCounter: true,
+    'overleaf.history.id': 1,
   })
   if (project.deferredTpdsFlushCounter > 0) {
     await _flushProjectToTpds(project)
@@ -44,6 +47,11 @@ async function flushProjectToTpdsIfNeeded(projectId) {
 }
 
 async function _flushProjectToTpds(project) {
+  const historyId = project?.overleaf?.history?.id
+  if (!historyId) {
+    const projectId = project._id
+    throw new OError('project does not have a history id', { projectId })
+  }
   logger.debug({ projectId: project._id }, 'flushing project to TPDS')
   logger.debug({ projectId: project._id }, 'finished flushing project to TPDS')
   await DocumentUpdaterHandler.promises.flushProjectToMongo(project._id)
@@ -64,7 +72,9 @@ async function _flushProjectToTpds(project) {
   for (const [filePath, file] of Object.entries(files)) {
     await TpdsUpdateSender.promises.addFile({
       projectId: project._id,
+      historyId,
       fileId: file._id,
+      hash: file.hash,
       path: filePath,
       projectName: project.name,
       rev: file.rev,

services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateSender.js

@@ -41,6 +41,7 @@ async function addEntity(params) {
     rev,
     folderId,
     streamOrigin,
+    streamFallback,
     entityId,
     entityType,
   } = params
@@ -60,6 +61,7 @@ async function addEntity(params) {
       uri: buildTpdsUrl(userId, projectName, path),
       title: 'addFile',
       streamOrigin,
+      streamFallback,
     }
 
     await enqueue(userId, 'pipeStreamFrom', job)
@@ -68,8 +70,21 @@ async function addEntity(params) {
 
 async function addFile(params) {
   metrics.inc('tpds.add-file')
-  const { projectId, fileId, path, projectName, rev, folderId } = params
+  const {
+    projectId,
+    historyId,
+    fileId,
+    hash,
+    path,
+    projectName,
+    rev,
+    folderId,
+  } = params
+  // Go through project-history to avoid the need for handling history-v1 authentication.
   const streamOrigin =
+    settings.apis.project_history.url +
+    Path.join(`/project/${historyId}/blob/${hash}`)
+  const streamFallback =
     settings.apis.filestore.url +
     Path.join(`/project/${projectId}`, `/file/${fileId}`)
 
@@ -80,6 +95,7 @@ async function addFile(params) {
     rev,
     folderId,
     streamOrigin,
+    streamFallback,
     entityId: fileId,
     entityType: 'file',
   })