Merge pull request #33341 from overleaf/mj-two-step-export-web

[web] Add two-step pandoc conversion download

GitOrigin-RevId: 093f435a497a7583d2b4d23558415cc442f84553

services/web/app/src/Features/Compile/ClsiManager.mjs

@@ -444,7 +444,7 @@ async function _makeRequest(
   timer.done()
   let newClsiServerId
   if (CLSI_COOKIES_ENABLED) {
-    newClsiServerId = _getClsiServerIdFromResponse(response)
+    newClsiServerId = getClsiServerIdFromResponse(response)
     await ClsiCookieManager.promises.setServerId(
       projectId,
       userId,
@@ -603,7 +603,7 @@ async function _makeNewBackendRequest(
   timer.done()
   let newClsiServerId
   if (CLSI_COOKIES_ENABLED) {
-    newClsiServerId = _getClsiServerIdFromResponse(response)
+    newClsiServerId = getClsiServerIdFromResponse(response)
     await NewBackendCloudClsiCookieManager.promises.setServerId(
       projectId,
       userId,
@@ -1268,7 +1268,7 @@ async function syncTeX(
   }
 }
 
-function _getClsiServerIdFromResponse(response) {
+function getClsiServerIdFromResponse(response) {
   const setCookieHeaders = response.headers.raw()['set-cookie'] ?? []
   for (const header of setCookieHeaders) {
     const cookie = Cookie.parse(header)
@@ -1307,6 +1307,8 @@ export default {
   getOutputFileStream: callbackify(getOutputFileStream),
   wordCount: callbackify(wordCount),
   syncTeX: callbackify(syncTeX),
+  getClsiServerIdFromResponse,
+  CLSI_COOKIES_ENABLED,
   promises: {
     sendRequest,
     sendExternalRequest,

services/web/app/src/Features/Downloads/ProjectDownloadsController.mjs

@@ -1,4 +1,5 @@
 import Metrics from '@overleaf/metrics'
+import Settings from '@overleaf/settings'
 import ProjectGetter from '../Project/ProjectGetter.mjs'
 import ProjectZipStreamManager from './ProjectZipStreamManager.mjs'
 import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
@@ -6,56 +7,130 @@ import { prepareZipAttachment } from '../../infrastructure/Response.mjs'
 import SessionManager from '../Authentication/SessionManager.mjs'
 import ProjectAuditLogHandler from '../Project/ProjectAuditLogHandler.mjs'
 import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs'
+import Validation from '../../infrastructure/Validation.mjs'
 import { expressify } from '@overleaf/promise-utils'
 import { pipeline } from 'node:stream/promises'
 
+const { z, zz, parseReq } = Validation
+
 const SUPPORTED_CONVERSION_TYPES = new Map([
   ['docx', 'docx'],
   ['markdown', 'zip'],
 ])
 
+const exportProjectConversionSchema = z.object({
+  params: z.object({
+    Project_id: zz.objectId(),
+    type: z.enum([...SUPPORTED_CONVERSION_TYPES.keys()]),
+  }),
+  query: z.object({
+    responseFormat: z.enum(['json', 'stream']).optional().default('stream'),
+  }),
+})
+
+const downloadPreparedProjectExportSchema = z.object({
+  params: z.object({
+    Project_id: zz.objectId(),
+    buildId: zz.buildId(),
+    conversionId: z.uuid(),
+    file: zz.filepath(),
+    type: z.enum([...SUPPORTED_CONVERSION_TYPES.keys()]),
+  }),
+  query: z.object({
+    clsiserverid: zz.clsiServerId().optional(),
+  }),
+})
+
 // Keep in sync with the logic for PDF files in CompileController
 function getSafeProjectName(project) {
   return project.name.replace(/[^\p{L}\p{Nd}]/gu, '_')
 }
 
-async function exportProjectConversion(req, res) {
-  const type = req.params.type
+async function _streamConvertedDocumentToResponse(
+  res,
+  { projectId, type, conversionId, buildId, clsiServerId, file }
+) {
   const extension = SUPPORTED_CONVERSION_TYPES.get(type)
-  if (!extension) {
-    return res.sendStatus(400)
-  }
-  const userId = SessionManager.getLoggedInUserId(req.session)
-  const projectId = req.params.Project_id
-  Metrics.inc('document-exports', 1, { type })
-
   const project = await ProjectGetter.promises.getProject(projectId, {
     name: true,
   })
-
   const safeFileName = getSafeProjectName(project)
 
   const { stream, contentLength } =
+    await DocumentConversionManager.promises.streamConvertedProjectDocument({
+      conversionId,
+      buildId,
+      clsiServerId,
+      file,
+    })
+  res.setHeader('Content-Length', contentLength)
+  res.attachment(`${safeFileName}.${extension}`)
+  res.setHeader('X-Content-Type-Options', 'nosniff')
+  res.setHeader('X-Accel-Buffering', 'no')
+  await pipeline(stream, res)
+}
+
+async function exportProjectConversion(req, res) {
+  const { params, query } = parseReq(req, exportProjectConversionSchema)
+  const { Project_id: projectId, type } = params
+  const { responseFormat } = query
+  const userId = SessionManager.getLoggedInUserId(req.session)
+  Metrics.inc('document-exports', 1, { type })
+
+  const { conversionId, buildId, clsiServerId, file } =
     await DocumentConversionManager.promises.convertProjectToDocument(
       projectId,
       userId,
       type
     )
-  res.setHeader('Content-Length', contentLength)
-  res.attachment(`${safeFileName}.${extension}`)
-  res.setHeader('X-Content-Type-Options', 'nosniff')
-  res.setHeader('X-Accel-Buffering', 'no')
   ProjectAuditLogHandler.addEntryInBackground(
     projectId,
     `project-exported-${type}`,
     userId,
     req.ip
   )
-  await pipeline(stream, res)
+
+  if (responseFormat === 'json') {
+    const downloadUrl = new URL(
+      `/project/${projectId}/download/conversion/${conversionId}/${type}/build/${buildId}/output/${file}`,
+      Settings.siteUrl
+    )
+    if (clsiServerId) {
+      downloadUrl.searchParams.set('clsiserverid', clsiServerId)
+    }
+    return res.json({
+      downloadUrl: downloadUrl.pathname + downloadUrl.search,
+    })
+  }
+
+  await _streamConvertedDocumentToResponse(res, {
+    projectId,
+    type,
+    conversionId,
+    buildId,
+    clsiServerId,
+    file,
+  })
+}
+
+async function downloadPreparedProjectExport(req, res) {
+  const { params, query } = parseReq(req, downloadPreparedProjectExportSchema)
+  const { Project_id: projectId, conversionId, buildId, file, type } = params
+  const { clsiserverid: clsiServerId } = query
+
+  await _streamConvertedDocumentToResponse(res, {
+    projectId,
+    type,
+    conversionId,
+    buildId,
+    clsiServerId,
+    file,
+  })
 }
 
 export default {
   exportProjectConversion: expressify(exportProjectConversion),
+  downloadPreparedProjectExport: expressify(downloadPreparedProjectExport),
 
   downloadProject(req, res, next) {
     const userId = SessionManager.getLoggedInUserId(req.session)

services/web/app/src/Features/Uploads/DocumentConversionManager.mjs

@@ -1,11 +1,15 @@
 import Settings from '@overleaf/settings'
 import CompileManager from '../Compile/CompileManager.mjs'
 import ClsiManager from '../Compile/ClsiManager.mjs'
+import { getOutputFileURL } from '../Compile/ClsiURLHelpers.mjs'
 import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
 import logger from '@overleaf/logger'
 import Path from 'node:path'
-import { fetchStreamWithResponse } from '@overleaf/fetch-utils'
+import {
+  fetchJsonWithResponse,
+  fetchStreamWithResponse,
+} from '@overleaf/fetch-utils'
 import { pipeline } from 'node:stream/promises'
 import OError from '@overleaf/o-error'
 import FormData from 'form-data'
@@ -85,6 +89,7 @@ async function convertProjectToDocument(projectId, userId, type) {
   const clsiUrl = new URL(Settings.apis.clsi.url)
   clsiUrl.pathname = `/project/${projectId}/user/${userId}/download/project-to-document`
   clsiUrl.searchParams.set('type', type)
+  clsiUrl.searchParams.set('responseFormat', 'json')
   clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass)
   clsiUrl.searchParams.set('compileGroup', limits.compileGroup)
 
@@ -93,11 +98,33 @@ async function convertProjectToDocument(projectId, userId, type) {
     'sending project to CLSI for document conversion'
   )
 
-  const { stream, response } = await fetchStreamWithResponse(clsiUrl, {
+  const { json, response } = await fetchJsonWithResponse(clsiUrl, {
     method: 'POST',
     json: clsiRequest,
   })
+  const { conversionId, buildId, file } = json
+  const clsiServerId = ClsiManager.CLSI_COOKIES_ENABLED
+    ? ClsiManager.getClsiServerIdFromResponse(response)
+    : undefined
 
+  return { conversionId, buildId, clsiServerId, file }
+}
+
+async function streamConvertedProjectDocument({
+  conversionId,
+  buildId,
+  clsiServerId,
+  file,
+}) {
+  const downloadUrl = getOutputFileURL(
+    conversionId,
+    null,
+    buildId,
+    file,
+    clsiServerId ?? undefined
+  )
+
+  const { stream, response } = await fetchStreamWithResponse(downloadUrl)
   const contentLength = parseInt(response.headers.get('Content-Length'), 10)
 
   return { stream, contentLength }
@@ -107,5 +134,6 @@ export default {
   promises: {
     convertDocumentToLaTeXZipArchive,
     convertProjectToDocument,
+    streamConvertedProjectDocument,
   },
 }

services/web/app/src/router.mjs

@@ -197,6 +197,10 @@ const rateLimiters = {
     points: 5,
     duration: 60,
   }),
+  documentExportDownload: new RateLimiter('document-export-download', {
+    points: 30,
+    duration: 60,
+  }),
 }
 
 async function initialize(webRouter, privateApiRouter, publicApiRouter) {
@@ -759,6 +763,15 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) {
       AuthorizationMiddleware.ensureUserCanReadProject,
       ProjectDownloadsController.exportProjectConversion
     )
+    webRouter.get(
+      '/project/:Project_id/download/conversion/:conversionId/:type/build/:buildId/output/:file(.*)',
+      AuthenticationController.requireLogin(),
+      RateLimiterMiddleware.rateLimit(rateLimiters.documentExportDownload, {
+        params: ['Project_id'],
+      }),
+      AuthorizationMiddleware.ensureUserCanReadProject,
+      ProjectDownloadsController.downloadPreparedProjectExport
+    )
   }
 
   webRouter.get(