[WEB + CLSI] Download as docx file feature (#32851)
* using CLSI logic for fetching the project contents and skip the .zip export * Use unique conversion directory for project-to-docx export to avoid corrupting the shared compile directory when a compile runs concurrently * Remove X-Accel-Buffering header — not needed as CLSI does not run behind nginx * moving log before sending the data * Return CLSI stream directly instead of buffering to disk on web Previously convertProjectToDocx wrote the CLSI response to a temp file on disk, then the controller read it back to stream to the client. Now the stream is returned directly and piped to the response, avoiding unnecessary disk I/O on the web server. * Use href redirect for docx export instead of fetching blob into memory * making functions and files more generic so they can be used in future for other documents exports as well * adding export-docx split test * adding unit tests * adding cypress E2E test * format:fix * renaming the route to download from convert * adding new icon for export docx button * format:fix * remove unused showExportDocumentErrorToast export and adding guard against invalid Content-Length header from CLSI * format:fix * refactor(clsi): move promisify(parse) into RequestParser * refactor: generic conversion endpoint with type as route param * refactor: use type→extension map for validated conversion types * refactor(clsi): remove --standalone flag and fix rejection test * fixing the href in cypress test * renaming function * adding type to Metrics.inc * fix: rename exportProjectDocument, add WithLock wrapper and metrics type label * format:fix * fix: hide docx export from anonymous users and add WithLock wrapper * format fix * remove redundant Content-Length validation from DocumentConversionManager * format:fix * removing trailing icon GitOrigin-RevId: e9764fefac2c4b625d23be9e942ea4a8b283c70d
This commit is contained in:
Davinder Singh
@@ -130,6 +130,11 @@ app.post(
|
||||
FileUploadMiddleware.multerMiddleware,
|
||||
ConversionController.convertDocxToLaTeX
|
||||
)
|
||||
app.post(
|
||||
'/project/:project_id/user/:user_id/download/project-to-document',
|
||||
bodyParser.json({ limit: Settings.compileSizeLimit }),
|
||||
ConversionController.convertProjectToDocument
|
||||
)
|
||||
|
||||
if (process.env.NODE_ENV === 'development' && global.__coverage__) {
|
||||
app.get('/coverage', (req, res) => {
|
||||
|
||||
@@ -3,10 +3,14 @@ import { expressify } from '@overleaf/promise-utils'
|
||||
import fs from 'node:fs/promises'
|
||||
import fsSync from 'node:fs'
|
||||
import ConversionManager from './ConversionManager.js'
|
||||
import ResourceWriter from './ResourceWriter.js'
|
||||
import RequestParser from './RequestParser.js'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import Settings from '@overleaf/settings'
|
||||
import Path from 'node:path'
|
||||
|
||||
const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']])
|
||||
|
||||
async function convertDocxToLaTeX(req, res) {
|
||||
const { path } = req.file
|
||||
if (!Settings.enablePandocConversions) {
|
||||
@@ -41,6 +45,59 @@ async function convertDocxToLaTeX(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
async function convertProjectToDocument(req, res) {
|
||||
if (!Settings.enablePandocConversions) {
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
|
||||
const type = req.query.type
|
||||
const extension = SUPPORTED_CONVERSION_TYPES.get(type)
|
||||
if (!extension) {
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
|
||||
const request = await RequestParser.promises.parse(req.body)
|
||||
request.project_id = req.params.project_id
|
||||
request.user_id = req.params.user_id
|
||||
request.metricsOpts = {}
|
||||
|
||||
const conversionId = crypto.randomUUID()
|
||||
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
projectId: request.project_id,
|
||||
userId: request.user_id,
|
||||
rootResourcePath: request.rootResourcePath,
|
||||
type,
|
||||
},
|
||||
'syncing resources for project-to-document conversion'
|
||||
)
|
||||
|
||||
try {
|
||||
await ResourceWriter.promises.syncResourcesToDisk(request, conversionDir)
|
||||
|
||||
const documentPath =
|
||||
await ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
|
||||
conversionId,
|
||||
conversionDir,
|
||||
request.rootResourcePath,
|
||||
type,
|
||||
extension
|
||||
)
|
||||
|
||||
const documentStat = await fs.stat(documentPath)
|
||||
res.setHeader('Content-Length', documentStat.size)
|
||||
res.attachment(`output.${extension}`)
|
||||
res.setHeader('X-Content-Type-Options', 'nosniff')
|
||||
const readStream = fsSync.createReadStream(documentPath)
|
||||
await pipeline(readStream, res)
|
||||
} finally {
|
||||
await fs.rm(conversionDir, { recursive: true, force: true }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
convertDocxToLaTeX: expressify(convertDocxToLaTeX),
|
||||
convertProjectToDocument: expressify(convertProjectToDocument),
|
||||
}
|
||||
|
||||
@@ -94,8 +94,77 @@ async function convertDocxToLaTeX(conversionId, conversionDir, inputPath) {
|
||||
return Path.join(conversionDir, outputName)
|
||||
}
|
||||
|
||||
async function convertLaTeXToDocumentInDirWithLock(
|
||||
conversionId,
|
||||
compileDir,
|
||||
rootDocPath,
|
||||
type,
|
||||
extension
|
||||
) {
|
||||
const lock = LockManager.acquire(compileDir)
|
||||
try {
|
||||
return await convertLaTeXToDocumentInDir(
|
||||
conversionId,
|
||||
compileDir,
|
||||
rootDocPath,
|
||||
type,
|
||||
extension
|
||||
)
|
||||
} finally {
|
||||
lock.release()
|
||||
}
|
||||
}
|
||||
|
||||
async function convertLaTeXToDocumentInDir(
|
||||
conversionId,
|
||||
compileDir,
|
||||
rootDocPath = 'main.tex',
|
||||
type,
|
||||
extension
|
||||
) {
|
||||
const outputName = crypto.randomUUID() + '.' + extension
|
||||
const timeoutMs = Settings.conversionTimeoutSeconds * 1000
|
||||
|
||||
logger.debug(
|
||||
{ compileDir, rootDocPath, type },
|
||||
'running pandoc latex-to-document in compile dir'
|
||||
)
|
||||
|
||||
const { exitCode, stdout, stderr } = await CommandRunner.promises.run(
|
||||
conversionId,
|
||||
[
|
||||
'pandoc',
|
||||
rootDocPath,
|
||||
'--output',
|
||||
outputName,
|
||||
'--from',
|
||||
'latex',
|
||||
'--to',
|
||||
type,
|
||||
'--resource-path=.',
|
||||
],
|
||||
compileDir,
|
||||
Settings.pandocImage,
|
||||
timeoutMs,
|
||||
{},
|
||||
'conversions'
|
||||
)
|
||||
|
||||
if (exitCode !== 0) {
|
||||
throw new OError('pandoc latex-to-document conversion failed', {
|
||||
type,
|
||||
exitCode,
|
||||
stdout,
|
||||
stderr,
|
||||
})
|
||||
}
|
||||
|
||||
return Path.join(compileDir, outputName)
|
||||
}
|
||||
|
||||
export default {
|
||||
promises: {
|
||||
convertDocxToLaTeXWithLock,
|
||||
convertLaTeXToDocumentInDirWithLock,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { promisify } from 'node:util'
|
||||
import settings from '@overleaf/settings'
|
||||
import OutputCacheManager from './OutputCacheManager.js'
|
||||
|
||||
@@ -285,4 +286,4 @@ function _checkPath(path) {
|
||||
return path
|
||||
}
|
||||
|
||||
export default { parse, MAX_TIMEOUT }
|
||||
export default { parse, MAX_TIMEOUT, promises: { parse: promisify(parse) } }
|
||||
|
||||
@@ -13,12 +13,29 @@ describe('ConversionController', function () {
|
||||
ctx.conversionDir = '/path/to/conversion/result'
|
||||
ctx.zipPath = '/path/to/conversion/result/output.zip'
|
||||
ctx.zipStat = { size: 1234 }
|
||||
ctx.documentPath = '/compiles/output-uuid/output-uuid.docx'
|
||||
ctx.documentStat = { size: 5678 }
|
||||
ctx.Settings = {
|
||||
enablePandocConversions: true,
|
||||
path: { compilesDir: '/compiles' },
|
||||
}
|
||||
ctx.parsedRequest = { rootResourcePath: 'main.tex' }
|
||||
ctx.ConversionManager = {
|
||||
promises: {
|
||||
convertDocxToLaTeXWithLock: sinon.stub().resolves(ctx.zipPath),
|
||||
convertLaTeXToDocumentInDirWithLock: sinon
|
||||
.stub()
|
||||
.resolves(ctx.documentPath),
|
||||
},
|
||||
}
|
||||
ctx.ResourceWriter = {
|
||||
promises: {
|
||||
syncResourcesToDisk: sinon.stub().resolves(),
|
||||
},
|
||||
}
|
||||
ctx.RequestParser = {
|
||||
promises: {
|
||||
parse: sinon.stub().resolves(ctx.parsedRequest),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -54,6 +71,14 @@ describe('ConversionController', function () {
|
||||
default: ctx.ConversionManager,
|
||||
}))
|
||||
|
||||
vi.doMock('../../../app/js/ResourceWriter', () => ({
|
||||
default: ctx.ResourceWriter,
|
||||
}))
|
||||
|
||||
vi.doMock('../../../app/js/RequestParser', () => ({
|
||||
default: ctx.RequestParser,
|
||||
}))
|
||||
|
||||
ctx.res = new PassThrough()
|
||||
ctx.res.attachment = sinon.stub()
|
||||
ctx.res.setHeader = sinon.stub()
|
||||
@@ -155,4 +180,156 @@ describe('ConversionController', function () {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('convertProjectToDocument', function () {
|
||||
beforeEach(function (ctx) {
|
||||
ctx.req = {
|
||||
body: {},
|
||||
params: { project_id: 'test-project-id', user_id: 'test-user-id' },
|
||||
query: { type: 'docx' },
|
||||
}
|
||||
ctx.fs.stat.resolves(ctx.documentStat)
|
||||
})
|
||||
|
||||
describe('when conversions are disabled', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.Settings.enablePandocConversions = false
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
sinon.stub()
|
||||
)
|
||||
})
|
||||
|
||||
it('should return 404', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.sendStatus, 404)
|
||||
})
|
||||
|
||||
it('should not sync resources or call the conversion manager', function (ctx) {
|
||||
sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk)
|
||||
sinon.assert.notCalled(
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when an unsupported type is requested', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.query = { type: 'unsupported' }
|
||||
ctx.res.sendStatus = sinon.stub()
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
sinon.stub()
|
||||
)
|
||||
})
|
||||
|
||||
it('should return 400', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.sendStatus, 400)
|
||||
})
|
||||
|
||||
it('should not sync resources or call the conversion manager', function (ctx) {
|
||||
sinon.assert.notCalled(ctx.ResourceWriter.promises.syncResourcesToDisk)
|
||||
sinon.assert.notCalled(
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
const uuidDirPattern =
|
||||
/^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
|
||||
|
||||
describe('successfully', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
sinon.stub()
|
||||
)
|
||||
})
|
||||
|
||||
it('should sync resources to a unique conversion directory', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.ResourceWriter.promises.syncResourcesToDisk,
|
||||
sinon.match({ rootResourcePath: 'main.tex' }),
|
||||
sinon.match(uuidDirPattern)
|
||||
)
|
||||
})
|
||||
|
||||
it('should call convertLaTeXToDocumentInDirWithLock with docx type and extension', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
|
||||
sinon.match(
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
|
||||
),
|
||||
sinon.match(uuidDirPattern),
|
||||
'main.tex',
|
||||
'docx',
|
||||
'docx'
|
||||
)
|
||||
})
|
||||
|
||||
it('should set the Content-Length header from the document stat', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.res.setHeader,
|
||||
'Content-Length',
|
||||
ctx.documentStat.size
|
||||
)
|
||||
})
|
||||
|
||||
it('should set the attachment filename', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.attachment, 'output.docx')
|
||||
})
|
||||
|
||||
it('should set X-Content-Type-Options header', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.res.setHeader,
|
||||
'X-Content-Type-Options',
|
||||
'nosniff'
|
||||
)
|
||||
})
|
||||
|
||||
it('should stream the document to the response', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.fsSync.createReadStream, ctx.documentPath)
|
||||
sinon.assert.calledWith(ctx.pipeline, ctx.readStream, ctx.res)
|
||||
})
|
||||
|
||||
it('should clean up the conversion directory', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when conversion fails', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.next = sinon.stub()
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock.rejects(
|
||||
new Error('mock conversion error')
|
||||
)
|
||||
|
||||
await ctx.ConversionController.convertProjectToDocument(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
ctx.next
|
||||
)
|
||||
})
|
||||
|
||||
it('should pass the error to next', function (ctx) {
|
||||
sinon.assert.calledOnce(ctx.next)
|
||||
expect(ctx.next.firstCall.args[0]).to.be.instanceOf(Error)
|
||||
})
|
||||
|
||||
it('should still clean up the conversion directory', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.fs.rm, sinon.match(uuidDirPattern), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -250,4 +250,89 @@ describe('ConversionManager', function () {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('convertLaTeXToDocumentInDirWithLock', function () {
|
||||
describe('successfully', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.compileDir = '/compiles/test-compile-dir'
|
||||
ctx.rootDocPath = 'main.tex'
|
||||
ctx.type = 'docx'
|
||||
ctx.extension = 'docx'
|
||||
|
||||
ctx.result =
|
||||
await ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
|
||||
ctx.conversionId,
|
||||
ctx.compileDir,
|
||||
ctx.rootDocPath,
|
||||
ctx.type,
|
||||
ctx.extension
|
||||
)
|
||||
})
|
||||
|
||||
it('should acquire a lock on the compile dir', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.LockManager.acquire, ctx.compileDir)
|
||||
})
|
||||
|
||||
it('should release the lock', function (ctx) {
|
||||
sinon.assert.called(ctx.lock.release)
|
||||
})
|
||||
|
||||
it('should run pandoc with correct arguments', function (ctx) {
|
||||
expect(ctx.CommandRunner.promises.run.callCount).toBe(1)
|
||||
expect(ctx.CommandRunner.promises.run.firstCall.args).toEqual([
|
||||
ctx.conversionId,
|
||||
[
|
||||
'pandoc',
|
||||
ctx.rootDocPath,
|
||||
'--output',
|
||||
`output-uuid.${ctx.extension}`,
|
||||
'--from',
|
||||
'latex',
|
||||
'--to',
|
||||
ctx.type,
|
||||
'--resource-path=.',
|
||||
],
|
||||
ctx.compileDir,
|
||||
ctx.Settings.pandocImage,
|
||||
60_000,
|
||||
{},
|
||||
'conversions',
|
||||
])
|
||||
})
|
||||
|
||||
it('should convert conversion timeout to milliseconds', function (ctx) {
|
||||
expect(ctx.CommandRunner.promises.run.firstCall.args[4]).toBe(60_000)
|
||||
})
|
||||
|
||||
it('should return path to the output document', function (ctx) {
|
||||
expect(ctx.result).toBe(
|
||||
Path.join(ctx.compileDir, `output-uuid.${ctx.extension}`)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when pandoc fails (non-zero exit code)', function () {
|
||||
it('should reject with an error and release the lock', async function (ctx) {
|
||||
ctx.compileDir = '/compiles/test-compile-dir'
|
||||
|
||||
ctx.CommandRunner.promises.run.resolves({
|
||||
stdout: 'mock-stdout',
|
||||
stderr: 'mock-stderr',
|
||||
exitCode: 1,
|
||||
})
|
||||
|
||||
await expect(
|
||||
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock(
|
||||
ctx.conversionId,
|
||||
ctx.compileDir,
|
||||
'main.tex',
|
||||
'docx',
|
||||
'docx'
|
||||
)
|
||||
).to.be.rejectedWith('pandoc latex-to-document conversion failed')
|
||||
|
||||
sinon.assert.called(ctx.lock.release)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1171,6 +1171,21 @@ function _finaliseRequest(projectId, options, project, docs, files) {
|
||||
}
|
||||
}
|
||||
|
||||
async function buildDocumentConversionRequest(projectId) {
|
||||
const project = await ProjectGetter.promises.getProject(projectId, {
|
||||
compiler: 1,
|
||||
imageName: 1,
|
||||
'overleaf.history.id': 1,
|
||||
rootDoc_id: 1,
|
||||
rootFolder: 1,
|
||||
})
|
||||
if (project == null) {
|
||||
throw new Errors.NotFoundError(`project does not exist: ${projectId}`)
|
||||
}
|
||||
const projectStateHash = ClsiStateManager.computeHash(project, {})
|
||||
return _buildRequestFromMongo(projectId, {}, project, projectStateHash)
|
||||
}
|
||||
|
||||
async function wordCount(projectId, userId, file, limits, clsiserverid) {
|
||||
const { compileBackendClass, compileGroup } = limits
|
||||
const req = await _buildRequest(projectId, userId, limits)
|
||||
@@ -1297,5 +1312,6 @@ export default {
|
||||
getOutputFileStream,
|
||||
wordCount,
|
||||
syncTeX,
|
||||
buildDocumentConversionRequest,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -5,13 +5,55 @@ import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mj
|
||||
import { prepareZipAttachment } from '../../infrastructure/Response.mjs'
|
||||
import SessionManager from '../Authentication/SessionManager.mjs'
|
||||
import ProjectAuditLogHandler from '../Project/ProjectAuditLogHandler.mjs'
|
||||
import DocumentConversionManager from '../Uploads/DocumentConversionManager.mjs'
|
||||
import { expressify } from '@overleaf/promise-utils'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
|
||||
const SUPPORTED_CONVERSION_TYPES = new Map([['docx', 'docx']])
|
||||
|
||||
// Keep in sync with the logic for PDF files in CompileController
|
||||
function getSafeProjectName(project) {
|
||||
return project.name.replace(/[^\p{L}\p{Nd}]/gu, '_')
|
||||
}
|
||||
|
||||
async function exportProjectConversion(req, res) {
|
||||
const type = req.params.type
|
||||
const extension = SUPPORTED_CONVERSION_TYPES.get(type)
|
||||
if (!extension) {
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
const userId = SessionManager.getLoggedInUserId(req.session)
|
||||
const projectId = req.params.Project_id
|
||||
Metrics.inc('document-exports', 1, { type })
|
||||
|
||||
const project = await ProjectGetter.promises.getProject(projectId, {
|
||||
name: true,
|
||||
})
|
||||
|
||||
const { stream, contentLength } =
|
||||
await DocumentConversionManager.promises.convertProjectToDocument(
|
||||
projectId,
|
||||
userId,
|
||||
type
|
||||
)
|
||||
|
||||
const safeFileName = getSafeProjectName(project)
|
||||
res.setHeader('Content-Length', contentLength)
|
||||
res.attachment(`${safeFileName}.${extension}`)
|
||||
res.setHeader('X-Content-Type-Options', 'nosniff')
|
||||
res.setHeader('X-Accel-Buffering', 'no')
|
||||
ProjectAuditLogHandler.addEntryInBackground(
|
||||
projectId,
|
||||
`project-exported-${type}`,
|
||||
userId,
|
||||
req.ip
|
||||
)
|
||||
await pipeline(stream, res)
|
||||
}
|
||||
|
||||
export default {
|
||||
exportProjectConversion: expressify(exportProjectConversion),
|
||||
|
||||
downloadProject(req, res, next) {
|
||||
const userId = SessionManager.getLoggedInUserId(req.session)
|
||||
const projectId = req.params.Project_id
|
||||
|
||||
@@ -480,6 +480,7 @@ const _ProjectController = {
|
||||
'wf-fake-non-english-suggestions',
|
||||
'editor-tabs',
|
||||
'overleaf-code',
|
||||
'export-docx',
|
||||
].filter(Boolean)
|
||||
|
||||
const getUserValues = async userId =>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import Settings from '@overleaf/settings'
|
||||
import CompileManager from '../Compile/CompileManager.mjs'
|
||||
import ClsiManager from '../Compile/ClsiManager.mjs'
|
||||
import fs from 'node:fs'
|
||||
import fsPromises from 'node:fs/promises'
|
||||
import logger from '@overleaf/logger'
|
||||
@@ -38,14 +39,7 @@ async function convertDocxToLaTeXZipArchive(path, userId) {
|
||||
signal: abortController.signal,
|
||||
})
|
||||
|
||||
const contentLengthHeader = response.headers.get('Content-Length')
|
||||
if (contentLengthHeader == null) {
|
||||
logger.warn(
|
||||
'CLSI did not provide Content-Length header for converted document'
|
||||
)
|
||||
throw new OError('CLSI response missing Content-Length header')
|
||||
}
|
||||
const contentLength = parseInt(contentLengthHeader, 10)
|
||||
const contentLength = parseInt(response.headers.get('Content-Length'), 10)
|
||||
if (contentLength > Settings.maxUploadSize) {
|
||||
abortController.abort()
|
||||
stream.destroy()
|
||||
@@ -77,8 +71,35 @@ async function convertDocxToLaTeXZipArchive(path, userId) {
|
||||
return outputPath
|
||||
}
|
||||
|
||||
async function convertProjectToDocument(projectId, userId, type) {
|
||||
const limits = await CompileManager.promises._getUserCompileLimits(userId)
|
||||
const clsiRequest =
|
||||
await ClsiManager.promises.buildDocumentConversionRequest(projectId)
|
||||
|
||||
const clsiUrl = new URL(Settings.apis.clsi.url)
|
||||
clsiUrl.pathname = `/project/${projectId}/user/${userId}/download/project-to-document`
|
||||
clsiUrl.searchParams.set('type', type)
|
||||
clsiUrl.searchParams.set('compileBackendClass', limits.compileBackendClass)
|
||||
clsiUrl.searchParams.set('compileGroup', limits.compileGroup)
|
||||
|
||||
logger.debug(
|
||||
{ clsiUrl: clsiUrl.toString(), projectId, userId, type },
|
||||
'sending project to CLSI for document conversion'
|
||||
)
|
||||
|
||||
const { stream, response } = await fetchStreamWithResponse(clsiUrl, {
|
||||
method: 'POST',
|
||||
json: clsiRequest,
|
||||
})
|
||||
|
||||
const contentLength = parseInt(response.headers.get('Content-Length'), 10)
|
||||
|
||||
return { stream, contentLength }
|
||||
}
|
||||
|
||||
export default {
|
||||
promises: {
|
||||
convertDocxToLaTeXZipArchive,
|
||||
convertProjectToDocument,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -193,6 +193,10 @@ const rateLimiters = {
|
||||
points: 10,
|
||||
duration: 60,
|
||||
}),
|
||||
documentExport: new RateLimiter('document-export', {
|
||||
points: 5,
|
||||
duration: 60,
|
||||
}),
|
||||
}
|
||||
|
||||
async function initialize(webRouter, privateApiRouter, publicApiRouter) {
|
||||
@@ -764,6 +768,18 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) {
|
||||
ExportsController.exportDownload
|
||||
)
|
||||
|
||||
if (Settings.enablePandocConversions) {
|
||||
webRouter.get(
|
||||
'/project/:Project_id/download/conversion/:type',
|
||||
AuthenticationController.requireLogin(),
|
||||
RateLimiterMiddleware.rateLimit(rateLimiters.documentExport, {
|
||||
params: ['Project_id'],
|
||||
}),
|
||||
AuthorizationMiddleware.ensureUserCanReadProject,
|
||||
ProjectDownloadsController.exportProjectConversion
|
||||
)
|
||||
}
|
||||
|
||||
webRouter.get(
|
||||
'/Project/:Project_id/download/zip',
|
||||
RateLimiterMiddleware.rateLimit(rateLimiters.zipDownload, {
|
||||
|
||||
@@ -618,7 +618,9 @@
|
||||
"expires": "",
|
||||
"expires_in_days": "",
|
||||
"expires_on": "",
|
||||
"export_as_docx": "",
|
||||
"export_csv": "",
|
||||
"export_document_error": "",
|
||||
"export_project_to_github": "",
|
||||
"failed": "",
|
||||
"failed_to_consent_to_workbench_terms": "",
|
||||
|
||||
@@ -7,6 +7,7 @@ import importOverleafModules from '../../../../macros/import-overleaf-module.mac
|
||||
import { OLToastContainer } from '@/shared/components/ol/ol-toast-container'
|
||||
import clipboardToastGenerators from '@/features/source-editor/components/clipboard-toasts'
|
||||
import importDocxFeedbackToastGenerators from '@/features/project-list/components/new-project-button/import-docx-feedback-toast'
|
||||
import exportDocumentToastGenerators from '@/features/ide-react/components/toolbar/export-document-toasts'
|
||||
|
||||
const moduleGeneratorsImport = importOverleafModules('toastGenerators') as {
|
||||
import: { default: GlobalToastGeneratorEntry[] }
|
||||
@@ -29,6 +30,7 @@ const GENERATOR_LIST: GlobalToastGeneratorEntry[] = [
|
||||
...moduleGenerators.flat(),
|
||||
...clipboardToastGenerators,
|
||||
...importDocxFeedbackToastGenerators,
|
||||
...exportDocumentToastGenerators,
|
||||
]
|
||||
const GENERATOR_MAP: Map<string, GlobalToastGenerator> = new Map(
|
||||
GENERATOR_LIST.map(({ key, generator }) => [key, generator])
|
||||
|
||||
@@ -7,6 +7,8 @@ import { useProjectContext } from '@/shared/context/project-context'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useEditorAnalytics } from '@/shared/hooks/use-editor-analytics'
|
||||
import getMeta from '@/utils/meta'
|
||||
import { useFeatureFlag } from '@/shared/context/split-test-context'
|
||||
|
||||
export const DownloadProjectZip = () => {
|
||||
const { t } = useTranslation()
|
||||
@@ -100,3 +102,43 @@ export const DownloadProjectPDF = () => {
|
||||
return button
|
||||
}
|
||||
}
|
||||
|
||||
export const ExportProjectDocx = () => {
|
||||
const { t } = useTranslation()
|
||||
const { projectId } = useProjectContext()
|
||||
const exportDocxEnabled = useFeatureFlag('export-docx')
|
||||
const enablePandocConversions =
|
||||
getMeta('ol-ExposedSettings')?.enablePandocConversions
|
||||
const anonymous = getMeta('ol-anonymous')
|
||||
|
||||
const showExportDocx =
|
||||
exportDocxEnabled && enablePandocConversions && !anonymous
|
||||
|
||||
useCommandProvider(
|
||||
() =>
|
||||
showExportDocx
|
||||
? [
|
||||
{
|
||||
id: 'export-as-docx',
|
||||
href: `/project/${projectId}/download/conversion/docx`,
|
||||
label: t('export_as_docx'),
|
||||
},
|
||||
]
|
||||
: [],
|
||||
[t, showExportDocx, projectId]
|
||||
)
|
||||
|
||||
if (!showExportDocx) {
|
||||
return null
|
||||
}
|
||||
|
||||
return (
|
||||
<OLDropdownMenuItem
|
||||
href={`/project/${projectId}/download/conversion/docx`}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
{t('export_as_docx')}
|
||||
</OLDropdownMenuItem>
|
||||
)
|
||||
}
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import { GlobalToastGeneratorEntry } from '@/features/ide-react/components/global-toasts'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
const ExportDocumentErrorToast = () => {
|
||||
const { t } = useTranslation()
|
||||
return <span>{t('export_document_error')}</span>
|
||||
}
|
||||
|
||||
const generators: GlobalToastGeneratorEntry[] = [
|
||||
{
|
||||
key: 'export-document:error',
|
||||
generator: () => ({
|
||||
content: <ExportDocumentErrorToast />,
|
||||
type: 'error',
|
||||
autoHide: true,
|
||||
delay: 5000,
|
||||
isDismissible: true,
|
||||
}),
|
||||
},
|
||||
]
|
||||
|
||||
export default generators
|
||||
@@ -87,7 +87,7 @@ export const ToolbarMenuBar = () => {
|
||||
{ id: 'submit', children: ['submit-project', 'manage-template'] },
|
||||
{
|
||||
id: 'file-download',
|
||||
children: ['download-as-source-zip', 'download-pdf'],
|
||||
children: ['download-as-source-zip', 'download-pdf', 'export-as-docx'],
|
||||
},
|
||||
{
|
||||
id: 'settings',
|
||||
|
||||
@@ -10,7 +10,11 @@ import { useTranslation } from 'react-i18next'
|
||||
import importOverleafModules from '../../../../../macros/import-overleaf-module.macro'
|
||||
import { useEditorContext } from '@/shared/context/editor-context'
|
||||
import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
|
||||
import { DownloadProjectPDF, DownloadProjectZip } from './download-project'
|
||||
import {
|
||||
DownloadProjectPDF,
|
||||
DownloadProjectZip,
|
||||
ExportProjectDocx,
|
||||
} from './download-project'
|
||||
import { useCallback, useState } from 'react'
|
||||
import OLDropdownMenuItem from '@/shared/components/ol/ol-dropdown-menu-item'
|
||||
import EditableLabel from './editable-label'
|
||||
@@ -76,6 +80,7 @@ export const ToolbarProjectTitle = () => {
|
||||
)}
|
||||
<DownloadProjectPDF />
|
||||
<DownloadProjectZip />
|
||||
<ExportProjectDocx />
|
||||
<DropdownDivider />
|
||||
<DuplicateProject />
|
||||
<OLDropdownMenuItem
|
||||
|
||||
@@ -822,7 +822,9 @@
|
||||
"expires_on": "Expires: __date__",
|
||||
"expiry": "Expiry Date",
|
||||
"explore_all_plans": "Explore all plans",
|
||||
"export_as_docx": "Export as Word document (.docx)",
|
||||
"export_csv": "Export CSV",
|
||||
"export_document_error": "Export failed. Please try again.",
|
||||
"export_project_to_github": "Export Project to GitHub",
|
||||
"failed": "Failed",
|
||||
"failed_to_consent_to_workbench_terms": "Failed to consent to workbench terms. Please try again later.",
|
||||
|
||||
@@ -28,7 +28,11 @@ describe('ProjectDownloadsController', function () {
|
||||
)
|
||||
|
||||
vi.doMock('../../../../app/src/Features/Project/ProjectGetter.mjs', () => ({
|
||||
default: (ctx.ProjectGetter = {}),
|
||||
default: (ctx.ProjectGetter = {
|
||||
promises: {
|
||||
getProject: sinon.stub(),
|
||||
},
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.doMock(
|
||||
@@ -47,6 +51,32 @@ describe('ProjectDownloadsController', function () {
|
||||
})
|
||||
)
|
||||
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Authentication/SessionManager.mjs',
|
||||
() => ({
|
||||
default: (ctx.SessionManager = {
|
||||
getLoggedInUserId: sinon
|
||||
.stub()
|
||||
.callsFake(session => session?.user?._id),
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
||||
vi.doMock(
|
||||
'../../../../app/src/Features/Uploads/DocumentConversionManager.mjs',
|
||||
() => ({
|
||||
default: (ctx.DocumentConversionManager = {
|
||||
promises: {
|
||||
convertProjectToDocument: sinon.stub(),
|
||||
},
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
||||
vi.doMock('node:stream/promises', () => ({
|
||||
pipeline: (ctx.pipeline = sinon.stub().resolves()),
|
||||
}))
|
||||
|
||||
ctx.ProjectDownloadsController = (await import(modulePath)).default
|
||||
})
|
||||
|
||||
@@ -208,4 +238,107 @@ describe('ProjectDownloadsController', function () {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('exportProjectConversion', function () {
|
||||
describe('when an unsupported type is requested', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.req.params = { Project_id: 'test-project-id', type: 'unsupported' }
|
||||
ctx.req.session = { user: { _id: 'test-user-id' } }
|
||||
|
||||
await ctx.ProjectDownloadsController.exportProjectConversion(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
ctx.next
|
||||
)
|
||||
})
|
||||
|
||||
it('should return 400', function (ctx) {
|
||||
expect(ctx.res.statusCode).to.equal(400)
|
||||
})
|
||||
|
||||
it('should not call the conversion manager', function (ctx) {
|
||||
sinon.assert.notCalled(
|
||||
ctx.DocumentConversionManager.promises.convertProjectToDocument
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a supported type', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.projectId = 'test-project-id'
|
||||
ctx.userId = 'test-user-id'
|
||||
ctx.projectName = 'My Test Project'
|
||||
ctx.exportStream = { pipe: sinon.stub() }
|
||||
ctx.contentLength = 9876
|
||||
|
||||
ctx.req.params = { Project_id: ctx.projectId, type: 'docx' }
|
||||
ctx.req.session = { user: { _id: ctx.userId } }
|
||||
ctx.req.ip = '192.168.1.1'
|
||||
|
||||
ctx.res.attachment = sinon.stub().returns(ctx.res)
|
||||
|
||||
ctx.SessionManager.getLoggedInUserId.returns(ctx.userId)
|
||||
ctx.ProjectGetter.promises.getProject.resolves({
|
||||
name: ctx.projectName,
|
||||
})
|
||||
ctx.DocumentConversionManager.promises.convertProjectToDocument.resolves(
|
||||
{
|
||||
stream: ctx.exportStream,
|
||||
contentLength: ctx.contentLength,
|
||||
}
|
||||
)
|
||||
|
||||
await ctx.ProjectDownloadsController.exportProjectConversion(
|
||||
ctx.req,
|
||||
ctx.res,
|
||||
ctx.next
|
||||
)
|
||||
})
|
||||
|
||||
it('should call convertProjectToDocument with the docx type', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.DocumentConversionManager.promises.convertProjectToDocument,
|
||||
ctx.projectId,
|
||||
ctx.userId,
|
||||
'docx'
|
||||
)
|
||||
})
|
||||
|
||||
it('should set the Content-Length header', function (ctx) {
|
||||
expect(ctx.res.headers['Content-Length']).to.equal(ctx.contentLength)
|
||||
})
|
||||
|
||||
it('should set the attachment filename with safe project name', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.res.attachment, 'My_Test_Project.docx')
|
||||
})
|
||||
|
||||
it('should set the X-Content-Type-Options header', function (ctx) {
|
||||
expect(ctx.res.headers['X-Content-Type-Options']).to.equal('nosniff')
|
||||
})
|
||||
|
||||
it('should set the X-Accel-Buffering header', function (ctx) {
|
||||
expect(ctx.res.headers['X-Accel-Buffering']).to.equal('no')
|
||||
})
|
||||
|
||||
it('should add an audit log entry', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.ProjectAuditLogHandler.addEntryInBackground,
|
||||
ctx.projectId,
|
||||
'project-exported-docx',
|
||||
ctx.userId,
|
||||
ctx.req.ip
|
||||
)
|
||||
})
|
||||
|
||||
it('should record the action via Metrics', function (ctx) {
|
||||
ctx.Metrics.inc
|
||||
.calledWith('document-exports', 1, { type: 'docx' })
|
||||
.should.equal(true)
|
||||
})
|
||||
|
||||
it('should stream the document to the response', function (ctx) {
|
||||
sinon.assert.calledWith(ctx.pipeline, ctx.exportStream, ctx.res)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -73,6 +73,18 @@ describe('DocumentConversionManager', function () {
|
||||
})
|
||||
)
|
||||
|
||||
ctx.ClsiManager = {
|
||||
promises: {
|
||||
buildDocumentConversionRequest: sinon
|
||||
.stub()
|
||||
.resolves({ some: 'clsi-request' }),
|
||||
},
|
||||
}
|
||||
|
||||
vi.doMock('../../../../app/src/Features/Compile/ClsiManager.mjs', () => ({
|
||||
default: ctx.ClsiManager,
|
||||
}))
|
||||
|
||||
ctx.DocumentConversionManager = (await import(MODULE_PATH)).default
|
||||
})
|
||||
|
||||
@@ -215,43 +227,64 @@ describe('DocumentConversionManager', function () {
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the Content-Length header is missing', function () {
|
||||
describe('convertProjectToDocument', function () {
|
||||
beforeEach(function (ctx) {
|
||||
ctx.projectId = 'test-project-id'
|
||||
ctx.userId = 'test-user-id'
|
||||
ctx.type = 'docx'
|
||||
ctx.mockStream = { destroy: sinon.stub() }
|
||||
ctx.response = {
|
||||
headers: { get: sinon.stub().returns(null) },
|
||||
}
|
||||
ctx.response.headers.get.withArgs('Content-Length').returns('50')
|
||||
ctx.fetchUtils.fetchStreamWithResponse.resolves({
|
||||
stream: ctx.mockStream,
|
||||
response: ctx.response,
|
||||
})
|
||||
})
|
||||
|
||||
describe('successfully', function () {
|
||||
beforeEach(async function (ctx) {
|
||||
ctx.path = '/path/to/input.docx'
|
||||
ctx.userId = 'test-user-id'
|
||||
ctx.response = {
|
||||
headers: {
|
||||
get: sinon.stub().returns(null),
|
||||
},
|
||||
}
|
||||
|
||||
ctx.fetchUtils.fetchStreamWithResponse.resolves({
|
||||
stream: 'mocked-fetch-stream',
|
||||
response: ctx.response,
|
||||
})
|
||||
|
||||
await expect(
|
||||
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
|
||||
ctx.path,
|
||||
ctx.userId
|
||||
ctx.result =
|
||||
await ctx.DocumentConversionManager.promises.convertProjectToDocument(
|
||||
ctx.projectId,
|
||||
ctx.userId,
|
||||
ctx.type
|
||||
)
|
||||
).to.be.rejectedWith('document conversion failed')
|
||||
})
|
||||
|
||||
it('should not write the archive to disk', function (ctx) {
|
||||
sinon.assert.notCalled(ctx.fs.createWriteStream)
|
||||
sinon.assert.notCalled(ctx.nodeStream.pipeline)
|
||||
})
|
||||
|
||||
it('should attempt to clean up the output path', function (ctx) {
|
||||
it('should build the CLSI document conversion request', function (ctx) {
|
||||
sinon.assert.calledWith(
|
||||
ctx.fsPromises.unlink,
|
||||
sinon.match(
|
||||
/\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/
|
||||
)
|
||||
ctx.ClsiManager.promises.buildDocumentConversionRequest,
|
||||
ctx.projectId
|
||||
)
|
||||
})
|
||||
|
||||
it('should call CLSI with the correct URL', function (ctx) {
|
||||
const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
|
||||
expectedUrl.pathname = `/project/${ctx.projectId}/user/${ctx.userId}/download/project-to-document`
|
||||
expectedUrl.searchParams.set('type', ctx.type)
|
||||
expectedUrl.searchParams.set(
|
||||
'compileBackendClass',
|
||||
'test-backend-class'
|
||||
)
|
||||
expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
|
||||
|
||||
sinon.assert.calledWith(
|
||||
ctx.fetchUtils.fetchStreamWithResponse,
|
||||
sinon.match(url => url.toString() === expectedUrl.toString()),
|
||||
{ method: 'POST', json: { some: 'clsi-request' } }
|
||||
)
|
||||
})
|
||||
|
||||
it('should return the stream and content length', function (ctx) {
|
||||
expect(ctx.result).to.deep.equal({
|
||||
stream: ctx.mockStream,
|
||||
contentLength: 50,
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user