alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #32857 from overleaf/ds-pandoc-import-md

Browse Source 
[WEB + CLSI] Import markdown files using pandoc

GitOrigin-RevId: adad7831ddb13a8fcb8063871166bde13cbbf1b6
This commit is contained in:
Mathias Jakobsen
2026-05-08 08:09:02 +00:00
committed by Copybot
parent 44efc9d745
commit eddcc5a42e
26 changed files with 813 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/test/unit/src/Uploads/DocumentConversionManager.test.mjs
+78 -12
View File
@@ -88,12 +88,74 @@ describe('DocumentConversionManager', function () {
ctx.DocumentConversionManager = (await import(MODULE_PATH)).default
})
describe('convertDocxToLaTeXZipArchive', function () {
describe('successfully', function () {
describe('convertDocumentToLaTeXZipArchive', function () {
describe('with conversionType=docx', function () {
describe('successfully', function () {
beforeEach(async function (ctx) {
ctx.path = '/path/to/input.docx'
ctx.userId = 'test-user-id'
ctx.response = {
headers: {
get: sinon.stub().returns(null),
},
}
ctx.response.headers.get.withArgs('Content-Length').returns('50')
ctx.fetchUtils.fetchStreamWithResponse.resolves({
stream: 'mocked-fetch-stream',
response: ctx.response,
})
ctx.result =
await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId,
'docx'
)
})
it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
// TODO: revert this to '/convert/document-to-latex' once the deploy is done (PR #32857)
expectedUrl.pathname = '/convert/docx-to-latex'
expectedUrl.searchParams.set(
'compileBackendClass',
'test-backend-class'
)
expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
expectedUrl.searchParams.set('type', 'docx')
sinon.assert.calledWith(
ctx.fetchUtils.fetchStreamWithResponse,
sinon.match(url => url.toString() === expectedUrl.toString()),
{
method: 'POST',
body: sinon.match.instanceOf(FormData),
signal: sinon.match.instanceOf(AbortSignal),
}
)
})
it('should pipe result into the output file', function (ctx) {
sinon.assert.calledWith(
ctx.nodeStream.pipeline,
'mocked-fetch-stream',
'mocked-write-stream'
)
})
it('should return a path to the output file', function (ctx) {
expect(ctx.result).to.match(
/\/path\/to\/dump\/folder\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}_document-conversion\.zip/
)
})
})
})
describe('with conversionType=markdown', function () {
beforeEach(async function (ctx) {
ctx.path = '/path/to/input.docx'
ctx.path = '/path/to/input.md'
ctx.userId = 'test-user-id'
ctx.outputPath = '/path/to/output.zip'
ctx.response = {
headers: {
get: sinon.stub().returns(null),
@@ -107,20 +169,22 @@ describe('DocumentConversionManager', function () {
})
ctx.result =
await ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
await ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'markdown'
)
})
it('should call fetchStreamWithResponse with the correct URL and form data', function (ctx) {
it('should call fetchStreamWithResponse with the correct URL including markdown type', function (ctx) {
const expectedUrl = new URL(ctx.Settings.apis.clsi.url)
expectedUrl.pathname = '/convert/docx-to-latex'
expectedUrl.pathname = '/convert/document-to-latex'
expectedUrl.searchParams.set(
'compileBackendClass',
'test-backend-class'
)
expectedUrl.searchParams.set('compileGroup', 'test-compile-group')
expectedUrl.searchParams.set('type', 'markdown')
sinon.assert.calledWith(
ctx.fetchUtils.fetchStreamWithResponse,
@@ -158,9 +222,10 @@ describe('DocumentConversionManager', function () {
)
await expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'docx'
)
).to.be.rejectedWith('document conversion failed')
})
@@ -195,9 +260,10 @@ describe('DocumentConversionManager', function () {
})
await expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive(
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive(
ctx.path,
ctx.userId
ctx.userId,
'docx'
)
).to.be.rejectedWith(sinon.match.instanceOf(FileTooLargeError))
})
services/web/test/unit/src/Uploads/ProjectUploadController.test.mjs
+104 -14
View File
@@ -48,7 +48,7 @@ describe('ProjectUploadController', function () {
}
ctx.DocumentConversionManager = {
promises: {
convertDocxToLaTeXZipArchive: sinon.stub(),
convertDocumentToLaTeXZipArchive: sinon.stub(),
},
}
@@ -463,7 +463,7 @@ describe('ProjectUploadController', function () {
})
})
describe('importDocx', function () {
describe('importDocument', function () {
beforeEach(async function (ctx) {
ctx.req.file = {
path: '/path/to/uploaded/file.docx',
@@ -471,13 +471,73 @@ describe('ProjectUploadController', function () {
ctx.req.body = {
name: 'file.docx',
}
ctx.req.query = { type: 'docx' }
ctx.archivePath = '/path/to/archive.zip'
ctx.fsPromises.unlink = sinon.stub().resolves()
})
describe('successfully', async function () {
describe('with conversionType=docx', async function () {
describe('successfully', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().resolves(ctx.archivePath)
ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
.stub()
.resolves({
_id: 'new-project-id',
})
await new Promise(resolve => {
ctx.res.json = data => {
expect(data.success).to.be.true
expect(data.project_id).to.equal('new-project-id')
resolve()
}
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager with file path and type', function (ctx) {
expect(
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
})
it('should use the resulting archive to create a new project', function (ctx) {
expect(
ctx.ProjectUploadManager.promises.createProjectFromZipArchive
).to.have.been.calledWith(ctx.user_id, 'file', ctx.archivePath)
})
it('should set the compiler to lualatex', function (ctx) {
expect(
ctx.ProjectOptionsHandler.promises.setCompiler
).to.have.been.calledWith('new-project-id', 'lualatex')
})
it('should unlink the archive after creating the project', function (ctx) {
expect(ctx.fsPromises.unlink).to.have.been.calledWith(ctx.archivePath)
})
it('should unlink the uploaded file', function (ctx) {
expect(ctx.fsPromises.unlink).to.have.been.calledWith(
ctx.req.file.path
)
})
})
})
describe('with conversionType=markdown', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.req.file = {
path: '/path/to/uploaded/file.md',
}
ctx.req.body = {
name: 'file.md',
}
ctx.req.query = { type: 'markdown' }
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().resolves(ctx.archivePath)
ctx.ProjectUploadManager.promises.createProjectFromZipArchive = sinon
.stub()
@@ -491,14 +551,15 @@ describe('ProjectUploadController', function () {
expect(data.project_id).to.equal('new-project-id')
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager to convert the file', function (ctx) {
it('should call the DocumentConversionManager with file path and markdown type', function (ctx) {
expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'markdown')
})
it('should use the resulting archive to create a new project', function (ctx) {
@@ -522,9 +583,37 @@ describe('ProjectUploadController', function () {
})
})
describe('with an invalid conversionType', async function () {
beforeEach(async function (ctx) {
ctx.req.query = { type: 'invalid' }
await new Promise(resolve => {
ctx.res.json = data => {
expect(data).to.deep.equal({
success: false,
error: 'invalid_type',
})
resolve()
}
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should return http 400', function (ctx) {
expect(ctx.res.statusCode).to.equal(400)
})
it('should not call DocumentConversionManager', function (ctx) {
expect(
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).not.to.have.been.called
})
})
describe('unsuccessfully', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().rejects(new Error('Conversion failed'))
await new Promise(resolve => {
@@ -532,14 +621,15 @@ describe('ProjectUploadController', function () {
expect(data.success).to.be.false
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})
it('should call the DocumentConversionManager to convert the file', function (ctx) {
expect(
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id)
ctx.DocumentConversionManager.promises
.convertDocumentToLaTeXZipArchive
).to.have.been.calledWith(ctx.req.file.path, ctx.user_id, 'docx')
})
it('should unlink the uploaded file', function (ctx) {
@@ -553,7 +643,7 @@ describe('ProjectUploadController', function () {
describe('when the converted archive is too large', async function () {
beforeEach(async function (ctx) {
ctx.DocumentConversionManager.promises.convertDocxToLaTeXZipArchive =
ctx.DocumentConversionManager.promises.convertDocumentToLaTeXZipArchive =
sinon.stub().rejects(new FileTooLargeError('file too large'))
await new Promise(resolve => {
@@ -564,7 +654,7 @@ describe('ProjectUploadController', function () {
})
resolve()
}
ctx.ProjectUploadController.importDocx(ctx.req, ctx.res)
ctx.ProjectUploadController.importDocument(ctx.req, ctx.res)
})
})