alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #28533 from overleaf/mj-blob-ranges-maxsize

Browse Source 
[web+history-v1] Use range request for binary blobs in project snapshot

GitOrigin-RevId: e72a2d481bf163b3e85361b42e30e82cf867f1cf
This commit is contained in:
Mathias Jakobsen
2025-09-22 08:05:04 +00:00
committed by Copybot
parent 6ed14c56ae
commit 6c79ce97cc
9 changed files with 313 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app/src/Features/History/HistoryController.mjs
+7 -3
View File
@@ -66,9 +66,9 @@ async function requestBlob(method, req, res) {
}
const range = req.get('Range')
let stream, contentLength
let stream, contentLength, contentRange
try {
;({ stream, contentLength } =
;({ stream, contentLength, contentRange } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
hash,
@@ -80,7 +80,11 @@ async function requestBlob(method, req, res) {
throw err
}
if (contentLength) res.setHeader('Content-Length', contentLength) // set on HEAD
if (contentLength) res.setHeader('Content-Length', contentLength)
if (contentRange) {
res.setHeader('Content-Range', contentRange)
res.status(206) // Partial Content
}
res.setHeader('Content-Type', 'application/octet-stream')
setBlobCacheHeaders(res, hash)
services/web/app/src/Features/History/HistoryManager.js
+1
View File
@@ -207,6 +207,7 @@ async function requestBlob(historyId, hash, method = 'GET', range = '') {
url,
stream,
contentLength: parseInt(response.headers.get('Content-Length'), 10),
contentRange: response.headers.get('Content-Range'),
}
}
services/web/frontend/js/infrastructure/project-snapshot.ts
+20 -36
View File
@@ -143,10 +143,18 @@ export class ProjectSnapshot {
): Promise<any> {
const file = this.snapshot.getFile(path)
const hash = file?.getHash()
const byteLength = file?.getByteLength()
if (hash == null) {
return null
}
return await this.blobStore.getString(hash, options)
if (byteLength == null) {
return null
}
let blobStoreOptions
if (options?.maxSize != null && byteLength > options?.maxSize) {
blobStoreOptions = { maxSize: options.maxSize }
}
return await this.blobStore.getString(hash, blobStoreOptions)
}
getDocs(): Map<string, File> {
@@ -323,44 +331,20 @@ async function fetchBlob(
options?: { maxSize?: number }
): Promise<string> {
const url = `/project/${projectId}/blob/${hash}`
if (options?.maxSize) {
return await fetchTextFileWithSizeLimit(url, options.maxSize)
let fetchOpts
if (options?.maxSize === 0) {
return ''
}
const res = await fetch(url)
if (options?.maxSize) {
fetchOpts = {
headers: {
Range: `bytes=0-${options.maxSize - 1}`,
},
}
}
const res = await fetch(url, fetchOpts)
if (!res.ok) {
throw new FetchError('Failed to fetch blob', url, undefined, res)
}
return await res.text()
}
async function fetchTextFileWithSizeLimit(url: string, maxSize: number) {
let result = ''
try {
const abortController = new AbortController()
const response = await fetch(url, {
signal: abortController.signal,
})
if (!response.ok) {
throw new Error('Failed to fetch blob')
}
if (!response.body) {
throw new Error('Response body is empty')
}
const reader = response.body.pipeThrough(new TextDecoderStream())
for await (const chunk of reader) {
result += chunk
if (result.length > maxSize) {
abortController.abort()
}
}
} catch (error: any) {
if (error?.name === 'AbortError') {
// This is fine, we just return the result we have so far
} else {
throw error
}
}
return result.slice(0, maxSize)
}
services/web/test/frontend/infrastructure/project-snapshot.test.ts
+44 -8
View File
@@ -44,6 +44,10 @@ describe('ProjectSnapshot', function () {
), // 6.5MB
hash: 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
},
'empty.png': {
contents: '',
hash: 'ffffffffffffffffffffffffffffffffffffffff',
},
}
const chunk = {
@@ -82,6 +86,13 @@ describe('ProjectSnapshot', function () {
byteLength: files['bibliography.bib'].contents.length,
},
},
{
pathname: 'empty.png',
file: {
hash: files['empty.png'].hash,
byteLength: files['empty.png'].contents.length,
},
},
],
timestamp: '2025-01-01T12:00:00.000Z',
},
@@ -145,10 +156,24 @@ describe('ProjectSnapshot', function () {
})
}
// fetch-mock doesn't seem to expose the header to the response function,
// so we just use a constant here
const MOCKED_MAX_SIZE = 100
function mockBlobs(paths = Object.keys(files) as (keyof typeof files)[]) {
for (const path of paths) {
const file = files[path]
fetchMock.get(`/project/${projectId}/blob/${file.hash}`, file.contents)
fetchMock
.get({
url: `/project/${projectId}/blob/${file.hash}`,
missingHeaders: ['Range'],
response: file.contents,
})
.get({
url: `/project/${projectId}/blob/${file.hash}`,
headers: { Range: `bytes=0-${MOCKED_MAX_SIZE - 1}` },
response: file.contents.slice(0, MOCKED_MAX_SIZE),
})
}
}
@@ -243,13 +268,18 @@ describe('ProjectSnapshot', function () {
hash: files['bibliography.bib'].hash,
size: files['bibliography.bib'].contents.length,
},
{
path: 'empty.png',
hash: 'ffffffffffffffffffffffffffffffffffffffff',
size: 0,
},
])
})
})
describe('getBinaryFileContents', function () {
beforeEach(function () {
mockBlobs(['bibliography.bib'])
mockBlobs(['bibliography.bib', 'empty.png'])
})
it('can fetch whole file', async function () {
@@ -257,14 +287,20 @@ describe('ProjectSnapshot', function () {
expect(blob).to.equal(files['bibliography.bib'].contents)
})
// NOTE: fetch-mock does not support the .response.body.pipeThrough API,
// so this test is skipped for now.
// eslint-disable-next-line mocha/no-skipped-tests
it.skip('can fetch part of file', async function () {
it('can fetch part of file', async function () {
const blob = await snapshot.getBinaryFileContents('bibliography.bib', {
maxSize: 100,
maxSize: MOCKED_MAX_SIZE,
})
expect(blob).to.equal(files['bibliography.bib'].contents.slice(0, 100))
expect(blob).to.equal(
files['bibliography.bib'].contents.slice(0, MOCKED_MAX_SIZE)
)
})
it('can fetch empty file with maxSize', async function () {
const blob = await snapshot.getBinaryFileContents('empty.png', {
maxSize: 200,
})
expect(blob).to.equal(files['empty.png'].contents)
})
})
})
services/web/test/unit/src/History/HistoryController.test.mjs
+78
View File
@@ -32,6 +32,7 @@ describe('HistoryController', function () {
ctx.HistoryManager = {
promises: {
injectUserDetails: sinon.stub(),
requestBlobWithProjectId: sinon.stub(),
},
}
@@ -297,4 +298,81 @@ describe('HistoryController', function () {
})
})
})
describe('requestBlob', function () {
describe('With Range header', function () {
beforeEach(async function (ctx) {
ctx.req = {
params: { project_id: ctx.project_id, hash: 'foo' },
body: {},
get: sinon.stub(),
}
ctx.req.get.withArgs('Range').returns('bytes=0-42')
ctx.res = { setHeader: sinon.stub(), status: sinon.stub() }
ctx.HistoryManager.promises.requestBlobWithProjectId.resolves({
stream: null,
contentLength: '43',
contentRange: 'bytes 0-42/100',
})
await ctx.HistoryController.getBlob(ctx.req, ctx.res, ctx.next)
})
it('should forward the range request', function (ctx) {
ctx.HistoryManager.promises.requestBlobWithProjectId.should.have.been.calledWith(
ctx.project_id,
'foo',
'GET',
'bytes=0-42'
)
})
it('should forward the Content-Range header', function (ctx) {
ctx.res.setHeader.should.have.been.calledWith(
'Content-Range',
'bytes 0-42/100'
)
})
it('should forward the Content-Length header', function (ctx) {
ctx.res.setHeader.should.have.been.calledWith('Content-Length', '43')
})
it('should have status 206', function (ctx) {
ctx.res.status.should.have.been.calledWith(206)
})
})
describe('Without Range header', function () {
beforeEach(async function (ctx) {
ctx.req = {
params: { project_id: ctx.project_id, hash: 'foo' },
body: {},
get: sinon.stub(),
}
ctx.req.get.withArgs('Range').returns(null)
ctx.res = { setHeader: sinon.stub(), status: sinon.stub() }
ctx.HistoryManager.promises.requestBlobWithProjectId.resolves({
stream: null,
contentLength: '100',
range: null,
})
await ctx.HistoryController.getBlob(ctx.req, ctx.res, ctx.next)
})
it('should not have a Content-Range header', function (ctx) {
expect(ctx.res.setHeader).to.not.have.been.calledWith(
'Content-Range',
sinon.match.string
)
})
it('should forward the Content-Length header', function (ctx) {
ctx.res.setHeader.should.have.been.calledWith('Content-Length', '100')
})
it('should not have status 206', function (ctx) {
ctx.res.status.should.not.have.been.calledWith(206)
})
})
})
})