alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] display project history blob size on admin pages (#28351)

Browse Source 
* [history-v1] add endpoint for getting blob stats of projects

* [web] display project history blob size on admin pages

* [web] break down history storage size by text and binary blob

GitOrigin-RevId: bfa4d56cf2c503d03005c13a0f7ac38158156cd2
This commit is contained in:
Jakob Ackermann
2025-09-09 16:05:47 +02:00
committed by Copybot
parent 452d77cabc
commit f8d69d05af
9 changed files with 294 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/history-v1/api/controllers/projects.js
+37
View File
@@ -25,6 +25,8 @@ const render = require('./render')
const expressify = require('./expressify')
const withTmpDir = require('./with_tmp_dir')
const StreamSizeLimit = require('./stream_size_limit')
const { getProjectBlobsBatch } = require('../../storage/lib/blob_store')
const assert = require('../../storage/lib/assert')
const pipeline = promisify(Stream.pipeline)
@@ -378,6 +380,40 @@ async function getSnapshotAtVersion(projectId, version) {
return snapshot
}
function sumUpByteLength(blobs) {
return blobs.reduce((sum, blob) => sum + blob.getByteLength(), 0)
}
async function getProjectBlobsStats(req, res) {
const projectIds = req.swagger.params.body.value.projectIds
const { blobs } = await getProjectBlobsBatch(
projectIds.map(id => {
if (assert.POSTGRES_ID_REGEXP.test(id)) {
return parseInt(id, 10)
} else {
return id
}
})
)
const sizes = []
for (const projectId of projectIds) {
const projectBlobs = blobs.get(projectId) || []
const textBlobs = projectBlobs.filter(b => b.getStringLength() !== null)
const binaryBlobs = projectBlobs.filter(b => b.getStringLength() === null)
const textBlobBytes = sumUpByteLength(textBlobs)
const binaryBlobBytes = sumUpByteLength(binaryBlobs)
sizes.push({
projectId,
textBlobBytes,
binaryBlobBytes,
totalBytes: textBlobBytes + binaryBlobBytes,
nTextBlobs: textBlobs.length,
nBinaryBlobs: binaryBlobs.length,
})
}
res.json(sizes)
}
module.exports = {
initializeProject: expressify(initializeProject),
getLatestContent: expressify(getLatestContent),
@@ -396,4 +432,5 @@ module.exports = {
getProjectBlob: expressify(getProjectBlob),
headProjectBlob: expressify(headProjectBlob),
copyProjectBlob: expressify(copyProjectBlob),
getProjectBlobsStats: expressify(getProjectBlobsStats),
}
services/history-v1/api/swagger/index.js
+22
View File
@@ -32,6 +32,28 @@ module.exports = {
},
required: ['projectId'],
},
ProjectBlobStats: {
properties: {
projectId: {
type: 'string',
},
textBlobBytes: {
type: 'integer',
},
binaryBlobBytes: {
type: 'integer',
},
totalBytes: {
type: 'integer',
},
nTextBlobs: {
type: 'integer',
},
nBinaryBlobs: {
type: 'integer',
},
},
},
File: {
properties: {
hash: {
services/history-v1/api/swagger/projects.js
+40
View File
@@ -37,6 +37,46 @@ exports.paths = {
],
},
},
'/projects/blob-stats': {
post: {
'x-swagger-router-controller': 'projects',
operationId: 'getProjectBlobsStats',
tags: ['Project'],
description: 'Get Blob stats for projects.',
consumes: ['application/json'],
parameters: [
{
name: 'body',
in: 'body',
schema: {
type: 'object',
properties: {
projectIds: {
type: 'array',
items: { type: 'string' },
},
},
},
},
],
responses: {
200: {
description: 'Initialized',
schema: {
type: 'array',
items: {
$ref: '#/definitions/ProjectBlobStats',
},
},
},
},
security: [
{
basic: [],
},
],
},
},
'/projects/{project_id}': {
delete: {
'x-swagger-router-controller': 'projects',
services/history-v1/test/acceptance/js/api/projects.test.js
+160
View File
@@ -30,6 +30,7 @@ const {
TextOperation,
} = require('overleaf-editor-core')
const testProjects = require('./support/test_projects')
const { ObjectId } = require('mongodb')
describe('project controller', function () {
beforeEach(cleanup.everything)
@@ -104,6 +105,165 @@ describe('project controller', function () {
})
})
describe('blob stats', function () {
let populatedPostgresProjectId,
populatedMongoProjectId,
emptyPostgresProjectId,
emptyMongoProjectId
async function populateProject(projectId) {
const files = {
[testFiles.GRAPH_PNG_HASH]: testFiles.path('graph.png'),
[testFiles.HELLO_TXT_HASH]: testFiles.path('hello.txt'),
}
for (const [hash, path] of Object.entries(files)) {
const response = await fetch(
testServer.url(`/api/projects/${projectId}/blobs/${hash}`),
{
method: 'PUT',
body: fs.createReadStream(path),
headers: {
Authorization: testServer.basicAuthHeader,
},
}
)
expect(response.status).to.equal(201)
}
}
beforeEach(async function () {
emptyPostgresProjectId = await testProjects.createEmptyProject()
emptyMongoProjectId = await testProjects.createEmptyProject(
new ObjectId().toString()
)
populatedPostgresProjectId = await testProjects.createEmptyProject()
await populateProject(populatedPostgresProjectId)
populatedMongoProjectId = await testProjects.createEmptyProject(
new ObjectId().toString()
)
await populateProject(populatedMongoProjectId)
})
it('handles empty postgres project', async function () {
const { body } =
await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
body: { projectIds: [emptyPostgresProjectId] },
})
expect(body).to.deep.equal([
{
projectId: emptyPostgresProjectId,
textBlobBytes: 0,
binaryBlobBytes: 0,
totalBytes: 0,
nTextBlobs: 0,
nBinaryBlobs: 0,
},
])
})
it('handles populated postgres project', async function () {
const { body } =
await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
body: { projectIds: [populatedPostgresProjectId] },
})
expect(body).to.deep.equal([
{
projectId: populatedPostgresProjectId,
textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
totalBytes:
testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
nTextBlobs: 1,
nBinaryBlobs: 1,
},
])
})
it('handles empty mongo project', async function () {
const { body } =
await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
body: { projectIds: [emptyMongoProjectId] },
})
expect(body).to.deep.equal([
{
projectId: emptyMongoProjectId,
textBlobBytes: 0,
binaryBlobBytes: 0,
totalBytes: 0,
nTextBlobs: 0,
nBinaryBlobs: 0,
},
])
})
it('handles populated mongo project', async function () {
const { body } =
await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
body: { projectIds: [populatedMongoProjectId] },
})
expect(body).to.deep.equal([
{
projectId: populatedMongoProjectId,
textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
totalBytes:
testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
nTextBlobs: 1,
nBinaryBlobs: 1,
},
])
})
it('handles batch of projects', async function () {
const { body } =
await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({
body: {
projectIds: [
populatedPostgresProjectId,
populatedMongoProjectId,
emptyPostgresProjectId,
emptyMongoProjectId,
],
},
})
expect(body).to.deep.equal([
{
projectId: populatedPostgresProjectId,
textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
totalBytes:
testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
nTextBlobs: 1,
nBinaryBlobs: 1,
},
{
projectId: populatedMongoProjectId,
textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH,
binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH,
totalBytes:
testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH,
nTextBlobs: 1,
nBinaryBlobs: 1,
},
{
projectId: emptyPostgresProjectId,
textBlobBytes: 0,
binaryBlobBytes: 0,
totalBytes: 0,
nTextBlobs: 0,
nBinaryBlobs: 0,
},
{
projectId: emptyMongoProjectId,
textBlobBytes: 0,
binaryBlobBytes: 0,
totalBytes: 0,
nTextBlobs: 0,
nBinaryBlobs: 0,
},
])
})
})
// eslint-disable-next-line mocha/no-skipped-tests
describe.skip('getLatestContent', function () {
// TODO: remove this endpoint entirely, see
services/web/app/src/Features/History/HistoryManager.js
+9
View File
@@ -307,6 +307,14 @@ async function getHistoryId(projectId) {
return historyId
}
async function getProjectBlobStats(historyIds) {
return await fetchJson(`${HISTORY_V1_URL}/projects/blob-stats`, {
method: 'POST',
basicAuth: HISTORY_V1_BASIC_AUTH,
json: { projectIds: historyIds.map(id => id.toString()) },
})
}
async function injectUserDetails(data) {
// data can be either:
// {
@@ -418,5 +426,6 @@ module.exports = {
requestBlobWithProjectId,
getLatestHistory,
getChanges,
getProjectBlobStats,
},
}
services/web/frontend/extracted-translations.json
+1
View File
@@ -1612,6 +1612,7 @@
"sign_up": "",
"simple_search_mode": "",
"single_sign_on_sso": "",
"size": "",
"something_not_right": "",
"something_went_wrong": "",
"something_went_wrong_canceling_your_subscription": "",
services/web/frontend/js/utils/meta.ts
+9
View File
@@ -151,6 +151,15 @@ export interface Meta {
'ol-hasTrackChangesFeature': boolean
'ol-hasWriteAccess': boolean
'ol-hideLinkingWidgets': boolean // CI only
'ol-historyBlobStats': {
projectId: string
textBlobsBytes: number
binaryBlobsBytes: number
totalBytes: number
nTextBlobs: number
nBinaryBlobs: number
owned?: boolean
}
'ol-i18n': { currentLangCode: string }
'ol-inactiveTutorials': string[]
'ol-institutionEmailNonCanonical': string | undefined
services/web/locales/en.json
+1
View File
@@ -2089,6 +2089,7 @@
"site_wide_option_available": "Site-wide option available",
"six_collaborators_per_project": "6 collaborators per project",
"six_per_project": "6 per project",
"size": "Size",
"skip": "Skip",
"skip_to_content": "Skip to content",
"solutions": "Solutions",
services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs
+15
View File
@@ -16,6 +16,21 @@ class MockV1HistoryApi extends AbstractMockApi {
}
applyRoutes() {
this.app.post('/api/projects/blob-stats', (req, res, next) => {
res.json(
req.body.projectIds.map(projectId => {
return {
projectId,
textBlobBytes: 7331,
binaryBlobBytes: 1337,
totalBytes: 7331 + 1337,
nTextBlobs: 13,
nBinaryBlobs: 42,
}
})
)
})
this.app.get(
'/api/projects/:project_id/version/:version/zip',
(req, res, next) => {