diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js index b7f07c4834..1f27379259 100644 --- a/services/history-v1/api/controllers/projects.js +++ b/services/history-v1/api/controllers/projects.js @@ -25,6 +25,8 @@ const render = require('./render') const expressify = require('./expressify') const withTmpDir = require('./with_tmp_dir') const StreamSizeLimit = require('./stream_size_limit') +const { getProjectBlobsBatch } = require('../../storage/lib/blob_store') +const assert = require('../../storage/lib/assert') const pipeline = promisify(Stream.pipeline) @@ -378,6 +380,40 @@ async function getSnapshotAtVersion(projectId, version) { return snapshot } +function sumUpByteLength(blobs) { + return blobs.reduce((sum, blob) => sum + blob.getByteLength(), 0) +} + +async function getProjectBlobsStats(req, res) { + const projectIds = req.swagger.params.body.value.projectIds + const { blobs } = await getProjectBlobsBatch( + projectIds.map(id => { + if (assert.POSTGRES_ID_REGEXP.test(id)) { + return parseInt(id, 10) + } else { + return id + } + }) + ) + const sizes = [] + for (const projectId of projectIds) { + const projectBlobs = blobs.get(projectId) || [] + const textBlobs = projectBlobs.filter(b => b.getStringLength() !== null) + const binaryBlobs = projectBlobs.filter(b => b.getStringLength() === null) + const textBlobBytes = sumUpByteLength(textBlobs) + const binaryBlobBytes = sumUpByteLength(binaryBlobs) + sizes.push({ + projectId, + textBlobBytes, + binaryBlobBytes, + totalBytes: textBlobBytes + binaryBlobBytes, + nTextBlobs: textBlobs.length, + nBinaryBlobs: binaryBlobs.length, + }) + } + res.json(sizes) +} + module.exports = { initializeProject: expressify(initializeProject), getLatestContent: expressify(getLatestContent), @@ -396,4 +432,5 @@ module.exports = { getProjectBlob: expressify(getProjectBlob), headProjectBlob: expressify(headProjectBlob), copyProjectBlob: expressify(copyProjectBlob), + getProjectBlobsStats: expressify(getProjectBlobsStats), } diff --git a/services/history-v1/api/swagger/index.js b/services/history-v1/api/swagger/index.js index 3702c6ec07..b18e93a1c7 100644 --- a/services/history-v1/api/swagger/index.js +++ b/services/history-v1/api/swagger/index.js @@ -32,6 +32,28 @@ module.exports = { }, required: ['projectId'], }, + ProjectBlobStats: { + properties: { + projectId: { + type: 'string', + }, + textBlobBytes: { + type: 'integer', + }, + binaryBlobBytes: { + type: 'integer', + }, + totalBytes: { + type: 'integer', + }, + nTextBlobs: { + type: 'integer', + }, + nBinaryBlobs: { + type: 'integer', + }, + }, + }, File: { properties: { hash: { diff --git a/services/history-v1/api/swagger/projects.js b/services/history-v1/api/swagger/projects.js index cd4d2338fa..f008670ad8 100644 --- a/services/history-v1/api/swagger/projects.js +++ b/services/history-v1/api/swagger/projects.js @@ -37,6 +37,46 @@ exports.paths = { ], }, }, + '/projects/blob-stats': { + post: { + 'x-swagger-router-controller': 'projects', + operationId: 'getProjectBlobsStats', + tags: ['Project'], + description: 'Get Blob stats for projects.', + consumes: ['application/json'], + parameters: [ + { + name: 'body', + in: 'body', + schema: { + type: 'object', + properties: { + projectIds: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + }, + ], + responses: { + 200: { + description: 'Initialized', + schema: { + type: 'array', + items: { + $ref: '#/definitions/ProjectBlobStats', + }, + }, + }, + }, + security: [ + { + basic: [], + }, + ], + }, + }, '/projects/{project_id}': { delete: { 'x-swagger-router-controller': 'projects', diff --git a/services/history-v1/test/acceptance/js/api/projects.test.js b/services/history-v1/test/acceptance/js/api/projects.test.js index 22220ae8bd..fbae11fb25 100644 --- a/services/history-v1/test/acceptance/js/api/projects.test.js +++ b/services/history-v1/test/acceptance/js/api/projects.test.js @@ -30,6 +30,7 @@ const { TextOperation, } = require('overleaf-editor-core') const testProjects = require('./support/test_projects') +const { ObjectId } = require('mongodb') describe('project controller', function () { beforeEach(cleanup.everything) @@ -104,6 +105,165 @@ describe('project controller', function () { }) }) + describe('blob stats', function () { + let populatedPostgresProjectId, + populatedMongoProjectId, + emptyPostgresProjectId, + emptyMongoProjectId + + async function populateProject(projectId) { + const files = { + [testFiles.GRAPH_PNG_HASH]: testFiles.path('graph.png'), + [testFiles.HELLO_TXT_HASH]: testFiles.path('hello.txt'), + } + for (const [hash, path] of Object.entries(files)) { + const response = await fetch( + testServer.url(`/api/projects/${projectId}/blobs/${hash}`), + { + method: 'PUT', + body: fs.createReadStream(path), + headers: { + Authorization: testServer.basicAuthHeader, + }, + } + ) + expect(response.status).to.equal(201) + } + } + + beforeEach(async function () { + emptyPostgresProjectId = await testProjects.createEmptyProject() + emptyMongoProjectId = await testProjects.createEmptyProject( + new ObjectId().toString() + ) + + populatedPostgresProjectId = await testProjects.createEmptyProject() + await populateProject(populatedPostgresProjectId) + populatedMongoProjectId = await testProjects.createEmptyProject( + new ObjectId().toString() + ) + await populateProject(populatedMongoProjectId) + }) + + it('handles empty postgres project', async function () { + const { body } = + await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({ + body: { projectIds: [emptyPostgresProjectId] }, + }) + expect(body).to.deep.equal([ + { + projectId: emptyPostgresProjectId, + textBlobBytes: 0, + binaryBlobBytes: 0, + totalBytes: 0, + nTextBlobs: 0, + nBinaryBlobs: 0, + }, + ]) + }) + it('handles populated postgres project', async function () { + const { body } = + await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({ + body: { projectIds: [populatedPostgresProjectId] }, + }) + expect(body).to.deep.equal([ + { + projectId: populatedPostgresProjectId, + textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH, + binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH, + totalBytes: + testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH, + nTextBlobs: 1, + nBinaryBlobs: 1, + }, + ]) + }) + + it('handles empty mongo project', async function () { + const { body } = + await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({ + body: { projectIds: [emptyMongoProjectId] }, + }) + expect(body).to.deep.equal([ + { + projectId: emptyMongoProjectId, + textBlobBytes: 0, + binaryBlobBytes: 0, + totalBytes: 0, + nTextBlobs: 0, + nBinaryBlobs: 0, + }, + ]) + }) + it('handles populated mongo project', async function () { + const { body } = + await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({ + body: { projectIds: [populatedMongoProjectId] }, + }) + expect(body).to.deep.equal([ + { + projectId: populatedMongoProjectId, + textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH, + binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH, + totalBytes: + testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH, + nTextBlobs: 1, + nBinaryBlobs: 1, + }, + ]) + }) + + it('handles batch of projects', async function () { + const { body } = + await testServer.basicAuthClient.apis.Project.getProjectBlobsStats({ + body: { + projectIds: [ + populatedPostgresProjectId, + populatedMongoProjectId, + emptyPostgresProjectId, + emptyMongoProjectId, + ], + }, + }) + expect(body).to.deep.equal([ + { + projectId: populatedPostgresProjectId, + textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH, + binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH, + totalBytes: + testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH, + nTextBlobs: 1, + nBinaryBlobs: 1, + }, + { + projectId: populatedMongoProjectId, + textBlobBytes: testFiles.HELLO_TXT_BYTE_LENGTH, + binaryBlobBytes: testFiles.GRAPH_PNG_BYTE_LENGTH, + totalBytes: + testFiles.HELLO_TXT_BYTE_LENGTH + testFiles.GRAPH_PNG_BYTE_LENGTH, + nTextBlobs: 1, + nBinaryBlobs: 1, + }, + { + projectId: emptyPostgresProjectId, + textBlobBytes: 0, + binaryBlobBytes: 0, + totalBytes: 0, + nTextBlobs: 0, + nBinaryBlobs: 0, + }, + { + projectId: emptyMongoProjectId, + textBlobBytes: 0, + binaryBlobBytes: 0, + totalBytes: 0, + nTextBlobs: 0, + nBinaryBlobs: 0, + }, + ]) + }) + }) + // eslint-disable-next-line mocha/no-skipped-tests describe.skip('getLatestContent', function () { // TODO: remove this endpoint entirely, see diff --git a/services/web/app/src/Features/History/HistoryManager.js b/services/web/app/src/Features/History/HistoryManager.js index 34130f10ce..39911a25cb 100644 --- a/services/web/app/src/Features/History/HistoryManager.js +++ b/services/web/app/src/Features/History/HistoryManager.js @@ -307,6 +307,14 @@ async function getHistoryId(projectId) { return historyId } +async function getProjectBlobStats(historyIds) { + return await fetchJson(`${HISTORY_V1_URL}/projects/blob-stats`, { + method: 'POST', + basicAuth: HISTORY_V1_BASIC_AUTH, + json: { projectIds: historyIds.map(id => id.toString()) }, + }) +} + async function injectUserDetails(data) { // data can be either: // { @@ -418,5 +426,6 @@ module.exports = { requestBlobWithProjectId, getLatestHistory, getChanges, + getProjectBlobStats, }, } diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 9ba6c48010..6af3dfd775 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -1612,6 +1612,7 @@ "sign_up": "", "simple_search_mode": "", "single_sign_on_sso": "", + "size": "", "something_not_right": "", "something_went_wrong": "", "something_went_wrong_canceling_your_subscription": "", diff --git a/services/web/frontend/js/utils/meta.ts b/services/web/frontend/js/utils/meta.ts index 17cfdb690d..6d5b40c40a 100644 --- a/services/web/frontend/js/utils/meta.ts +++ b/services/web/frontend/js/utils/meta.ts @@ -151,6 +151,15 @@ export interface Meta { 'ol-hasTrackChangesFeature': boolean 'ol-hasWriteAccess': boolean 'ol-hideLinkingWidgets': boolean // CI only + 'ol-historyBlobStats': { + projectId: string + textBlobsBytes: number + binaryBlobsBytes: number + totalBytes: number + nTextBlobs: number + nBinaryBlobs: number + owned?: boolean + } 'ol-i18n': { currentLangCode: string } 'ol-inactiveTutorials': string[] 'ol-institutionEmailNonCanonical': string | undefined diff --git a/services/web/locales/en.json b/services/web/locales/en.json index fb4511a4e0..b7589f920a 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -2089,6 +2089,7 @@ "site_wide_option_available": "Site-wide option available", "six_collaborators_per_project": "6 collaborators per project", "six_per_project": "6 per project", + "size": "Size", "skip": "Skip", "skip_to_content": "Skip to content", "solutions": "Solutions", diff --git a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs index d190f61398..2dfdf4478c 100644 --- a/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs +++ b/services/web/test/acceptance/src/mocks/MockV1HistoryApi.mjs @@ -16,6 +16,21 @@ class MockV1HistoryApi extends AbstractMockApi { } applyRoutes() { + this.app.post('/api/projects/blob-stats', (req, res, next) => { + res.json( + req.body.projectIds.map(projectId => { + return { + projectId, + textBlobBytes: 7331, + binaryBlobBytes: 1337, + totalBytes: 7331 + 1337, + nTextBlobs: 13, + nBinaryBlobs: 42, + } + }) + ) + }) + this.app.get( '/api/projects/:project_id/version/:version/zip', (req, res, next) => {