alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9563 from overleaf/em-tpds-merge-metadata

Browse Source 
Return metadata from TPDS update endpoint in web

GitOrigin-RevId: 9154be67f7f975807c6e986a5d6fb66013c9a384
This commit is contained in:
Eric Mc Sween
2022-09-13 08:05:50 +00:00
committed by Copybot
parent ceb000eb2b
commit 19c73cbd73
13 changed files with 236 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app/src/Features/Editor/EditorController.js
+2 -2
View File
@@ -213,7 +213,7 @@ const EditorController = {
userId
)
}
callback()
callback(null, doc)
}
)
}
@@ -273,7 +273,7 @@ const EditorController = {
linkedFileData,
userId
)
callback()
callback(null, newFile)
}
)
}
services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js
+4 -1
View File
@@ -43,6 +43,7 @@ module.exports = {
'project',
'path',
'newProject',
'newFileRef',
]),
replaceDocWithFile: callbackify(replaceDocWithFile),
replaceFileWithDoc: callbackify(replaceFileWithDoc),
@@ -188,7 +189,9 @@ async function replaceFileWithNew(projectId, fileId, newFileRef) {
path,
})
}
return { oldFileRef: fileRef, project, path, newProject }
// Refresh newFileRef with the version returned from the database
newFileRef = ProjectLocator.findElementByMongoPath(newProject, path.mongo)
return { oldFileRef: fileRef, project, path, newProject, newFileRef }
}
async function replaceDocWithFile(projectId, docId, fileRef) {
services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js
+11 -10
View File
@@ -343,6 +343,7 @@ const ProjectEntityUpdateHandler = {
if (err != null) {
return callback(err)
}
doc.rev = rev
next(
projectId,
folderId,
@@ -576,7 +577,7 @@ const ProjectEntityUpdateHandler = {
projectId,
fileId,
newFileRef,
(err, oldFileRef, project, path, newProject) => {
(err, oldFileRef, project, path, newProject, newFileRef) => {
if (err != null) {
return callback(err)
}
@@ -597,18 +598,12 @@ const ProjectEntityUpdateHandler = {
project.overleaf &&
project.overleaf.history &&
project.overleaf.history.id
// Increment the rev for an in-place update (with the same path) so the third-party-datastore
// knows this is a new file.
// Ideally we would get this from ProjectEntityMongoUpdateHandler.replaceFileWithNew
// but it returns the original oldFileRef (after incrementing the rev value in mongo),
// so we add 1 to the rev from that. This isn't atomic and relies on the lock
// but it is acceptable for now.
TpdsUpdateSender.addFile(
{
projectId: project._id,
fileId: newFileRef._id,
path: path.fileSystem,
rev: oldFileRef.rev + 1,
rev: newFileRef.rev,
projectName: project.name,
folderId,
},
@@ -624,7 +619,12 @@ const ProjectEntityUpdateHandler = {
userId,
{ oldFiles, newFiles, newProject },
source,
callback
err => {
if (err) {
return callback(err)
}
callback(null, newFileRef)
}
)
}
)
@@ -670,6 +670,7 @@ const ProjectEntityUpdateHandler = {
if (err != null) {
return callback(err)
}
doc.rev = rev
ProjectEntityMongoUpdateHandler.replaceFileWithDoc(
projectId,
existingFile._id,
@@ -936,7 +937,7 @@ const ProjectEntityUpdateHandler = {
fileStoreUrl,
folderId,
source,
err => {
(err, newFileRef) => {
if (err != null) {
return callback(err)
}
services/web/app/src/Features/Project/ProjectLocator.js
+22
View File
@@ -1,5 +1,6 @@
const _ = require('underscore')
const logger = require('@overleaf/logger')
const OError = require('@overleaf/o-error')
const async = require('async')
const ProjectGetter = require('./ProjectGetter')
const Errors = require('../Errors/Errors')
@@ -282,10 +283,31 @@ function getIndexOf(searchEntity, id) {
}
}
/**
* Follow the given Mongo path (as returned by findElement) and return the
* entity at the end of it.
*/
function findElementByMongoPath(project, mongoPath) {
const components = mongoPath.split('.')
let node = project
for (const component of components) {
const key = Array.isArray(node) ? parseInt(component, 10) : component
node = node[key]
if (node == null) {
throw new OError('entity not found', {
projectId: project._id,
mongoPath,
})
}
}
return node
}
module.exports = {
findElement,
findElementByPath,
findRootDoc,
findElementByMongoPath,
promises: {
findElement: promisifyMultiResult(findElement, [
'element',
services/web/app/src/Features/ThirdPartyDataStore/TpdsController.js
+24 -9
View File
@@ -9,17 +9,19 @@ const NotificationsBuilder = require('../Notifications/NotificationsBuilder')
const SessionManager = require('../Authentication/SessionManager')
const TpdsQueueManager = require('./TpdsQueueManager')
// mergeUpdate and deleteUpdate are used by Dropbox, where the project is only passed as the name, as the
// first part of the file path. They have to check the project exists, find it, and create it if not.
// They also ignore 'noisy' files like .DS_Store, .gitignore, etc.
// mergeUpdate and deleteUpdate are used by Dropbox, where the project is only
// passed as the name, as the first part of the file path. They have to check
// the project exists, find it, and create it if not. They also ignore 'noisy'
// files like .DS_Store, .gitignore, etc.
async function mergeUpdate(req, res) {
metrics.inc('tpds.merge-update')
const { filePath, userId, projectName } = parseParams(req)
const source = req.headers['x-sl-update-source'] || 'unknown'
let metadata
try {
await TpdsUpdateHandler.promises.newUpdate(
metadata = await TpdsUpdateHandler.promises.newUpdate(
userId,
projectName,
filePath,
@@ -45,7 +47,19 @@ async function mergeUpdate(req, res) {
}
}
res.sendStatus(200)
if (metadata == null) {
return res.json({ status: 'rejected' })
}
const payload = {
status: 'applied',
entityId: metadata.entityId.toString(),
entityType: metadata.entityType,
}
if (metadata.rev != null) {
payload.rev = metadata.rev
}
res.json(payload)
}
async function deleteUpdate(req, res) {
@@ -62,10 +76,11 @@ async function deleteUpdate(req, res) {
res.sendStatus(200)
}
// updateProjectContents and deleteProjectContents are used by GitHub. The project_id is known so we
// can skip right ahead to creating/updating/deleting the file. These methods will not ignore noisy
// files like .DS_Store, .gitignore, etc because people are generally more explicit with the files they
// want in git.
// updateProjectContents and deleteProjectContents are used by GitHub. The
// project_id is known so we can skip right ahead to creating/updating/deleting
// the file. These methods will not ignore noisy files like .DS_Store,
// .gitignore, etc because people are generally more explicit with the files
// they want in git.
async function updateProjectContents(req, res, next) {
const projectId = req.params.project_id
services/web/app/src/Features/ThirdPartyDataStore/TpdsUpdateHandler.js
+4 -3
View File
@@ -22,7 +22,7 @@ const rootDocResets = new BackgroundTaskTracker('root doc resets')
async function newUpdate(userId, projectName, path, updateRequest, source) {
const project = await getOrCreateProject(userId, projectName)
if (project == null) {
return
return null
}
const projectIsOnCooldown =
@@ -33,16 +33,17 @@ async function newUpdate(userId, projectName, path, updateRequest, source) {
const shouldIgnore = await FileTypeManager.promises.shouldIgnore(path)
if (shouldIgnore) {
return
return null
}
await UpdateMerger.promises.mergeUpdate(
const metadata = await UpdateMerger.promises.mergeUpdate(
userId,
project._id,
path,
updateRequest,
source
)
return metadata
}
async function deleteUpdate(userId, projectName, path, source) {
services/web/app/src/Features/ThirdPartyDataStore/UpdateMerger.js
+28 -42
View File
@@ -13,7 +13,8 @@ async function mergeUpdate(userId, projectId, path, updateRequest, source) {
updateRequest
)
try {
await _mergeUpdate(userId, projectId, path, fsPath, source)
const metadata = await _mergeUpdate(userId, projectId, path, fsPath, source)
return metadata
} finally {
try {
await fsPromises.unlink(fsPath)
@@ -52,54 +53,37 @@ async function _determineFileType(projectId, path, fsPath) {
// sync, so we'll treat non-utf8 files as binary
const isBinary = binary || encoding !== 'utf-8'
// Existing | Update | Action
// ---------|-----------|-------
// file | isBinary | existing-file
// file | !isBinary | existing-file
// doc | isBinary | new-file, delete-existing-doc
// doc | !isBinary | existing-doc
// null | isBinary | new-file
// null | !isBinary | new-doc
// Existing | Update | Resulting file type
// ---------|-----------|--------------------
// file | isBinary | file
// file | !isBinary | file
// doc | isBinary | file
// doc | !isBinary | doc
// null | isBinary | file
// null | !isBinary | doc
// if a binary file already exists, always keep it as a binary file
// even if the update looks like a text file
if (existingFileType === 'file') {
return { fileType: 'existing-file' }
return 'file'
} else {
return isBinary ? 'file' : 'doc'
}
// if there is an existing doc, keep it as a doc except when the
// incoming update is binary. In that case delete the doc and replace
// it with a new file.
if (existingFileType === 'doc') {
if (isBinary) {
return {
fileType: 'new-file',
deleteOriginalEntity: 'delete-existing-doc',
}
} else {
return { fileType: 'existing-doc' }
}
}
// if there no existing file, create a file or doc as needed
return { fileType: isBinary ? 'new-file' : 'new-doc' }
}
async function _mergeUpdate(userId, projectId, path, fsPath, source) {
const { fileType, deleteOriginalEntity } = await _determineFileType(
projectId,
path,
fsPath
)
const fileType = await _determineFileType(projectId, path, fsPath)
if (deleteOriginalEntity) {
await deleteUpdate(userId, projectId, path, source)
}
if (['existing-file', 'new-file'].includes(fileType)) {
await _processFile(projectId, fsPath, path, source, userId)
} else if (['existing-doc', 'new-doc'].includes(fileType)) {
await _processDoc(projectId, userId, fsPath, path, source)
if (fileType === 'file') {
const file = await _processFile(projectId, fsPath, path, source, userId)
return { entityType: 'file', entityId: file._id, rev: file.rev }
} else if (fileType === 'doc') {
const doc = await _processDoc(projectId, userId, fsPath, path, source)
// The doc entry doesn't have a rev. Since the document is set in
// docupdater, it's possible that the next rev contains a merge of changes
// in Dropbox and changes from docupdater.
const metadata = { entityType: 'doc', entityId: doc._id, rev: doc.rev }
return metadata
} else {
throw new Error('unrecognized file')
}
@@ -124,17 +108,18 @@ async function deleteUpdate(userId, projectId, path, source) {
async function _processDoc(projectId, userId, fsPath, path, source) {
const docLines = await _readFileIntoTextArray(fsPath)
logger.debug({ docLines }, 'processing doc update from tpds')
await EditorController.promises.upsertDocWithPath(
const doc = await EditorController.promises.upsertDocWithPath(
projectId,
path,
docLines,
source,
userId
)
return doc
}
async function _processFile(projectId, fsPath, path, source, userId) {
await EditorController.promises.upsertFileWithPath(
const file = await EditorController.promises.upsertFileWithPath(
projectId,
path,
fsPath,
@@ -142,6 +127,7 @@ async function _processFile(projectId, fsPath, path, source, userId) {
source,
userId
)
return file
}
async function _readFileIntoTextArray(path) {