From e739e86c48cccfc828a538bb6ff6da3b57555f6f Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 28 Nov 2016 10:14:42 +0000 Subject: [PATCH 01/25] Get basic ChangeTracker hooked up. WIP --- services/document-updater/app.coffee | 1 + .../app/coffee/ChangesTracker.coffee | 457 ++++++++++++++++++ .../app/coffee/DocumentManager.coffee | 35 +- .../app/coffee/HistoryManager.coffee | 44 ++ .../app/coffee/HttpController.coffee | 12 + .../app/coffee/PersistenceManager.coffee | 14 +- .../app/coffee/ProjectManager.coffee | 26 + .../app/coffee/RedisKeyBuilder.coffee | 4 + .../app/coffee/RedisManager.coffee | 20 +- .../app/coffee/ShareJsDB.coffee | 27 +- .../app/coffee/ShareJsUpdateManager.coffee | 28 +- .../app/coffee/TrackChangesManager.coffee | 52 +- .../app/coffee/UpdateManager.coffee | 19 +- .../app/coffee/WebRedisManager.coffee | 5 +- .../config/settings.defaults.coffee | 4 + .../coffee/TrackChangesTests.coffee | 96 ++++ .../coffee/helpers/DocUpdaterClient.coffee | 19 +- 17 files changed, 751 insertions(+), 112 deletions(-) create mode 100644 services/document-updater/app/coffee/ChangesTracker.coffee create mode 100644 services/document-updater/app/coffee/HistoryManager.coffee create mode 100644 services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee index af0ad242c2..df26b81a2f 100644 --- a/services/document-updater/app.coffee +++ b/services/document-updater/app.coffee @@ -46,6 +46,7 @@ app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLo app.delete '/project/:project_id/doc/:doc_id', HttpController.flushAndDeleteDoc app.delete '/project/:project_id', HttpController.deleteProject app.post '/project/:project_id/flush', HttpController.flushProject +app.post '/project/:project_id/track_changes', HttpController.setTrackChanges app.get '/total', (req, res)-> timer = new Metrics.Timer("http.allDocList") diff --git a/services/document-updater/app/coffee/ChangesTracker.coffee b/services/document-updater/app/coffee/ChangesTracker.coffee new file mode 100644 index 0000000000..8bc4cf9380 --- /dev/null +++ b/services/document-updater/app/coffee/ChangesTracker.coffee @@ -0,0 +1,457 @@ +load = (EventEmitter) -> + class ChangesTracker extends EventEmitter + # The purpose of this class is to track a set of inserts and deletes to a document, like + # track changes in Word. We store these as a set of ShareJs style ranges: + # {i: "foo", p: 42} # Insert 'foo' at offset 42 + # {d: "bar", p: 37} # Delete 'bar' at offset 37 + # We only track the inserts and deletes, not the whole document, but by being given all + # updates that are applied to a document, we can update these appropriately. + # + # Note that the set of inserts and deletes we store applies to the document as-is at the moment. + # So inserts correspond to text which is in the document, while deletes correspond to text which + # is no longer there, so their lengths do not affect the position of later offsets. + # E.g. + # this is the current text of the document + # |-----| | + # {i: "current ", p:12} -^ ^- {d: "old ", p: 31} + # + # Track changes rules (should be consistent with Word): + # * When text is inserted at a delete, the text goes to the left of the delete + # I.e. "foo|bar" -> "foobaz|bar", where | is the delete, and 'baz' is inserted + # * Deleting content flagged as 'inserted' does not create a new delete marker, it only + # removes the insert marker. E.g. + # * "abdefghijkl" -> "abfghijkl" when 'de' is deleted. No delete marker added + # |---| <- inserted |-| <- inserted + # * Deletes overlapping regular text and inserted text will insert a delete marker for the + # regular text: + # "abcdefghijkl" -> "abcdejkl" when 'fghi' is deleted + # |----| |--|| + # ^- inserted 'bcdefg' \ ^- deleted 'hi' + # \--inserted 'bcde' + # * Deletes overlapping other deletes are merged. E.g. + # "abcghijkl" -> "ahijkl" when 'bcg is deleted' + # | <- delete 'def' | <- delete 'bcdefg' + # * Deletes by another user will consume deletes by the first user + # * Inserts by another user will not combine with inserts by the first user. If they are in the + # middle of a previous insert by the first user, the original insert will be split into two. + constructor: (@changes = [], @comments = []) -> + # Change objects have the following structure: + # { + # id: ... # Uniquely generated by us + # op: { # ShareJs style op tracking the offset (p) and content inserted (i) or deleted (d) + # i: "..." + # p: 42 + # } + # } + # + # Ids are used to uniquely identify a change, e.g. for updating it in the database, or keeping in + # sync with Ace ranges. + @id = 0 + + addComment: (offset, length, metadata) -> + # TODO: Don't allow overlapping comments? + @comments.push comment = { + id: @_newId() + offset, length, metadata + } + @emit "comment:added", comment + return comment + + getComment: (comment_id) -> + comment = null + for c in @comments + if c.id == comment_id + comment = c + break + return comment + + resolveCommentId: (comment_id, resolved_data) -> + comment = @getComment(comment_id) + return if !comment? + comment.metadata.resolved = true + comment.metadata.resolved_data = resolved_data + @emit "comment:resolved", comment + + unresolveCommentId: (comment_id) -> + comment = @getComment(comment_id) + return if !comment? + comment.metadata.resolved = false + @emit "comment:unresolved", comment + + removeCommentId: (comment_id) -> + comment = @getComment(comment_id) + return if !comment? + @comments = @comments.filter (c) -> c.id != comment_id + @emit "comment:removed", comment + + getChange: (change_id) -> + change = null + for c in @changes + if c.id == change_id + change = c + break + return change + + removeChangeId: (change_id) -> + change = @getChange(change_id) + return if !change? + @_removeChange(change) + + applyOp: (op, metadata = {}) -> + metadata.ts ?= new Date() + # Apply an op that has been applied to the document to our changes to keep them up to date + if op.i? + @applyInsertToChanges(op, metadata) + @applyInsertToComments(op) + else if op.d? + @applyDeleteToChanges(op, metadata) + @applyDeleteToComments(op) + + applyInsertToComments: (op) -> + for comment in @comments + if op.p <= comment.offset + comment.offset += op.i.length + @emit "comment:moved", comment + else if op.p < comment.offset + comment.length + comment.length += op.i.length + @emit "comment:moved", comment + + applyDeleteToComments: (op) -> + op_start = op.p + op_length = op.d.length + op_end = op.p + op_length + for comment in @comments + comment_end = comment.offset + comment.length + if op_end <= comment.offset + # delete is fully before comment + comment.offset -= op_length + @emit "comment:moved", comment + else if op_start >= comment_end + # delete is fully after comment, nothing to do + else + # delete and comment overlap + delete_length_before = Math.max(0, comment.offset - op_start) + delete_length_after = Math.max(0, op_end - comment_end) + delete_length_overlapping = op_length - delete_length_before - delete_length_after + comment.offset = Math.min(comment.offset, op_start) + comment.length -= delete_length_overlapping + @emit "comment:moved", comment + + applyInsertToChanges: (op, metadata) -> + op_start = op.p + op_length = op.i.length + op_end = op.p + op_length + + already_merged = false + previous_change = null + moved_changes = [] + remove_changes = [] + new_changes = [] + for change in @changes + change_start = change.op.p + + if change.op.d? + # Shift any deletes after this along by the length of this insert + if op_start < change_start + change.op.p += op_length + moved_changes.push change + else if op_start == change_start + # If the insert matches the start of the delete, just remove it from the delete instead + if change.op.d.length >= op.i.length and change.op.d.slice(0, op.i.length) == op.i + change.op.d = change.op.d.slice(op.i.length) + change.op.p += op.i.length + if change.op.d == "" + remove_changes.push change + else + moved_changes.push change + already_merged = true + else + change.op.p += op_length + moved_changes.push change + else if change.op.i? + change_end = change_start + change.op.i.length + is_change_overlapping = (op_start >= change_start and op_start <= change_end) + + # Only merge inserts if they are from the same user + is_same_user = metadata.user_id == change.metadata.user_id + + # If there is a delete at the start of the insert, and we're inserting + # at the start, we SHOULDN'T merge since the delete acts as a partition. + # The previous op will be the delete, but it's already been shifted by this insert + # + # I.e. + # Originally: |-- existing insert --| + # | <- existing delete at same offset + # + # Now: |-- existing insert --| <- not shifted yet + # |-- this insert --|| <- existing delete shifted along to end of this op + # + # After: |-- existing insert --| + # |-- this insert --|| <- existing delete + # + # Without the delete, the inserts would be merged. + is_insert_blocked_by_delete = (previous_change? and previous_change.op.d? and previous_change.op.p == op_end) + + # If the insert is overlapping another insert, either at the beginning in the middle or touching the end, + # then we merge them into one. + if @track_changes and + is_change_overlapping and + !is_insert_blocked_by_delete and + !already_merged and + is_same_user + offset = op_start - change_start + change.op.i = change.op.i.slice(0, offset) + op.i + change.op.i.slice(offset) + change.metadata.ts = metadata.ts + already_merged = true + moved_changes.push change + else if op_start <= change_start + # If we're fully before the other insert we can just shift the other insert by our length. + # If they are touching, and should have been merged, they will have been above. + # If not merged above, then it must be blocked by a delete, and will be after this insert, so we shift it along as well + change.op.p += op_length + moved_changes.push change + else if (!is_same_user or !@track_changes) and change_start < op_start < change_end + # This user is inserting inside a change by another user, so we need to split the + # other user's change into one before and after this one. + offset = op_start - change_start + before_content = change.op.i.slice(0, offset) + after_content = change.op.i.slice(offset) + + # The existing change can become the 'before' change + change.op.i = before_content + moved_changes.push change + + # Create a new op afterwards + after_change = { + op: { + i: after_content + p: change_start + offset + op_length + } + metadata: {} + } + after_change.metadata[key] = value for key, value of change.metadata + new_changes.push after_change + + previous_change = change + + if @track_changes and !already_merged + @_addOp op, metadata + for {op, metadata} in new_changes + @_addOp op, metadata + + for change in remove_changes + @_removeChange change + + if moved_changes.length > 0 + @emit "changes:moved", moved_changes + + applyDeleteToChanges: (op, metadata) -> + op_start = op.p + op_length = op.d.length + op_end = op.p + op_length + remove_changes = [] + moved_changes = [] + + # We might end up modifying our delete op if it merges with existing deletes, or cancels out + # with an existing insert. Since we might do multiple modifications, we record them and do + # all the modifications after looping through the existing changes, so as not to mess up the + # offset indexes as we go. + op_modifications = [] + for change in @changes + if change.op.i? + change_start = change.op.p + change_end = change_start + change.op.i.length + if op_end <= change_start + # Shift ops after us back by our length + change.op.p -= op_length + moved_changes.push change + else if op_start >= change_end + # Delete is after insert, nothing to do + else + # When the new delete overlaps an insert, we should remove the part of the insert that + # is now deleted, and also remove the part of the new delete that overlapped. I.e. + # the two cancel out where they overlap. + if op_start >= change_start + # |-- existing insert --| + # insert_remaining_before -> |.....||-- new delete --| + delete_remaining_before = "" + insert_remaining_before = change.op.i.slice(0, op_start - change_start) + else + # delete_remaining_before -> |.....||-- existing insert --| + # |-- new delete --| + delete_remaining_before = op.d.slice(0, change_start - op_start) + insert_remaining_before = "" + + if op_end <= change_end + # |-- existing insert --| + # |-- new delete --||.....| <- insert_remaining_after + delete_remaining_after = "" + insert_remaining_after = change.op.i.slice(op_end - change_start) + else + # |-- existing insert --||.....| <- delete_remaining_after + # |-- new delete --| + delete_remaining_after = op.d.slice(change_end - op_start) + insert_remaining_after = "" + + insert_remaining = insert_remaining_before + insert_remaining_after + if insert_remaining.length > 0 + change.op.i = insert_remaining + change.op.p = Math.min(change_start, op_start) + change.metadata.ts = metadata.ts + moved_changes.push change + else + remove_changes.push change + + # We know what we want to preserve of our delete op before (delete_remaining_before) and what we want to preserve + # afterwards (delete_remaining_before). Now we need to turn that into a modification which deletes the + # chunk in the middle not covered by these. + delete_removed_length = op.d.length - delete_remaining_before.length - delete_remaining_after.length + delete_removed_start = delete_remaining_before.length + modification = { + d: op.d.slice(delete_removed_start, delete_removed_start + delete_removed_length) + p: delete_removed_start + } + if modification.d.length > 0 + op_modifications.push modification + else if change.op.d? + change_start = change.op.p + if op_end < change_start or (!@track_changes and op_end == change_start) + # Shift ops after us back by our length. + # If we're tracking changes, it must be strictly before, since we'll merge + # below if they are touching. Otherwise, touching is fine. + change.op.p -= op_length + moved_changes.push change + else if op_start <= change_start <= op_end + if @track_changes + # If we overlap a delete, add it in our content, and delete the existing change. + # It's easier to do it this way, rather than modifying the existing delete in case + # we overlap many deletes and we'd need to track that. We have a workaround to + # update the delete in place if possible below. + offset = change_start - op_start + op_modifications.push { i: change.op.d, p: offset } + remove_changes.push change + else + change.op.p = op_start + moved_changes.push change + + # Copy rather than modify because we still need to apply it to comments + op = { + p: op.p + d: @_applyOpModifications(op.d, op_modifications) + } + + for change in remove_changes + # This is a bit of hack to avoid removing one delete and replacing it with another. + # If we don't do this, it causes the UI to flicker + if op.d.length > 0 and change.op.d? and op.p <= change.op.p <= op.p + op.d.length + change.op.p = op.p + change.op.d = op.d + change.metadata = metadata + moved_changes.push change + op.d = "" # stop it being added + else + @_removeChange change + + if @track_changes and op.d.length > 0 + @_addOp op, metadata + else + # It's possible that we deleted an insert between two other inserts. I.e. + # If we delete 'user_2 insert' in: + # |-- user_1 insert --||-- user_2 insert --||-- user_1 insert --| + # it becomes: + # |-- user_1 insert --||-- user_1 insert --| + # We need to merge these together again + results = @_scanAndMergeAdjacentUpdates() + moved_changes = moved_changes.concat(results.moved_changes) + for change in results.remove_changes + @_removeChange change + moved_changes = moved_changes.filter (c) -> c != change + + if moved_changes.length > 0 + @emit "changes:moved", moved_changes + + _newId: () -> + (@id++).toString() + + _addOp: (op, metadata) -> + change = { + id: @_newId() + op: op + metadata: metadata + } + @changes.push change + + # Keep ops in order of offset, with deletes before inserts + @changes.sort (c1, c2) -> + result = c1.op.p - c2.op.p + if result != 0 + return result + else if c1.op.i? and c2.op.d? + return 1 + else + return -1 + + if op.d? + @emit "delete:added", change + else if op.i? + @emit "insert:added", change + + _removeChange: (change) -> + @changes = @changes.filter (c) -> c.id != change.id + if change.op.d? + @emit "delete:removed", change + else if change.op.i? + @emit "insert:removed", change + + _applyOpModifications: (content, op_modifications) -> + # Put in descending position order, with deleting first if at the same offset + # (Inserting first would modify the content that the delete will delete) + op_modifications.sort (a, b) -> + result = b.p - a.p + if result != 0 + return result + else if a.i? and b.d? + return 1 + else + return -1 + + for modification in op_modifications + if modification.i? + content = content.slice(0, modification.p) + modification.i + content.slice(modification.p) + else if modification.d? + if content.slice(modification.p, modification.p + modification.d.length) != modification.d + throw new Error("deleted content does not match. content: #{JSON.stringify(content)}; modification: #{JSON.stringify(modification)}") + content = content.slice(0, modification.p) + content.slice(modification.p + modification.d.length) + return content + + _scanAndMergeAdjacentUpdates: () -> + # This should only need calling when deleting an update between two + # other updates. There's no other way to get two adjacent updates from the + # same user, since they would be merged on insert. + previous_change = null + remove_changes = [] + moved_changes = [] + for change in @changes + if previous_change?.op.i? and change.op.i? + previous_change_end = previous_change.op.p + previous_change.op.i.length + previous_change_user_id = previous_change.metadata.user_id + change_start = change.op.p + change_user_id = change.metadata.user_id + if previous_change_end == change_start and previous_change_user_id == change_user_id + remove_changes.push change + previous_change.op.i += change.op.i + moved_changes.push previous_change + else if previous_change?.op.d? and change.op.d? and previous_change.op.p == change.op.p + # Merge adjacent deletes + previous_change.op.d += change.op.d + remove_changes.push change + moved_changes.push previous_change + else # Only update to the current change if we haven't removed it. + previous_change = change + return { moved_changes, remove_changes } + +if define? + define ["utils/EventEmitter"], load +else + EventEmitter = require("events").EventEmitter + module.exports = load(EventEmitter) \ No newline at end of file diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee index ebbdc3a66e..9c7277c469 100644 --- a/services/document-updater/app/coffee/DocumentManager.coffee +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -3,7 +3,8 @@ PersistenceManager = require "./PersistenceManager" DiffCodec = require "./DiffCodec" logger = require "logger-sharelatex" Metrics = require "./Metrics" -TrackChangesManager = require "./TrackChangesManager" +HistoryManager = require "./HistoryManager" +WebRedisManager = require "./WebRedisManager" module.exports = DocumentManager = getDoc: (project_id, doc_id, _callback = (error, lines, version, alreadyLoaded) ->) -> @@ -12,18 +13,18 @@ module.exports = DocumentManager = timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> return callback(error) if error? if !lines? or !version? - logger.log project_id: project_id, doc_id: doc_id, "doc not in redis so getting from persistence API" - PersistenceManager.getDoc project_id, doc_id, (error, lines, version) -> + logger.log {project_id, doc_id, track_changes}, "doc not in redis so getting from persistence API" + PersistenceManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> return callback(error) if error? - logger.log project_id: project_id, doc_id: doc_id, lines: lines, version: version, "got doc from persistence API" - RedisManager.putDocInMemory project_id, doc_id, lines, version, (error) -> + logger.log {project_id, doc_id, lines, version, track_changes}, "got doc from persistence API" + RedisManager.putDocInMemory project_id, doc_id, lines, version, track_changes, track_changes_entries, (error) -> return callback(error) if error? - callback null, lines, version, false + callback null, lines, version, track_changes, track_changes_entries, false else - callback null, lines, version, true + callback null, lines, version, track_changes, track_changes_entries, true getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps) ->) -> timer = new Metrics.Timer("docManager.getDocAndRecentOps") @@ -50,7 +51,7 @@ module.exports = DocumentManager = return callback(new Error("No lines were provided to setDoc")) UpdateManager = require "./UpdateManager" - DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, alreadyLoaded) -> + DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, track_changes, alreadyLoaded) -> return callback(error) if error? if oldLines? and oldLines.length > 0 and oldLines[0].text? @@ -89,14 +90,14 @@ module.exports = DocumentManager = callback = (args...) -> timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> return callback(error) if error? if !lines? or !version? logger.log project_id: project_id, doc_id: doc_id, "doc is not loaded so not flushing" callback null # TODO: return a flag to bail out, as we go on to remove doc from memory? else logger.log project_id: project_id, doc_id: doc_id, version: version, "flushing doc" - PersistenceManager.setDoc project_id, doc_id, lines, version, (error) -> + PersistenceManager.setDoc project_id, doc_id, lines, version, track_changes, track_changes_entries, (error) -> return callback(error) if error? callback null @@ -111,13 +112,19 @@ module.exports = DocumentManager = # Flush in the background since it requires and http request # to track changes - TrackChangesManager.flushDocChanges project_id, doc_id, (err) -> + HistoryManager.flushDocChanges project_id, doc_id, (err) -> if err? logger.err {err, project_id, doc_id}, "error flushing to track changes" RedisManager.removeDocFromMemory project_id, doc_id, (error) -> return callback(error) if error? callback null + + setTrackChanges: (project_id, doc_id, track_changes_on, callback = (error) ->) -> + RedisManager.setTrackChanges project_id, doc_id, track_changes_on, (error) -> + return callback(error) if error? + WebRedisManager.sendData {project_id, doc_id, track_changes_on} + callback() getDocWithLock: (project_id, doc_id, callback = (error, lines, version) ->) -> UpdateManager = require "./UpdateManager" @@ -138,3 +145,7 @@ module.exports = DocumentManager = flushAndDeleteDocWithLock: (project_id, doc_id, callback = (error) ->) -> UpdateManager = require "./UpdateManager" UpdateManager.lockUpdatesAndDo DocumentManager.flushAndDeleteDoc, project_id, doc_id, callback + + setTrackChangesWithLock: (project_id, doc_id, track_changes_on, callback = (error) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.setTrackChanges, project_id, doc_id, track_changes_on, callback \ No newline at end of file diff --git a/services/document-updater/app/coffee/HistoryManager.coffee b/services/document-updater/app/coffee/HistoryManager.coffee new file mode 100644 index 0000000000..637fd2cb5f --- /dev/null +++ b/services/document-updater/app/coffee/HistoryManager.coffee @@ -0,0 +1,44 @@ +settings = require "settings-sharelatex" +request = require "request" +logger = require "logger-sharelatex" +async = require "async" +WebRedisManager = require "./WebRedisManager" + +module.exports = HistoryManager = + flushDocChanges: (project_id, doc_id, callback = (error) ->) -> + if !settings.apis?.trackchanges? + logger.warn doc_id: doc_id, "track changes API is not configured, so not flushing" + return callback() + + url = "#{settings.apis.trackchanges.url}/project/#{project_id}/doc/#{doc_id}/flush" + logger.log project_id: project_id, doc_id: doc_id, url: url, "flushing doc in track changes api" + request.post url, (error, res, body)-> + if error? + return callback(error) + else if res.statusCode >= 200 and res.statusCode < 300 + return callback(null) + else + error = new Error("track changes api returned a failure status code: #{res.statusCode}") + return callback(error) + + FLUSH_EVERY_N_OPS: 50 + pushUncompressedHistoryOps: (project_id, doc_id, ops = [], callback = (error) ->) -> + if ops.length == 0 + return callback() + WebRedisManager.pushUncompressedHistoryOps project_id, doc_id, ops, (error, length) -> + return callback(error) if error? + # We want to flush every 50 ops, i.e. 50, 100, 150, etc + # Find out which 'block' (i.e. 0-49, 50-99) we were in before and after pushing these + # ops. If we've changed, then we've gone over a multiple of 50 and should flush. + # (Most of the time, we will only hit 50 and then flushing will put us back to 0) + previousLength = length - ops.length + prevBlock = Math.floor(previousLength / HistoryManager.FLUSH_EVERY_N_OPS) + newBlock = Math.floor(length / HistoryManager.FLUSH_EVERY_N_OPS) + if newBlock != prevBlock + # Do this in the background since it uses HTTP and so may be too + # slow to wait for when processing a doc update. + logger.log length: length, doc_id: doc_id, project_id: project_id, "flushing track changes api" + HistoryManager.flushDocChanges project_id, doc_id, (error) -> + if error? + logger.error err: error, doc_id: doc_id, project_id: project_id, "error flushing doc to track changes api" + callback() \ No newline at end of file diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee index ee6359b104..0366746d56 100644 --- a/services/document-updater/app/coffee/HttpController.coffee +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -96,3 +96,15 @@ module.exports = HttpController = return next(error) if error? logger.log project_id: project_id, "deleted project via http" res.send 204 # No Content + + setTrackChanges: (req, res, next = (error) ->) -> + project_id = req.params.project_id + track_changes_on = req.body.on + if !track_changes_on? + return res.send 400 + track_changes_on = !!track_changes_on # Make boolean + logger.log {project_id, track_changes_on}, "turning on track changes via http" + ProjectManager.setTrackChangesWithLocks project_id, track_changes_on, (error) -> + return next(error) if error? + res.send 204 + diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index 605425eb5e..8d5578b4cf 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -12,14 +12,14 @@ MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds module.exports = PersistenceManager = getDoc: (project_id, doc_id, callback = (error, lines, version) ->) -> - PersistenceManager.getDocFromWeb project_id, doc_id, (error, lines) -> + PersistenceManager.getDocFromWeb project_id, doc_id, (error, lines, track_changes, track_changes_entries) -> return callback(error) if error? PersistenceManager.getDocVersionInMongo doc_id, (error, version) -> return callback(error) if error? - callback null, lines, version + callback null, lines, version, track_changes, track_changes_entries - setDoc: (project_id, doc_id, lines, version, callback = (error) ->) -> - PersistenceManager.setDocInWeb project_id, doc_id, lines, (error) -> + setDoc: (project_id, doc_id, lines, version, track_changes, track_changes_entries, callback = (error) ->) -> + PersistenceManager.setDocInWeb project_id, doc_id, lines, track_changes, track_changes_entries, (error) -> return callback(error) if error? PersistenceManager.setDocVersionInMongo doc_id, version, (error) -> return callback(error) if error? @@ -50,13 +50,13 @@ module.exports = PersistenceManager = body = JSON.parse body catch e return callback(e) - return callback null, body.lines + return callback null, body.lines, body.track_changes, body.track_changes_entries else if res.statusCode == 404 return callback(new Errors.NotFoundError("doc not not found: #{url}")) else return callback(new Error("error accessing web API: #{url} #{res.statusCode}")) - setDocInWeb: (project_id, doc_id, lines, _callback = (error) ->) -> + setDocInWeb: (project_id, doc_id, lines, track_changes, track_changes_entries, _callback = (error) ->) -> timer = new Metrics.Timer("persistenceManager.setDoc") callback = (args...) -> timer.done() @@ -68,6 +68,8 @@ module.exports = PersistenceManager = method: "POST" body: JSON.stringify lines: lines + track_changes: track_changes + track_changes_entries: track_changes_entries headers: "content-type": "application/json" auth: diff --git a/services/document-updater/app/coffee/ProjectManager.coffee b/services/document-updater/app/coffee/ProjectManager.coffee index f0f62b6d1b..a38fe08397 100644 --- a/services/document-updater/app/coffee/ProjectManager.coffee +++ b/services/document-updater/app/coffee/ProjectManager.coffee @@ -57,4 +57,30 @@ module.exports = ProjectManager = else callback(null) + setTrackChangesWithLocks: (project_id, track_changes_on, _callback = (error) ->) -> + timer = new Metrics.Timer("projectManager.toggleTrackChangesWithLocks") + callback = (args...) -> + timer.done() + _callback(args...) + + RedisManager.getDocIdsInProject project_id, (error, doc_ids) -> + return callback(error) if error? + jobs = [] + errors = [] + for doc_id in (doc_ids or []) + do (doc_id) -> + jobs.push (callback) -> + DocumentManager.setTrackChangesWithLock project_id, doc_id, track_changes_on, (error) -> + if error? + logger.error {err: error, project_id, doc_ids, track_changes_on}, "error toggle track changes for doc" + errors.push(error) + callback() + # TODO: If no docs, turn on track changes in Mongo manually + + logger.log {project_id, doc_ids, track_changes_on}, "toggling track changes for docs" + async.series jobs, () -> + if errors.length > 0 + callback new Error("Errors toggling track changes for docs. See log for details") + else + callback(null) diff --git a/services/document-updater/app/coffee/RedisKeyBuilder.coffee b/services/document-updater/app/coffee/RedisKeyBuilder.coffee index 0e9e59e8f1..c09fb43f00 100644 --- a/services/document-updater/app/coffee/RedisKeyBuilder.coffee +++ b/services/document-updater/app/coffee/RedisKeyBuilder.coffee @@ -34,6 +34,10 @@ module.exports = RedisKeyBuilder = return (key_schema) -> key_schema.uncompressedHistoryOp({doc_id}) pendingUpdates: ({doc_id}) -> return (key_schema) -> key_schema.pendingUpdates({doc_id}) + trackChangesEnabled: ({doc_id}) -> + return (key_schema) -> key_schema.trackChangesEnabled({doc_id}) + trackChangesEntries: ({doc_id}) -> + return (key_schema) -> key_schema.trackChangesEntries({doc_id}) docsInProject: ({project_id}) -> return (key_schema) -> key_schema.docsInProject({project_id}) docsWithHistoryOps: ({project_id}) -> diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 87e31b7826..6ee764cb7e 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -13,7 +13,7 @@ minutes = 60 # seconds for Redis expire module.exports = RedisManager = rclient: rclient - putDocInMemory : (project_id, doc_id, docLines, version, _callback)-> + putDocInMemory : (project_id, doc_id, docLines, version, track_changes, track_changes_entries, _callback)-> timer = new metrics.Timer("redis.put-doc") callback = (error) -> timer.done() @@ -23,6 +23,8 @@ module.exports = RedisManager = multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines) multi.set keys.projectKey({doc_id:doc_id}), project_id multi.set keys.docVersion(doc_id:doc_id), version + multi.set keys.trackChangesEnabled(doc_id:doc_id), if track_changes then "1" else "0" + multi.set keys.trackChangesEntries(doc_id:doc_id), JSON.stringify(track_changes_entries) multi.exec (error) -> return callback(error) if error? rclient.sadd keys.docsInProject(project_id:project_id), doc_id, callback @@ -41,30 +43,36 @@ module.exports = RedisManager = multi.del keys.docLines(doc_id:doc_id) multi.del keys.projectKey(doc_id:doc_id) multi.del keys.docVersion(doc_id:doc_id) + multi.del keys.trackChangesEnabled(doc_id:doc_id) + multi.del keys.trackChangesEntries(doc_id:doc_id) multi.exec (error) -> return callback(error) if error? rclient.srem keys.docsInProject(project_id:project_id), doc_id, callback - getDoc : (project_id, doc_id, callback = (error, lines, version, project_id) ->)-> + getDoc : (project_id, doc_id, callback = (error, lines, version, track_changes, track_changes_entries) ->)-> timer = new metrics.Timer("redis.get-doc") multi = rclient.multi() multi.get keys.docLines(doc_id:doc_id) multi.get keys.docVersion(doc_id:doc_id) multi.get keys.projectKey(doc_id:doc_id) + multi.get keys.trackChangesEnabled(doc_id:doc_id) + multi.get keys.trackChangesEntries(doc_id:doc_id) multi.exec (error, result)-> timer.done() return callback(error) if error? try docLines = JSON.parse result[0] + track_changes_entries = JSON.parse result[4] catch e return callback(e) version = parseInt(result[1] or 0, 10) doc_project_id = result[2] + track_changes = (result[3] == "1") # check doc is in requested project if doc_project_id? and doc_project_id isnt project_id logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, "doc not in project" return callback(new Errors.NotFoundError("document not found")) - callback null, docLines, version, project_id + callback null, docLines, version, track_changes, track_changes_entries getDocVersion: (doc_id, callback = (error, version) ->) -> rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> @@ -104,7 +112,7 @@ module.exports = RedisManager = DOC_OPS_TTL: 60 * minutes DOC_OPS_MAX_LENGTH: 100 - updateDocument : (doc_id, docLines, newVersion, appliedOps = [], callback = (error) ->)-> + updateDocument : (doc_id, docLines, newVersion, appliedOps = [], track_changes_entries, callback = (error) ->)-> RedisManager.getDocVersion doc_id, (error, currentVersion) -> return callback(error) if error? if currentVersion + appliedOps.length != newVersion @@ -119,6 +127,7 @@ module.exports = RedisManager = multi.rpush keys.docOps(doc_id: doc_id), jsonOps... multi.expire keys.docOps(doc_id: doc_id), RedisManager.DOC_OPS_TTL multi.ltrim keys.docOps(doc_id: doc_id), -RedisManager.DOC_OPS_MAX_LENGTH, -1 + multi.set keys.trackChangesEntries(doc_id:doc_id), JSON.stringify(track_changes_entries) multi.exec (error, replys) -> return callback(error) if error? return callback() @@ -126,3 +135,6 @@ module.exports = RedisManager = getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) -> rclient.smembers keys.docsInProject(project_id: project_id), callback + setTrackChanges: (project_id, doc_id, track_changes_on, callback = (error) ->) -> + value = (if track_changes_on then "1" else "0") + rclient.set keys.trackChangesEnabled({doc_id}), value, callback diff --git a/services/document-updater/app/coffee/ShareJsDB.coffee b/services/document-updater/app/coffee/ShareJsDB.coffee index 3d80c680cb..a21c8aea7f 100644 --- a/services/document-updater/app/coffee/ShareJsDB.coffee +++ b/services/document-updater/app/coffee/ShareJsDB.coffee @@ -1,12 +1,11 @@ Keys = require('./UpdateKeys') Settings = require('settings-sharelatex') -DocumentManager = require "./DocumentManager" RedisManager = require "./RedisManager" Errors = require "./Errors" logger = require "logger-sharelatex" module.exports = class ShareJsDB - constructor: () -> + constructor: (@project_id, @doc_id, @lines, @version) -> @appliedOps = {} # ShareJS calls this detacted from the instance, so we need # bind it to keep our context that can access @appliedOps @@ -31,22 +30,14 @@ module.exports = class ShareJsDB callback() getSnapshot: (doc_key, callback) -> - [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) - DocumentManager.getDoc project_id, doc_id, (error, lines, version) -> - return callback(error) if error? - if !lines? or !version? - return callback(new Errors.NotFoundError("document not found: #{doc_id}")) - - if lines.length > 0 and lines[0].text? - type = "json" - snapshot = lines: lines - else - type = "text" - snapshot = lines.join("\n") - callback null, - snapshot: snapshot - v: parseInt(version, 10) - type: type + if doc_key != Keys.combineProjectIdAndDocId(@project_id, @doc_id) + return callback(new Errors.NotFoundError("unexpected doc_key #{doc_key}, expected #{Keys.combineProjectIdAndDocId(@project_id, @doc_id)}")) + else + return callback null, { + snapshot: @lines.join("\n") + v: parseInt(@version, 10) + type: "text" + } # To be able to remove a doc from the ShareJS memory # we need to called Model::delete, which calls this diff --git a/services/document-updater/app/coffee/ShareJsUpdateManager.coffee b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee index 985d03094a..876d56e71b 100644 --- a/services/document-updater/app/coffee/ShareJsUpdateManager.coffee +++ b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee @@ -6,22 +6,19 @@ Settings = require('settings-sharelatex') Keys = require "./UpdateKeys" {EventEmitter} = require "events" util = require "util" - -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) - +WebRedisManager = require "./WebRedisManager" ShareJsModel:: = {} util.inherits ShareJsModel, EventEmitter module.exports = ShareJsUpdateManager = - getNewShareJsModel: () -> - db = new ShareJsDB() + getNewShareJsModel: (project_id, doc_id, lines, version) -> + db = new ShareJsDB(project_id, doc_id, lines, version) model = new ShareJsModel(db, maxDocLength: Settings.max_doc_length) model.db = db return model - applyUpdate: (project_id, doc_id, update, callback = (error, updatedDocLines) ->) -> + applyUpdate: (project_id, doc_id, update, lines, version, callback = (error, updatedDocLines) ->) -> logger.log project_id: project_id, doc_id: doc_id, update: update, "applying sharejs updates" jobs = [] @@ -30,7 +27,7 @@ module.exports = ShareJsUpdateManager = # getting stuck due to queued callbacks (line 260 of sharejs/server/model.coffee) # This adds a small but hopefully acceptable overhead (~12ms per 1000 updates on # my 2009 MBP). - model = @getNewShareJsModel() + model = @getNewShareJsModel(project_id, doc_id, lines, version) @_listenForOps(model) doc_key = Keys.combineProjectIdAndDocId(project_id, doc_id) model.applyOp doc_key, update, (error) -> @@ -55,18 +52,9 @@ module.exports = ShareJsUpdateManager = [project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key) ShareJsUpdateManager._sendOp(project_id, doc_id, opData) - _sendOp: (project_id, doc_id, opData) -> - data = - project_id: project_id - doc_id: doc_id - op: opData - data = JSON.stringify data - rclient.publish "applied-ops", data + _sendOp: (project_id, doc_id, op) -> + WebRedisManager.sendData {project_id, doc_id, op} _sendError: (project_id, doc_id, error) -> - data = JSON.stringify - project_id: project_id - doc_id: doc_id - error: error.message || error - rclient.publish "applied-ops", data + WebRedisManager.sendData {project_id, doc_id, error: error.message || error} diff --git a/services/document-updater/app/coffee/TrackChangesManager.coffee b/services/document-updater/app/coffee/TrackChangesManager.coffee index 9aa1c0ad47..94f8a11ca1 100644 --- a/services/document-updater/app/coffee/TrackChangesManager.coffee +++ b/services/document-updater/app/coffee/TrackChangesManager.coffee @@ -1,44 +1,12 @@ -settings = require "settings-sharelatex" -request = require "request" -logger = require "logger-sharelatex" -async = require "async" -WebRedisManager = require "./WebRedisManager" +ChangesTracker = require "./ChangesTracker" module.exports = TrackChangesManager = - flushDocChanges: (project_id, doc_id, callback = (error) ->) -> - if !settings.apis?.trackchanges? - logger.warn doc_id: doc_id, "track changes API is not configured, so not flushing" - return callback() - - url = "#{settings.apis.trackchanges.url}/project/#{project_id}/doc/#{doc_id}/flush" - logger.log project_id: project_id, doc_id: doc_id, url: url, "flushing doc in track changes api" - request.post url, (error, res, body)-> - if error? - return callback(error) - else if res.statusCode >= 200 and res.statusCode < 300 - return callback(null) - else - error = new Error("track changes api returned a failure status code: #{res.statusCode}") - return callback(error) - - FLUSH_EVERY_N_OPS: 50 - pushUncompressedHistoryOps: (project_id, doc_id, ops = [], callback = (error) ->) -> - if ops.length == 0 - return callback() - WebRedisManager.pushUncompressedHistoryOps project_id, doc_id, ops, (error, length) -> - return callback(error) if error? - # We want to flush every 50 ops, i.e. 50, 100, 150, etc - # Find out which 'block' (i.e. 0-49, 50-99) we were in before and after pushing these - # ops. If we've changed, then we've gone over a multiple of 50 and should flush. - # (Most of the time, we will only hit 50 and then flushing will put us back to 0) - previousLength = length - ops.length - prevBlock = Math.floor(previousLength / TrackChangesManager.FLUSH_EVERY_N_OPS) - newBlock = Math.floor(length / TrackChangesManager.FLUSH_EVERY_N_OPS) - if newBlock != prevBlock - # Do this in the background since it uses HTTP and so may be too - # slow to wait for when processing a doc update. - logger.log length: length, doc_id: doc_id, project_id: project_id, "flushing track changes api" - TrackChangesManager.flushDocChanges project_id, doc_id, (error) -> - if error? - logger.error err: error, doc_id: doc_id, project_id: project_id, "error flushing doc to track changes api" - callback() \ No newline at end of file + applyUpdate: (project_id, doc_id, entries = {}, updates = [], track_changes, callback = (error, new_entries) ->) -> + {changes, comments} = entries + changesTracker = new ChangesTracker(changes, comments) + changesTracker.track_changes = track_changes + for update in updates + for op in update.op + changesTracker.applyOp(op, { user_id: update.meta?.user_id, }) + {changes, comments} = changesTracker + callback null, {changes, comments} \ No newline at end of file diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee index f35bc1a9b7..d08d9a62f3 100644 --- a/services/document-updater/app/coffee/UpdateManager.coffee +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -2,11 +2,14 @@ LockManager = require "./LockManager" RedisManager = require "./RedisManager" WebRedisManager = require "./WebRedisManager" ShareJsUpdateManager = require "./ShareJsUpdateManager" -TrackChangesManager = require "./TrackChangesManager" +HistoryManager = require "./HistoryManager" Settings = require('settings-sharelatex') async = require("async") logger = require('logger-sharelatex') Metrics = require "./Metrics" +Errors = require "./Errors" +DocumentManager = require "./DocumentManager" +TrackChangesManager = require "./TrackChangesManager" module.exports = UpdateManager = processOutstandingUpdates: (project_id, doc_id, callback = (error) ->) -> @@ -45,12 +48,18 @@ module.exports = UpdateManager = applyUpdate: (project_id, doc_id, update, callback = (error) ->) -> UpdateManager._sanitizeUpdate update - ShareJsUpdateManager.applyUpdate project_id, doc_id, update, (error, updatedDocLines, version, appliedOps) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> return callback(error) if error? - logger.log doc_id: doc_id, version: version, "updating doc in redis" - RedisManager.updateDocument doc_id, updatedDocLines, version, appliedOps, (error) -> + if !lines? or !version? + return callback(new Errors.NotFoundError("document not found: #{doc_id}")) + ShareJsUpdateManager.applyUpdate project_id, doc_id, update, lines, version, (error, updatedDocLines, version, appliedOps) -> return callback(error) if error? - TrackChangesManager.pushUncompressedHistoryOps project_id, doc_id, appliedOps, callback + TrackChangesManager.applyUpdate project_id, doc_id, track_changes_entries, appliedOps, track_changes, (error, new_track_changes_entries) -> + return callback(error) if error? + logger.log doc_id: doc_id, version: version, "updating doc in redis" + RedisManager.updateDocument doc_id, updatedDocLines, version, appliedOps, new_track_changes_entries, (error) -> + return callback(error) if error? + HistoryManager.pushUncompressedHistoryOps project_id, doc_id, appliedOps, callback lockUpdatesAndDo: (method, project_id, doc_id, args..., callback) -> LockManager.getLock doc_id, (error, lockValue) -> diff --git a/services/document-updater/app/coffee/WebRedisManager.coffee b/services/document-updater/app/coffee/WebRedisManager.coffee index 73c099f9da..eb3b6a583c 100644 --- a/services/document-updater/app/coffee/WebRedisManager.coffee +++ b/services/document-updater/app/coffee/WebRedisManager.coffee @@ -32,4 +32,7 @@ module.exports = WebRedisManager = ], (error, results) -> return callback(error) if error? [length, _] = results - callback(error, length) \ No newline at end of file + callback(error, length) + + sendData: (data) -> + rclient.publish "applied-ops", JSON.stringify(data) \ No newline at end of file diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index 15456db932..edb8c56ad3 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -32,6 +32,8 @@ module.exports = docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" + trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:#{doc_id}" + trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:#{doc_id}" # }, { # cluster: [{ # port: "7000" @@ -44,6 +46,8 @@ module.exports = # docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" # projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" # docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" + # trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:{#{doc_id}}" + # trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:{#{doc_id}}" }] max_doc_length: 2 * 1024 * 1024 # 2mb diff --git a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee new file mode 100644 index 0000000000..406f46b430 --- /dev/null +++ b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee @@ -0,0 +1,96 @@ +sinon = require "sinon" +chai = require("chai") +chai.should() +async = require "async" +rclient = require("redis").createClient() + +MockWebApi = require "./helpers/MockWebApi" +DocUpdaterClient = require "./helpers/DocUpdaterClient" + +describe "Track changes", -> + describe "turning on track changes", -> + before (done) -> + DocUpdaterClient.subscribeToAppliedOps @appliedOpsListener = sinon.stub() + @project_id = DocUpdaterClient.randomId() + @docs = [{ + id: doc_id0 = DocUpdaterClient.randomId() + lines: ["one", "two", "three"] + updatedLines: ["one", "one and a half", "two", "three"] + }, { + id: doc_id1 = DocUpdaterClient.randomId() + lines: ["four", "five", "six"] + updatedLines: ["four", "four and a half", "five", "six"] + }] + for doc in @docs + MockWebApi.insertDoc @project_id, doc.id, { + lines: doc.lines + version: 0 + } + async.series @docs.map((doc) => + (callback) => + DocUpdaterClient.preloadDoc @project_id, doc.id, callback + ), (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.setTrackChangesOn @project_id, (error, res, body) => + @statusCode = res.statusCode + done() + , 200 + + it "should return a 204 status code", -> + @statusCode.should.equal 204 + + it "should send a track changes message to real-time for each doc", -> + @appliedOpsListener.calledWith("applied-ops", JSON.stringify({ + project_id: @project_id, doc_id: @docs[0].id, track_changes_on: true + })).should.equal true + @appliedOpsListener.calledWith("applied-ops", JSON.stringify({ + project_id: @project_id, doc_id: @docs[1].id, track_changes_on: true + })).should.equal true + + it "should set the track changes key in redis", (done) -> + rclient.get "TrackChangesEnabled:#{@docs[0].id}", (error, value) => + throw error if error? + value.should.equal "1" + rclient.get "TrackChangesEnabled:#{@docs[1].id}", (error, value) -> + throw error if error? + value.should.equal "1" + done() + + describe "tracking changes", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @doc = { + id: doc_id0 = DocUpdaterClient.randomId() + lines: ["one", "two", "three"] + } + @update = + doc: @doc.id + op: [{ + i: "one and a half\n" + p: 4 + }] + v: 0 + meta: + user_id: @user_id = DocUpdaterClient.randomId() + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + DocUpdaterClient.setTrackChangesOn @project_id, (error, res, body) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc.id, @update, (error) -> + throw error if error? + setTimeout done, 200 + + it "should set the updated track changes entries in redis", (done) -> + rclient.get "TrackChangesEntries:#{@doc.id}", (error, value) => + throw error if error? + entries = JSON.parse(value) + change = entries.changes[0] + change.op.should.deep.equal @update.op[0] + change.metadata.user_id.should.equal @user_id + done() + diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee index a14f6f9364..b90e7ea82e 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee @@ -72,7 +72,18 @@ module.exports = DocUpdaterClient = deleteProject: (project_id, callback = () ->) -> request.del "http://localhost:3003/project/#{project_id}", callback - - - - + + setTrackChangesOn: (project_id, callback = () ->) -> + request.post { + url: "http://localhost:3003/project/#{project_id}/track_changes" + json: + on: true + }, callback + + setTrackChangesOff: (project_id, callback = () ->) -> + request.post { + url: "http://localhost:3003/project/#{project_id}/track_changes" + json: + on: false + }, callback + From fb39e37fe0ebf446a61043c113e592668e90d182 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 16:27:40 +0000 Subject: [PATCH 02/25] Update DocumentManager tests --- .../DocumentManagerTests.coffee | 263 ++++++++++++++++++ .../flushAndDeleteDocTests.coffee | 48 ---- .../flushDocIfLoadedTests.coffee | 68 ----- .../getDocAndRecentOpsTests.coffee | 67 ----- .../coffee/DocumentManager/getDocTests.coffee | 70 ----- .../coffee/DocumentManager/setDocTests.coffee | 107 ------- 6 files changed, 263 insertions(+), 360 deletions(-) create mode 100644 services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/DocumentManager/flushDocIfLoadedTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee diff --git a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee new file mode 100644 index 0000000000..2b0dce169b --- /dev/null +++ b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee @@ -0,0 +1,263 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/DocumentManager.js" +SandboxedModule = require('sandboxed-module') + +describe "DocumentManager", -> + beforeEach -> + @DocumentManager = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + "./PersistenceManager": @PersistenceManager = {} + "./HistoryManager": @HistoryManager = {} + "logger-sharelatex": @logger = {log: sinon.stub()} + "./DocOpsManager": @DocOpsManager = {} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + "./WebRedisManager": @WebRedisManager = {} + "./DiffCodec": @DiffCodec = {} + "./UpdateManager": @UpdateManager = {} + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + @lines = ["one", "two", "three"] + @version = 42 + @track_changes_entries = { comments: "mock", entries: "mock" } + @track_changes_on = true + + describe "flushAndDeleteDoc", -> + describe "successfully", -> + beforeEach -> + @RedisManager.removeDocFromMemory = sinon.stub().callsArg(2) + @DocumentManager.flushDocIfLoaded = sinon.stub().callsArgWith(2) + @HistoryManager.flushDocChanges = sinon.stub().callsArg(2) + @DocumentManager.flushAndDeleteDoc @project_id, @doc_id, @callback + + it "should flush the doc", -> + @DocumentManager.flushDocIfLoaded + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should remove the doc from redis", -> + @RedisManager.removeDocFromMemory + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + it "should flush to track changes", -> + @HistoryManager.flushDocChanges + .calledWith(@project_id, @doc_id) + .should.equal true + + describe "flushDocIfLoaded", -> + describe "when the doc is in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @PersistenceManager.setDoc = sinon.stub().yields() + @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback + + it "should get the doc from redis", -> + @RedisManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should write the doc lines to the persistence layer", -> + @PersistenceManager.setDoc + .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the document is not in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null) + @PersistenceManager.setDoc = sinon.stub().yields() + @DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2) + @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback + + it "should get the doc from redis", -> + @RedisManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should not write anything to the persistence layer", -> + @PersistenceManager.setDoc.called.should.equal false + @DocOpsManager.flushDocOpsToMongo.called.should.equal false + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "getDocAndRecentOps", -> + describe "with a previous version specified", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should get the doc ops", -> + @RedisManager.getPreviousDocOps + .calledWith(@doc_id, @fromVersion, @version) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, @ops).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "with no previous version specified", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback + + it "should get the doc", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should not need to get the doc ops", -> + @RedisManager.getPreviousDocOps.called.should.equal false + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, []).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "getDoc", -> + describe "when the doc exists in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @DocumentManager.getDoc @project_id, @doc_id, @callback + + it "should get the doc from Redis", -> + @RedisManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries, true).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the doc does not exist in Redis", -> + beforeEach -> + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null) + @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @RedisManager.putDocInMemory = sinon.stub().yields() + @DocumentManager.getDoc @project_id, @doc_id, @callback + + it "should try to get the doc from Redis", -> + @RedisManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should get the doc from the PersistenceManager", -> + @PersistenceManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should set the doc in Redis", -> + @RedisManager.putDocInMemory + .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries) + .should.equal true + + it "should call the callback with the doc info", -> + @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries, false).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "setDoc", -> + describe "with plain tex lines", -> + beforeEach -> + @beforeLines = ["before", "lines"] + @afterLines = ["after", "lines"] + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @track_changes_on, @track_changes_entries, true) + @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) + @UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null) + @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) + @DocumentManager.flushAndDeleteDoc = sinon.stub().callsArg(2) + + describe "when already loaded", -> + beforeEach -> + @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @source, @user_id, @callback + + it "should get the current doc lines", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a diff of the old and new lines", -> + @DiffCodec.diffAsShareJsOp + .calledWith(@beforeLines, @afterLines) + .should.equal true + + it "should apply the diff as a ShareJS op", -> + @UpdateManager.applyUpdate + .calledWith( + @project_id, + @doc_id, + { + doc: @doc_id, + v: @version, + op: @ops, + meta: { + type: "external" + source: @source + user_id: @user_id + } + } + ) + .should.equal true + + it "should flush the doc to Mongo", -> + @DocumentManager.flushDocIfLoaded + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the callback", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when not already loaded", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, false) + @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @source, @user_id, @callback + + it "should flush and delete the doc from the doc updater", -> + @DocumentManager.flushAndDeleteDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + describe "without new lines", -> + beforeEach -> + @DocumentManager.setDoc @project_id, @doc_id, null, @source, @user_id, @callback + + it "should return the callback with an error", -> + @callback.calledWith(new Error("No lines were passed to setDoc")) + + it "should not try to get the doc lines", -> + @DocumentManager.getDoc.called.should.equal false \ No newline at end of file diff --git a/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee deleted file mode 100644 index 911efce1ba..0000000000 --- a/services/document-updater/test/unit/coffee/DocumentManager/flushAndDeleteDocTests.coffee +++ /dev/null @@ -1,48 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/DocumentManager.js" -SandboxedModule = require('sandboxed-module') - -describe "DocumentUpdater.flushAndDeleteDoc", -> - beforeEach -> - @DocumentManager = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./PersistenceManager": @PersistenceManager = {} - "./TrackChangesManager": @TrackChangesManager = {} - "logger-sharelatex": @logger = {log: sinon.stub()} - "./DocOpsManager" :{} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @callback = sinon.stub() - - describe "successfully", -> - beforeEach -> - @RedisManager.removeDocFromMemory = sinon.stub().callsArg(2) - @DocumentManager.flushDocIfLoaded = sinon.stub().callsArgWith(2) - @TrackChangesManager.flushDocChanges = sinon.stub().callsArg(2) - @DocumentManager.flushAndDeleteDoc @project_id, @doc_id, @callback - - it "should flush the doc", -> - @DocumentManager.flushDocIfLoaded - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should remove the doc from redis", -> - @RedisManager.removeDocFromMemory - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should call the callback without error", -> - @callback.calledWith(null).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - it "should flush to track changes", -> - @TrackChangesManager.flushDocChanges - .calledWith(@project_id, @doc_id) - .should.equal true diff --git a/services/document-updater/test/unit/coffee/DocumentManager/flushDocIfLoadedTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/flushDocIfLoadedTests.coffee deleted file mode 100644 index 4a17e2b84c..0000000000 --- a/services/document-updater/test/unit/coffee/DocumentManager/flushDocIfLoadedTests.coffee +++ /dev/null @@ -1,68 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/DocumentManager.js" -SandboxedModule = require('sandboxed-module') - -describe "DocumentManager.flushDocIfLoaded", -> - beforeEach -> - @DocumentManager = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./PersistenceManager": @PersistenceManager = {} - "./DocOpsManager": @DocOpsManager = {} - "logger-sharelatex": @logger = {log: sinon.stub()} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "./TrackChangesManager": {} - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @callback = sinon.stub() - - describe "when the doc is in Redis", -> - beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @PersistenceManager.setDoc = sinon.stub().callsArgWith(4) - @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback - - it "should get the doc from redis", -> - @RedisManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should write the doc lines to the persistence layer", -> - @PersistenceManager.setDoc - .calledWith(@project_id, @doc_id, @lines, @version) - .should.equal true - - it "should call the callback without error", -> - @callback.calledWith(null).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the document is not in Redis", -> - beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null) - @PersistenceManager.setDoc = sinon.stub().callsArgWith(4) - @DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2) - @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback - - it "should get the doc from redis", -> - @RedisManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should not write anything to the persistence layer", -> - @PersistenceManager.setDoc.called.should.equal false - @DocOpsManager.flushDocOpsToMongo.called.should.equal false - - it "should call the callback without error", -> - @callback.calledWith(null).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - diff --git a/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee deleted file mode 100644 index c77af9a77c..0000000000 --- a/services/document-updater/test/unit/coffee/DocumentManager/getDocAndRecentOpsTests.coffee +++ /dev/null @@ -1,67 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/DocumentManager.js" -SandboxedModule = require('sandboxed-module') - -describe "DocumentManager.getDocAndRecentOps", -> - beforeEach -> - @DocumentManager = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./PersistenceManager": @PersistenceManager = {} - "logger-sharelatex": @logger = {log: sinon.stub()} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "./TrackChangesManager": {} - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @fromVersion = 40 - @ops = ["mock-op-1", "mock-op-2"] - @callback = sinon.stub() - - describe "with a previous version specified", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) - @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback - - it "should get the doc", -> - @DocumentManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should get the doc ops", -> - @RedisManager.getPreviousDocOps - .calledWith(@doc_id, @fromVersion, @version) - .should.equal true - - it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @ops).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "with no previous version specified", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) - @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback - - it "should get the doc", -> - @DocumentManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should not need to get the doc ops", -> - @RedisManager.getPreviousDocOps.called.should.equal false - - it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, []).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - diff --git a/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee deleted file mode 100644 index 3edf4cb67d..0000000000 --- a/services/document-updater/test/unit/coffee/DocumentManager/getDocTests.coffee +++ /dev/null @@ -1,70 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/DocumentManager.js" -SandboxedModule = require('sandboxed-module') - -describe "DocumentUpdater.getDoc", -> - beforeEach -> - @DocumentManager = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./PersistenceManager": @PersistenceManager = {} - "./DocOpsManager": @DocOpsManager = {} - "logger-sharelatex": @logger = {log: sinon.stub()} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "./TrackChangesManager": {} - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @callback = sinon.stub() - - describe "when the doc exists in Redis", -> - beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @DocumentManager.getDoc @project_id, @doc_id, @callback - - it "should get the doc from Redis", -> - @RedisManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, true).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the doc does not exist in Redis", -> - beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null) - @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @RedisManager.putDocInMemory = sinon.stub().callsArg(4) - @DocumentManager.getDoc @project_id, @doc_id, @callback - - it "should try to get the doc from Redis", -> - @RedisManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should get the doc from the PersistenceManager", -> - @PersistenceManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should set the doc in Redis", -> - @RedisManager.putDocInMemory - .calledWith(@project_id, @doc_id, @lines, @version) - .should.equal true - - it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, false).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - - diff --git a/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee deleted file mode 100644 index 360d939b9f..0000000000 --- a/services/document-updater/test/unit/coffee/DocumentManager/setDocTests.coffee +++ /dev/null @@ -1,107 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/DocumentManager.js" -SandboxedModule = require('sandboxed-module') - -describe "DocumentManager.setDoc", -> - beforeEach -> - @DocumentManager = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./PersistenceManager": @PersistenceManager = {} - "./DiffCodec": @DiffCodec = {} - "./DocOpsManager":{} - "./UpdateManager": @UpdateManager = {} - "logger-sharelatex": @logger = {log: sinon.stub()} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "./TrackChangesManager": {} - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @version = 42 - @ops = ["mock-ops"] - @callback = sinon.stub() - @source = "dropbox" - @user_id = "mock-user-id" - - describe "with plain tex lines", -> - beforeEach -> - @beforeLines = ["before", "lines"] - @afterLines = ["after", "lines"] - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, true) - @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) - @UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null) - @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) - @DocumentManager.flushAndDeleteDoc = sinon.stub().callsArg(2) - - describe "when already loaded", -> - beforeEach -> - @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @source, @user_id, @callback - - it "should get the current doc lines", -> - @DocumentManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should return a diff of the old and new lines", -> - @DiffCodec.diffAsShareJsOp - .calledWith(@beforeLines, @afterLines) - .should.equal true - - it "should apply the diff as a ShareJS op", -> - @UpdateManager.applyUpdate - .calledWith( - @project_id, - @doc_id, - { - doc: @doc_id, - v: @version, - op: @ops, - meta: { - type: "external" - source: @source - user_id: @user_id - } - } - ) - .should.equal true - - it "should flush the doc to Mongo", -> - @DocumentManager.flushDocIfLoaded - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should call the callback", -> - @callback.calledWith(null).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when not already loaded", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, false) - @DocumentManager.setDoc @project_id, @doc_id, @afterLines, @source, @user_id, @callback - - it "should flush and delete the doc from the doc updater", -> - @DocumentManager.flushAndDeleteDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - describe "without new lines", -> - beforeEach -> - @DocumentManager.setDoc @project_id, @doc_id, null, @source, @user_id, @callback - - it "should return the callback with an error", -> - @callback.calledWith(new Error("No lines were passed to setDoc")) - - it "should not try to get the doc lines", -> - @DocumentManager.getDoc.called.should.equal false - - - - - - - From 9ee913be39a5ccac287187c23167a337a715ee8b Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 16:40:15 +0000 Subject: [PATCH 03/25] Update PersistenceManagerTests --- .../app/coffee/PersistenceManager.coffee | 6 +- .../PersistenceManagerTests.coffee | 176 ++++++++++++++++++ .../PersistenceManager/getDocTests.coffee | 104 ----------- .../PersistenceManager/setDocTests.coffee | 90 --------- 4 files changed, 178 insertions(+), 198 deletions(-) create mode 100644 services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index fff037f6fc..25eb1f32a2 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -39,7 +39,7 @@ module.exports = PersistenceManager = return callback(new Error("web API response had no doc lines")) if !body.version? or not body.version instanceof Number return callback(new Error("web API response had no valid doc version")) - return callback null, body.lines, body.track_changes, body.track_changes_entries + return callback null, body.lines, body.version, body.track_changes, body.track_changes_entries else if res.statusCode == 404 return callback(new Errors.NotFoundError("doc not not found: #{url}")) else @@ -55,13 +55,11 @@ module.exports = PersistenceManager = request { url: url method: "POST" - body: JSON.stringify + json: lines: lines track_changes: track_changes track_changes_entries: track_changes_entries version: version - headers: - "content-type": "application/json" auth: user: Settings.apis.web.user pass: Settings.apis.web.pass diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee new file mode 100644 index 0000000000..e9f2cf212e --- /dev/null +++ b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee @@ -0,0 +1,176 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/PersistenceManager.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" + +describe "PersistenceManager", -> + beforeEach -> + @PersistenceManager = SandboxedModule.require modulePath, requires: + "request": @request = sinon.stub() + "settings-sharelatex": @Settings = {} + "./Metrics": @Metrics = + Timer: class Timer + done: sinon.stub() + "logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()} + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @lines = ["one", "two", "three"] + @version = 42 + @callback = sinon.stub() + @track_changes_on = true + @track_changes_entries = { comments: "mock", entries: "mock" } + @Settings.apis = + web: + url: @url = "www.example.com" + user: @user = "sharelatex" + pass: @pass = "password" + + describe "getDoc", -> + + describe "with a successful response from the web api", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify({ + lines: @lines, + version: @version, + track_changes: @track_changes_on, + track_changes_entries: @track_changes_entries + })) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should call the web api", -> + @request + .calledWith({ + url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" + method: "GET" + headers: + "accept": "application/json" + auth: + user: @user + pass: @pass + sendImmediately: true + jar: false + timeout: 5000 + }) + .should.equal true + + it "should call the callback with the doc lines, version and track changes state", -> + @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when request returns an error", -> + beforeEach -> + @request.callsArgWith(1, @error = new Error("oops"), null, null) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return the error", -> + @callback.calledWith(@error).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns 404", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 404}, "") + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns an error status code", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 500}, "") + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return an error", -> + @callback.calledWith(new Error("web api error")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when request returns an doc without lines", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(version: @version)) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return and error", -> + @callback.calledWith(new Error("web API response had no doc lines")).should.equal true + + describe "when request returns an doc without a version", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines)) + @PersistenceManager.getDoc(@project_id, @doc_id, @callback) + + it "should return and error", -> + @callback.calledWith(new Error("web API response had no valid doc version")).should.equal true + + describe "setDoc", -> + describe "with a successful response from the web api", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 200}) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + + it "should call the web api", -> + @request + .calledWith({ + url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" + json: + lines: @lines + version: @version + track_changes: @track_changes_on + track_changes_entries: @track_changes_entries + method: "POST" + auth: + user: @user + pass: @pass + sendImmediately: true + jar: false + timeout: 5000 + }) + .should.equal true + + it "should call the callback without error", -> + @callback.calledWith(null).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when request returns an error", -> + beforeEach -> + @request.callsArgWith(1, @error = new Error("oops"), null, null) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + + it "should return the error", -> + @callback.calledWith(@error).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns 404", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 404}, "") + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + + it "should return a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the request returns an error status code", -> + beforeEach -> + @request.callsArgWith(1, null, {statusCode: 500}, "") + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + + it "should return an error", -> + @callback.calledWith(new Error("web api error")).should.equal true + + it "should time the execution", -> + @Metrics.Timer::done.called.should.equal true + diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee deleted file mode 100644 index d4f44afa46..0000000000 --- a/services/document-updater/test/unit/coffee/PersistenceManager/getDocTests.coffee +++ /dev/null @@ -1,104 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/PersistenceManager.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors" - -describe "PersistenceManager.getDoc", -> - beforeEach -> - @PersistenceManager = SandboxedModule.require modulePath, requires: - "request": @request = sinon.stub() - "settings-sharelatex": @Settings = {} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()} - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @callback = sinon.stub() - @Settings.apis = - web: - url: @url = "www.example.com" - user: @user = "sharelatex" - pass: @pass = "password" - - describe "with a successful response from the web api", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines, version: @version)) - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should call the web api", -> - @request - .calledWith({ - url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" - method: "GET" - headers: - "accept": "application/json" - auth: - user: @user - pass: @pass - sendImmediately: true - jar: false - timeout: 5000 - }) - .should.equal true - - it "should call the callback with the doc lines and version", -> - @callback.calledWith(null, @lines, @version).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when request returns an error", -> - beforeEach -> - @request.callsArgWith(1, @error = new Error("oops"), null, null) - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should return the error", -> - @callback.calledWith(@error).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the request returns 404", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 404}, "") - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should return a NotFoundError", -> - @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the request returns an error status code", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 500}, "") - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should return an error", -> - @callback.calledWith(new Error("web api error")).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when request returns an doc without lines", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(version: @version)) - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should return and error", -> - @callback.calledWith(new Error("web API response had no doc lines")).should.equal true - - describe "when request returns an doc without a version", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines)) - @PersistenceManager.getDoc(@project_id, @doc_id, @callback) - - it "should return and error", -> - @callback.calledWith(new Error("web API response had no valid doc version")).should.equal true - - diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee deleted file mode 100644 index 98f252a35d..0000000000 --- a/services/document-updater/test/unit/coffee/PersistenceManager/setDocTests.coffee +++ /dev/null @@ -1,90 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/PersistenceManager.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors" - -describe "PersistenceManager.setDoc", -> - beforeEach -> - @PersistenceManager = SandboxedModule.require modulePath, requires: - "request": @request = sinon.stub() - "settings-sharelatex": @Settings = {} - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()} - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @callback = sinon.stub() - @Settings.apis = - web: - url: @url = "www.example.com" - user: @user = "sharelatex" - pass: @pass = "password" - - describe "with a successful response from the web api", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify(lines: @lines, version: @version)) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback) - - it "should call the web api", -> - @request - .calledWith({ - url: "#{@url}/project/#{@project_id}/doc/#{@doc_id}" - body: JSON.stringify - lines: @lines - version: @version - method: "POST" - headers: - "content-type": "application/json" - auth: - user: @user - pass: @pass - sendImmediately: true - jar: false - timeout: 5000 - }) - .should.equal true - - it "should call the callback without error", -> - @callback.calledWith(null).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when request returns an error", -> - beforeEach -> - @request.callsArgWith(1, @error = new Error("oops"), null, null) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback) - - it "should return the error", -> - @callback.calledWith(@error).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the request returns 404", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 404}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback) - - it "should return a NotFoundError", -> - @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the request returns an error status code", -> - beforeEach -> - @request.callsArgWith(1, null, {statusCode: 500}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @callback) - - it "should return an error", -> - @callback.calledWith(new Error("web api error")).should.equal true - - it "should time the execution", -> - @Metrics.Timer::done.called.should.equal true - From d878dd575826bc7d01746268f22ccc5d640dcf55 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 16:49:53 +0000 Subject: [PATCH 04/25] Fix RedisManagerTests --- .../RedisManager/RedisManagerTests.coffee | 99 ++++++++++++------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index 205692d634..6f9afc29d3 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -22,6 +22,8 @@ describe "RedisManager", -> projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" + trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:#{doc_id}" + trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:#{doc_id}" "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "./Metrics": @metrics = inc: sinon.stub() @@ -37,39 +39,52 @@ describe "RedisManager", -> @lines = ["one", "two", "three"] @jsonlines = JSON.stringify @lines @version = 42 + @track_changes_on = true + @redis_track_changes_on = "1" + @track_changes_entries = { comments: "mock", entries: "mock" } + @json_track_changes_entries = JSON.stringify @track_changes_entries @rclient.get = sinon.stub() - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id]) - @RedisManager.getDoc @project_id, @doc_id, @callback - - it "should get the lines from redis", -> - @rclient.get - .calledWith("doclines:#{@doc_id}") - .should.equal true - - it "should get the version from", -> - @rclient.get - .calledWith("DocVersion:#{@doc_id}") - .should.equal true + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @redis_track_changes_on, @json_track_changes_entries]) - it 'should return the document', -> - @callback - .calledWith(null, @lines, @version) - .should.equal true + describe "successfully", -> + beforeEach -> + @RedisManager.getDoc @project_id, @doc_id, @callback - describe "getDoc with an invalid project id", -> - beforeEach -> - @lines = ["one", "two", "three"] - @jsonlines = JSON.stringify @lines - @version = 42 - @another_project_id = "project-id-456" - @rclient.get = sinon.stub() - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id]) - @RedisManager.getDoc @project_id, @doc_id, @callback + it "should get the lines from redis", -> + @rclient.get + .calledWith("doclines:#{@doc_id}") + .should.equal true + + it "should get the version from", -> + @rclient.get + .calledWith("DocVersion:#{@doc_id}") + .should.equal true + + it "should get the track changes state", -> + @rclient.get + .calledWith("TrackChangesEnabled:#{@doc_id}") + .should.equal true + + it "should get the track changes entries", -> + @rclient.get + .calledWith("TrackChangesEntries:#{@doc_id}") + .should.equal true - it 'should return an error', -> - @callback - .calledWith(new Errors.NotFoundError("not found")) - .should.equal true + it 'should return the document', -> + @callback + .calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries) + .should.equal true + + describe "getDoc with an invalid project id", -> + beforeEach -> + @another_project_id = "project-id-456" + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id, @redis_track_changes_on, @json_track_changes_entries]) + @RedisManager.getDoc @project_id, @doc_id, @callback + + it 'should return an error', -> + @callback + .calledWith(new Errors.NotFoundError("not found")) + .should.equal true describe "getPreviousDocOpsTests", -> describe "with a start and an end value", -> @@ -166,13 +181,14 @@ describe "RedisManager", -> @lines = ["one", "two", "three"] @ops = [{ op: [{ i: "foo", p: 4 }] },{ op: [{ i: "bar", p: 8 }] }] @version = 42 + @track_changes_entries = { comments: "mock", entries: "mock" } @rclient.exec = sinon.stub().callsArg(0) describe "with a consistent version", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length) - @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @callback + @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @track_changes_entries, @callback it "should get the current doc version to check for consistency", -> @RedisManager.getDocVersion @@ -188,6 +204,11 @@ describe "RedisManager", -> @rclient.set .calledWith("DocVersion:#{@doc_id}", @version) .should.equal true + + it "should set the track changes entries", -> + @rclient.set + .calledWith("TrackChangesEntries:#{@doc_id}", JSON.stringify(@track_changes_entries)) + .should.equal true it "should push the doc op into the doc ops list", -> @rclient.rpush @@ -210,7 +231,7 @@ describe "RedisManager", -> describe "with an inconsistent version", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length - 1) - @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @callback + @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @track_changes_entries, @callback it "should not call multi.exec", -> @rclient.exec.called.should.equal false @@ -223,7 +244,7 @@ describe "RedisManager", -> describe "with no updates", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version) - @RedisManager.updateDocument @doc_id, @lines, @version, [], @callback + @RedisManager.updateDocument @doc_id, @lines, @version, [], @track_changes_entries, @callback it "should not do an rpush", -> @rclient.rpush @@ -242,7 +263,9 @@ describe "RedisManager", -> @rclient.exec.yields() @lines = ["one", "two", "three"] @version = 42 - @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, done + @track_changes_on = true + @track_changes_entries = { comments: "mock", entries: "mock" } + @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, done it "should set the lines", -> @rclient.set @@ -253,6 +276,16 @@ describe "RedisManager", -> @rclient.set .calledWith("DocVersion:#{@doc_id}", @version) .should.equal true + + it "should set the track changes entries", -> + @rclient.set + .calledWith("TrackChangesEntries:#{@doc_id}", JSON.stringify(@track_changes_entries)) + .should.equal true + + it "should set the track changes state", -> + @rclient.set + .calledWith("TrackChangesEnabled:#{@doc_id}", "1") + .should.equal true it "should set the project_id for the doc", -> @rclient.set From b6c93c718d77e0cba2ad96b6e4445afe8a772bdd Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 16:50:55 +0000 Subject: [PATCH 05/25] Update TrackChangesManagerTests -> HistoryManagerTests --- .../HistoryManagerTests.coffee} | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) rename services/document-updater/test/unit/coffee/{TrackChangesManager/TrackChangesManagerTests.coffee => HistoryManager/HistoryManagerTests.coffee} (68%) diff --git a/services/document-updater/test/unit/coffee/TrackChangesManager/TrackChangesManagerTests.coffee b/services/document-updater/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee similarity index 68% rename from services/document-updater/test/unit/coffee/TrackChangesManager/TrackChangesManagerTests.coffee rename to services/document-updater/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee index 03106e2c2e..c33a18d4e6 100644 --- a/services/document-updater/test/unit/coffee/TrackChangesManager/TrackChangesManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/HistoryManager/HistoryManagerTests.coffee @@ -1,11 +1,11 @@ SandboxedModule = require('sandboxed-module') sinon = require('sinon') require('chai').should() -modulePath = require('path').join __dirname, '../../../../app/js/TrackChangesManager' +modulePath = require('path').join __dirname, '../../../../app/js/HistoryManager' -describe "TrackChangesManager", -> +describe "HistoryManager", -> beforeEach -> - @TrackChangesManager = SandboxedModule.require modulePath, requires: + @HistoryManager = SandboxedModule.require modulePath, requires: "request": @request = {} "settings-sharelatex": @Settings = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } @@ -22,7 +22,7 @@ describe "TrackChangesManager", -> describe "successfully", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, statusCode: 204) - @TrackChangesManager.flushDocChanges @project_id, @doc_id, @callback + @HistoryManager.flushDocChanges @project_id, @doc_id, @callback it "should send a request to the track changes api", -> @request.post @@ -35,7 +35,7 @@ describe "TrackChangesManager", -> describe "when the track changes api returns an error", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, statusCode: 500) - @TrackChangesManager.flushDocChanges @project_id, @doc_id, @callback + @HistoryManager.flushDocChanges @project_id, @doc_id, @callback it "should return the callback with an error", -> @callback.calledWith(new Error("track changes api return non-success code: 500")).should.equal true @@ -43,12 +43,12 @@ describe "TrackChangesManager", -> describe "pushUncompressedHistoryOps", -> beforeEach -> @ops = ["mock-ops"] - @TrackChangesManager.flushDocChanges = sinon.stub().callsArg(2) + @HistoryManager.flushDocChanges = sinon.stub().callsArg(2) describe "pushing the op", -> beforeEach -> @WebRedisManager.pushUncompressedHistoryOps = sinon.stub().callsArgWith(3, null, 1) - @TrackChangesManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback + @HistoryManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback it "should push the ops into redis", -> @WebRedisManager.pushUncompressedHistoryOps @@ -59,16 +59,16 @@ describe "TrackChangesManager", -> @callback.called.should.equal true it "should not try to flush the op", -> - @TrackChangesManager.flushDocChanges.called.should.equal false + @HistoryManager.flushDocChanges.called.should.equal false describe "when we hit a multiple of FLUSH_EVERY_N_OPS ops", -> beforeEach -> @WebRedisManager.pushUncompressedHistoryOps = - sinon.stub().callsArgWith(3, null, 2 * @TrackChangesManager.FLUSH_EVERY_N_OPS) - @TrackChangesManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback + sinon.stub().callsArgWith(3, null, 2 * @HistoryManager.FLUSH_EVERY_N_OPS) + @HistoryManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback it "should tell the track changes api to flush", -> - @TrackChangesManager.flushDocChanges + @HistoryManager.flushDocChanges .calledWith(@project_id, @doc_id) .should.equal true @@ -76,20 +76,20 @@ describe "TrackChangesManager", -> beforeEach -> @ops = ["op1", "op2", "op3"] @WebRedisManager.pushUncompressedHistoryOps = - sinon.stub().callsArgWith(3, null, 2 * @TrackChangesManager.FLUSH_EVERY_N_OPS + 1) - @TrackChangesManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback + sinon.stub().callsArgWith(3, null, 2 * @HistoryManager.FLUSH_EVERY_N_OPS + 1) + @HistoryManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback it "should tell the track changes api to flush", -> - @TrackChangesManager.flushDocChanges + @HistoryManager.flushDocChanges .calledWith(@project_id, @doc_id) .should.equal true - describe "when TrackChangesManager errors", -> + describe "when HistoryManager errors", -> beforeEach -> @WebRedisManager.pushUncompressedHistoryOps = - sinon.stub().callsArgWith(3, null, 2 * @TrackChangesManager.FLUSH_EVERY_N_OPS) - @TrackChangesManager.flushDocChanges = sinon.stub().callsArgWith(2, @error = new Error("oops")) - @TrackChangesManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback + sinon.stub().callsArgWith(3, null, 2 * @HistoryManager.FLUSH_EVERY_N_OPS) + @HistoryManager.flushDocChanges = sinon.stub().callsArgWith(2, @error = new Error("oops")) + @HistoryManager.pushUncompressedHistoryOps @project_id, @doc_id, @ops, @callback it "should log out the error", -> @logger.error @@ -104,7 +104,7 @@ describe "TrackChangesManager", -> describe "with no ops", -> beforeEach -> @WebRedisManager.pushUncompressedHistoryOps = sinon.stub().callsArgWith(3, null, 1) - @TrackChangesManager.pushUncompressedHistoryOps @project_id, @doc_id, [], @callback + @HistoryManager.pushUncompressedHistoryOps @project_id, @doc_id, [], @callback it "should not call WebRedisManager.pushUncompressedHistoryOps", -> @WebRedisManager.pushUncompressedHistoryOps.called.should.equal false From 889f5fdf9f0a95a6ba1f03e51c0f29ecfacb1c5e Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 17:14:40 +0000 Subject: [PATCH 06/25] Fix ShareJsDB tests --- .../app/coffee/ShareJsDB.coffee | 2 - .../unit/coffee/ShareJsDB/GetOpsTests.coffee | 55 ----------- .../coffee/ShareJsDB/GetSnapshotTests.coffee | 87 ----------------- .../coffee/ShareJsDB/ShareJsDBTests.coffee | 93 +++++++++++++++++++ .../coffee/ShareJsDB/WriteOpsTests.coffee | 39 -------- 5 files changed, 93 insertions(+), 183 deletions(-) delete mode 100644 services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee create mode 100644 services/document-updater/test/unit/coffee/ShareJsDB/ShareJsDBTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee diff --git a/services/document-updater/app/coffee/ShareJsDB.coffee b/services/document-updater/app/coffee/ShareJsDB.coffee index a21c8aea7f..3e5dfe303f 100644 --- a/services/document-updater/app/coffee/ShareJsDB.coffee +++ b/services/document-updater/app/coffee/ShareJsDB.coffee @@ -1,8 +1,6 @@ Keys = require('./UpdateKeys') -Settings = require('settings-sharelatex') RedisManager = require "./RedisManager" Errors = require "./Errors" -logger = require "logger-sharelatex" module.exports = class ShareJsDB constructor: (@project_id, @doc_id, @lines, @version) -> diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee deleted file mode 100644 index 5621f39a85..0000000000 --- a/services/document-updater/test/unit/coffee/ShareJsDB/GetOpsTests.coffee +++ /dev/null @@ -1,55 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/ShareJsDB.js" -SandboxedModule = require('sandboxed-module') - -describe "ShareJsDB.getOps", -> - beforeEach -> - @doc_id = "document-id" - @project_id = "project-id" - @doc_key = "#{@project_id}:#{@doc_id}" - @callback = sinon.stub() - @ops = [{p: 20, t: "foo"}] - @redis_ops = (JSON.stringify(op) for op in @ops) - @ShareJsDB = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./DocumentManager":{} - "logger-sharelatex": {} - @db = new @ShareJsDB() - - describe "with start == end", -> - beforeEach -> - @start = @end = 42 - @db.getOps @doc_key, @start, @end, @callback - - it "should return an empty array", -> - @callback.calledWith(null, []).should.equal true - - describe "with a non empty range", -> - beforeEach -> - @start = 35 - @end = 42 - @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) - @db.getOps @doc_key, @start, @end, @callback - - it "should get the range from redis", -> - @RedisManager.getPreviousDocOps - .calledWith(@doc_id, @start, @end-1) - .should.equal true - - it "should return the ops", -> - @callback.calledWith(null, @ops).should.equal true - - describe "with no specified end", -> - beforeEach -> - @start = 35 - @end = null - @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) - @db.getOps @doc_key, @start, @end, @callback - - it "should get until the end of the list", -> - @RedisManager.getPreviousDocOps - .calledWith(@doc_id, @start, -1) - .should.equal true - diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee deleted file mode 100644 index f2527b01a2..0000000000 --- a/services/document-updater/test/unit/coffee/ShareJsDB/GetSnapshotTests.coffee +++ /dev/null @@ -1,87 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../../app/js/ShareJsDB.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors" - -describe "ShareJsDB.getSnapshot", -> - beforeEach -> - @doc_id = "document-id" - @project_id = "project-id" - @doc_key = "#{@project_id}:#{@doc_id}" - @callback = sinon.stub() - @ShareJsDB = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./RedisManager": {} - "./DocOpsManager": {} - "logger-sharelatex": {} - @db = new @ShareJsDB() - - @version = 42 - - describe "with a text document", -> - beforeEach -> - @lines = ["one", "two", "three"] - - describe "successfully", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @db.getSnapshot @doc_key, @callback - - it "should get the doc", -> - @DocumentManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should return the doc lines", -> - @callback.args[0][1].snapshot.should.equal @lines.join("\n") - - it "should return the doc version", -> - @callback.args[0][1].v.should.equal @version - - it "should return the type as text", -> - @callback.args[0][1].type.should.equal "text" - - describe "when the doclines do not exist", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, null, null) - @db.getSnapshot @doc_key, @callback - - it "should return the callback with a NotFoundError", -> - @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true - - describe "when getDoc returns an error", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, @error = new Error("oops"), null, null) - @db.getSnapshot @doc_key, @callback - - it "should return the callback with an error", -> - @callback.calledWith(@error).should.equal true - - describe "with a JSON document", -> - beforeEach -> - @lines = [{text: "one"}, {text:"two"}, {text:"three"}] - - describe "successfully", -> - beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version) - @db.getSnapshot @doc_key, @callback - - it "should get the doc", -> - @DocumentManager.getDoc - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should return the doc lines", -> - expect(@callback.args[0][1].snapshot).to.deep.equal lines: @lines - - it "should return the doc version", -> - @callback.args[0][1].v.should.equal @version - - it "should return the type as text", -> - @callback.args[0][1].type.should.equal "json" - - - diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/ShareJsDBTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/ShareJsDBTests.coffee new file mode 100644 index 0000000000..aa03d9fb1e --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJsDB/ShareJsDBTests.coffee @@ -0,0 +1,93 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = "../../../../app/js/ShareJsDB.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" + +describe "ShareJsDB", -> + beforeEach -> + @doc_id = "document-id" + @project_id = "project-id" + @doc_key = "#{@project_id}:#{@doc_id}" + @callback = sinon.stub() + @ShareJsDB = SandboxedModule.require modulePath, requires: + "./RedisManager": @RedisManager = {} + + @version = 42 + @lines = ["one", "two", "three"] + @db = new @ShareJsDB(@project_id, @doc_id, @lines, @version) + + describe "getSnapshot", -> + describe "successfully", -> + beforeEach -> + @db.getSnapshot @doc_key, @callback + + it "should return the doc lines", -> + @callback.args[0][1].snapshot.should.equal @lines.join("\n") + + it "should return the doc version", -> + @callback.args[0][1].v.should.equal @version + + it "should return the type as text", -> + @callback.args[0][1].type.should.equal "text" + + describe "when the key does not match", -> + beforeEach -> + @db.getSnapshot "bad:key", @callback + + it "should return the callback with a NotFoundError", -> + @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true + + describe "getOps", -> + describe "with start == end", -> + beforeEach -> + @start = @end = 42 + @db.getOps @doc_key, @start, @end, @callback + + it "should return an empty array", -> + @callback.calledWith(null, []).should.equal true + + describe "with a non empty range", -> + beforeEach -> + @start = 35 + @end = 42 + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @db.getOps @doc_key, @start, @end, @callback + + it "should get the range from redis", -> + @RedisManager.getPreviousDocOps + .calledWith(@doc_id, @start, @end-1) + .should.equal true + + it "should return the ops", -> + @callback.calledWith(null, @ops).should.equal true + + describe "with no specified end", -> + beforeEach -> + @start = 35 + @end = null + @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) + @db.getOps @doc_key, @start, @end, @callback + + it "should get until the end of the list", -> + @RedisManager.getPreviousDocOps + .calledWith(@doc_id, @start, -1) + .should.equal true + + describe "writeOps", -> + describe "writing an op", -> + beforeEach -> + @opData = + op: {p: 20, t: "foo"} + meta: {source: "bar"} + v: @version + @db.writeOp @doc_key, @opData, @callback + + it "should write into appliedOps", -> + expect(@db.appliedOps[@doc_key]).to.deep.equal [@opData] + + it "should call the callback without an error", -> + @callback.called.should.equal true + (@callback.args[0][0]?).should.equal false diff --git a/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee b/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee deleted file mode 100644 index 838f63034e..0000000000 --- a/services/document-updater/test/unit/coffee/ShareJsDB/WriteOpsTests.coffee +++ /dev/null @@ -1,39 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -expect = chai.expect -should = chai.should() -modulePath = "../../../../app/js/ShareJsDB.js" -SandboxedModule = require('sandboxed-module') - -describe "ShareJsDB.writeOps", -> - beforeEach -> - @project_id = "project-id" - @doc_id = "document-id" - @doc_key = "#{@project_id}:#{@doc_id}" - @callback = sinon.stub() - @opData = - op: {p: 20, t: "foo"} - meta: {source: "bar"} - @ShareJsDB = SandboxedModule.require modulePath, requires: - "./RedisManager": @RedisManager = {} - "./DocOpsManager": @DocOpsManager = {} - "./DocumentManager": {} - "logger-sharelatex": @logger = {error: sinon.stub()} - @db = new @ShareJsDB() - - describe "writing an op", -> - beforeEach -> - @version = 42 - @opData.v = @version - @db.writeOp @doc_key, @opData, @callback - - it "should write into appliedOps", -> - expect(@db.appliedOps[@doc_key]).to.deep.equal [@opData] - - it "should call the callback without an error", -> - @callback.called.should.equal true - (@callback.args[0][0]?).should.equal false - - - - From e7ff05e79220b6ace4d2119a3ec870f1553f0b1c Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 18:06:33 +0000 Subject: [PATCH 07/25] Fix UpdateManager tests --- .../UpdateManager/ApplyingUpdates.coffee | 79 +++++++++++++++- .../lockUpdatesAndDoTests.coffee | 90 ------------------- 2 files changed, 76 insertions(+), 93 deletions(-) delete mode 100644 services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee diff --git a/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee b/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee index 43786f4b98..d1898378d2 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee @@ -14,12 +14,14 @@ describe "UpdateManager", -> "./RedisManager" : @RedisManager = {} "./WebRedisManager" : @WebRedisManager = {} "./ShareJsUpdateManager" : @ShareJsUpdateManager = {} - "./TrackChangesManager" : @TrackChangesManager = {} + "./HistoryManager" : @HistoryManager = {} "logger-sharelatex": @logger = { log: sinon.stub() } "./Metrics": @Metrics = Timer: class Timer done: sinon.stub() "settings-sharelatex": Settings = {} + "./DocumentManager": @DocumentManager = {} + "./TrackChangesManager": @TrackChangesManager = {} describe "processOutstandingUpdates", -> beforeEach -> @@ -158,7 +160,7 @@ describe "UpdateManager", -> @appliedOps = ["mock-applied-ops"] @ShareJsUpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null, @updatedDocLines, @version, @appliedOps) @RedisManager.updateDocument = sinon.stub().callsArg(4) - @TrackChangesManager.pushUncompressedHistoryOps = sinon.stub().callsArg(3) + @HistoryManager.pushUncompressedHistoryOps = sinon.stub().callsArg(3) describe "normally", -> beforeEach -> @@ -175,7 +177,7 @@ describe "UpdateManager", -> .should.equal true it "should push the applied ops into the track changes queue", -> - @TrackChangesManager.pushUncompressedHistoryOps + @HistoryManager.pushUncompressedHistoryOps .calledWith(@project_id, @doc_id, @appliedOps) .should.equal true @@ -195,3 +197,74 @@ describe "UpdateManager", -> # \uFFFD is 'replacement character' @update.op[0].i.should.equal "\uFFFD\uFFFD" + describe "lockUpdatesAndDo", -> + beforeEach -> + @method = sinon.stub().callsArgWith(3, null, @response_arg1) + @callback = sinon.stub() + @arg1 = "argument 1" + @response_arg1 = "response argument 1" + @lockValue = "mock-lock-value" + @LockManager.getLock = sinon.stub().callsArgWith(1, null, @lockValue) + @LockManager.releaseLock = sinon.stub().callsArg(2) + + describe "successfully", -> + beforeEach -> + @UpdateManager.continueProcessingUpdatesWithLock = sinon.stub() + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should lock the doc", -> + @LockManager.getLock + .calledWith(@doc_id) + .should.equal true + + it "should process any outstanding updates", -> + @UpdateManager.processOutstandingUpdates + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should call the method", -> + @method + .calledWith(@project_id, @doc_id, @arg1) + .should.equal true + + it "should return the method response to the callback", -> + @callback + .calledWith(null, @response_arg1) + .should.equal true + + it "should release the lock", -> + @LockManager.releaseLock + .calledWith(@doc_id, @lockValue) + .should.equal true + + it "should continue processing updates", -> + @UpdateManager.continueProcessingUpdatesWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + describe "when processOutstandingUpdates returns an error", -> + beforeEach -> + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArgWith(2, @error = new Error("Something went wrong")) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id, @lockValue).should.equal true + + it "should return the error in the callback", -> + @callback.calledWith(@error).should.equal true + + describe "when the method returns an error", -> + beforeEach -> + @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) + @method = sinon.stub().callsArgWith(3, @error = new Error("something went wrong"), @response_arg1) + @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback + + it "should free the lock", -> + @LockManager.releaseLock.calledWith(@doc_id, @lockValue).should.equal true + + it "should return the error in the callback", -> + @callback.calledWith(@error).should.equal true + + + diff --git a/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee deleted file mode 100644 index a4b455d219..0000000000 --- a/services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee +++ /dev/null @@ -1,90 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/UpdateManager.js" -SandboxedModule = require('sandboxed-module') - -describe 'UpdateManager - lockUpdatesAndDo', -> - beforeEach -> - @UpdateManager = SandboxedModule.require modulePath, requires: - "./LockManager" : @LockManager = {} - "./RedisManager" : @RedisManager = {} - "./WebRedisManager" : @WebRedisManager = {} - "./ShareJsUpdateManager" : @ShareJsUpdateManager = {} - "./TrackChangesManager" : @TrackChangesManager = {} - "logger-sharelatex": @logger = { log: sinon.stub() } - "./Metrics": @Metrics = - Timer: class Timer - done: sinon.stub() - "settings-sharelatex": Settings = {} - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @method = sinon.stub().callsArgWith(3, null, @response_arg1) - @callback = sinon.stub() - @arg1 = "argument 1" - @response_arg1 = "response argument 1" - @lockValue = "mock-lock-value" - @LockManager.getLock = sinon.stub().callsArgWith(1, null, @lockValue) - @LockManager.releaseLock = sinon.stub().callsArg(2) - - describe "successfully", -> - beforeEach -> - @UpdateManager.continueProcessingUpdatesWithLock = sinon.stub() - @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) - @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback - - it "should lock the doc", -> - @LockManager.getLock - .calledWith(@doc_id) - .should.equal true - - it "should process any outstanding updates", -> - @UpdateManager.processOutstandingUpdates - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should call the method", -> - @method - .calledWith(@project_id, @doc_id, @arg1) - .should.equal true - - it "should return the method response to the callback", -> - @callback - .calledWith(null, @response_arg1) - .should.equal true - - it "should release the lock", -> - @LockManager.releaseLock - .calledWith(@doc_id, @lockValue) - .should.equal true - - it "should continue processing updates", -> - @UpdateManager.continueProcessingUpdatesWithLock - .calledWith(@project_id, @doc_id) - .should.equal true - - describe "when processOutstandingUpdates returns an error", -> - beforeEach -> - @UpdateManager.processOutstandingUpdates = sinon.stub().callsArgWith(2, @error = new Error("Something went wrong")) - @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback - - it "should free the lock", -> - @LockManager.releaseLock.calledWith(@doc_id, @lockValue).should.equal true - - it "should return the error in the callback", -> - @callback.calledWith(@error).should.equal true - - describe "when the method returns an error", -> - beforeEach -> - @UpdateManager.processOutstandingUpdates = sinon.stub().callsArg(2) - @method = sinon.stub().callsArgWith(3, @error = new Error("something went wrong"), @response_arg1) - @UpdateManager.lockUpdatesAndDo @method, @project_id, @doc_id, @arg1, @callback - - it "should free the lock", -> - @LockManager.releaseLock.calledWith(@doc_id, @lockValue).should.equal true - - it "should return the error in the callback", -> - @callback.calledWith(@error).should.equal true - - - From ce93a76e7f3f653d95a5b77a754462366b5387e8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 18:11:03 +0000 Subject: [PATCH 08/25] Fix ShareJsUpdateManager tests --- .../ShareJsUpdateManagerTests.coffee | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee index 94806a1a9d..f3b871149d 100644 --- a/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee @@ -17,12 +17,16 @@ describe "ShareJsUpdateManager", -> "./ShareJsDB" : @ShareJsDB = { mockDB: true } "redis-sharelatex" : createClient: () => @rclient = auth:-> "logger-sharelatex": @logger = { log: sinon.stub() } + "./WebRedisManager": @WebRedisManager = {} globals: clearTimeout: @clearTimeout = sinon.stub() describe "applyUpdate", -> beforeEach -> + @lines = ["one", "two"] @version = 34 + @update = {p: 4, t: "foo"} + @updatedDocLines = ["onefoo", "two"] @model = applyOp: sinon.stub().callsArg(2) getSnapshot: sinon.stub() @@ -31,20 +35,19 @@ describe "ShareJsUpdateManager", -> @ShareJsUpdateManager.getNewShareJsModel = sinon.stub().returns(@model) @ShareJsUpdateManager._listenForOps = sinon.stub() @ShareJsUpdateManager.removeDocFromCache = sinon.stub().callsArg(1) - @update = {p: 4, t: "foo"} - @updatedDocLines = ["one", "two"] describe "successfully", -> beforeEach (done) -> @model.getSnapshot.callsArgWith(1, null, {snapshot: @updatedDocLines.join("\n"), v: @version}) @model.db.appliedOps["#{@project_id}:#{@doc_id}"] = @appliedOps = ["mock-ops"] - @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, (err, docLines, version, appliedOps) => + @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, @lines, @version, (err, docLines, version, appliedOps) => @callback(err, docLines, version, appliedOps) done() it "should create a new ShareJs model", -> @ShareJsUpdateManager.getNewShareJsModel - .called.should.equal true + .calledWith(@project_id, @doc_id, @lines, @version) + .should.equal true it "should listen for ops on the model", -> @ShareJsUpdateManager._listenForOps @@ -69,7 +72,7 @@ describe "ShareJsUpdateManager", -> @error = new Error("Something went wrong") @ShareJsUpdateManager._sendError = sinon.stub() @model.applyOp = sinon.stub().callsArgWith(2, @error) - @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, (err, docLines, version) => + @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, @lines, @version, (err, docLines, version) => @callback(err, docLines, version) done() @@ -86,7 +89,7 @@ describe "ShareJsUpdateManager", -> @error = new Error("Something went wrong") @ShareJsUpdateManager._sendError = sinon.stub() @model.getSnapshot.callsArgWith(1, @error) - @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, (err, docLines, version) => + @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, @lines, @version, (err, docLines, version) => @callback(err, docLines, version) done() @@ -114,22 +117,22 @@ describe "ShareJsUpdateManager", -> @opData = op: {t: "foo", p: 1} meta: source: "bar" - @rclient.publish = sinon.stub() + @WebRedisManager.sendData = sinon.stub() @callback("#{@project_id}:#{@doc_id}", @opData) it "should publish the op to redis", -> - @rclient.publish - .calledWith("applied-ops", JSON.stringify(project_id: @project_id, doc_id: @doc_id, op: @opData)) + @WebRedisManager.sendData + .calledWith({project_id: @project_id, doc_id: @doc_id, op: @opData}) .should.equal true describe "_sendError", -> beforeEach -> @error_text = "Something went wrong" - @rclient.publish = sinon.stub() + @WebRedisManager.sendData = sinon.stub() @ShareJsUpdateManager._sendError(@project_id, @doc_id, new Error(@error_text)) it "should publish the error to the redis stream", -> - @rclient.publish - .calledWith("applied-ops", JSON.stringify(project_id: @project_id, doc_id: @doc_id, error: @error_text)) + @WebRedisManager.sendData + .calledWith({project_id: @project_id, doc_id: @doc_id, error: @error_text}) .should.equal true From f43355b74dce812328584f689f982eed1b472567 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 1 Dec 2016 18:19:47 +0000 Subject: [PATCH 09/25] Fix UpdateManager tests --- ...dates.coffee => UpdateManagerTests.coffee} | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) rename services/document-updater/test/unit/coffee/UpdateManager/{ApplyingUpdates.coffee => UpdateManagerTests.coffee} (91%) diff --git a/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee similarity index 91% rename from services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee rename to services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee index d1898378d2..9969e42d61 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee @@ -157,9 +157,15 @@ describe "UpdateManager", -> @update = {op: [{p: 42, i: "foo"}]} @updatedDocLines = ["updated", "lines"] @version = 34 + @lines = ["original", "lines"] + @track_changes_on = true + @track_changes_entries = { entries: "mock", comments: "mock" } + @updated_track_changes_entries = { entries: "updated", comments: "updated" } @appliedOps = ["mock-applied-ops"] - @ShareJsUpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null, @updatedDocLines, @version, @appliedOps) - @RedisManager.updateDocument = sinon.stub().callsArg(4) + @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @track_changes_on, @track_changes_entries) + @TrackChangesManager.applyUpdate = sinon.stub().yields(null, @updated_track_changes_entries) + @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) + @RedisManager.updateDocument = sinon.stub().yields() @HistoryManager.pushUncompressedHistoryOps = sinon.stub().callsArg(3) describe "normally", -> @@ -168,12 +174,17 @@ describe "UpdateManager", -> it "should apply the updates via ShareJS", -> @ShareJsUpdateManager.applyUpdate - .calledWith(@project_id, @doc_id, @update) + .calledWith(@project_id, @doc_id, @update, @lines, @version) + .should.equal true + + it "should update the track changes entries", -> + @TrackChangesManager.applyUpdate + .calledWith(@project_id, @doc_id, @track_changes_entries, @appliedOps, @track_changes_on) .should.equal true it "should save the document", -> @RedisManager.updateDocument - .calledWith(@doc_id, @updatedDocLines, @version, @appliedOps) + .calledWith(@doc_id, @updatedDocLines, @version, @appliedOps, @updated_track_changes_entries) .should.equal true it "should push the applied ops into the track changes queue", -> From 4fadd75ef3da82bfb2fffd8eb413a9923bb91bfa Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 2 Dec 2016 11:04:21 +0000 Subject: [PATCH 10/25] Track changes based on flag on op, not global setting --- services/document-updater/app.coffee | 1 - .../app/coffee/DocumentManager.coffee | 28 ++---- .../app/coffee/HttpController.coffee | 11 --- .../app/coffee/PersistenceManager.coffee | 7 +- .../app/coffee/RedisManager.coffee | 23 ++--- .../app/coffee/TrackChangesManager.coffee | 6 +- .../app/coffee/UpdateManager.coffee | 4 +- .../coffee/TrackChangesTests.coffee | 88 ++++++------------- .../DocumentManagerTests.coffee | 23 +++-- .../PersistenceManagerTests.coffee | 13 ++- .../RedisManager/RedisManagerTests.coffee | 21 +---- .../UpdateManager/UpdateManagerTests.coffee | 5 +- 12 files changed, 71 insertions(+), 159 deletions(-) diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee index f9099660c8..004b9f77bc 100644 --- a/services/document-updater/app.coffee +++ b/services/document-updater/app.coffee @@ -45,7 +45,6 @@ app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLo app.delete '/project/:project_id/doc/:doc_id', HttpController.flushAndDeleteDoc app.delete '/project/:project_id', HttpController.deleteProject app.post '/project/:project_id/flush', HttpController.flushProject -app.post '/project/:project_id/track_changes', HttpController.setTrackChanges app.get '/total', (req, res)-> timer = new Metrics.Timer("http.allDocList") diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee index 9c7277c469..258d55bb65 100644 --- a/services/document-updater/app/coffee/DocumentManager.coffee +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -13,18 +13,18 @@ module.exports = DocumentManager = timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> return callback(error) if error? if !lines? or !version? - logger.log {project_id, doc_id, track_changes}, "doc not in redis so getting from persistence API" - PersistenceManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> + logger.log {project_id, doc_id}, "doc not in redis so getting from persistence API" + PersistenceManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> return callback(error) if error? - logger.log {project_id, doc_id, lines, version, track_changes}, "got doc from persistence API" - RedisManager.putDocInMemory project_id, doc_id, lines, version, track_changes, track_changes_entries, (error) -> + logger.log {project_id, doc_id, lines, version}, "got doc from persistence API" + RedisManager.putDocInMemory project_id, doc_id, lines, version, track_changes_entries, (error) -> return callback(error) if error? - callback null, lines, version, track_changes, track_changes_entries, false + callback null, lines, version, track_changes_entries, false else - callback null, lines, version, track_changes, track_changes_entries, true + callback null, lines, version, track_changes_entries, true getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps) ->) -> timer = new Metrics.Timer("docManager.getDocAndRecentOps") @@ -90,14 +90,14 @@ module.exports = DocumentManager = callback = (args...) -> timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> return callback(error) if error? if !lines? or !version? logger.log project_id: project_id, doc_id: doc_id, "doc is not loaded so not flushing" callback null # TODO: return a flag to bail out, as we go on to remove doc from memory? else logger.log project_id: project_id, doc_id: doc_id, version: version, "flushing doc" - PersistenceManager.setDoc project_id, doc_id, lines, version, track_changes, track_changes_entries, (error) -> + PersistenceManager.setDoc project_id, doc_id, lines, version, track_changes_entries, (error) -> return callback(error) if error? callback null @@ -119,12 +119,6 @@ module.exports = DocumentManager = RedisManager.removeDocFromMemory project_id, doc_id, (error) -> return callback(error) if error? callback null - - setTrackChanges: (project_id, doc_id, track_changes_on, callback = (error) ->) -> - RedisManager.setTrackChanges project_id, doc_id, track_changes_on, (error) -> - return callback(error) if error? - WebRedisManager.sendData {project_id, doc_id, track_changes_on} - callback() getDocWithLock: (project_id, doc_id, callback = (error, lines, version) ->) -> UpdateManager = require "./UpdateManager" @@ -145,7 +139,3 @@ module.exports = DocumentManager = flushAndDeleteDocWithLock: (project_id, doc_id, callback = (error) ->) -> UpdateManager = require "./UpdateManager" UpdateManager.lockUpdatesAndDo DocumentManager.flushAndDeleteDoc, project_id, doc_id, callback - - setTrackChangesWithLock: (project_id, doc_id, track_changes_on, callback = (error) ->) -> - UpdateManager = require "./UpdateManager" - UpdateManager.lockUpdatesAndDo DocumentManager.setTrackChanges, project_id, doc_id, track_changes_on, callback \ No newline at end of file diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee index 0366746d56..dc74833697 100644 --- a/services/document-updater/app/coffee/HttpController.coffee +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -96,15 +96,4 @@ module.exports = HttpController = return next(error) if error? logger.log project_id: project_id, "deleted project via http" res.send 204 # No Content - - setTrackChanges: (req, res, next = (error) ->) -> - project_id = req.params.project_id - track_changes_on = req.body.on - if !track_changes_on? - return res.send 400 - track_changes_on = !!track_changes_on # Make boolean - logger.log {project_id, track_changes_on}, "turning on track changes via http" - ProjectManager.setTrackChangesWithLocks project_id, track_changes_on, (error) -> - return next(error) if error? - res.send 204 diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index 25eb1f32a2..ee7674d80a 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -10,7 +10,7 @@ logger = require "logger-sharelatex" MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds module.exports = PersistenceManager = - getDoc: (project_id, doc_id, _callback = (error, lines, version, track_changes, track_changes_entries) ->) -> + getDoc: (project_id, doc_id, _callback = (error, lines, version, track_changes_entries) ->) -> timer = new Metrics.Timer("persistenceManager.getDoc") callback = (args...) -> timer.done() @@ -39,13 +39,13 @@ module.exports = PersistenceManager = return callback(new Error("web API response had no doc lines")) if !body.version? or not body.version instanceof Number return callback(new Error("web API response had no valid doc version")) - return callback null, body.lines, body.version, body.track_changes, body.track_changes_entries + return callback null, body.lines, body.version, body.track_changes_entries else if res.statusCode == 404 return callback(new Errors.NotFoundError("doc not not found: #{url}")) else return callback(new Error("error accessing web API: #{url} #{res.statusCode}")) - setDoc: (project_id, doc_id, lines, version, track_changes, track_changes_entries, _callback = (error) ->) -> + setDoc: (project_id, doc_id, lines, version, track_changes_entries, _callback = (error) ->) -> timer = new Metrics.Timer("persistenceManager.setDoc") callback = (args...) -> timer.done() @@ -57,7 +57,6 @@ module.exports = PersistenceManager = method: "POST" json: lines: lines - track_changes: track_changes track_changes_entries: track_changes_entries version: version auth: diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 6ee764cb7e..cad5bd9f04 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -13,7 +13,7 @@ minutes = 60 # seconds for Redis expire module.exports = RedisManager = rclient: rclient - putDocInMemory : (project_id, doc_id, docLines, version, track_changes, track_changes_entries, _callback)-> + putDocInMemory : (project_id, doc_id, docLines, version, track_changes_entries, _callback)-> timer = new metrics.Timer("redis.put-doc") callback = (error) -> timer.done() @@ -23,7 +23,6 @@ module.exports = RedisManager = multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines) multi.set keys.projectKey({doc_id:doc_id}), project_id multi.set keys.docVersion(doc_id:doc_id), version - multi.set keys.trackChangesEnabled(doc_id:doc_id), if track_changes then "1" else "0" multi.set keys.trackChangesEntries(doc_id:doc_id), JSON.stringify(track_changes_entries) multi.exec (error) -> return callback(error) if error? @@ -43,36 +42,32 @@ module.exports = RedisManager = multi.del keys.docLines(doc_id:doc_id) multi.del keys.projectKey(doc_id:doc_id) multi.del keys.docVersion(doc_id:doc_id) - multi.del keys.trackChangesEnabled(doc_id:doc_id) multi.del keys.trackChangesEntries(doc_id:doc_id) multi.exec (error) -> return callback(error) if error? rclient.srem keys.docsInProject(project_id:project_id), doc_id, callback - getDoc : (project_id, doc_id, callback = (error, lines, version, track_changes, track_changes_entries) ->)-> + getDoc : (project_id, doc_id, callback = (error, lines, version, track_changes_entries) ->)-> timer = new metrics.Timer("redis.get-doc") multi = rclient.multi() multi.get keys.docLines(doc_id:doc_id) multi.get keys.docVersion(doc_id:doc_id) multi.get keys.projectKey(doc_id:doc_id) - multi.get keys.trackChangesEnabled(doc_id:doc_id) multi.get keys.trackChangesEntries(doc_id:doc_id) - multi.exec (error, result)-> + multi.exec (error, [docLines, version, doc_project_id, track_changes_entries])-> timer.done() return callback(error) if error? try - docLines = JSON.parse result[0] - track_changes_entries = JSON.parse result[4] + docLines = JSON.parse docLines + track_changes_entries = JSON.parse track_changes_entries catch e return callback(e) - version = parseInt(result[1] or 0, 10) - doc_project_id = result[2] - track_changes = (result[3] == "1") + version = parseInt(version or 0, 10) # check doc is in requested project if doc_project_id? and doc_project_id isnt project_id logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, "doc not in project" return callback(new Errors.NotFoundError("document not found")) - callback null, docLines, version, track_changes, track_changes_entries + callback null, docLines, version, track_changes_entries getDocVersion: (doc_id, callback = (error, version) ->) -> rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> @@ -134,7 +129,3 @@ module.exports = RedisManager = getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) -> rclient.smembers keys.docsInProject(project_id: project_id), callback - - setTrackChanges: (project_id, doc_id, track_changes_on, callback = (error) ->) -> - value = (if track_changes_on then "1" else "0") - rclient.set keys.trackChangesEnabled({doc_id}), value, callback diff --git a/services/document-updater/app/coffee/TrackChangesManager.coffee b/services/document-updater/app/coffee/TrackChangesManager.coffee index 94f8a11ca1..65f1931bb4 100644 --- a/services/document-updater/app/coffee/TrackChangesManager.coffee +++ b/services/document-updater/app/coffee/TrackChangesManager.coffee @@ -1,12 +1,12 @@ ChangesTracker = require "./ChangesTracker" module.exports = TrackChangesManager = - applyUpdate: (project_id, doc_id, entries = {}, updates = [], track_changes, callback = (error, new_entries) ->) -> + applyUpdate: (project_id, doc_id, entries = {}, updates = [], callback = (error, new_entries) ->) -> {changes, comments} = entries changesTracker = new ChangesTracker(changes, comments) - changesTracker.track_changes = track_changes for update in updates + changesTracker.track_changes = !!update.meta.tc for op in update.op - changesTracker.applyOp(op, { user_id: update.meta?.user_id, }) + changesTracker.applyOp(op, { user_id: update.meta?.user_id }) {changes, comments} = changesTracker callback null, {changes, comments} \ No newline at end of file diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee index d08d9a62f3..7c98b97eee 100644 --- a/services/document-updater/app/coffee/UpdateManager.coffee +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -48,13 +48,13 @@ module.exports = UpdateManager = applyUpdate: (project_id, doc_id, update, callback = (error) ->) -> UpdateManager._sanitizeUpdate update - DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes, track_changes_entries) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> return callback(error) if error? if !lines? or !version? return callback(new Errors.NotFoundError("document not found: #{doc_id}")) ShareJsUpdateManager.applyUpdate project_id, doc_id, update, lines, version, (error, updatedDocLines, version, appliedOps) -> return callback(error) if error? - TrackChangesManager.applyUpdate project_id, doc_id, track_changes_entries, appliedOps, track_changes, (error, new_track_changes_entries) -> + TrackChangesManager.applyUpdate project_id, doc_id, track_changes_entries, appliedOps, (error, new_track_changes_entries) -> return callback(error) if error? logger.log doc_id: doc_id, version: version, "updating doc in redis" RedisManager.updateDocument doc_id, updatedDocLines, version, appliedOps, new_track_changes_entries, (error) -> diff --git a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee index 406f46b430..43401cef6d 100644 --- a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee @@ -8,89 +8,51 @@ MockWebApi = require "./helpers/MockWebApi" DocUpdaterClient = require "./helpers/DocUpdaterClient" describe "Track changes", -> - describe "turning on track changes", -> - before (done) -> - DocUpdaterClient.subscribeToAppliedOps @appliedOpsListener = sinon.stub() - @project_id = DocUpdaterClient.randomId() - @docs = [{ - id: doc_id0 = DocUpdaterClient.randomId() - lines: ["one", "two", "three"] - updatedLines: ["one", "one and a half", "two", "three"] - }, { - id: doc_id1 = DocUpdaterClient.randomId() - lines: ["four", "five", "six"] - updatedLines: ["four", "four and a half", "five", "six"] - }] - for doc in @docs - MockWebApi.insertDoc @project_id, doc.id, { - lines: doc.lines - version: 0 - } - async.series @docs.map((doc) => - (callback) => - DocUpdaterClient.preloadDoc @project_id, doc.id, callback - ), (error) => - throw error if error? - setTimeout () => - DocUpdaterClient.setTrackChangesOn @project_id, (error, res, body) => - @statusCode = res.statusCode - done() - , 200 - - it "should return a 204 status code", -> - @statusCode.should.equal 204 - - it "should send a track changes message to real-time for each doc", -> - @appliedOpsListener.calledWith("applied-ops", JSON.stringify({ - project_id: @project_id, doc_id: @docs[0].id, track_changes_on: true - })).should.equal true - @appliedOpsListener.calledWith("applied-ops", JSON.stringify({ - project_id: @project_id, doc_id: @docs[1].id, track_changes_on: true - })).should.equal true - - it "should set the track changes key in redis", (done) -> - rclient.get "TrackChangesEnabled:#{@docs[0].id}", (error, value) => - throw error if error? - value.should.equal "1" - rclient.get "TrackChangesEnabled:#{@docs[1].id}", (error, value) -> - throw error if error? - value.should.equal "1" - done() - describe "tracking changes", -> before (done) -> @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() @doc = { - id: doc_id0 = DocUpdaterClient.randomId() - lines: ["one", "two", "three"] + id: DocUpdaterClient.randomId() + lines: ["aaa"] } - @update = + @updates = [{ doc: @doc.id - op: [{ - i: "one and a half\n" - p: 4 - }] + op: [{ i: "123", p: 1 }] v: 0 - meta: - user_id: @user_id = DocUpdaterClient.randomId() + meta: { user_id: @user_id } + }, { + doc: @doc.id + op: [{ i: "456", p: 5 }] + v: 1 + meta: { user_id: @user_id, tc: 1 } + }, { + doc: @doc.id + op: [{ d: "12", p: 1 }] + v: 2 + meta: { user_id: @user_id } + }] MockWebApi.insertDoc @project_id, @doc.id, { lines: @doc.lines version: 0 } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => throw error if error? - DocUpdaterClient.setTrackChangesOn @project_id, (error, res, body) => + async.series jobs, (error) -> throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc.id, @update, (error) -> - throw error if error? - setTimeout done, 200 + setTimeout done, 200 it "should set the updated track changes entries in redis", (done) -> + console.log "TODO: GET ME FROM HTTP REQUEST" rclient.get "TrackChangesEntries:#{@doc.id}", (error, value) => throw error if error? entries = JSON.parse(value) change = entries.changes[0] - change.op.should.deep.equal @update.op[0] + change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id done() diff --git a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee index 2b0dce169b..d29a569d46 100644 --- a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee @@ -24,7 +24,6 @@ describe "DocumentManager", -> @lines = ["one", "two", "three"] @version = 42 @track_changes_entries = { comments: "mock", entries: "mock" } - @track_changes_on = true describe "flushAndDeleteDoc", -> describe "successfully", -> @@ -58,7 +57,7 @@ describe "DocumentManager", -> describe "flushDocIfLoaded", -> describe "when the doc is in Redis", -> beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) @PersistenceManager.setDoc = sinon.stub().yields() @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback @@ -69,7 +68,7 @@ describe "DocumentManager", -> it "should write the doc lines to the persistence layer", -> @PersistenceManager.setDoc - .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries) + .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_entries) .should.equal true it "should call the callback without error", -> @@ -80,7 +79,7 @@ describe "DocumentManager", -> describe "when the document is not in Redis", -> beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null) + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null) @PersistenceManager.setDoc = sinon.stub().yields() @DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2) @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback @@ -103,7 +102,7 @@ describe "DocumentManager", -> describe "getDocAndRecentOps", -> describe "with a previous version specified", -> beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback @@ -125,7 +124,7 @@ describe "DocumentManager", -> describe "with no previous version specified", -> beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback @@ -146,7 +145,7 @@ describe "DocumentManager", -> describe "getDoc", -> describe "when the doc exists in Redis", -> beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) @DocumentManager.getDoc @project_id, @doc_id, @callback it "should get the doc from Redis", -> @@ -155,7 +154,7 @@ describe "DocumentManager", -> .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries, true).should.equal true + @callback.calledWith(null, @lines, @version, @track_changes_entries, true).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -163,7 +162,7 @@ describe "DocumentManager", -> describe "when the doc does not exist in Redis", -> beforeEach -> @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null) - @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_on, @track_changes_entries) + @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) @RedisManager.putDocInMemory = sinon.stub().yields() @DocumentManager.getDoc @project_id, @doc_id, @callback @@ -179,11 +178,11 @@ describe "DocumentManager", -> it "should set the doc in Redis", -> @RedisManager.putDocInMemory - .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries) + .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_entries) .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries, false).should.equal true + @callback.calledWith(null, @lines, @version, @track_changes_entries, false).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -193,7 +192,7 @@ describe "DocumentManager", -> beforeEach -> @beforeLines = ["before", "lines"] @afterLines = ["after", "lines"] - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @track_changes_on, @track_changes_entries, true) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @track_changes_entries, true) @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) @UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null) @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee index e9f2cf212e..35c276a4f2 100644 --- a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee @@ -19,7 +19,6 @@ describe "PersistenceManager", -> @lines = ["one", "two", "three"] @version = 42 @callback = sinon.stub() - @track_changes_on = true @track_changes_entries = { comments: "mock", entries: "mock" } @Settings.apis = web: @@ -34,7 +33,6 @@ describe "PersistenceManager", -> @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify({ lines: @lines, version: @version, - track_changes: @track_changes_on, track_changes_entries: @track_changes_entries })) @PersistenceManager.getDoc(@project_id, @doc_id, @callback) @@ -56,7 +54,7 @@ describe "PersistenceManager", -> .should.equal true it "should call the callback with the doc lines, version and track changes state", -> - @callback.calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries).should.equal true + @callback.calledWith(null, @lines, @version, @track_changes_entries).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -114,7 +112,7 @@ describe "PersistenceManager", -> describe "with a successful response from the web api", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 200}) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) it "should call the web api", -> @request @@ -123,7 +121,6 @@ describe "PersistenceManager", -> json: lines: @lines version: @version - track_changes: @track_changes_on track_changes_entries: @track_changes_entries method: "POST" auth: @@ -144,7 +141,7 @@ describe "PersistenceManager", -> describe "when request returns an error", -> beforeEach -> @request.callsArgWith(1, @error = new Error("oops"), null, null) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) it "should return the error", -> @callback.calledWith(@error).should.equal true @@ -155,7 +152,7 @@ describe "PersistenceManager", -> describe "when the request returns 404", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 404}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) it "should return a NotFoundError", -> @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true @@ -166,7 +163,7 @@ describe "PersistenceManager", -> describe "when the request returns an error status code", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 500}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) it "should return an error", -> @callback.calledWith(new Error("web api error")).should.equal true diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index 6f9afc29d3..901af153c1 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -22,7 +22,6 @@ describe "RedisManager", -> projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" - trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:#{doc_id}" trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:#{doc_id}" "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "./Metrics": @metrics = @@ -40,11 +39,10 @@ describe "RedisManager", -> @jsonlines = JSON.stringify @lines @version = 42 @track_changes_on = true - @redis_track_changes_on = "1" @track_changes_entries = { comments: "mock", entries: "mock" } @json_track_changes_entries = JSON.stringify @track_changes_entries @rclient.get = sinon.stub() - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @redis_track_changes_on, @json_track_changes_entries]) + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @json_track_changes_entries]) describe "successfully", -> beforeEach -> @@ -60,11 +58,6 @@ describe "RedisManager", -> .calledWith("DocVersion:#{@doc_id}") .should.equal true - it "should get the track changes state", -> - @rclient.get - .calledWith("TrackChangesEnabled:#{@doc_id}") - .should.equal true - it "should get the track changes entries", -> @rclient.get .calledWith("TrackChangesEntries:#{@doc_id}") @@ -72,13 +65,13 @@ describe "RedisManager", -> it 'should return the document', -> @callback - .calledWith(null, @lines, @version, @track_changes_on, @track_changes_entries) + .calledWith(null, @lines, @version, @track_changes_entries) .should.equal true describe "getDoc with an invalid project id", -> beforeEach -> @another_project_id = "project-id-456" - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id, @redis_track_changes_on, @json_track_changes_entries]) + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id, @json_track_changes_entries]) @RedisManager.getDoc @project_id, @doc_id, @callback it 'should return an error', -> @@ -263,9 +256,8 @@ describe "RedisManager", -> @rclient.exec.yields() @lines = ["one", "two", "three"] @version = 42 - @track_changes_on = true @track_changes_entries = { comments: "mock", entries: "mock" } - @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @track_changes_on, @track_changes_entries, done + @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @track_changes_entries, done it "should set the lines", -> @rclient.set @@ -282,11 +274,6 @@ describe "RedisManager", -> .calledWith("TrackChangesEntries:#{@doc_id}", JSON.stringify(@track_changes_entries)) .should.equal true - it "should set the track changes state", -> - @rclient.set - .calledWith("TrackChangesEnabled:#{@doc_id}", "1") - .should.equal true - it "should set the project_id for the doc", -> @rclient.set .calledWith("ProjectId:#{@doc_id}", @project_id) diff --git a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee index 9969e42d61..fb9bc18eb1 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee @@ -158,11 +158,10 @@ describe "UpdateManager", -> @updatedDocLines = ["updated", "lines"] @version = 34 @lines = ["original", "lines"] - @track_changes_on = true @track_changes_entries = { entries: "mock", comments: "mock" } @updated_track_changes_entries = { entries: "updated", comments: "updated" } @appliedOps = ["mock-applied-ops"] - @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @track_changes_on, @track_changes_entries) + @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @track_changes_entries) @TrackChangesManager.applyUpdate = sinon.stub().yields(null, @updated_track_changes_entries) @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) @RedisManager.updateDocument = sinon.stub().yields() @@ -179,7 +178,7 @@ describe "UpdateManager", -> it "should update the track changes entries", -> @TrackChangesManager.applyUpdate - .calledWith(@project_id, @doc_id, @track_changes_entries, @appliedOps, @track_changes_on) + .calledWith(@project_id, @doc_id, @track_changes_entries, @appliedOps) .should.equal true it "should save the document", -> From 418405e8b954003e848180831ce56203318bfbcb Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 2 Dec 2016 11:37:27 +0000 Subject: [PATCH 11/25] Return track changes entries in HTTP request --- .../document-updater/app/coffee/DocumentManager.coffee | 8 ++++---- .../document-updater/app/coffee/HttpController.coffee | 3 ++- .../test/acceptance/coffee/TrackChangesTests.coffee | 5 ++--- .../coffee/DocumentManager/DocumentManagerTests.coffee | 4 ++-- .../test/unit/coffee/HttpController/getDocTests.coffee | 4 +++- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee index 258d55bb65..85a7a4263c 100644 --- a/services/document-updater/app/coffee/DocumentManager.coffee +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -26,20 +26,20 @@ module.exports = DocumentManager = else callback null, lines, version, track_changes_entries, true - getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps) ->) -> + getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps, track_changes_entries) ->) -> timer = new Metrics.Timer("docManager.getDocAndRecentOps") callback = (args...) -> timer.done() _callback(args...) - DocumentManager.getDoc project_id, doc_id, (error, lines, version) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> return callback(error) if error? if fromVersion == -1 - callback null, lines, version, [] + callback null, lines, version, [], track_changes_entries else RedisManager.getPreviousDocOps doc_id, fromVersion, version, (error, ops) -> return callback(error) if error? - callback null, lines, version, ops + callback null, lines, version, ops, track_changes_entries setDoc: (project_id, doc_id, newLines, source, user_id, _callback = (error) ->) -> timer = new Metrics.Timer("docManager.setDoc") diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee index dc74833697..fb916da045 100644 --- a/services/document-updater/app/coffee/HttpController.coffee +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -18,7 +18,7 @@ module.exports = HttpController = else fromVersion = -1 - DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops) -> + DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops, track_changes_entries) -> timer.done() return next(error) if error? logger.log project_id: project_id, doc_id: doc_id, "got doc via http" @@ -29,6 +29,7 @@ module.exports = HttpController = lines: lines version: version ops: ops + track_changes_entries: track_changes_entries _getTotalSizeOfLines: (lines) -> size = 0 diff --git a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee index 43401cef6d..e3577fd6d7 100644 --- a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee @@ -47,10 +47,9 @@ describe "Track changes", -> setTimeout done, 200 it "should set the updated track changes entries in redis", (done) -> - console.log "TODO: GET ME FROM HTTP REQUEST" - rclient.get "TrackChangesEntries:#{@doc.id}", (error, value) => + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => throw error if error? - entries = JSON.parse(value) + entries = data.track_changes_entries change = entries.changes[0] change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id diff --git a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee index d29a569d46..3a1db1961c 100644 --- a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee @@ -117,7 +117,7 @@ describe "DocumentManager", -> .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @ops).should.equal true + @callback.calledWith(null, @lines, @version, @ops, @track_changes_entries).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -137,7 +137,7 @@ describe "DocumentManager", -> @RedisManager.getPreviousDocOps.called.should.equal false it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, []).should.equal true + @callback.calledWith(null, @lines, @version, [], @track_changes_entries).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true diff --git a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee index 17e5ad8e08..8ad2966b23 100644 --- a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee +++ b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee @@ -22,6 +22,7 @@ describe "HttpController.getDoc", -> @ops = ["mock-op-1", "mock-op-2"] @version = 42 @fromVersion = 42 + @track_changes_entries = { changes: "mock", comments: "mock" } @res = send: sinon.stub() @req = @@ -32,7 +33,7 @@ describe "HttpController.getDoc", -> describe "when the document exists and no recent ops are requested", -> beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, []) + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @track_changes_entries) @HttpController.getDoc(@req, @res, @next) it "should get the doc", -> @@ -47,6 +48,7 @@ describe "HttpController.getDoc", -> lines: @lines version: @version ops: [] + track_changes_entries: @track_changes_entries })) .should.equal true From 3ea2e079938424ac52c2b3ee7470f32422f7ac63 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 2 Dec 2016 12:01:23 +0000 Subject: [PATCH 12/25] Add tests for fetching and flushing track changes entries to persistence layer --- .../app/coffee/TrackChangesManager.coffee | 13 ++++- .../coffee/TrackChangesTests.coffee | 50 ++++++++++++++++++- .../coffee/helpers/DocUpdaterClient.coffee | 15 ------ .../coffee/helpers/MockWebApi.coffee | 5 +- 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/services/document-updater/app/coffee/TrackChangesManager.coffee b/services/document-updater/app/coffee/TrackChangesManager.coffee index 65f1931bb4..126b9ec7e0 100644 --- a/services/document-updater/app/coffee/TrackChangesManager.coffee +++ b/services/document-updater/app/coffee/TrackChangesManager.coffee @@ -8,5 +8,14 @@ module.exports = TrackChangesManager = changesTracker.track_changes = !!update.meta.tc for op in update.op changesTracker.applyOp(op, { user_id: update.meta?.user_id }) - {changes, comments} = changesTracker - callback null, {changes, comments} \ No newline at end of file + + # Return the minimal data structure needed, since most documents won't have any + # changes or comments + response = null + if changesTracker.changes?.length > 0 + response ?= {} + response.changes = changesTracker.changes + if changesTracker.comments?.length > 0 + response ?= {} + response.comments = changesTracker.comments + callback null, response \ No newline at end of file diff --git a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee index e3577fd6d7..88250f82a7 100644 --- a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee @@ -46,7 +46,7 @@ describe "Track changes", -> throw error if error? setTimeout done, 200 - it "should set the updated track changes entries in redis", (done) -> + it "should update the tracked entries", (done) -> DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => throw error if error? entries = data.track_changes_entries @@ -55,3 +55,51 @@ describe "Track changes", -> change.metadata.user_id.should.equal @user_id done() + describe "Loading changes from persistence layer", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["a123aa"] + } + @update = { + doc: @doc.id + op: [{ i: "456", p: 5 }] + v: 0 + meta: { user_id: @user_id, tc: 1 } + } + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + track_changes_entries: { + changes: [{ + op: { i: "123", p: 1 } + metadata: + user_id: @user_id + ts: new Date() + }] + } + } + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc.id, @update, (error) -> + throw error if error? + setTimeout done, 200 + + it "should have preloaded the existing changes", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + {changes} = data.track_changes_entries + changes[0].op.should.deep.equal { i: "123", p: 1 } + changes[1].op.should.deep.equal { i: "456", p: 5 } + done() + + it "should flush the changes to the persistence layer again", (done) -> + DocUpdaterClient.flushDoc @project_id, @doc.id, (error) => + throw error if error? + MockWebApi.getDocument @project_id, @doc.id, (error, doc) => + {changes} = doc.track_changes_entries + changes[0].op.should.deep.equal { i: "123", p: 1 } + changes[1].op.should.deep.equal { i: "456", p: 5 } + done() diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee index b90e7ea82e..d704daefd1 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee @@ -72,18 +72,3 @@ module.exports = DocUpdaterClient = deleteProject: (project_id, callback = () ->) -> request.del "http://localhost:3003/project/#{project_id}", callback - - setTrackChangesOn: (project_id, callback = () ->) -> - request.post { - url: "http://localhost:3003/project/#{project_id}/track_changes" - json: - on: true - }, callback - - setTrackChangesOff: (project_id, callback = () ->) -> - request.post { - url: "http://localhost:3003/project/#{project_id}/track_changes" - json: - on: false - }, callback - diff --git a/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee index e77a18c0ea..4e9d073cc4 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee @@ -11,10 +11,11 @@ module.exports = MockWebApi = doc.lines ?= [] @docs["#{project_id}:#{doc_id}"] = doc - setDocument: (project_id, doc_id, lines, version, callback = (error) ->) -> + setDocument: (project_id, doc_id, lines, version, track_changes_entries, callback = (error) ->) -> doc = @docs["#{project_id}:#{doc_id}"] ||= {} doc.lines = lines doc.version = version + doc.track_changes_entries = track_changes_entries callback null getDocument: (project_id, doc_id, callback = (error, doc) ->) -> @@ -31,7 +32,7 @@ module.exports = MockWebApi = res.send 404 app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) => - MockWebApi.setDocument req.params.project_id, req.params.doc_id, req.body.lines, req.body.version, (error) -> + MockWebApi.setDocument req.params.project_id, req.params.doc_id, req.body.lines, req.body.version, req.body.track_changes_entries, (error) -> if error? res.send 500 else From e3fee1a1d13c82675884e9c452e8a32141a6ba57 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 8 Dec 2016 12:31:43 +0000 Subject: [PATCH 13/25] Rename 'track changes entries' -> 'ranges' --- .../app/coffee/DocumentManager.coffee | 24 ++-- .../app/coffee/HttpController.coffee | 4 +- .../app/coffee/PersistenceManager.coffee | 8 +- .../app/coffee/ProjectManager.coffee | 28 ----- .../app/coffee/RangesManager.coffee | 24 ++++ ...gesTracker.coffee => RangesTracker.coffee} | 22 +++- .../app/coffee/RedisKeyBuilder.coffee | 6 +- .../app/coffee/RedisManager.coffee | 42 +++++-- .../app/coffee/TrackChangesManager.coffee | 21 ---- .../app/coffee/UpdateManager.coffee | 8 +- .../config/settings.defaults.coffee | 6 +- ...ChangesTests.coffee => RangesTests.coffee} | 22 ++-- .../coffee/helpers/MockWebApi.coffee | 6 +- .../DocumentManagerTests.coffee | 28 ++--- .../coffee/HttpController/getDocTests.coffee | 6 +- .../PersistenceManagerTests.coffee | 18 +-- .../RedisManager/RedisManagerTests.coffee | 115 +++++++++++------- .../UpdateManager/UpdateManagerTests.coffee | 20 +-- 18 files changed, 225 insertions(+), 183 deletions(-) create mode 100644 services/document-updater/app/coffee/RangesManager.coffee rename services/document-updater/app/coffee/{ChangesTracker.coffee => RangesTracker.coffee} (95%) delete mode 100644 services/document-updater/app/coffee/TrackChangesManager.coffee rename services/document-updater/test/acceptance/coffee/{TrackChangesTests.coffee => RangesTests.coffee} (83%) diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee index 85a7a4263c..c6d4773036 100644 --- a/services/document-updater/app/coffee/DocumentManager.coffee +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -13,33 +13,33 @@ module.exports = DocumentManager = timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? if !lines? or !version? logger.log {project_id, doc_id}, "doc not in redis so getting from persistence API" - PersistenceManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> + PersistenceManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? logger.log {project_id, doc_id, lines, version}, "got doc from persistence API" - RedisManager.putDocInMemory project_id, doc_id, lines, version, track_changes_entries, (error) -> + RedisManager.putDocInMemory project_id, doc_id, lines, version, ranges, (error) -> return callback(error) if error? - callback null, lines, version, track_changes_entries, false + callback null, lines, version, ranges, false else - callback null, lines, version, track_changes_entries, true + callback null, lines, version, ranges, true - getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps, track_changes_entries) ->) -> + getDocAndRecentOps: (project_id, doc_id, fromVersion, _callback = (error, lines, version, recentOps, ranges) ->) -> timer = new Metrics.Timer("docManager.getDocAndRecentOps") callback = (args...) -> timer.done() _callback(args...) - DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? if fromVersion == -1 - callback null, lines, version, [], track_changes_entries + callback null, lines, version, [], ranges else RedisManager.getPreviousDocOps doc_id, fromVersion, version, (error, ops) -> return callback(error) if error? - callback null, lines, version, ops, track_changes_entries + callback null, lines, version, ops, ranges setDoc: (project_id, doc_id, newLines, source, user_id, _callback = (error) ->) -> timer = new Metrics.Timer("docManager.setDoc") @@ -51,7 +51,7 @@ module.exports = DocumentManager = return callback(new Error("No lines were provided to setDoc")) UpdateManager = require "./UpdateManager" - DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, track_changes, alreadyLoaded) -> + DocumentManager.getDoc project_id, doc_id, (error, oldLines, version, ranges, alreadyLoaded) -> return callback(error) if error? if oldLines? and oldLines.length > 0 and oldLines[0].text? @@ -90,14 +90,14 @@ module.exports = DocumentManager = callback = (args...) -> timer.done() _callback(args...) - RedisManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> + RedisManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? if !lines? or !version? logger.log project_id: project_id, doc_id: doc_id, "doc is not loaded so not flushing" callback null # TODO: return a flag to bail out, as we go on to remove doc from memory? else logger.log project_id: project_id, doc_id: doc_id, version: version, "flushing doc" - PersistenceManager.setDoc project_id, doc_id, lines, version, track_changes_entries, (error) -> + PersistenceManager.setDoc project_id, doc_id, lines, version, ranges, (error) -> return callback(error) if error? callback null diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee index fb916da045..e138fe8bc4 100644 --- a/services/document-updater/app/coffee/HttpController.coffee +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -18,7 +18,7 @@ module.exports = HttpController = else fromVersion = -1 - DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops, track_changes_entries) -> + DocumentManager.getDocAndRecentOpsWithLock project_id, doc_id, fromVersion, (error, lines, version, ops, ranges) -> timer.done() return next(error) if error? logger.log project_id: project_id, doc_id: doc_id, "got doc via http" @@ -29,7 +29,7 @@ module.exports = HttpController = lines: lines version: version ops: ops - track_changes_entries: track_changes_entries + ranges: ranges _getTotalSizeOfLines: (lines) -> size = 0 diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index ee7674d80a..457627982f 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -10,7 +10,7 @@ logger = require "logger-sharelatex" MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds module.exports = PersistenceManager = - getDoc: (project_id, doc_id, _callback = (error, lines, version, track_changes_entries) ->) -> + getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges) ->) -> timer = new Metrics.Timer("persistenceManager.getDoc") callback = (args...) -> timer.done() @@ -39,13 +39,13 @@ module.exports = PersistenceManager = return callback(new Error("web API response had no doc lines")) if !body.version? or not body.version instanceof Number return callback(new Error("web API response had no valid doc version")) - return callback null, body.lines, body.version, body.track_changes_entries + return callback null, body.lines, body.version, body.ranges else if res.statusCode == 404 return callback(new Errors.NotFoundError("doc not not found: #{url}")) else return callback(new Error("error accessing web API: #{url} #{res.statusCode}")) - setDoc: (project_id, doc_id, lines, version, track_changes_entries, _callback = (error) ->) -> + setDoc: (project_id, doc_id, lines, version, ranges, _callback = (error) ->) -> timer = new Metrics.Timer("persistenceManager.setDoc") callback = (args...) -> timer.done() @@ -57,7 +57,7 @@ module.exports = PersistenceManager = method: "POST" json: lines: lines - track_changes_entries: track_changes_entries + ranges: ranges version: version auth: user: Settings.apis.web.user diff --git a/services/document-updater/app/coffee/ProjectManager.coffee b/services/document-updater/app/coffee/ProjectManager.coffee index a38fe08397..cd4c66ae8d 100644 --- a/services/document-updater/app/coffee/ProjectManager.coffee +++ b/services/document-updater/app/coffee/ProjectManager.coffee @@ -56,31 +56,3 @@ module.exports = ProjectManager = callback new Error("Errors deleting docs. See log for details") else callback(null) - - setTrackChangesWithLocks: (project_id, track_changes_on, _callback = (error) ->) -> - timer = new Metrics.Timer("projectManager.toggleTrackChangesWithLocks") - callback = (args...) -> - timer.done() - _callback(args...) - - RedisManager.getDocIdsInProject project_id, (error, doc_ids) -> - return callback(error) if error? - jobs = [] - errors = [] - for doc_id in (doc_ids or []) - do (doc_id) -> - jobs.push (callback) -> - DocumentManager.setTrackChangesWithLock project_id, doc_id, track_changes_on, (error) -> - if error? - logger.error {err: error, project_id, doc_ids, track_changes_on}, "error toggle track changes for doc" - errors.push(error) - callback() - # TODO: If no docs, turn on track changes in Mongo manually - - logger.log {project_id, doc_ids, track_changes_on}, "toggling track changes for docs" - async.series jobs, () -> - if errors.length > 0 - callback new Error("Errors toggling track changes for docs. See log for details") - else - callback(null) - diff --git a/services/document-updater/app/coffee/RangesManager.coffee b/services/document-updater/app/coffee/RangesManager.coffee new file mode 100644 index 0000000000..1e19a63b0d --- /dev/null +++ b/services/document-updater/app/coffee/RangesManager.coffee @@ -0,0 +1,24 @@ +RangesTracker = require "./RangesTracker" +logger = require "logger-sharelatex" + +module.exports = RangesManager = + applyUpdate: (project_id, doc_id, entries = {}, updates = [], callback = (error, new_entries) ->) -> + {changes, comments} = entries + logger.log {changes, comments, updates}, "appliyng updates to ranges" + rangesTracker = new RangesTracker(changes, comments) + for update in updates + rangesTracker.track_changes = !!update.meta.tc + for op in update.op + rangesTracker.applyOp(op, { user_id: update.meta?.user_id }) + + # Return the minimal data structure needed, since most documents won't have any + # changes or comments + response = {} + if rangesTracker.changes?.length > 0 + response ?= {} + response.changes = rangesTracker.changes + if rangesTracker.comments?.length > 0 + response ?= {} + response.comments = rangesTracker.comments + logger.log {response}, "applied updates to ranges" + callback null, response \ No newline at end of file diff --git a/services/document-updater/app/coffee/ChangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee similarity index 95% rename from services/document-updater/app/coffee/ChangesTracker.coffee rename to services/document-updater/app/coffee/RangesTracker.coffee index 8bc4cf9380..6a3625fd09 100644 --- a/services/document-updater/app/coffee/ChangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -1,5 +1,5 @@ load = (EventEmitter) -> - class ChangesTracker extends EventEmitter + class RangesTracker extends EventEmitter # The purpose of this class is to track a set of inserts and deletes to a document, like # track changes in Word. We store these as a set of ShareJs style ranges: # {i: "foo", p: 42} # Insert 'foo' at offset 42 @@ -97,7 +97,7 @@ load = (EventEmitter) -> return if !change? @_removeChange(change) - applyOp: (op, metadata = {}) -> + applyOp: (op, metadata) -> metadata.ts ?= new Date() # Apply an op that has been applied to the document to our changes to keep them up to date if op.i? @@ -371,7 +371,23 @@ load = (EventEmitter) -> @emit "changes:moved", moved_changes _newId: () -> - (@id++).toString() + # Generate a Mongo ObjectId + # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js + @_pid ?= Math.floor(Math.random() * (32767)) + @_machine ?= Math.floor(Math.random() * (16777216)) + timestamp = Math.floor(new Date().valueOf() / 1000) + @_increment ?= 0 + @_increment++ + + timestamp = timestamp.toString(16) + machine = @_machine.toString(16) + pid = @_pid.toString(16) + increment = @_increment.toString(16) + + return '00000000'.substr(0, 8 - timestamp.length) + timestamp + + '000000'.substr(0, 6 - machine.length) + machine + + '0000'.substr(0, 4 - pid.length) + pid + + '000000'.substr(0, 6 - increment.length) + increment; _addOp: (op, metadata) -> change = { diff --git a/services/document-updater/app/coffee/RedisKeyBuilder.coffee b/services/document-updater/app/coffee/RedisKeyBuilder.coffee index c09fb43f00..1b5e548809 100644 --- a/services/document-updater/app/coffee/RedisKeyBuilder.coffee +++ b/services/document-updater/app/coffee/RedisKeyBuilder.coffee @@ -34,10 +34,8 @@ module.exports = RedisKeyBuilder = return (key_schema) -> key_schema.uncompressedHistoryOp({doc_id}) pendingUpdates: ({doc_id}) -> return (key_schema) -> key_schema.pendingUpdates({doc_id}) - trackChangesEnabled: ({doc_id}) -> - return (key_schema) -> key_schema.trackChangesEnabled({doc_id}) - trackChangesEntries: ({doc_id}) -> - return (key_schema) -> key_schema.trackChangesEntries({doc_id}) + ranges: ({doc_id}) -> + return (key_schema) -> key_schema.ranges({doc_id}) docsInProject: ({project_id}) -> return (key_schema) -> key_schema.docsInProject({project_id}) docsWithHistoryOps: ({project_id}) -> diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index cad5bd9f04..be5166b94c 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -13,17 +13,21 @@ minutes = 60 # seconds for Redis expire module.exports = RedisManager = rclient: rclient - putDocInMemory : (project_id, doc_id, docLines, version, track_changes_entries, _callback)-> + putDocInMemory : (project_id, doc_id, docLines, version, ranges, _callback)-> timer = new metrics.Timer("redis.put-doc") callback = (error) -> timer.done() _callback(error) logger.log project_id:project_id, doc_id:doc_id, version: version, "putting doc in redis" + ranges = RedisManager._serializeRanges(ranges) multi = rclient.multi() multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines) multi.set keys.projectKey({doc_id:doc_id}), project_id multi.set keys.docVersion(doc_id:doc_id), version - multi.set keys.trackChangesEntries(doc_id:doc_id), JSON.stringify(track_changes_entries) + if ranges? + multi.set keys.ranges(doc_id:doc_id), ranges + else + multi.del keys.ranges(doc_id:doc_id) multi.exec (error) -> return callback(error) if error? rclient.sadd keys.docsInProject(project_id:project_id), doc_id, callback @@ -42,32 +46,33 @@ module.exports = RedisManager = multi.del keys.docLines(doc_id:doc_id) multi.del keys.projectKey(doc_id:doc_id) multi.del keys.docVersion(doc_id:doc_id) - multi.del keys.trackChangesEntries(doc_id:doc_id) + multi.del keys.ranges(doc_id:doc_id) multi.exec (error) -> return callback(error) if error? rclient.srem keys.docsInProject(project_id:project_id), doc_id, callback - getDoc : (project_id, doc_id, callback = (error, lines, version, track_changes_entries) ->)-> + getDoc : (project_id, doc_id, callback = (error, lines, version, ranges) ->)-> timer = new metrics.Timer("redis.get-doc") multi = rclient.multi() multi.get keys.docLines(doc_id:doc_id) multi.get keys.docVersion(doc_id:doc_id) multi.get keys.projectKey(doc_id:doc_id) - multi.get keys.trackChangesEntries(doc_id:doc_id) - multi.exec (error, [docLines, version, doc_project_id, track_changes_entries])-> + multi.get keys.ranges(doc_id:doc_id) + multi.exec (error, [docLines, version, doc_project_id, ranges])-> timer.done() return callback(error) if error? try docLines = JSON.parse docLines - track_changes_entries = JSON.parse track_changes_entries + ranges = RedisManager._deserializeRanges(ranges) catch e return callback(e) + version = parseInt(version or 0, 10) # check doc is in requested project if doc_project_id? and doc_project_id isnt project_id logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, "doc not in project" return callback(new Errors.NotFoundError("document not found")) - callback null, docLines, version, track_changes_entries + callback null, docLines, version, ranges getDocVersion: (doc_id, callback = (error, version) ->) -> rclient.get keys.docVersion(doc_id: doc_id), (error, version) -> @@ -107,7 +112,7 @@ module.exports = RedisManager = DOC_OPS_TTL: 60 * minutes DOC_OPS_MAX_LENGTH: 100 - updateDocument : (doc_id, docLines, newVersion, appliedOps = [], track_changes_entries, callback = (error) ->)-> + updateDocument : (doc_id, docLines, newVersion, appliedOps = [], ranges, callback = (error) ->)-> RedisManager.getDocVersion doc_id, (error, currentVersion) -> return callback(error) if error? if currentVersion + appliedOps.length != newVersion @@ -122,10 +127,27 @@ module.exports = RedisManager = multi.rpush keys.docOps(doc_id: doc_id), jsonOps... multi.expire keys.docOps(doc_id: doc_id), RedisManager.DOC_OPS_TTL multi.ltrim keys.docOps(doc_id: doc_id), -RedisManager.DOC_OPS_MAX_LENGTH, -1 - multi.set keys.trackChangesEntries(doc_id:doc_id), JSON.stringify(track_changes_entries) + ranges = RedisManager._serializeRanges(ranges) + if ranges? + multi.set keys.ranges(doc_id:doc_id), ranges + else + multi.del keys.ranges(doc_id:doc_id) multi.exec (error, replys) -> return callback(error) if error? return callback() getDocIdsInProject: (project_id, callback = (error, doc_ids) ->) -> rclient.smembers keys.docsInProject(project_id: project_id), callback + + _serializeRanges: (ranges) -> + jsonRanges = JSON.stringify(ranges) + if jsonRanges == '{}' + # Most doc will have empty ranges so don't fill redis with lots of '{}' keys + jsonRanges = null + return jsonRanges + + _deserializeRanges: (ranges) -> + if !ranges? or ranges == "" + return {} + else + return JSON.parse(ranges) \ No newline at end of file diff --git a/services/document-updater/app/coffee/TrackChangesManager.coffee b/services/document-updater/app/coffee/TrackChangesManager.coffee deleted file mode 100644 index 126b9ec7e0..0000000000 --- a/services/document-updater/app/coffee/TrackChangesManager.coffee +++ /dev/null @@ -1,21 +0,0 @@ -ChangesTracker = require "./ChangesTracker" - -module.exports = TrackChangesManager = - applyUpdate: (project_id, doc_id, entries = {}, updates = [], callback = (error, new_entries) ->) -> - {changes, comments} = entries - changesTracker = new ChangesTracker(changes, comments) - for update in updates - changesTracker.track_changes = !!update.meta.tc - for op in update.op - changesTracker.applyOp(op, { user_id: update.meta?.user_id }) - - # Return the minimal data structure needed, since most documents won't have any - # changes or comments - response = null - if changesTracker.changes?.length > 0 - response ?= {} - response.changes = changesTracker.changes - if changesTracker.comments?.length > 0 - response ?= {} - response.comments = changesTracker.comments - callback null, response \ No newline at end of file diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee index 7c98b97eee..1678c4d4c0 100644 --- a/services/document-updater/app/coffee/UpdateManager.coffee +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -9,7 +9,7 @@ logger = require('logger-sharelatex') Metrics = require "./Metrics" Errors = require "./Errors" DocumentManager = require "./DocumentManager" -TrackChangesManager = require "./TrackChangesManager" +RangesManager = require "./RangesManager" module.exports = UpdateManager = processOutstandingUpdates: (project_id, doc_id, callback = (error) ->) -> @@ -48,16 +48,16 @@ module.exports = UpdateManager = applyUpdate: (project_id, doc_id, update, callback = (error) ->) -> UpdateManager._sanitizeUpdate update - DocumentManager.getDoc project_id, doc_id, (error, lines, version, track_changes_entries) -> + DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? if !lines? or !version? return callback(new Errors.NotFoundError("document not found: #{doc_id}")) ShareJsUpdateManager.applyUpdate project_id, doc_id, update, lines, version, (error, updatedDocLines, version, appliedOps) -> return callback(error) if error? - TrackChangesManager.applyUpdate project_id, doc_id, track_changes_entries, appliedOps, (error, new_track_changes_entries) -> + RangesManager.applyUpdate project_id, doc_id, ranges, appliedOps, (error, new_ranges) -> return callback(error) if error? logger.log doc_id: doc_id, version: version, "updating doc in redis" - RedisManager.updateDocument doc_id, updatedDocLines, version, appliedOps, new_track_changes_entries, (error) -> + RedisManager.updateDocument doc_id, updatedDocLines, version, appliedOps, new_ranges, (error) -> return callback(error) if error? HistoryManager.pushUncompressedHistoryOps project_id, doc_id, appliedOps, callback diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index edb8c56ad3..42daf505d9 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -32,8 +32,7 @@ module.exports = docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" - trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:#{doc_id}" - trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:#{doc_id}" + ranges: ({doc_id}) -> "Ranges:#{doc_id}" # }, { # cluster: [{ # port: "7000" @@ -46,8 +45,7 @@ module.exports = # docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" # projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" # docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" - # trackChangesEnabled: ({doc_id}) -> "TrackChangesEnabled:{#{doc_id}}" - # trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:{#{doc_id}}" + # ranges: ({doc_id}) -> "Ranges:{#{doc_id}}" }] max_doc_length: 2 * 1024 * 1024 # 2mb diff --git a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee similarity index 83% rename from services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee rename to services/document-updater/test/acceptance/coffee/RangesTests.coffee index 88250f82a7..95a7fae9d4 100644 --- a/services/document-updater/test/acceptance/coffee/TrackChangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -7,8 +7,8 @@ rclient = require("redis").createClient() MockWebApi = require "./helpers/MockWebApi" DocUpdaterClient = require "./helpers/DocUpdaterClient" -describe "Track changes", -> - describe "tracking changes", -> +describe "Ranges", -> + describe "tracking changes from ops", -> before (done) -> @project_id = DocUpdaterClient.randomId() @user_id = DocUpdaterClient.randomId() @@ -46,16 +46,16 @@ describe "Track changes", -> throw error if error? setTimeout done, 200 - it "should update the tracked entries", (done) -> + it "should update the ranges", (done) -> DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => throw error if error? - entries = data.track_changes_entries - change = entries.changes[0] + ranges = data.ranges + change = ranges.changes[0] change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id done() - describe "Loading changes from persistence layer", -> + describe "Loading ranges from persistence layer", -> before (done) -> @project_id = DocUpdaterClient.randomId() @user_id = DocUpdaterClient.randomId() @@ -72,7 +72,7 @@ describe "Track changes", -> MockWebApi.insertDoc @project_id, @doc.id, { lines: @doc.lines version: 0 - track_changes_entries: { + ranges: { changes: [{ op: { i: "123", p: 1 } metadata: @@ -87,19 +87,19 @@ describe "Track changes", -> throw error if error? setTimeout done, 200 - it "should have preloaded the existing changes", (done) -> + it "should have preloaded the existing ranges", (done) -> DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => throw error if error? - {changes} = data.track_changes_entries + {changes} = data.ranges changes[0].op.should.deep.equal { i: "123", p: 1 } changes[1].op.should.deep.equal { i: "456", p: 5 } done() - it "should flush the changes to the persistence layer again", (done) -> + it "should flush the ranges to the persistence layer again", (done) -> DocUpdaterClient.flushDoc @project_id, @doc.id, (error) => throw error if error? MockWebApi.getDocument @project_id, @doc.id, (error, doc) => - {changes} = doc.track_changes_entries + {changes} = doc.ranges changes[0].op.should.deep.equal { i: "123", p: 1 } changes[1].op.should.deep.equal { i: "456", p: 5 } done() diff --git a/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee index 4e9d073cc4..f2b8bce318 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee @@ -11,11 +11,11 @@ module.exports = MockWebApi = doc.lines ?= [] @docs["#{project_id}:#{doc_id}"] = doc - setDocument: (project_id, doc_id, lines, version, track_changes_entries, callback = (error) ->) -> + setDocument: (project_id, doc_id, lines, version, ranges, callback = (error) ->) -> doc = @docs["#{project_id}:#{doc_id}"] ||= {} doc.lines = lines doc.version = version - doc.track_changes_entries = track_changes_entries + doc.ranges = ranges callback null getDocument: (project_id, doc_id, callback = (error, doc) ->) -> @@ -32,7 +32,7 @@ module.exports = MockWebApi = res.send 404 app.post "/project/:project_id/doc/:doc_id", express.bodyParser(), (req, res, next) => - MockWebApi.setDocument req.params.project_id, req.params.doc_id, req.body.lines, req.body.version, req.body.track_changes_entries, (error) -> + MockWebApi.setDocument req.params.project_id, req.params.doc_id, req.body.lines, req.body.version, req.body.ranges, (error) -> if error? res.send 500 else diff --git a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee index 3a1db1961c..5966843f5a 100644 --- a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee @@ -23,7 +23,7 @@ describe "DocumentManager", -> @callback = sinon.stub() @lines = ["one", "two", "three"] @version = 42 - @track_changes_entries = { comments: "mock", entries: "mock" } + @ranges = { comments: "mock", entries: "mock" } describe "flushAndDeleteDoc", -> describe "successfully", -> @@ -49,7 +49,7 @@ describe "DocumentManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true - it "should flush to track changes", -> + it "should flush to the history api", -> @HistoryManager.flushDocChanges .calledWith(@project_id, @doc_id) .should.equal true @@ -57,7 +57,7 @@ describe "DocumentManager", -> describe "flushDocIfLoaded", -> describe "when the doc is in Redis", -> beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges) @PersistenceManager.setDoc = sinon.stub().yields() @DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback @@ -68,7 +68,7 @@ describe "DocumentManager", -> it "should write the doc lines to the persistence layer", -> @PersistenceManager.setDoc - .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_entries) + .calledWith(@project_id, @doc_id, @lines, @version, @ranges) .should.equal true it "should call the callback without error", -> @@ -102,7 +102,7 @@ describe "DocumentManager", -> describe "getDocAndRecentOps", -> describe "with a previous version specified", -> beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @DocumentManager.getDocAndRecentOps @project_id, @doc_id, @fromVersion, @callback @@ -117,14 +117,14 @@ describe "DocumentManager", -> .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @ops, @track_changes_entries).should.equal true + @callback.calledWith(null, @lines, @version, @ops, @ranges).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true describe "with no previous version specified", -> beforeEach -> - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges) @RedisManager.getPreviousDocOps = sinon.stub().callsArgWith(3, null, @ops) @DocumentManager.getDocAndRecentOps @project_id, @doc_id, -1, @callback @@ -137,7 +137,7 @@ describe "DocumentManager", -> @RedisManager.getPreviousDocOps.called.should.equal false it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, [], @track_changes_entries).should.equal true + @callback.calledWith(null, @lines, @version, [], @ranges).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -145,7 +145,7 @@ describe "DocumentManager", -> describe "getDoc", -> describe "when the doc exists in Redis", -> beforeEach -> - @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) + @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges) @DocumentManager.getDoc @project_id, @doc_id, @callback it "should get the doc from Redis", -> @@ -154,7 +154,7 @@ describe "DocumentManager", -> .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @track_changes_entries, true).should.equal true + @callback.calledWith(null, @lines, @version, @ranges, true).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -162,7 +162,7 @@ describe "DocumentManager", -> describe "when the doc does not exist in Redis", -> beforeEach -> @RedisManager.getDoc = sinon.stub().callsArgWith(2, null, null, null, null, null) - @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @track_changes_entries) + @PersistenceManager.getDoc = sinon.stub().callsArgWith(2, null, @lines, @version, @ranges) @RedisManager.putDocInMemory = sinon.stub().yields() @DocumentManager.getDoc @project_id, @doc_id, @callback @@ -178,11 +178,11 @@ describe "DocumentManager", -> it "should set the doc in Redis", -> @RedisManager.putDocInMemory - .calledWith(@project_id, @doc_id, @lines, @version, @track_changes_entries) + .calledWith(@project_id, @doc_id, @lines, @version, @ranges) .should.equal true it "should call the callback with the doc info", -> - @callback.calledWith(null, @lines, @version, @track_changes_entries, false).should.equal true + @callback.calledWith(null, @lines, @version, @ranges, false).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -192,7 +192,7 @@ describe "DocumentManager", -> beforeEach -> @beforeLines = ["before", "lines"] @afterLines = ["after", "lines"] - @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @track_changes_entries, true) + @DocumentManager.getDoc = sinon.stub().callsArgWith(2, null, @beforeLines, @version, @ranges, true) @DiffCodec.diffAsShareJsOp = sinon.stub().callsArgWith(2, null, @ops) @UpdateManager.applyUpdate = sinon.stub().callsArgWith(3, null) @DocumentManager.flushDocIfLoaded = sinon.stub().callsArg(2) diff --git a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee index 8ad2966b23..8fa3931d65 100644 --- a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee +++ b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee @@ -22,7 +22,7 @@ describe "HttpController.getDoc", -> @ops = ["mock-op-1", "mock-op-2"] @version = 42 @fromVersion = 42 - @track_changes_entries = { changes: "mock", comments: "mock" } + @ranges = { changes: "mock", comments: "mock" } @res = send: sinon.stub() @req = @@ -33,7 +33,7 @@ describe "HttpController.getDoc", -> describe "when the document exists and no recent ops are requested", -> beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @track_changes_entries) + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @ranges) @HttpController.getDoc(@req, @res, @next) it "should get the doc", -> @@ -48,7 +48,7 @@ describe "HttpController.getDoc", -> lines: @lines version: @version ops: [] - track_changes_entries: @track_changes_entries + ranges: @ranges })) .should.equal true diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee index 35c276a4f2..19a3d547a2 100644 --- a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee @@ -19,7 +19,7 @@ describe "PersistenceManager", -> @lines = ["one", "two", "three"] @version = 42 @callback = sinon.stub() - @track_changes_entries = { comments: "mock", entries: "mock" } + @ranges = { comments: "mock", entries: "mock" } @Settings.apis = web: url: @url = "www.example.com" @@ -33,7 +33,7 @@ describe "PersistenceManager", -> @request.callsArgWith(1, null, {statusCode: 200}, JSON.stringify({ lines: @lines, version: @version, - track_changes_entries: @track_changes_entries + ranges: @ranges })) @PersistenceManager.getDoc(@project_id, @doc_id, @callback) @@ -53,8 +53,8 @@ describe "PersistenceManager", -> }) .should.equal true - it "should call the callback with the doc lines, version and track changes state", -> - @callback.calledWith(null, @lines, @version, @track_changes_entries).should.equal true + it "should call the callback with the doc lines, version and ranges", -> + @callback.calledWith(null, @lines, @version, @ranges).should.equal true it "should time the execution", -> @Metrics.Timer::done.called.should.equal true @@ -112,7 +112,7 @@ describe "PersistenceManager", -> describe "with a successful response from the web api", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 200}) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @ranges, @callback) it "should call the web api", -> @request @@ -121,7 +121,7 @@ describe "PersistenceManager", -> json: lines: @lines version: @version - track_changes_entries: @track_changes_entries + ranges: @ranges method: "POST" auth: user: @user @@ -141,7 +141,7 @@ describe "PersistenceManager", -> describe "when request returns an error", -> beforeEach -> @request.callsArgWith(1, @error = new Error("oops"), null, null) - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @ranges, @callback) it "should return the error", -> @callback.calledWith(@error).should.equal true @@ -152,7 +152,7 @@ describe "PersistenceManager", -> describe "when the request returns 404", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 404}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @ranges, @callback) it "should return a NotFoundError", -> @callback.calledWith(new Errors.NotFoundError("not found")).should.equal true @@ -163,7 +163,7 @@ describe "PersistenceManager", -> describe "when the request returns an error status code", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 500}, "") - @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @track_changes_entries, @callback) + @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @ranges, @callback) it "should return an error", -> @callback.calledWith(new Error("web api error")).should.equal true diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index 901af153c1..420d2039b4 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -22,7 +22,7 @@ describe "RedisManager", -> projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" docsInProject: ({project_id}) -> "DocsIn:#{project_id}" - trackChangesEntries: ({doc_id}) -> "TrackChangesEntries:#{doc_id}" + ranges: ({doc_id}) -> "Ranges:#{doc_id}" "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "./Metrics": @metrics = inc: sinon.stub() @@ -38,11 +38,10 @@ describe "RedisManager", -> @lines = ["one", "two", "three"] @jsonlines = JSON.stringify @lines @version = 42 - @track_changes_on = true - @track_changes_entries = { comments: "mock", entries: "mock" } - @json_track_changes_entries = JSON.stringify @track_changes_entries + @ranges = { comments: "mock", entries: "mock" } + @json_ranges = JSON.stringify @ranges @rclient.get = sinon.stub() - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @json_track_changes_entries]) + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @json_ranges]) describe "successfully", -> beforeEach -> @@ -58,20 +57,20 @@ describe "RedisManager", -> .calledWith("DocVersion:#{@doc_id}") .should.equal true - it "should get the track changes entries", -> + it "should get the ranges", -> @rclient.get - .calledWith("TrackChangesEntries:#{@doc_id}") + .calledWith("Ranges:#{@doc_id}") .should.equal true it 'should return the document', -> @callback - .calledWith(null, @lines, @version, @track_changes_entries) + .calledWith(null, @lines, @version, @ranges) .should.equal true describe "getDoc with an invalid project id", -> beforeEach -> @another_project_id = "project-id-456" - @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id, @json_track_changes_entries]) + @rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @another_project_id, @json_ranges]) @RedisManager.getDoc @project_id, @doc_id, @callback it 'should return an error', -> @@ -169,19 +168,20 @@ describe "RedisManager", -> @rclient.rpush = sinon.stub() @rclient.expire = sinon.stub() @rclient.ltrim = sinon.stub() + @rclient.del = sinon.stub() @RedisManager.getDocVersion = sinon.stub() @lines = ["one", "two", "three"] @ops = [{ op: [{ i: "foo", p: 4 }] },{ op: [{ i: "bar", p: 8 }] }] @version = 42 - @track_changes_entries = { comments: "mock", entries: "mock" } + @ranges = { comments: "mock", entries: "mock" } @rclient.exec = sinon.stub().callsArg(0) describe "with a consistent version", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length) - @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @track_changes_entries, @callback + @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @ranges, @callback it "should get the current doc version to check for consistency", -> @RedisManager.getDocVersion @@ -198,9 +198,9 @@ describe "RedisManager", -> .calledWith("DocVersion:#{@doc_id}", @version) .should.equal true - it "should set the track changes entries", -> + it "should set the ranges", -> @rclient.set - .calledWith("TrackChangesEntries:#{@doc_id}", JSON.stringify(@track_changes_entries)) + .calledWith("Ranges:#{@doc_id}", JSON.stringify(@ranges)) .should.equal true it "should push the doc op into the doc ops list", -> @@ -224,7 +224,7 @@ describe "RedisManager", -> describe "with an inconsistent version", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length - 1) - @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @track_changes_entries, @callback + @RedisManager.updateDocument @doc_id, @lines, @version, @ops, @ranges, @callback it "should not call multi.exec", -> @rclient.exec.called.should.equal false @@ -237,7 +237,7 @@ describe "RedisManager", -> describe "with no updates", -> beforeEach -> @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version) - @RedisManager.updateDocument @doc_id, @lines, @version, [], @track_changes_entries, @callback + @RedisManager.updateDocument @doc_id, @lines, @version, [], @ranges, @callback it "should not do an rpush", -> @rclient.rpush @@ -248,42 +248,75 @@ describe "RedisManager", -> @rclient.set .calledWith("doclines:#{@doc_id}", JSON.stringify(@lines)) .should.equal true + + describe "with empty ranges", -> + beforeEach -> + @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length) + @RedisManager.updateDocument @doc_id, @lines, @version, @ops, {}, @callback + + it "should not set the ranges", -> + @rclient.set + .calledWith("Ranges:#{@doc_id}", JSON.stringify(@ranges)) + .should.equal false + + it "should delete the ranges key", -> + @rclient.del + .calledWith("Ranges:#{@doc_id}") + .should.equal true describe "putDocInMemory", -> - beforeEach (done) -> + beforeEach -> @rclient.set = sinon.stub() @rclient.sadd = sinon.stub().yields() + @rclient.del = sinon.stub() @rclient.exec.yields() @lines = ["one", "two", "three"] @version = 42 - @track_changes_entries = { comments: "mock", entries: "mock" } - @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @track_changes_entries, done + @ranges = { comments: "mock", entries: "mock" } - it "should set the lines", -> - @rclient.set - .calledWith("doclines:#{@doc_id}", JSON.stringify @lines) - .should.equal true - - it "should set the version", -> - @rclient.set - .calledWith("DocVersion:#{@doc_id}", @version) - .should.equal true + describe "with non-empty ranges", -> + beforeEach (done) -> + @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, @ranges, done - it "should set the track changes entries", -> - @rclient.set - .calledWith("TrackChangesEntries:#{@doc_id}", JSON.stringify(@track_changes_entries)) - .should.equal true + it "should set the lines", -> + @rclient.set + .calledWith("doclines:#{@doc_id}", JSON.stringify @lines) + .should.equal true - it "should set the project_id for the doc", -> - @rclient.set - .calledWith("ProjectId:#{@doc_id}", @project_id) - .should.equal true - - it "should add the doc_id to the project set", -> - @rclient.sadd - .calledWith("DocsIn:#{@project_id}", @doc_id) - .should.equal true - + it "should set the version", -> + @rclient.set + .calledWith("DocVersion:#{@doc_id}", @version) + .should.equal true + + it "should set the ranges", -> + @rclient.set + .calledWith("Ranges:#{@doc_id}", JSON.stringify(@ranges)) + .should.equal true + + it "should set the project_id for the doc", -> + @rclient.set + .calledWith("ProjectId:#{@doc_id}", @project_id) + .should.equal true + + it "should add the doc_id to the project set", -> + @rclient.sadd + .calledWith("DocsIn:#{@project_id}", @doc_id) + .should.equal true + + describe "with empty ranges", -> + beforeEach (done) -> + @RedisManager.putDocInMemory @project_id, @doc_id, @lines, @version, {}, done + + it "should delete the ranges key", -> + @rclient.del + .calledWith("Ranges:#{@doc_id}") + .should.equal true + + it "should not set the ranges", -> + @rclient.set + .calledWith("Ranges:#{@doc_id}", JSON.stringify(@ranges)) + .should.equal false + describe "removeDocFromMemory", -> beforeEach (done) -> @rclient.del = sinon.stub() diff --git a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee index fb9bc18eb1..e87391af44 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee @@ -21,7 +21,7 @@ describe "UpdateManager", -> done: sinon.stub() "settings-sharelatex": Settings = {} "./DocumentManager": @DocumentManager = {} - "./TrackChangesManager": @TrackChangesManager = {} + "./RangesManager": @RangesManager = {} describe "processOutstandingUpdates", -> beforeEach -> @@ -158,11 +158,11 @@ describe "UpdateManager", -> @updatedDocLines = ["updated", "lines"] @version = 34 @lines = ["original", "lines"] - @track_changes_entries = { entries: "mock", comments: "mock" } - @updated_track_changes_entries = { entries: "updated", comments: "updated" } + @ranges = { entries: "mock", comments: "mock" } + @updated_ranges = { entries: "updated", comments: "updated" } @appliedOps = ["mock-applied-ops"] - @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @track_changes_entries) - @TrackChangesManager.applyUpdate = sinon.stub().yields(null, @updated_track_changes_entries) + @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges) + @RangesManager.applyUpdate = sinon.stub().yields(null, @updated_ranges) @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) @RedisManager.updateDocument = sinon.stub().yields() @HistoryManager.pushUncompressedHistoryOps = sinon.stub().callsArg(3) @@ -176,17 +176,17 @@ describe "UpdateManager", -> .calledWith(@project_id, @doc_id, @update, @lines, @version) .should.equal true - it "should update the track changes entries", -> - @TrackChangesManager.applyUpdate - .calledWith(@project_id, @doc_id, @track_changes_entries, @appliedOps) + it "should update the ranges", -> + @RangesManager.applyUpdate + .calledWith(@project_id, @doc_id, @ranges, @appliedOps) .should.equal true it "should save the document", -> @RedisManager.updateDocument - .calledWith(@doc_id, @updatedDocLines, @version, @appliedOps, @updated_track_changes_entries) + .calledWith(@doc_id, @updatedDocLines, @version, @appliedOps, @updated_ranges) .should.equal true - it "should push the applied ops into the track changes queue", -> + it "should push the applied ops into the history queue", -> @HistoryManager.pushUncompressedHistoryOps .calledWith(@project_id, @doc_id, @appliedOps) .should.equal true From 47b19818ff40b10fa181e6f11fa31b4dfb4e809f Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 12 Dec 2016 17:53:43 +0000 Subject: [PATCH 14/25] Add in new comment op type --- .../app/coffee/sharejs/types/text.coffee | 66 +++- .../coffee/ShareJS/TextTransformTests.coffee | 284 ++++++++++++++++++ 2 files changed, 344 insertions(+), 6 deletions(-) create mode 100644 services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee diff --git a/services/document-updater/app/coffee/sharejs/types/text.coffee b/services/document-updater/app/coffee/sharejs/types/text.coffee index c64b4dfa68..dcf2ef4cfe 100644 --- a/services/document-updater/app/coffee/sharejs/types/text.coffee +++ b/services/document-updater/app/coffee/sharejs/types/text.coffee @@ -31,7 +31,8 @@ checkValidComponent = (c) -> i_type = typeof c.i d_type = typeof c.d - throw new Error 'component needs an i or d field' unless (i_type == 'string') ^ (d_type == 'string') + c_type = typeof c.c + throw new Error 'component needs an i, d or c field' unless (i_type == 'string') ^ (d_type == 'string') ^ (c_type == 'string') throw new Error 'position cannot be negative' unless c.p >= 0 @@ -44,11 +45,15 @@ text.apply = (snapshot, op) -> for component in op if component.i? snapshot = strInject snapshot, component.p, component.i - else + else if component.d? deleted = snapshot[component.p...(component.p + component.d.length)] throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..] - + else if component.c? + comment = snapshot[component.p...(component.p + component.c.length)] + throw new Error "Comment component '#{component.c}' does not match commented text '#{comment}'" unless component.c == comment + else + throw new Error "Unknown op type" snapshot @@ -112,7 +117,7 @@ transformPosition = (pos, c, insertAfter) -> pos + c.i.length else pos - else + else if c.d? # I think this could also be written as: Math.min(c.p, Math.min(c.p - otherC.p, otherC.d.length)) # but I think its harder to read that way, and it compiles using ternary operators anyway # so its no slower written like this. @@ -122,6 +127,10 @@ transformPosition = (pos, c, insertAfter) -> c.p else pos - c.d.length + else if c.c? + pos + else + throw new Error("unknown op type") # Helper method to transform a cursor position as a result of an op. # @@ -143,7 +152,7 @@ text._tc = transformComponent = (dest, c, otherC, side) -> if c.i? append dest, {i:c.i, p:transformPosition(c.p, otherC, side == 'right')} - else # Delete + else if c.d? # Delete if otherC.i? # delete vs insert s = c.d if c.p < otherC.p @@ -152,7 +161,7 @@ text._tc = transformComponent = (dest, c, otherC, side) -> if s != '' append dest, {d:s, p:c.p + otherC.i.length} - else # Delete vs delete + else if otherC.d? # Delete vs delete if c.p >= otherC.p + otherC.d.length append dest, {d:c.d, p:c.p - otherC.d.length} else if c.p + c.d.length <= otherC.p @@ -177,6 +186,51 @@ text._tc = transformComponent = (dest, c, otherC, side) -> # This could be rewritten similarly to insert v delete, above. newC.p = transformPosition newC.p, otherC append dest, newC + + else if otherC.c? + append dest, c + + else + throw new Error("unknown op type") + + else if c.c? # Comment + if otherC.i? + if c.p < otherC.p < c.p + c.c.length + offset = otherC.p - c.p + new_c = (c.c[0..(offset-1)] + otherC.i + c.c[offset...]) + append dest, {c:new_c, p:c.p} + else + append dest, {c:c.c, p:transformPosition(c.p, otherC, true)} + + else if otherC.d? + if c.p >= otherC.p + otherC.d.length + append dest, {c:c.c, p:c.p - otherC.d.length} + else if c.p + c.c.length <= otherC.p + append dest, c + else # Delete overlaps comment + # They overlap somewhere. + newC = {c:'', p:c.p} + if c.p < otherC.p + newC.c = c.c[...(otherC.p - c.p)] + if c.p + c.c.length > otherC.p + otherC.d.length + newC.c += c.c[(otherC.p + otherC.d.length - c.p)..] + + # This is entirely optional - just for a check that the deleted + # text in the two ops matches + intersectStart = Math.max c.p, otherC.p + intersectEnd = Math.min c.p + c.c.length, otherC.p + otherC.d.length + cIntersect = c.c[intersectStart - c.p...intersectEnd - c.p] + otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p] + throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect + + newC.p = transformPosition newC.p, otherC + append dest, newC + + else if otherC.c? + append dest, c + + else + throw new Error("unknown op type") dest diff --git a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee new file mode 100644 index 0000000000..e0f75d2756 --- /dev/null +++ b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee @@ -0,0 +1,284 @@ +text = require "../../../../app/js/sharejs/types/text" +require("chai").should() +RangesTracker = require "../../../../app/js/RangesTracker" + +describe "ShareJS text type", -> + describe "transform", -> + describe "insert / insert", -> + it "with an insert before", -> + dest = [] + text._tc(dest, { i: "foo", p: 9 }, { i: "bar", p: 3 }) + dest.should.deep.equal [{ i: "foo", p: 12 }] + + it "with an insert after", -> + dest = [] + text._tc(dest, { i: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ i: "foo", p: 3 }] + + it "with an insert at the same place with side == 'right'", -> + dest = [] + text._tc(dest, { i: "foo", p: 3 }, { i: "bar", p: 3 }, 'right') + dest.should.deep.equal [{ i: "foo", p: 6 }] + + it "with an insert at the same place with side == 'left'", -> + dest = [] + text._tc(dest, { i: "foo", p: 3 }, { i: "bar", p: 3 }, 'left') + dest.should.deep.equal [{ i: "foo", p: 3 }] + + describe "insert / delete", -> + it "with a delete before", -> + dest = [] + text._tc(dest, { i: "foo", p: 9 }, { d: "bar", p: 3 }) + dest.should.deep.equal [{ i: "foo", p: 6 }] + + it "with a delete after", -> + dest = [] + text._tc(dest, { i: "foo", p: 3 }, { d: "bar", p: 9 }) + dest.should.deep.equal [{ i: "foo", p: 3 }] + + it "with a delete at the same place with side == 'right'", -> + dest = [] + text._tc(dest, { i: "foo", p: 3 }, { d: "bar", p: 3 }, 'right') + dest.should.deep.equal [{ i: "foo", p: 3 }] + + it "with a delete at the same place with side == 'left'", -> + dest = [] + + text._tc(dest, { i: "foo", p: 3 }, { d: "bar", p: 3 }, 'left') + dest.should.deep.equal [{ i: "foo", p: 3 }] + + describe "delete / insert", -> + it "with an insert before", -> + dest = [] + text._tc(dest, { d: "foo", p: 9 }, { i: "bar", p: 3 }) + dest.should.deep.equal [{ d: "foo", p: 12 }] + + it "with an insert after", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ d: "foo", p: 3 }] + + it "with an insert at the same place with side == 'right'", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { i: "bar", p: 3 }, 'right') + dest.should.deep.equal [{ d: "foo", p: 6 }] + + it "with an insert at the same place with side == 'left'", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { i: "bar", p: 3 }, 'left') + dest.should.deep.equal [{ d: "foo", p: 6 }] + + it "with a delete that overlaps the insert location", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { i: "bar", p: 4 }) + dest.should.deep.equal [{ d: "f", p: 3 }, { d: "oo", p: 6 }] + + + describe "delete / delete", -> + it "with a delete before", -> + dest = [] + text._tc(dest, { d: "foo", p: 9 }, { d: "bar", p: 3 }) + dest.should.deep.equal [{ d: "foo", p: 6 }] + + it "with a delete after", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { d: "bar", p: 9 }) + dest.should.deep.equal [{ d: "foo", p: 3 }] + + it "with deleting the same content", -> + dest = [] + text._tc(dest, { d: "foo", p: 3 }, { d: "foo", p: 3 }, 'right') + dest.should.deep.equal [] + + it "with the delete overlapping before", -> + dest = [] + text._tc(dest, { d: "foobar", p: 3 }, { d: "abcfoo", p: 0 }, 'right') + dest.should.deep.equal [{ d: "bar", p: 0 }] + + it "with the delete overlapping after", -> + dest = [] + text._tc(dest, { d: "abcfoo", p: 3 }, { d: "foobar", p: 6 }) + dest.should.deep.equal [{ d: "abc", p: 3 }] + + it "with the delete overlapping the whole delete", -> + dest = [] + text._tc(dest, { d: "abcfoo123", p: 3 }, { d: "foo", p: 6 }) + dest.should.deep.equal [{ d: "abc123", p: 3 }] + + it "with the delete inside the whole delete", -> + dest = [] + text._tc(dest, { d: "foo", p: 6 }, { d: "abcfoo123", p: 3 }) + dest.should.deep.equal [] + + describe "comment / insert", -> + it "with an insert before", -> + dest = [] + text._tc(dest, { c: "foo", p: 9 }, { i: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 12 }] + + it "with an insert after", -> + dest = [] + text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 3 }] + + it "with an insert at the left edge", -> + dest = [] + text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 3 }) + # RangesTracker doesn't inject inserts into comments on edges, so neither should we + dest.should.deep.equal [{ c: "foo", p: 6 }] + + it "with an insert at the right edge", -> + dest = [] + text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 6 }) + # RangesTracker doesn't inject inserts into comments on edges, so neither should we + dest.should.deep.equal [{ c: "foo", p: 3 }] + + it "with an insert in the middle", -> + dest = [] + text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 5 }) + dest.should.deep.equal [{ c: "fobaro", p: 3 }] + + describe "comment / delete", -> + it "with a delete before", -> + dest = [] + text._tc(dest, { c: "foo", p: 9 }, { d: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 6 }] + + it "with a delete after", -> + dest = [] + text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 3 }] + + it "with a delete overlapping the comment content before", -> + dest = [] + text._tc(dest, { c: "foobar", p: 6 }, { d: "123foo", p: 3 }) + dest.should.deep.equal [{ c: "bar", p: 3 }] + + it "with a delete overlapping the comment content after", -> + dest = [] + text._tc(dest, { c: "foobar", p: 6 }, { d: "bar123", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 6 }] + + it "with a delete overlapping the comment content in the middle", -> + dest = [] + text._tc(dest, { c: "foo123bar", p: 6 }, { d: "123", p: 9 }) + dest.should.deep.equal [{ c: "foobar", p: 6 }] + + it "with a delete overlapping the whole comment", -> + dest = [] + text._tc(dest, { c: "foo", p: 6 }, { d: "123foo456", p: 3 }) + dest.should.deep.equal [{ c: "", p: 3 }] + + describe "comment / insert", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { i: "foo", p: 6 }, { c: "bar", p: 3 }) + dest.should.deep.equal [{ i: "foo", p: 6 }] + + describe "comment / delete", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { d: "foo", p: 6 }, { c: "bar", p: 3 }) + dest.should.deep.equal [{ d: "foo", p: 6 }] + + describe "comment / comment", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { c: "foo", p: 6 }, { c: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 6 }] + + describe "apply", -> + it "should apply an insert", -> + text.apply("foo", [{ i: "bar", p: 2 }]).should.equal "fobaro" + + it "should apply a delete", -> + text.apply("foo123bar", [{ d: "123", p: 3 }]).should.equal "foobar" + + it "should do nothing with a comment", -> + text.apply("foo123bar", [{ c: "123", p: 3 }]).should.equal "foo123bar" + + it "should throw an error when deleted content does not match", -> + (() -> + text.apply("foo123bar", [{ d: "456", p: 3 }]) + ).should.throw(Error) + + it "should throw an error when comment content does not match", -> + (() -> + text.apply("foo123bar", [{ c: "456", p: 3 }]) + ).should.throw(Error) + + describe "applying ops and comments in different orders", -> + it "should not matter which op or comment is applied first", -> + transform = (op1, op2, side) -> + d = [] + text._tc(d, op1, op2, side) + return d + + applySnapshot = (snapshot, op) -> + return text.apply(snapshot, op) + + applyRanges = (rangesTracker, ops) -> + for op in ops + if op.c? + rangesTracker.addComment(op.p, op.c.length, {}) + else + rangesTracker.applyOp(op, {}) + return rangesTracker + + commentsEqual = (comments1, comments2) -> + return false if comments1.length != comments2.length + comments1.sort (a,b) -> + if a.offset - b.offset == 0 + return a.length - b.length + else + return a.offset - b.offset + comments2.sort (a,b) -> + if a.offset - b.offset == 0 + return a.length - b.length + else + return a.offset - b.offset + for comment1, i in comments1 + comment2 = comments2[i] + if comment1.offset != comment2.offset or comment1.length != comment2.length + return false + return true + + SNAPSHOT = "123" + + OPS = [] + # Insert ops + for p in [0..SNAPSHOT.length] + OPS.push {i: "a", p: p} + OPS.push {i: "bc", p: p} + for p in [0..(SNAPSHOT.length-1)] + for length in [1..(SNAPSHOT.length - p)] + OPS.push {d: SNAPSHOT.slice(p, p+length), p} + for p in [0..(SNAPSHOT.length-1)] + for length in [1..(SNAPSHOT.length - p)] + OPS.push {c: SNAPSHOT.slice(p, p+length), p} + + for op1 in OPS + for op2 in OPS + op1_t = transform(op1, op2, "left") + op2_t = transform(op2, op1, "right") + + rt12 = new RangesTracker() + snapshot12 = applySnapshot(applySnapshot(SNAPSHOT, [op1]), op2_t) + applyRanges(rt12, [op1]) + applyRanges(rt12, op2_t) + + rt21 = new RangesTracker() + snapshot21 = applySnapshot(applySnapshot(SNAPSHOT, [op2]), op1_t) + applyRanges(rt21, [op2]) + applyRanges(rt21, op1_t) + + if snapshot12 != snapshot21 + console.error {op1, op2, op1_t, op2_t, snapshot12, snapshot21}, "Ops are not consistent" + throw new Error("OT is inconsistent") + + if !commentsEqual(rt12.comments, rt21.comments) + console.log rt12.comments + console.log rt21.comments + console.error {op1, op2, op1_t, op2_t, rt12_comments: rt12.comments, rt21_comments: rt21.comments}, "Comments are not consistent" + throw new Error("OT is inconsistent") + \ No newline at end of file From 59a06cd798f44c6370b66a5523dd7d077a97a9ab Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 13 Dec 2016 15:51:47 +0000 Subject: [PATCH 15/25] Accept comments with thread id as an op type --- .../app/coffee/RangesTracker.coffee | 66 +++++++++++----- .../app/coffee/sharejs/types/text.coffee | 8 +- .../test/acceptance/coffee/RangesTests.coffee | 76 +++++++++++++++++++ .../coffee/ShareJS/TextTransformTests.coffee | 57 +++++++------- 4 files changed, 153 insertions(+), 54 deletions(-) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 6a3625fd09..1b865a600d 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -48,15 +48,6 @@ load = (EventEmitter) -> # sync with Ace ranges. @id = 0 - addComment: (offset, length, metadata) -> - # TODO: Don't allow overlapping comments? - @comments.push comment = { - id: @_newId() - offset, length, metadata - } - @emit "comment:added", comment - return comment - getComment: (comment_id) -> comment = null for c in @comments @@ -106,14 +97,32 @@ load = (EventEmitter) -> else if op.d? @applyDeleteToChanges(op, metadata) @applyDeleteToComments(op) + else if op.c? + @addComment(op, metadata) + else + throw new Error("unknown op type") + + addComment: (op, metadata) -> + # TODO: Don't allow overlapping comments? + @comments.push comment = { + id: @_newId() + op: # Copy because we'll modify in place + c: op.c + p: op.p + t: op.t + metadata + } + @emit "comment:added", comment + return comment applyInsertToComments: (op) -> for comment in @comments - if op.p <= comment.offset - comment.offset += op.i.length + if op.p <= comment.op.p + comment.op.p += op.i.length @emit "comment:moved", comment - else if op.p < comment.offset + comment.length - comment.length += op.i.length + else if op.p < comment.op.p + comment.op.c.length + offset = op.p - comment.op.p + comment.op.c = comment.op.c[0..(offset-1)] + op.i + comment.op.c[offset...] @emit "comment:moved", comment applyDeleteToComments: (op) -> @@ -121,20 +130,35 @@ load = (EventEmitter) -> op_length = op.d.length op_end = op.p + op_length for comment in @comments - comment_end = comment.offset + comment.length - if op_end <= comment.offset + comment_start = comment.op.p + comment_end = comment.op.p + comment.op.c.length + comment_length = comment_end - comment_start + if op_end <= comment_start # delete is fully before comment - comment.offset -= op_length + comment.op.p -= op_length @emit "comment:moved", comment else if op_start >= comment_end # delete is fully after comment, nothing to do else # delete and comment overlap - delete_length_before = Math.max(0, comment.offset - op_start) - delete_length_after = Math.max(0, op_end - comment_end) - delete_length_overlapping = op_length - delete_length_before - delete_length_after - comment.offset = Math.min(comment.offset, op_start) - comment.length -= delete_length_overlapping + if op_start <= comment_start + remaining_before = "" + else + remaining_before = comment.op.c.slice(0, op_start - comment_start) + if op_end >= comment_end + remaining_after = "" + else + remaining_after = comment.op.c.slice(op_end - comment_start) + + # Check deleted content matches delete op + deleted_comment = comment.op.c.slice(remaining_before.length, comment_length - remaining_after.length) + offset = Math.max(0, comment_start - op_start) + deleted_op_content = op.d.slice(offset).slice(0, deleted_comment.length) + if deleted_comment != deleted_op_content + throw new Error("deleted content does not match comment content") + + comment.op.p = Math.min(comment_start, op_start) + comment.op.c = remaining_before + remaining_after @emit "comment:moved", comment applyInsertToChanges: (op, metadata) -> diff --git a/services/document-updater/app/coffee/sharejs/types/text.coffee b/services/document-updater/app/coffee/sharejs/types/text.coffee index dcf2ef4cfe..2a3b79997d 100644 --- a/services/document-updater/app/coffee/sharejs/types/text.coffee +++ b/services/document-updater/app/coffee/sharejs/types/text.coffee @@ -198,18 +198,18 @@ text._tc = transformComponent = (dest, c, otherC, side) -> if c.p < otherC.p < c.p + c.c.length offset = otherC.p - c.p new_c = (c.c[0..(offset-1)] + otherC.i + c.c[offset...]) - append dest, {c:new_c, p:c.p} + append dest, {c:new_c, p:c.p, t: c.t} else - append dest, {c:c.c, p:transformPosition(c.p, otherC, true)} + append dest, {c:c.c, p:transformPosition(c.p, otherC, true), t: c.t} else if otherC.d? if c.p >= otherC.p + otherC.d.length - append dest, {c:c.c, p:c.p - otherC.d.length} + append dest, {c:c.c, p:c.p - otherC.d.length, t: c.t} else if c.p + c.c.length <= otherC.p append dest, c else # Delete overlaps comment # They overlap somewhere. - newC = {c:'', p:c.p} + newC = {c:'', p:c.p, t: c.t} if c.p < otherC.p newC.c = c.c[...(otherC.p - c.p)] if c.p + c.c.length > otherC.p + otherC.d.length diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index 95a7fae9d4..0cee1598aa 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -54,6 +54,82 @@ describe "Ranges", -> change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id done() + + describe "Adding comments", -> + describe "standalone", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + async.series jobs, (error) -> + throw error if error? + setTimeout done, 200 + + it "should update the ranges", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 4, t: @tid } + done() + + describe "with conflicting ops needing OT", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ i: "ABC", p: 3 }] + v: 0 + meta: { user_id: @user_id } + }, { + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + async.series jobs, (error) -> + throw error if error? + setTimeout done, 200 + + it "should update the comments with the OT shifted comment", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 7, t: @tid } + done() describe "Loading ranges from persistence layer", -> before (done) -> diff --git a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee index e0f75d2756..81440bfe5b 100644 --- a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee +++ b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee @@ -3,6 +3,9 @@ require("chai").should() RangesTracker = require "../../../../app/js/RangesTracker" describe "ShareJS text type", -> + beforeEach -> + @t = "mock-thread-id" + describe "transform", -> describe "insert / insert", -> it "with an insert before", -> @@ -113,61 +116,61 @@ describe "ShareJS text type", -> describe "comment / insert", -> it "with an insert before", -> dest = [] - text._tc(dest, { c: "foo", p: 9 }, { i: "bar", p: 3 }) - dest.should.deep.equal [{ c: "foo", p: 12 }] + text._tc(dest, { c: "foo", p: 9, @t }, { i: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 12, @t }] it "with an insert after", -> dest = [] - text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 9 }) - dest.should.deep.equal [{ c: "foo", p: 3 }] + text._tc(dest, { c: "foo", p: 3, @t }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 3, @t }] it "with an insert at the left edge", -> dest = [] - text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 3 }) + text._tc(dest, { c: "foo", p: 3, @t }, { i: "bar", p: 3 }) # RangesTracker doesn't inject inserts into comments on edges, so neither should we - dest.should.deep.equal [{ c: "foo", p: 6 }] + dest.should.deep.equal [{ c: "foo", p: 6, @t }] it "with an insert at the right edge", -> dest = [] - text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 6 }) + text._tc(dest, { c: "foo", p: 3, @t }, { i: "bar", p: 6 }) # RangesTracker doesn't inject inserts into comments on edges, so neither should we - dest.should.deep.equal [{ c: "foo", p: 3 }] + dest.should.deep.equal [{ c: "foo", p: 3, @t }] it "with an insert in the middle", -> dest = [] - text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 5 }) - dest.should.deep.equal [{ c: "fobaro", p: 3 }] + text._tc(dest, { c: "foo", p: 3, @t }, { i: "bar", p: 5 }) + dest.should.deep.equal [{ c: "fobaro", p: 3, @t }] describe "comment / delete", -> it "with a delete before", -> dest = [] - text._tc(dest, { c: "foo", p: 9 }, { d: "bar", p: 3 }) - dest.should.deep.equal [{ c: "foo", p: 6 }] + text._tc(dest, { c: "foo", p: 9, @t }, { d: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 6, @t }] it "with a delete after", -> dest = [] - text._tc(dest, { c: "foo", p: 3 }, { i: "bar", p: 9 }) - dest.should.deep.equal [{ c: "foo", p: 3 }] + text._tc(dest, { c: "foo", p: 3, @t }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 3, @t }] it "with a delete overlapping the comment content before", -> dest = [] - text._tc(dest, { c: "foobar", p: 6 }, { d: "123foo", p: 3 }) - dest.should.deep.equal [{ c: "bar", p: 3 }] + text._tc(dest, { c: "foobar", p: 6, @t }, { d: "123foo", p: 3 }) + dest.should.deep.equal [{ c: "bar", p: 3, @t }] it "with a delete overlapping the comment content after", -> dest = [] - text._tc(dest, { c: "foobar", p: 6 }, { d: "bar123", p: 9 }) - dest.should.deep.equal [{ c: "foo", p: 6 }] + text._tc(dest, { c: "foobar", p: 6, @t }, { d: "bar123", p: 9 }) + dest.should.deep.equal [{ c: "foo", p: 6, @t }] it "with a delete overlapping the comment content in the middle", -> dest = [] - text._tc(dest, { c: "foo123bar", p: 6 }, { d: "123", p: 9 }) - dest.should.deep.equal [{ c: "foobar", p: 6 }] + text._tc(dest, { c: "foo123bar", p: 6, @t }, { d: "123", p: 9 }) + dest.should.deep.equal [{ c: "foobar", p: 6, @t }] it "with a delete overlapping the whole comment", -> dest = [] - text._tc(dest, { c: "foo", p: 6 }, { d: "123foo456", p: 3 }) - dest.should.deep.equal [{ c: "", p: 3 }] + text._tc(dest, { c: "foo", p: 6, @t }, { d: "123foo456", p: 3 }) + dest.should.deep.equal [{ c: "", p: 3, @t }] describe "comment / insert", -> it "should not do anything", -> @@ -219,10 +222,7 @@ describe "ShareJS text type", -> applyRanges = (rangesTracker, ops) -> for op in ops - if op.c? - rangesTracker.addComment(op.p, op.c.length, {}) - else - rangesTracker.applyOp(op, {}) + rangesTracker.applyOp(op, {}) return rangesTracker commentsEqual = (comments1, comments2) -> @@ -255,8 +255,8 @@ describe "ShareJS text type", -> OPS.push {d: SNAPSHOT.slice(p, p+length), p} for p in [0..(SNAPSHOT.length-1)] for length in [1..(SNAPSHOT.length - p)] - OPS.push {c: SNAPSHOT.slice(p, p+length), p} - + OPS.push {c: SNAPSHOT.slice(p, p+length), p, @t} + for op1 in OPS for op2 in OPS op1_t = transform(op1, op2, "left") @@ -281,4 +281,3 @@ describe "ShareJS text type", -> console.log rt21.comments console.error {op1, op2, op1_t, op2_t, rt12_comments: rt12.comments, rt21_comments: rt21.comments}, "Comments are not consistent" throw new Error("OT is inconsistent") - \ No newline at end of file From 0f13cb3aa7f91e6285393e44d5aeb5086cfe24af Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 6 Jan 2017 16:58:51 +0100 Subject: [PATCH 16/25] Support a {dr:...} op for deleting ranges --- .../app/coffee/RangesTracker.coffee | 80 ++++----- .../app/coffee/sharejs/types/text.coffee | 72 +++++--- .../test/acceptance/coffee/RangesTests.coffee | 168 ++++++++++-------- .../coffee/ShareJS/TextTransformTests.coffee | 92 ++++++++++ 4 files changed, 273 insertions(+), 139 deletions(-) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 1b865a600d..233f5ad989 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -35,18 +35,25 @@ load = (EventEmitter) -> # * Inserts by another user will not combine with inserts by the first user. If they are in the # middle of a previous insert by the first user, the original insert will be split into two. constructor: (@changes = [], @comments = []) -> - # Change objects have the following structure: - # { - # id: ... # Uniquely generated by us - # op: { # ShareJs style op tracking the offset (p) and content inserted (i) or deleted (d) - # i: "..." - # p: 42 - # } - # } - # - # Ids are used to uniquely identify a change, e.g. for updating it in the database, or keeping in - # sync with Ace ranges. - @id = 0 + + @_increment: 0 + @newId: () -> + # Generate a Mongo ObjectId + # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js + @_pid ?= Math.floor(Math.random() * (32767)) + @_machine ?= Math.floor(Math.random() * (16777216)) + timestamp = Math.floor(new Date().valueOf() / 1000) + @_increment++ + + timestamp = timestamp.toString(16) + machine = @_machine.toString(16) + pid = @_pid.toString(16) + increment = @_increment.toString(16) + + return '00000000'.substr(0, 8 - timestamp.length) + timestamp + + '000000'.substr(0, 6 - machine.length) + machine + + '0000'.substr(0, 4 - pid.length) + pid + + '000000'.substr(0, 6 - increment.length) + increment; getComment: (comment_id) -> comment = null @@ -56,19 +63,6 @@ load = (EventEmitter) -> break return comment - resolveCommentId: (comment_id, resolved_data) -> - comment = @getComment(comment_id) - return if !comment? - comment.metadata.resolved = true - comment.metadata.resolved_data = resolved_data - @emit "comment:resolved", comment - - unresolveCommentId: (comment_id) -> - comment = @getComment(comment_id) - return if !comment? - comment.metadata.resolved = false - @emit "comment:unresolved", comment - removeCommentId: (comment_id) -> comment = @getComment(comment_id) return if !comment? @@ -88,7 +82,7 @@ load = (EventEmitter) -> return if !change? @_removeChange(change) - applyOp: (op, metadata) -> + applyOp: (op, metadata = {}) -> metadata.ts ?= new Date() # Apply an op that has been applied to the document to our changes to keep them up to date if op.i? @@ -97,6 +91,8 @@ load = (EventEmitter) -> else if op.d? @applyDeleteToChanges(op, metadata) @applyDeleteToComments(op) + else if op.dr? + @applyDeleteRangeToChanges(op) else if op.c? @addComment(op, metadata) else @@ -105,7 +101,7 @@ load = (EventEmitter) -> addComment: (op, metadata) -> # TODO: Don't allow overlapping comments? @comments.push comment = { - id: @_newId() + id: RangesTracker.newId() op: # Copy because we'll modify in place c: op.c p: op.p @@ -394,28 +390,20 @@ load = (EventEmitter) -> if moved_changes.length > 0 @emit "changes:moved", moved_changes - _newId: () -> - # Generate a Mongo ObjectId - # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js - @_pid ?= Math.floor(Math.random() * (32767)) - @_machine ?= Math.floor(Math.random() * (16777216)) - timestamp = Math.floor(new Date().valueOf() / 1000) - @_increment ?= 0 - @_increment++ - - timestamp = timestamp.toString(16) - machine = @_machine.toString(16) - pid = @_pid.toString(16) - increment = @_increment.toString(16) - - return '00000000'.substr(0, 8 - timestamp.length) + timestamp + - '000000'.substr(0, 6 - machine.length) + machine + - '0000'.substr(0, 4 - pid.length) + pid + - '000000'.substr(0, 6 - increment.length) + increment; + applyDeleteRangeToChanges: (op) -> + remove_changes = [] + for change in @changes + change_text = change.op.i or change.op.d + if op.p == change.op.p and op.dr == change_text + remove_changes.push change + if remove_changes.length == 0 + throw new Error("no range to remove") + for change in remove_changes + @_removeChange(change) _addOp: (op, metadata) -> change = { - id: @_newId() + id: RangesTracker.newId() op: op metadata: metadata } diff --git a/services/document-updater/app/coffee/sharejs/types/text.coffee b/services/document-updater/app/coffee/sharejs/types/text.coffee index 2a3b79997d..84303b3307 100644 --- a/services/document-updater/app/coffee/sharejs/types/text.coffee +++ b/services/document-updater/app/coffee/sharejs/types/text.coffee @@ -32,7 +32,8 @@ checkValidComponent = (c) -> i_type = typeof c.i d_type = typeof c.d c_type = typeof c.c - throw new Error 'component needs an i, d or c field' unless (i_type == 'string') ^ (d_type == 'string') ^ (c_type == 'string') + dr_type = typeof c.dr + throw new Error 'component needs an i, d, c or dr field' unless (i_type == 'string') ^ (d_type == 'string') ^ (c_type == 'string') ^ (dr_type == 'string') throw new Error 'position cannot be negative' unless c.p >= 0 @@ -40,6 +41,26 @@ checkValidOp = (op) -> checkValidComponent(c) for c in op true +componentText = (c) -> + if c.c? + text = c.c + if c.dr? + text = c.dr + throw new Error("invalid component") if !text? + return text + +duplicateComponent = (c) -> + newC = {} + for key, value of c + newC[key] = value + return newC + +setComponentText = (c, text) -> + if c.c? + c.c = text + if c.dr? + c.dr = text + text.apply = (snapshot, op) -> checkValidOp op for component in op @@ -49,9 +70,10 @@ text.apply = (snapshot, op) -> deleted = snapshot[component.p...(component.p + component.d.length)] throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..] - else if component.c? - comment = snapshot[component.p...(component.p + component.c.length)] - throw new Error "Comment component '#{component.c}' does not match commented text '#{comment}'" unless component.c == comment + else if component.c? or component.dr? + c_text = componentText(component) + range = snapshot[component.p...(component.p + c_text.length)] + throw new Error "Range component '#{c_text}' does not match range text '#{range}'" unless c_text == range else throw new Error "Unknown op type" snapshot @@ -127,7 +149,7 @@ transformPosition = (pos, c, insertAfter) -> c.p else pos - c.d.length - else if c.c? + else if c.c? or c.dr? pos else throw new Error("unknown op type") @@ -187,46 +209,54 @@ text._tc = transformComponent = (dest, c, otherC, side) -> newC.p = transformPosition newC.p, otherC append dest, newC - else if otherC.c? + else if otherC.c? or otherC.dr? append dest, c else throw new Error("unknown op type") - else if c.c? # Comment + else if c.c? or c.dr? # Comment or delete range + c_text = componentText(c) if otherC.i? - if c.p < otherC.p < c.p + c.c.length + if c.p < otherC.p < c.p + c_text.length offset = otherC.p - c.p - new_c = (c.c[0..(offset-1)] + otherC.i + c.c[offset...]) - append dest, {c:new_c, p:c.p, t: c.t} + newText = (c_text[0..(offset-1)] + otherC.i + c_text[offset...]) + newC = duplicateComponent(c) + setComponentText(newC, newText) + append dest, newC else - append dest, {c:c.c, p:transformPosition(c.p, otherC, true), t: c.t} + newC = duplicateComponent(c) + newC.p = transformPosition(c.p, otherC, true) + append dest, newC else if otherC.d? if c.p >= otherC.p + otherC.d.length - append dest, {c:c.c, p:c.p - otherC.d.length, t: c.t} - else if c.p + c.c.length <= otherC.p + newC = duplicateComponent(c) + newC.p = c.p - otherC.d.length + append dest, newC + else if c.p + c_text.length <= otherC.p append dest, c else # Delete overlaps comment # They overlap somewhere. - newC = {c:'', p:c.p, t: c.t} + newC = duplicateComponent(c) + setComponentText(newC, '') if c.p < otherC.p - newC.c = c.c[...(otherC.p - c.p)] - if c.p + c.c.length > otherC.p + otherC.d.length - newC.c += c.c[(otherC.p + otherC.d.length - c.p)..] + setComponentText(newC, c_text[...(otherC.p - c.p)]) + if c.p + c_text.length > otherC.p + otherC.d.length + setComponentText(newC, componentText(newC) + c_text[(otherC.p + otherC.d.length - c.p)..]) # This is entirely optional - just for a check that the deleted # text in the two ops matches intersectStart = Math.max c.p, otherC.p - intersectEnd = Math.min c.p + c.c.length, otherC.p + otherC.d.length - cIntersect = c.c[intersectStart - c.p...intersectEnd - c.p] + intersectEnd = Math.min c.p + c_text.length, otherC.p + otherC.d.length + cIntersect = c_text[intersectStart - c.p...intersectEnd - c.p] otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p] - throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect + throw new Error 'Delete op text does not match range being modified' unless cIntersect == otherIntersect newC.p = transformPosition newC.p, otherC append dest, newC - else if otherC.c? + else if otherC.c? or otherC.dr? append dest, c else diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index 0cee1598aa..a5cbee7569 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -1,6 +1,7 @@ sinon = require "sinon" chai = require("chai") chai.should() +expect = chai.expect async = require "async" rclient = require("redis").createClient() @@ -54,82 +55,105 @@ describe "Ranges", -> change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id done() + + describe "removing ranges", -> + it "should delete the range (and perform OT)", (done) -> + @conflicting_update = { + doc: @doc.id + op: [{ i: "X", p: 1 }] + v: 3 + meta: { user_id: @user_id } + } + @delete_range = { + doc: @doc.id + op: [{ dr: "456", p: 3 }] + v: 3 + meta: { user_id: @user_id } + } + DocUpdaterClient.sendUpdate @project_id, @doc.id, @conflicting_update, (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc.id, @delete_range, (error) => + throw error if error? + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + expect(data.ranges.changes).to.be.undefined + done() - describe "Adding comments", -> - describe "standalone", -> - before (done) -> - @project_id = DocUpdaterClient.randomId() - @user_id = DocUpdaterClient.randomId() - @doc = { - id: DocUpdaterClient.randomId() - lines: ["foo bar baz"] - } - @updates = [{ - doc: @doc.id - op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] - v: 0 - }] - MockWebApi.insertDoc @project_id, @doc.id, { - lines: @doc.lines - version: 0 - } - jobs = [] - for update in @updates - do (update) => - jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback - DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + describe "Adding comments", -> + describe "standalone", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + async.series jobs, (error) -> throw error if error? - async.series jobs, (error) -> - throw error if error? - setTimeout done, 200 - - it "should update the ranges", (done) -> - DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => - throw error if error? - ranges = data.ranges - comment = ranges.comments[0] - comment.op.should.deep.equal { c: "bar", p: 4, t: @tid } - done() + setTimeout done, 200 + + it "should update the ranges", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 4, t: @tid } + done() - describe "with conflicting ops needing OT", -> - before (done) -> - @project_id = DocUpdaterClient.randomId() - @user_id = DocUpdaterClient.randomId() - @doc = { - id: DocUpdaterClient.randomId() - lines: ["foo bar baz"] - } - @updates = [{ - doc: @doc.id - op: [{ i: "ABC", p: 3 }] - v: 0 - meta: { user_id: @user_id } - }, { - doc: @doc.id - op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] - v: 0 - }] - MockWebApi.insertDoc @project_id, @doc.id, { - lines: @doc.lines - version: 0 - } - jobs = [] - for update in @updates - do (update) => - jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback - DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + describe "with conflicting ops needing OT", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ i: "ABC", p: 3 }] + v: 0 + meta: { user_id: @user_id } + }, { + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + async.series jobs, (error) -> throw error if error? - async.series jobs, (error) -> - throw error if error? - setTimeout done, 200 - - it "should update the comments with the OT shifted comment", (done) -> - DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => - throw error if error? - ranges = data.ranges - comment = ranges.comments[0] - comment.op.should.deep.equal { c: "bar", p: 7, t: @tid } - done() + setTimeout done, 200 + + it "should update the comments with the OT shifted comment", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 7, t: @tid } + done() describe "Loading ranges from persistence layer", -> before (done) -> diff --git a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee index 81440bfe5b..2d9dcf94a0 100644 --- a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee +++ b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee @@ -189,6 +189,95 @@ describe "ShareJS text type", -> dest = [] text._tc(dest, { c: "foo", p: 6 }, { c: "bar", p: 3 }) dest.should.deep.equal [{ c: "foo", p: 6 }] + + describe "comment / delete_range", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { c: "foo", p: 6 }, { dr: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 6 }] + + describe "delete_range / insert", -> + it "with an insert before", -> + dest = [] + text._tc(dest, { dr: "foo", p: 9 }, { i: "bar", p: 3 }) + dest.should.deep.equal [{ dr: "foo", p: 12 }] + + it "with an insert after", -> + dest = [] + text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ dr: "foo", p: 3 }] + + it "with an insert at the left edge", -> + dest = [] + text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 3 }) + # RangesTracker doesn't inject inserts into comments on edges, so neither should we + dest.should.deep.equal [{ dr: "foo", p: 6 }] + + it "with an insert at the right edge", -> + dest = [] + text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 6 }) + # RangesTracker doesn't inject inserts into comments on edges, so neither should we + dest.should.deep.equal [{ dr: "foo", p: 3 }] + + it "with an insert in the middle", -> + dest = [] + text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 5 }) + dest.should.deep.equal [{ dr: "fobaro", p: 3 }] + + describe "delete_range / delete", -> + it "with a delete before", -> + dest = [] + text._tc(dest, { dr: "foo", p: 9 }, { d: "bar", p: 3 }) + dest.should.deep.equal [{ dr: "foo", p: 6 }] + + it "with a delete after", -> + dest = [] + text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 9 }) + dest.should.deep.equal [{ dr: "foo", p: 3 }] + + it "with a delete overlapping the comment content before", -> + dest = [] + text._tc(dest, { dr: "foobar", p: 6 }, { d: "123foo", p: 3 }) + dest.should.deep.equal [{ dr: "bar", p: 3 }] + + it "with a delete overlapping the comment content after", -> + dest = [] + text._tc(dest, { dr: "foobar", p: 6 }, { d: "bar123", p: 9 }) + dest.should.deep.equal [{ dr: "foo", p: 6 }] + + it "with a delete overlapping the comment content in the middle", -> + dest = [] + text._tc(dest, { dr: "foo123bar", p: 6 }, { d: "123", p: 9 }) + dest.should.deep.equal [{ dr: "foobar", p: 6 }] + + it "with a delete overlapping the whole comment", -> + dest = [] + text._tc(dest, { dr: "foo", p: 6 }, { d: "123foo456", p: 3 }) + dest.should.deep.equal [{ dr: "", p: 3 }] + + describe "delete_range / insert", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { i: "foo", p: 6 }, { dr: "bar", p: 3 }) + dest.should.deep.equal [{ i: "foo", p: 6 }] + + describe "delete_range / delete", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { d: "foo", p: 6 }, { dr: "bar", p: 3 }) + dest.should.deep.equal [{ d: "foo", p: 6 }] + + describe "delete_range / comment", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { c: "foo", p: 6 }, { dr: "bar", p: 3 }) + dest.should.deep.equal [{ c: "foo", p: 6 }] + + describe "delete_range / delete_range", -> + it "should not do anything", -> + dest = [] + text._tc(dest, { dr: "foo", p: 6 }, { dr: "bar", p: 3 }) + dest.should.deep.equal [{ dr: "foo", p: 6 }] describe "apply", -> it "should apply an insert", -> @@ -199,6 +288,9 @@ describe "ShareJS text type", -> it "should do nothing with a comment", -> text.apply("foo123bar", [{ c: "123", p: 3 }]).should.equal "foo123bar" + + it "should do nothing with a delete_range", -> + text.apply("foo123bar", [{ dr: "123", p: 3 }]).should.equal "foo123bar" it "should throw an error when deleted content does not match", -> (() -> From 2c7029cc50612f00d21bbbe6633eda46f2c32031 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 9 Jan 2017 09:24:19 +0100 Subject: [PATCH 17/25] Revert "Support a {dr:...} op for deleting ranges" This reverts commit 24c58e5ad430e0240533cc1e5c21122859fe8dc9. --- .../app/coffee/RangesTracker.coffee | 80 +++++---- .../app/coffee/sharejs/types/text.coffee | 72 +++----- .../test/acceptance/coffee/RangesTests.coffee | 168 ++++++++---------- .../coffee/ShareJS/TextTransformTests.coffee | 92 ---------- 4 files changed, 139 insertions(+), 273 deletions(-) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 233f5ad989..1b865a600d 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -35,25 +35,18 @@ load = (EventEmitter) -> # * Inserts by another user will not combine with inserts by the first user. If they are in the # middle of a previous insert by the first user, the original insert will be split into two. constructor: (@changes = [], @comments = []) -> - - @_increment: 0 - @newId: () -> - # Generate a Mongo ObjectId - # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js - @_pid ?= Math.floor(Math.random() * (32767)) - @_machine ?= Math.floor(Math.random() * (16777216)) - timestamp = Math.floor(new Date().valueOf() / 1000) - @_increment++ - - timestamp = timestamp.toString(16) - machine = @_machine.toString(16) - pid = @_pid.toString(16) - increment = @_increment.toString(16) - - return '00000000'.substr(0, 8 - timestamp.length) + timestamp + - '000000'.substr(0, 6 - machine.length) + machine + - '0000'.substr(0, 4 - pid.length) + pid + - '000000'.substr(0, 6 - increment.length) + increment; + # Change objects have the following structure: + # { + # id: ... # Uniquely generated by us + # op: { # ShareJs style op tracking the offset (p) and content inserted (i) or deleted (d) + # i: "..." + # p: 42 + # } + # } + # + # Ids are used to uniquely identify a change, e.g. for updating it in the database, or keeping in + # sync with Ace ranges. + @id = 0 getComment: (comment_id) -> comment = null @@ -63,6 +56,19 @@ load = (EventEmitter) -> break return comment + resolveCommentId: (comment_id, resolved_data) -> + comment = @getComment(comment_id) + return if !comment? + comment.metadata.resolved = true + comment.metadata.resolved_data = resolved_data + @emit "comment:resolved", comment + + unresolveCommentId: (comment_id) -> + comment = @getComment(comment_id) + return if !comment? + comment.metadata.resolved = false + @emit "comment:unresolved", comment + removeCommentId: (comment_id) -> comment = @getComment(comment_id) return if !comment? @@ -82,7 +88,7 @@ load = (EventEmitter) -> return if !change? @_removeChange(change) - applyOp: (op, metadata = {}) -> + applyOp: (op, metadata) -> metadata.ts ?= new Date() # Apply an op that has been applied to the document to our changes to keep them up to date if op.i? @@ -91,8 +97,6 @@ load = (EventEmitter) -> else if op.d? @applyDeleteToChanges(op, metadata) @applyDeleteToComments(op) - else if op.dr? - @applyDeleteRangeToChanges(op) else if op.c? @addComment(op, metadata) else @@ -101,7 +105,7 @@ load = (EventEmitter) -> addComment: (op, metadata) -> # TODO: Don't allow overlapping comments? @comments.push comment = { - id: RangesTracker.newId() + id: @_newId() op: # Copy because we'll modify in place c: op.c p: op.p @@ -390,20 +394,28 @@ load = (EventEmitter) -> if moved_changes.length > 0 @emit "changes:moved", moved_changes - applyDeleteRangeToChanges: (op) -> - remove_changes = [] - for change in @changes - change_text = change.op.i or change.op.d - if op.p == change.op.p and op.dr == change_text - remove_changes.push change - if remove_changes.length == 0 - throw new Error("no range to remove") - for change in remove_changes - @_removeChange(change) + _newId: () -> + # Generate a Mongo ObjectId + # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js + @_pid ?= Math.floor(Math.random() * (32767)) + @_machine ?= Math.floor(Math.random() * (16777216)) + timestamp = Math.floor(new Date().valueOf() / 1000) + @_increment ?= 0 + @_increment++ + + timestamp = timestamp.toString(16) + machine = @_machine.toString(16) + pid = @_pid.toString(16) + increment = @_increment.toString(16) + + return '00000000'.substr(0, 8 - timestamp.length) + timestamp + + '000000'.substr(0, 6 - machine.length) + machine + + '0000'.substr(0, 4 - pid.length) + pid + + '000000'.substr(0, 6 - increment.length) + increment; _addOp: (op, metadata) -> change = { - id: RangesTracker.newId() + id: @_newId() op: op metadata: metadata } diff --git a/services/document-updater/app/coffee/sharejs/types/text.coffee b/services/document-updater/app/coffee/sharejs/types/text.coffee index 84303b3307..2a3b79997d 100644 --- a/services/document-updater/app/coffee/sharejs/types/text.coffee +++ b/services/document-updater/app/coffee/sharejs/types/text.coffee @@ -32,8 +32,7 @@ checkValidComponent = (c) -> i_type = typeof c.i d_type = typeof c.d c_type = typeof c.c - dr_type = typeof c.dr - throw new Error 'component needs an i, d, c or dr field' unless (i_type == 'string') ^ (d_type == 'string') ^ (c_type == 'string') ^ (dr_type == 'string') + throw new Error 'component needs an i, d or c field' unless (i_type == 'string') ^ (d_type == 'string') ^ (c_type == 'string') throw new Error 'position cannot be negative' unless c.p >= 0 @@ -41,26 +40,6 @@ checkValidOp = (op) -> checkValidComponent(c) for c in op true -componentText = (c) -> - if c.c? - text = c.c - if c.dr? - text = c.dr - throw new Error("invalid component") if !text? - return text - -duplicateComponent = (c) -> - newC = {} - for key, value of c - newC[key] = value - return newC - -setComponentText = (c, text) -> - if c.c? - c.c = text - if c.dr? - c.dr = text - text.apply = (snapshot, op) -> checkValidOp op for component in op @@ -70,10 +49,9 @@ text.apply = (snapshot, op) -> deleted = snapshot[component.p...(component.p + component.d.length)] throw new Error "Delete component '#{component.d}' does not match deleted text '#{deleted}'" unless component.d == deleted snapshot = snapshot[...component.p] + snapshot[(component.p + component.d.length)..] - else if component.c? or component.dr? - c_text = componentText(component) - range = snapshot[component.p...(component.p + c_text.length)] - throw new Error "Range component '#{c_text}' does not match range text '#{range}'" unless c_text == range + else if component.c? + comment = snapshot[component.p...(component.p + component.c.length)] + throw new Error "Comment component '#{component.c}' does not match commented text '#{comment}'" unless component.c == comment else throw new Error "Unknown op type" snapshot @@ -149,7 +127,7 @@ transformPosition = (pos, c, insertAfter) -> c.p else pos - c.d.length - else if c.c? or c.dr? + else if c.c? pos else throw new Error("unknown op type") @@ -209,54 +187,46 @@ text._tc = transformComponent = (dest, c, otherC, side) -> newC.p = transformPosition newC.p, otherC append dest, newC - else if otherC.c? or otherC.dr? + else if otherC.c? append dest, c else throw new Error("unknown op type") - else if c.c? or c.dr? # Comment or delete range - c_text = componentText(c) + else if c.c? # Comment if otherC.i? - if c.p < otherC.p < c.p + c_text.length + if c.p < otherC.p < c.p + c.c.length offset = otherC.p - c.p - newText = (c_text[0..(offset-1)] + otherC.i + c_text[offset...]) - newC = duplicateComponent(c) - setComponentText(newC, newText) - append dest, newC + new_c = (c.c[0..(offset-1)] + otherC.i + c.c[offset...]) + append dest, {c:new_c, p:c.p, t: c.t} else - newC = duplicateComponent(c) - newC.p = transformPosition(c.p, otherC, true) - append dest, newC + append dest, {c:c.c, p:transformPosition(c.p, otherC, true), t: c.t} else if otherC.d? if c.p >= otherC.p + otherC.d.length - newC = duplicateComponent(c) - newC.p = c.p - otherC.d.length - append dest, newC - else if c.p + c_text.length <= otherC.p + append dest, {c:c.c, p:c.p - otherC.d.length, t: c.t} + else if c.p + c.c.length <= otherC.p append dest, c else # Delete overlaps comment # They overlap somewhere. - newC = duplicateComponent(c) - setComponentText(newC, '') + newC = {c:'', p:c.p, t: c.t} if c.p < otherC.p - setComponentText(newC, c_text[...(otherC.p - c.p)]) - if c.p + c_text.length > otherC.p + otherC.d.length - setComponentText(newC, componentText(newC) + c_text[(otherC.p + otherC.d.length - c.p)..]) + newC.c = c.c[...(otherC.p - c.p)] + if c.p + c.c.length > otherC.p + otherC.d.length + newC.c += c.c[(otherC.p + otherC.d.length - c.p)..] # This is entirely optional - just for a check that the deleted # text in the two ops matches intersectStart = Math.max c.p, otherC.p - intersectEnd = Math.min c.p + c_text.length, otherC.p + otherC.d.length - cIntersect = c_text[intersectStart - c.p...intersectEnd - c.p] + intersectEnd = Math.min c.p + c.c.length, otherC.p + otherC.d.length + cIntersect = c.c[intersectStart - c.p...intersectEnd - c.p] otherIntersect = otherC.d[intersectStart - otherC.p...intersectEnd - otherC.p] - throw new Error 'Delete op text does not match range being modified' unless cIntersect == otherIntersect + throw new Error 'Delete ops delete different text in the same region of the document' unless cIntersect == otherIntersect newC.p = transformPosition newC.p, otherC append dest, newC - else if otherC.c? or otherC.dr? + else if otherC.c? append dest, c else diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index a5cbee7569..0cee1598aa 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -1,7 +1,6 @@ sinon = require "sinon" chai = require("chai") chai.should() -expect = chai.expect async = require "async" rclient = require("redis").createClient() @@ -55,105 +54,82 @@ describe "Ranges", -> change.op.should.deep.equal { i: "456", p: 3 } change.metadata.user_id.should.equal @user_id done() - - describe "removing ranges", -> - it "should delete the range (and perform OT)", (done) -> - @conflicting_update = { - doc: @doc.id - op: [{ i: "X", p: 1 }] - v: 3 - meta: { user_id: @user_id } - } - @delete_range = { - doc: @doc.id - op: [{ dr: "456", p: 3 }] - v: 3 - meta: { user_id: @user_id } - } - DocUpdaterClient.sendUpdate @project_id, @doc.id, @conflicting_update, (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc.id, @delete_range, (error) => - throw error if error? - DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => - throw error if error? - expect(data.ranges.changes).to.be.undefined - done() - describe "Adding comments", -> - describe "standalone", -> - before (done) -> - @project_id = DocUpdaterClient.randomId() - @user_id = DocUpdaterClient.randomId() - @doc = { - id: DocUpdaterClient.randomId() - lines: ["foo bar baz"] - } - @updates = [{ - doc: @doc.id - op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] - v: 0 - }] - MockWebApi.insertDoc @project_id, @doc.id, { - lines: @doc.lines - version: 0 - } - jobs = [] - for update in @updates - do (update) => - jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback - DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => - throw error if error? - async.series jobs, (error) -> + describe "Adding comments", -> + describe "standalone", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => throw error if error? - setTimeout done, 200 - - it "should update the ranges", (done) -> - DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => - throw error if error? - ranges = data.ranges - comment = ranges.comments[0] - comment.op.should.deep.equal { c: "bar", p: 4, t: @tid } - done() + async.series jobs, (error) -> + throw error if error? + setTimeout done, 200 + + it "should update the ranges", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 4, t: @tid } + done() - describe "with conflicting ops needing OT", -> - before (done) -> - @project_id = DocUpdaterClient.randomId() - @user_id = DocUpdaterClient.randomId() - @doc = { - id: DocUpdaterClient.randomId() - lines: ["foo bar baz"] - } - @updates = [{ - doc: @doc.id - op: [{ i: "ABC", p: 3 }] - v: 0 - meta: { user_id: @user_id } - }, { - doc: @doc.id - op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] - v: 0 - }] - MockWebApi.insertDoc @project_id, @doc.id, { - lines: @doc.lines - version: 0 - } - jobs = [] - for update in @updates - do (update) => - jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback - DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => - throw error if error? - async.series jobs, (error) -> + describe "with conflicting ops needing OT", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @doc = { + id: DocUpdaterClient.randomId() + lines: ["foo bar baz"] + } + @updates = [{ + doc: @doc.id + op: [{ i: "ABC", p: 3 }] + v: 0 + meta: { user_id: @user_id } + }, { + doc: @doc.id + op: [{ c: "bar", p: 4, t: @tid = DocUpdaterClient.randomId() }] + v: 0 + }] + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + jobs = [] + for update in @updates + do (update) => + jobs.push (callback) => DocUpdaterClient.sendUpdate @project_id, @doc.id, update, callback + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => throw error if error? - setTimeout done, 200 - - it "should update the comments with the OT shifted comment", (done) -> - DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => - throw error if error? - ranges = data.ranges - comment = ranges.comments[0] - comment.op.should.deep.equal { c: "bar", p: 7, t: @tid } - done() + async.series jobs, (error) -> + throw error if error? + setTimeout done, 200 + + it "should update the comments with the OT shifted comment", (done) -> + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + comment = ranges.comments[0] + comment.op.should.deep.equal { c: "bar", p: 7, t: @tid } + done() describe "Loading ranges from persistence layer", -> before (done) -> diff --git a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee index 2d9dcf94a0..81440bfe5b 100644 --- a/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee +++ b/services/document-updater/test/unit/coffee/ShareJS/TextTransformTests.coffee @@ -189,95 +189,6 @@ describe "ShareJS text type", -> dest = [] text._tc(dest, { c: "foo", p: 6 }, { c: "bar", p: 3 }) dest.should.deep.equal [{ c: "foo", p: 6 }] - - describe "comment / delete_range", -> - it "should not do anything", -> - dest = [] - text._tc(dest, { c: "foo", p: 6 }, { dr: "bar", p: 3 }) - dest.should.deep.equal [{ c: "foo", p: 6 }] - - describe "delete_range / insert", -> - it "with an insert before", -> - dest = [] - text._tc(dest, { dr: "foo", p: 9 }, { i: "bar", p: 3 }) - dest.should.deep.equal [{ dr: "foo", p: 12 }] - - it "with an insert after", -> - dest = [] - text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 9 }) - dest.should.deep.equal [{ dr: "foo", p: 3 }] - - it "with an insert at the left edge", -> - dest = [] - text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 3 }) - # RangesTracker doesn't inject inserts into comments on edges, so neither should we - dest.should.deep.equal [{ dr: "foo", p: 6 }] - - it "with an insert at the right edge", -> - dest = [] - text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 6 }) - # RangesTracker doesn't inject inserts into comments on edges, so neither should we - dest.should.deep.equal [{ dr: "foo", p: 3 }] - - it "with an insert in the middle", -> - dest = [] - text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 5 }) - dest.should.deep.equal [{ dr: "fobaro", p: 3 }] - - describe "delete_range / delete", -> - it "with a delete before", -> - dest = [] - text._tc(dest, { dr: "foo", p: 9 }, { d: "bar", p: 3 }) - dest.should.deep.equal [{ dr: "foo", p: 6 }] - - it "with a delete after", -> - dest = [] - text._tc(dest, { dr: "foo", p: 3 }, { i: "bar", p: 9 }) - dest.should.deep.equal [{ dr: "foo", p: 3 }] - - it "with a delete overlapping the comment content before", -> - dest = [] - text._tc(dest, { dr: "foobar", p: 6 }, { d: "123foo", p: 3 }) - dest.should.deep.equal [{ dr: "bar", p: 3 }] - - it "with a delete overlapping the comment content after", -> - dest = [] - text._tc(dest, { dr: "foobar", p: 6 }, { d: "bar123", p: 9 }) - dest.should.deep.equal [{ dr: "foo", p: 6 }] - - it "with a delete overlapping the comment content in the middle", -> - dest = [] - text._tc(dest, { dr: "foo123bar", p: 6 }, { d: "123", p: 9 }) - dest.should.deep.equal [{ dr: "foobar", p: 6 }] - - it "with a delete overlapping the whole comment", -> - dest = [] - text._tc(dest, { dr: "foo", p: 6 }, { d: "123foo456", p: 3 }) - dest.should.deep.equal [{ dr: "", p: 3 }] - - describe "delete_range / insert", -> - it "should not do anything", -> - dest = [] - text._tc(dest, { i: "foo", p: 6 }, { dr: "bar", p: 3 }) - dest.should.deep.equal [{ i: "foo", p: 6 }] - - describe "delete_range / delete", -> - it "should not do anything", -> - dest = [] - text._tc(dest, { d: "foo", p: 6 }, { dr: "bar", p: 3 }) - dest.should.deep.equal [{ d: "foo", p: 6 }] - - describe "delete_range / comment", -> - it "should not do anything", -> - dest = [] - text._tc(dest, { c: "foo", p: 6 }, { dr: "bar", p: 3 }) - dest.should.deep.equal [{ c: "foo", p: 6 }] - - describe "delete_range / delete_range", -> - it "should not do anything", -> - dest = [] - text._tc(dest, { dr: "foo", p: 6 }, { dr: "bar", p: 3 }) - dest.should.deep.equal [{ dr: "foo", p: 6 }] describe "apply", -> it "should apply an insert", -> @@ -288,9 +199,6 @@ describe "ShareJS text type", -> it "should do nothing with a comment", -> text.apply("foo123bar", [{ c: "123", p: 3 }]).should.equal "foo123bar" - - it "should do nothing with a delete_range", -> - text.apply("foo123bar", [{ dr: "123", p: 3 }]).should.equal "foo123bar" it "should throw an error when deleted content does not match", -> (() -> From 7cac2f7d76079150e078a33a2c9e8c4c7c597bb0 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 9 Jan 2017 10:46:58 +0100 Subject: [PATCH 18/25] Generate deterministic range ids based on seed --- .../app/coffee/RangesManager.coffee | 2 + .../app/coffee/RangesTracker.coffee | 63 +++++-------------- .../test/acceptance/coffee/RangesTests.coffee | 7 ++- 3 files changed, 23 insertions(+), 49 deletions(-) diff --git a/services/document-updater/app/coffee/RangesManager.coffee b/services/document-updater/app/coffee/RangesManager.coffee index 1e19a63b0d..ac1dcbe75f 100644 --- a/services/document-updater/app/coffee/RangesManager.coffee +++ b/services/document-updater/app/coffee/RangesManager.coffee @@ -8,6 +8,8 @@ module.exports = RangesManager = rangesTracker = new RangesTracker(changes, comments) for update in updates rangesTracker.track_changes = !!update.meta.tc + if !!update.meta.tc + rangesTracker.setIdSeed(update.meta.tc) for op in update.op rangesTracker.applyOp(op, { user_id: update.meta?.user_id }) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 1b865a600d..36ef621493 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -35,18 +35,19 @@ load = (EventEmitter) -> # * Inserts by another user will not combine with inserts by the first user. If they are in the # middle of a previous insert by the first user, the original insert will be split into two. constructor: (@changes = [], @comments = []) -> - # Change objects have the following structure: - # { - # id: ... # Uniquely generated by us - # op: { # ShareJs style op tracking the offset (p) and content inserted (i) or deleted (d) - # i: "..." - # p: 42 - # } - # } - # - # Ids are used to uniquely identify a change, e.g. for updating it in the database, or keeping in - # sync with Ace ranges. - @id = 0 + + getIdSeed: () -> + return @id_seed + + setIdSeed: (seed) -> + @id_seed = seed + @id_increment = 0 + + newId: () -> + @id_increment++ + increment = @id_increment.toString(16) + id = @id_seed + '000000'.substr(0, 6 - increment.length) + increment; + return id getComment: (comment_id) -> comment = null @@ -56,19 +57,6 @@ load = (EventEmitter) -> break return comment - resolveCommentId: (comment_id, resolved_data) -> - comment = @getComment(comment_id) - return if !comment? - comment.metadata.resolved = true - comment.metadata.resolved_data = resolved_data - @emit "comment:resolved", comment - - unresolveCommentId: (comment_id) -> - comment = @getComment(comment_id) - return if !comment? - comment.metadata.resolved = false - @emit "comment:unresolved", comment - removeCommentId: (comment_id) -> comment = @getComment(comment_id) return if !comment? @@ -88,7 +76,7 @@ load = (EventEmitter) -> return if !change? @_removeChange(change) - applyOp: (op, metadata) -> + applyOp: (op, metadata = {}) -> metadata.ts ?= new Date() # Apply an op that has been applied to the document to our changes to keep them up to date if op.i? @@ -105,7 +93,7 @@ load = (EventEmitter) -> addComment: (op, metadata) -> # TODO: Don't allow overlapping comments? @comments.push comment = { - id: @_newId() + id: @newId() op: # Copy because we'll modify in place c: op.c p: op.p @@ -394,28 +382,9 @@ load = (EventEmitter) -> if moved_changes.length > 0 @emit "changes:moved", moved_changes - _newId: () -> - # Generate a Mongo ObjectId - # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js - @_pid ?= Math.floor(Math.random() * (32767)) - @_machine ?= Math.floor(Math.random() * (16777216)) - timestamp = Math.floor(new Date().valueOf() / 1000) - @_increment ?= 0 - @_increment++ - - timestamp = timestamp.toString(16) - machine = @_machine.toString(16) - pid = @_pid.toString(16) - increment = @_increment.toString(16) - - return '00000000'.substr(0, 8 - timestamp.length) + timestamp + - '000000'.substr(0, 6 - machine.length) + machine + - '0000'.substr(0, 4 - pid.length) + pid + - '000000'.substr(0, 6 - increment.length) + increment; - _addOp: (op, metadata) -> change = { - id: @_newId() + id: @newId() op: op metadata: metadata } diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index 0cee1598aa..8da51c0899 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -12,6 +12,7 @@ describe "Ranges", -> before (done) -> @project_id = DocUpdaterClient.randomId() @user_id = DocUpdaterClient.randomId() + @id_seed = "587357bd35e64f6157" @doc = { id: DocUpdaterClient.randomId() lines: ["aaa"] @@ -25,7 +26,7 @@ describe "Ranges", -> doc: @doc.id op: [{ i: "456", p: 5 }] v: 1 - meta: { user_id: @user_id, tc: 1 } + meta: { user_id: @user_id, tc: @id_seed } }, { doc: @doc.id op: [{ d: "12", p: 1 }] @@ -52,6 +53,7 @@ describe "Ranges", -> ranges = data.ranges change = ranges.changes[0] change.op.should.deep.equal { i: "456", p: 3 } + change.id.should.equal @id_seed + "000001" change.metadata.user_id.should.equal @user_id done() @@ -135,6 +137,7 @@ describe "Ranges", -> before (done) -> @project_id = DocUpdaterClient.randomId() @user_id = DocUpdaterClient.randomId() + @id_seed = "587357bd35e64f6157" @doc = { id: DocUpdaterClient.randomId() lines: ["a123aa"] @@ -143,7 +146,7 @@ describe "Ranges", -> doc: @doc.id op: [{ i: "456", p: 5 }] v: 0 - meta: { user_id: @user_id, tc: 1 } + meta: { user_id: @user_id, tc: @id_seed } } MockWebApi.insertDoc @project_id, @doc.id, { lines: @doc.lines From 593e7260d44b4ad6063b6e0124fa8ad3fa592d9b Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 9 Jan 2017 10:52:06 +0100 Subject: [PATCH 19/25] Update RangesTracker --- services/document-updater/app/coffee/RangesTracker.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 36ef621493..09d471d476 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -35,6 +35,7 @@ load = (EventEmitter) -> # * Inserts by another user will not combine with inserts by the first user. If they are in the # middle of a previous insert by the first user, the original insert will be split into two. constructor: (@changes = [], @comments = []) -> + @setIdSeed("") getIdSeed: () -> return @id_seed From 65f4360738d7d3a6efef9b192f1790888f46bffd Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 9 Jan 2017 14:34:10 +0100 Subject: [PATCH 20/25] Consolidate HttpController tests into one file --- .../HttpController/HttpControllerTests.coffee | 335 ++++++++++++++++++ .../HttpController/deleteProjectTests.coffee | 63 ---- .../flushAndDeleteDocTests.coffee | 64 ---- .../flushDocIfLoadedTests.coffee | 65 ---- .../HttpController/flushProjectTests.coffee | 62 ---- .../coffee/HttpController/getDocTests.coffee | 112 ------ .../coffee/HttpController/setDocTests.coffee | 83 ----- 7 files changed, 335 insertions(+), 449 deletions(-) create mode 100644 services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee delete mode 100644 services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee diff --git a/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee b/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee new file mode 100644 index 0000000000..cf0f71a301 --- /dev/null +++ b/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee @@ -0,0 +1,335 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +modulePath = "../../../../app/js/HttpController.js" +SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors.js" + +describe "HttpController", -> + beforeEach -> + @HttpController = SandboxedModule.require modulePath, requires: + "./DocumentManager": @DocumentManager = {} + "./ProjectManager": @ProjectManager = {} + "logger-sharelatex" : @logger = { log: sinon.stub() } + "./Metrics": @Metrics = {} + + @Metrics.Timer = class Timer + done: sinon.stub() + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @next = sinon.stub() + @res = + send: sinon.stub() + + describe "getDoc", -> + beforeEach -> + @lines = ["one", "two", "three"] + @ops = ["mock-op-1", "mock-op-2"] + @version = 42 + @fromVersion = 42 + @ranges = { changes: "mock", comments: "mock" } + @req = + params: + project_id: @project_id + doc_id: @doc_id + + describe "when the document exists and no recent ops are requested", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @ranges) + @HttpController.getDoc(@req, @res, @next) + + it "should get the doc", -> + @DocumentManager.getDocAndRecentOpsWithLock + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc as JSON", -> + @res.send + .calledWith(JSON.stringify({ + id: @doc_id + lines: @lines + version: @version + ops: [] + ranges: @ranges + })) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when recent ops are requested", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, @ops) + @req.query = fromVersion: "#{@fromVersion}" + @HttpController.getDoc(@req, @res, @next) + + it "should get the doc", -> + @DocumentManager.getDocAndRecentOpsWithLock + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should return the doc as JSON", -> + @res.send + .calledWith(JSON.stringify({ + id: @doc_id + lines: @lines + version: @version + ops: @ops + })) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when the document does not exist", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, null, null) + @HttpController.getDoc(@req, @res, @next) + + it "should call next with NotFoundError", -> + @next + .calledWith(new Errors.NotFoundError("not found")) + .should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, new Error("oops"), null, null) + @HttpController.getDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + describe "setDoc", -> + beforeEach -> + @lines = ["one", "two", "three"] + @source = "dropbox" + @user_id = "user-id-123" + @req = + headers: {} + params: + project_id: @project_id + doc_id: @doc_id + body: + lines: @lines + source: @source + user_id: @user_id + + describe "successfully", -> + beforeEach -> + @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5) + @HttpController.setDoc(@req, @res, @next) + + it "should set the doc", -> + @DocumentManager.setDocWithLock + .calledWith(@project_id, @doc_id, @lines, @source, @user_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, lines: @lines, source: @source, user_id: @user_id, "setting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5, new Error("oops")) + @HttpController.setDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + describe "when the payload is too large", -> + beforeEach -> + lines = [] + for _ in [0..200000] + lines.push "test test test" + @req.body.lines = lines + @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5) + @HttpController.setDoc(@req, @res, @next) + + it 'should send back a 406 response', -> + @res.send.calledWith(406).should.equal true + + it 'should not call setDocWithLock', -> + @DocumentManager.setDocWithLock.callCount.should.equal 0 + + describe "flushProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + + describe "successfully", -> + beforeEach -> + @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1) + @HttpController.flushProject(@req, @res, @next) + + it "should flush the project", -> + @ProjectManager.flushProjectWithLocks + .calledWith(@project_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(project_id: @project_id, "flushing project via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) + @HttpController.flushProject(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + describe "flushDocIfLoaded", -> + beforeEach -> + @lines = ["one", "two", "three"] + @version = 42 + @req = + params: + project_id: @project_id + doc_id: @doc_id + + describe "successfully", -> + beforeEach -> + @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2) + @HttpController.flushDocIfLoaded(@req, @res, @next) + + it "should flush the doc", -> + @DocumentManager.flushDocIfLoadedWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "flushing doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2, new Error("oops")) + @HttpController.flushDocIfLoaded(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + describe "flushAndDeleteDoc", -> + beforeEach -> + @req = + params: + project_id: @project_id + doc_id: @doc_id + + describe "successfully", -> + beforeEach -> + @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2) + @HttpController.flushAndDeleteDoc(@req, @res, @next) + + it "should flush and delete the doc", -> + @DocumentManager.flushAndDeleteDocWithLock + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(doc_id: @doc_id, project_id: @project_id, "deleting doc via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2, new Error("oops")) + @HttpController.flushAndDeleteDoc(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true + + describe "deleteProject", -> + beforeEach -> + @req = + params: + project_id: @project_id + + describe "successfully", -> + beforeEach -> + @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1) + @HttpController.deleteProject(@req, @res, @next) + + it "should delete the project", -> + @ProjectManager.flushAndDeleteProjectWithLocks + .calledWith(@project_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith(project_id: @project_id, "deleting project via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) + @HttpController.deleteProject(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true \ No newline at end of file diff --git a/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee b/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee deleted file mode 100644 index 796df52e80..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/deleteProjectTests.coffee +++ /dev/null @@ -1,63 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.deleteProject", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager": @ProjectManager = {} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @res = - send: sinon.stub() - @req = - params: - project_id: @project_id - @next = sinon.stub() - - describe "successfully", -> - beforeEach -> - @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1) - @HttpController.deleteProject(@req, @res, @next) - - it "should delete the project", -> - @ProjectManager.flushAndDeleteProjectWithLocks - .calledWith(@project_id) - .should.equal true - - it "should return a successful No Content response", -> - @res.send - .calledWith(204) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(project_id: @project_id, "deleting project via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @ProjectManager.flushAndDeleteProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) - @HttpController.deleteProject(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - - - - diff --git a/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee deleted file mode 100644 index af09c2c1bd..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/flushAndDeleteDocTests.coffee +++ /dev/null @@ -1,64 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.flushAndDeleteDoc", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager":{} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @res = - send: sinon.stub() - @req = - params: - project_id: @project_id - doc_id: @doc_id - @next = sinon.stub() - - describe "successfully", -> - beforeEach -> - @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2) - @HttpController.flushAndDeleteDoc(@req, @res, @next) - - it "should flush and delete the doc", -> - @DocumentManager.flushAndDeleteDocWithLock - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should return a successful No Content response", -> - @res.send - .calledWith(204) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(doc_id: @doc_id, project_id: @project_id, "deleting doc via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArgWith(2, new Error("oops")) - @HttpController.flushAndDeleteDoc(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - - - diff --git a/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee deleted file mode 100644 index 3321030624..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/flushDocIfLoadedTests.coffee +++ /dev/null @@ -1,65 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.flushDocIfLoaded", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager": {} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @version = 42 - @res = - send: sinon.stub() - @req = - params: - project_id: @project_id - doc_id: @doc_id - @next = sinon.stub() - - describe "successfully", -> - beforeEach -> - @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2) - @HttpController.flushDocIfLoaded(@req, @res, @next) - - it "should flush the doc", -> - @DocumentManager.flushDocIfLoadedWithLock - .calledWith(@project_id, @doc_id) - .should.equal true - - it "should return a successful No Content response", -> - @res.send - .calledWith(204) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(doc_id: @doc_id, project_id: @project_id, "flushing doc via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArgWith(2, new Error("oops")) - @HttpController.flushDocIfLoaded(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - - diff --git a/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee b/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee deleted file mode 100644 index e45269ce6d..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/flushProjectTests.coffee +++ /dev/null @@ -1,62 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.flushProject", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager": @ProjectManager = {} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @res = - send: sinon.stub() - @req = - params: - project_id: @project_id - @next = sinon.stub() - - describe "successfully", -> - beforeEach -> - @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1) - @HttpController.flushProject(@req, @res, @next) - - it "should flush the project", -> - @ProjectManager.flushProjectWithLocks - .calledWith(@project_id) - .should.equal true - - it "should return a successful No Content response", -> - @res.send - .calledWith(204) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(project_id: @project_id, "flushing project via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @ProjectManager.flushProjectWithLocks = sinon.stub().callsArgWith(1, new Error("oops")) - @HttpController.flushProject(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - - - diff --git a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee deleted file mode 100644 index 8fa3931d65..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/getDocTests.coffee +++ /dev/null @@ -1,112 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.getDoc", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager": {} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @ops = ["mock-op-1", "mock-op-2"] - @version = 42 - @fromVersion = 42 - @ranges = { changes: "mock", comments: "mock" } - @res = - send: sinon.stub() - @req = - params: - project_id: @project_id - doc_id: @doc_id - @next = sinon.stub() - - describe "when the document exists and no recent ops are requested", -> - beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, [], @ranges) - @HttpController.getDoc(@req, @res, @next) - - it "should get the doc", -> - @DocumentManager.getDocAndRecentOpsWithLock - .calledWith(@project_id, @doc_id, -1) - .should.equal true - - it "should return the doc as JSON", -> - @res.send - .calledWith(JSON.stringify({ - id: @doc_id - lines: @lines - version: @version - ops: [] - ranges: @ranges - })) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when recent ops are requested", -> - beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, @lines, @version, @ops) - @req.query = fromVersion: "#{@fromVersion}" - @HttpController.getDoc(@req, @res, @next) - - it "should get the doc", -> - @DocumentManager.getDocAndRecentOpsWithLock - .calledWith(@project_id, @doc_id, @fromVersion) - .should.equal true - - it "should return the doc as JSON", -> - @res.send - .calledWith(JSON.stringify({ - id: @doc_id - lines: @lines - version: @version - ops: @ops - })) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(doc_id: @doc_id, project_id: @project_id, "getting doc via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when the document does not exist", -> - beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, null, null, null) - @HttpController.getDoc(@req, @res, @next) - - it "should call next with NotFoundError", -> - @next - .calledWith(new Errors.NotFoundError("not found")) - .should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @DocumentManager.getDocAndRecentOpsWithLock = sinon.stub().callsArgWith(3, new Error("oops"), null, null) - @HttpController.getDoc(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - diff --git a/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee b/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee deleted file mode 100644 index 385b8be044..0000000000 --- a/services/document-updater/test/unit/coffee/HttpController/setDocTests.coffee +++ /dev/null @@ -1,83 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -modulePath = "../../../../app/js/HttpController.js" -SandboxedModule = require('sandboxed-module') -Errors = require "../../../../app/js/Errors.js" - -describe "HttpController.setDoc", -> - beforeEach -> - @HttpController = SandboxedModule.require modulePath, requires: - "./DocumentManager": @DocumentManager = {} - "./ProjectManager": {} - "logger-sharelatex" : @logger = { log: sinon.stub() } - "./Metrics": @Metrics = {} - - @Metrics.Timer = class Timer - done: sinon.stub() - - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @lines = ["one", "two", "three"] - @source = "dropbox" - @user_id = "user-id-123" - @res = - send: sinon.stub() - @req = - headers: {} - params: - project_id: @project_id - doc_id: @doc_id - body: - lines: @lines - source: @source - user_id: @user_id - @next = sinon.stub() - - describe "successfully", -> - beforeEach -> - @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5) - @HttpController.setDoc(@req, @res, @next) - - it "should set the doc", -> - @DocumentManager.setDocWithLock - .calledWith(@project_id, @doc_id, @lines, @source, @user_id) - .should.equal true - - it "should return a successful No Content response", -> - @res.send - .calledWith(204) - .should.equal true - - it "should log the request", -> - @logger.log - .calledWith(doc_id: @doc_id, project_id: @project_id, lines: @lines, source: @source, user_id: @user_id, "setting doc via http") - .should.equal true - - it "should time the request", -> - @Metrics.Timer::done.called.should.equal true - - describe "when an errors occurs", -> - beforeEach -> - @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5, new Error("oops")) - @HttpController.setDoc(@req, @res, @next) - - it "should call next with the error", -> - @next - .calledWith(new Error("oops")) - .should.equal true - - describe "when the payload is too large", -> - beforeEach -> - lines = [] - for _ in [0..200000] - lines.push "test test test" - @req.body.lines = lines - @DocumentManager.setDocWithLock = sinon.stub().callsArgWith(5) - @HttpController.setDoc(@req, @res, @next) - - it 'should send back a 406 response', -> - @res.send.calledWith(406).should.equal true - - it 'should not call setDocWithLock', -> - @DocumentManager.setDocWithLock.callCount.should.equal 0 From be19532a1dfa8d281affbb9953649b4a373e8afb Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 9 Jan 2017 14:41:18 +0100 Subject: [PATCH 21/25] Add HTTP end point for accepting changes --- services/document-updater/app.coffee | 1 + .../app/coffee/DocumentManager.coffee | 23 ++++++++- .../app/coffee/HttpController.coffee | 11 ++++ .../app/coffee/RangesManager.coffee | 20 ++++++-- .../test/acceptance/coffee/RangesTests.coffee | 43 ++++++++++++++++ .../coffee/helpers/DocUpdaterClient.coffee | 3 ++ .../DocumentManagerTests.coffee | 50 ++++++++++++++++++- .../HttpController/HttpControllerTests.coffee | 43 +++++++++++++++- 8 files changed, 187 insertions(+), 7 deletions(-) diff --git a/services/document-updater/app.coffee b/services/document-updater/app.coffee index 004b9f77bc..20170cdd34 100644 --- a/services/document-updater/app.coffee +++ b/services/document-updater/app.coffee @@ -45,6 +45,7 @@ app.post '/project/:project_id/doc/:doc_id/flush', HttpController.flushDocIfLo app.delete '/project/:project_id/doc/:doc_id', HttpController.flushAndDeleteDoc app.delete '/project/:project_id', HttpController.deleteProject app.post '/project/:project_id/flush', HttpController.flushProject +app.post '/project/:project_id/doc/:doc_id/change/:change_id/accept', HttpController.acceptChange app.get '/total', (req, res)-> timer = new Metrics.Timer("http.allDocList") diff --git a/services/document-updater/app/coffee/DocumentManager.coffee b/services/document-updater/app/coffee/DocumentManager.coffee index c6d4773036..4f02bbcf5c 100644 --- a/services/document-updater/app/coffee/DocumentManager.coffee +++ b/services/document-updater/app/coffee/DocumentManager.coffee @@ -5,6 +5,8 @@ logger = require "logger-sharelatex" Metrics = require "./Metrics" HistoryManager = require "./HistoryManager" WebRedisManager = require "./WebRedisManager" +Errors = require "./Errors" +RangesManager = require "./RangesManager" module.exports = DocumentManager = getDoc: (project_id, doc_id, _callback = (error, lines, version, alreadyLoaded) ->) -> @@ -83,7 +85,6 @@ module.exports = DocumentManager = DocumentManager.flushAndDeleteDoc project_id, doc_id, (error) -> return callback(error) if error? callback null - flushDocIfLoaded: (project_id, doc_id, _callback = (error) ->) -> timer = new Metrics.Timer("docManager.flushDocIfLoaded") @@ -119,6 +120,22 @@ module.exports = DocumentManager = RedisManager.removeDocFromMemory project_id, doc_id, (error) -> return callback(error) if error? callback null + + acceptChange: (project_id, doc_id, change_id, _callback = (error) ->) -> + timer = new Metrics.Timer("docManager.acceptChange") + callback = (args...) -> + timer.done() + _callback(args...) + + DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> + return callback(error) if error? + if !lines? or !version? + return callback(new Errors.NotFoundError("document not found: #{doc_id}")) + RangesManager.acceptChange change_id, ranges, (error, new_ranges) -> + return callback(error) if error? + RedisManager.updateDocument doc_id, lines, version, [], new_ranges, (error) -> + return callback(error) if error? + callback() getDocWithLock: (project_id, doc_id, callback = (error, lines, version) ->) -> UpdateManager = require "./UpdateManager" @@ -139,3 +156,7 @@ module.exports = DocumentManager = flushAndDeleteDocWithLock: (project_id, doc_id, callback = (error) ->) -> UpdateManager = require "./UpdateManager" UpdateManager.lockUpdatesAndDo DocumentManager.flushAndDeleteDoc, project_id, doc_id, callback + + acceptChangeWithLock: (project_id, doc_id, change_id, callback = (error) ->) -> + UpdateManager = require "./UpdateManager" + UpdateManager.lockUpdatesAndDo DocumentManager.acceptChange, project_id, doc_id, change_id, callback diff --git a/services/document-updater/app/coffee/HttpController.coffee b/services/document-updater/app/coffee/HttpController.coffee index e138fe8bc4..683b94230f 100644 --- a/services/document-updater/app/coffee/HttpController.coffee +++ b/services/document-updater/app/coffee/HttpController.coffee @@ -97,4 +97,15 @@ module.exports = HttpController = return next(error) if error? logger.log project_id: project_id, "deleted project via http" res.send 204 # No Content + + acceptChange: (req, res, next = (error) ->) -> + {project_id, doc_id, change_id} = req.params + logger.log {project_id, doc_id, change_id}, "accepting change via http" + timer = new Metrics.Timer("http.acceptChange") + DocumentManager.acceptChangeWithLock project_id, doc_id, change_id, (error) -> + timer.done() + return next(error) if error? + logger.log {project_id, doc_id, change_id}, "accepted change via http" + res.send 204 # No Content + diff --git a/services/document-updater/app/coffee/RangesManager.coffee b/services/document-updater/app/coffee/RangesManager.coffee index ac1dcbe75f..64f7059399 100644 --- a/services/document-updater/app/coffee/RangesManager.coffee +++ b/services/document-updater/app/coffee/RangesManager.coffee @@ -4,7 +4,7 @@ logger = require "logger-sharelatex" module.exports = RangesManager = applyUpdate: (project_id, doc_id, entries = {}, updates = [], callback = (error, new_entries) ->) -> {changes, comments} = entries - logger.log {changes, comments, updates}, "appliyng updates to ranges" + logger.log {changes, comments, updates}, "applying updates to ranges" rangesTracker = new RangesTracker(changes, comments) for update in updates rangesTracker.track_changes = !!update.meta.tc @@ -12,7 +12,20 @@ module.exports = RangesManager = rangesTracker.setIdSeed(update.meta.tc) for op in update.op rangesTracker.applyOp(op, { user_id: update.meta?.user_id }) - + + response = RangesManager._getRanges rangesTracker + logger.log {response}, "applied updates to ranges" + callback null, response + + acceptChange: (change_id, ranges, callback = (error, ranges) ->) -> + {changes, comments} = ranges + logger.log {changes, comments, change_id}, "accepting change in ranges" + rangesTracker = new RangesTracker(changes, comments) + rangesTracker.removeChangeId(change_id) + response = RangesManager._getRanges(rangesTracker) + callback null, response + + _getRanges: (rangesTracker) -> # Return the minimal data structure needed, since most documents won't have any # changes or comments response = {} @@ -22,5 +35,4 @@ module.exports = RangesManager = if rangesTracker.comments?.length > 0 response ?= {} response.comments = rangesTracker.comments - logger.log {response}, "applied updates to ranges" - callback null, response \ No newline at end of file + return response \ No newline at end of file diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index 8da51c0899..7498a8087b 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -1,6 +1,7 @@ sinon = require "sinon" chai = require("chai") chai.should() +expect = chai.expect async = require "async" rclient = require("redis").createClient() @@ -182,3 +183,45 @@ describe "Ranges", -> changes[0].op.should.deep.equal { i: "123", p: 1 } changes[1].op.should.deep.equal { i: "456", p: 5 } done() + + describe "accepting a change", -> + before (done) -> + @project_id = DocUpdaterClient.randomId() + @user_id = DocUpdaterClient.randomId() + @id_seed = "587357bd35e64f6157" + @doc = { + id: DocUpdaterClient.randomId() + lines: ["aaa"] + } + @update = { + doc: @doc.id + op: [{ i: "456", p: 1 }] + v: 0 + meta: { user_id: @user_id, tc: @id_seed } + } + MockWebApi.insertDoc @project_id, @doc.id, { + lines: @doc.lines + version: 0 + } + DocUpdaterClient.preloadDoc @project_id, @doc.id, (error) => + throw error if error? + DocUpdaterClient.sendUpdate @project_id, @doc.id, @update, (error) => + throw error if error? + setTimeout () => + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + ranges = data.ranges + change = ranges.changes[0] + change.op.should.deep.equal { i: "456", p: 1 } + change.id.should.equal @id_seed + "000001" + change.metadata.user_id.should.equal @user_id + done() + , 200 + + it "should remove the change after accepting", (done) -> + DocUpdaterClient.acceptChange @project_id, @doc.id, @id_seed + "000001", (error) => + throw error if error? + DocUpdaterClient.getDoc @project_id, @doc.id, (error, res, data) => + throw error if error? + expect(data.ranges.changes).to.be.undefined + done() \ No newline at end of file diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee index d704daefd1..afcbfd4b45 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee @@ -72,3 +72,6 @@ module.exports = DocUpdaterClient = deleteProject: (project_id, callback = () ->) -> request.del "http://localhost:3003/project/#{project_id}", callback + + acceptChange: (project_id, doc_id, change_id, callback = () ->) -> + request.post "http://localhost:3003/project/#{project_id}/doc/#{doc_id}/change/#{change_id}/accept", callback \ No newline at end of file diff --git a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee index 5966843f5a..3f0279b5c7 100644 --- a/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/DocumentManager/DocumentManagerTests.coffee @@ -3,6 +3,7 @@ chai = require('chai') should = chai.should() modulePath = "../../../../app/js/DocumentManager.js" SandboxedModule = require('sandboxed-module') +Errors = require "../../../../app/js/Errors" describe "DocumentManager", -> beforeEach -> @@ -18,6 +19,7 @@ describe "DocumentManager", -> "./WebRedisManager": @WebRedisManager = {} "./DiffCodec": @DiffCodec = {} "./UpdateManager": @UpdateManager = {} + "./RangesManager": @RangesManager = {} @project_id = "project-id-123" @doc_id = "doc-id-123" @callback = sinon.stub() @@ -259,4 +261,50 @@ describe "DocumentManager", -> @callback.calledWith(new Error("No lines were passed to setDoc")) it "should not try to get the doc lines", -> - @DocumentManager.getDoc.called.should.equal false \ No newline at end of file + @DocumentManager.getDoc.called.should.equal false + + describe "acceptChanges", -> + beforeEach -> + @change_id = "mock-change-id" + @version = 34 + @lines = ["original", "lines"] + @ranges = { entries: "mock", comments: "mock" } + @updated_ranges = { entries: "updated", comments: "updated" } + @DocumentManager.getDoc = sinon.stub().yields(null, @lines, @version, @ranges) + @RangesManager.acceptChange = sinon.stub().yields(null, @updated_ranges) + @RedisManager.updateDocument = sinon.stub().yields() + + describe "successfully", -> + beforeEach -> + @DocumentManager.acceptChange @project_id, @doc_id, @change_id, @callback + + it "should get the document's current ranges", -> + @DocumentManager.getDoc + .calledWith(@project_id, @doc_id) + .should.equal true + + it "should apply the accept change to the ranges", -> + @RangesManager.acceptChange + .calledWith(@change_id, @ranges) + .should.equal true + + it "should save the updated ranges", -> + @RedisManager.updateDocument + .calledWith(@doc_id, @lines, @version, [], @updated_ranges) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "when the doc is not found", -> + beforeEach -> + @DocumentManager.getDoc = sinon.stub().yields(null, null, null, null) + @DocumentManager.acceptChange @project_id, @doc_id, @change_id, @callback + + it "should not save anything", -> + @RedisManager.updateDocument.called.should.equal false + + it "should call the callback with a not found error", -> + error = new Errors.NotFoundError("document not found: #{@doc_id}") + @callback.calledWith(error).should.equal true + \ No newline at end of file diff --git a/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee b/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee index cf0f71a301..4000b402aa 100644 --- a/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee +++ b/services/document-updater/test/unit/coffee/HttpController/HttpControllerTests.coffee @@ -332,4 +332,45 @@ describe "HttpController", -> it "should call next with the error", -> @next .calledWith(new Error("oops")) - .should.equal true \ No newline at end of file + .should.equal true + + describe "acceptChange", -> + beforeEach -> + @req = + params: + project_id: @project_id + doc_id: @doc_id + change_id: @change_id = "mock-change-od-1" + + describe "successfully", -> + beforeEach -> + @DocumentManager.acceptChangeWithLock = sinon.stub().callsArgWith(3) + @HttpController.acceptChange(@req, @res, @next) + + it "should accept the change", -> + @DocumentManager.acceptChangeWithLock + .calledWith(@project_id, @doc_id, @change_id) + .should.equal true + + it "should return a successful No Content response", -> + @res.send + .calledWith(204) + .should.equal true + + it "should log the request", -> + @logger.log + .calledWith({@project_id, @doc_id, @change_id}, "accepting change via http") + .should.equal true + + it "should time the request", -> + @Metrics.Timer::done.called.should.equal true + + describe "when an errors occurs", -> + beforeEach -> + @DocumentManager.acceptChangeWithLock = sinon.stub().callsArgWith(3, new Error("oops")) + @HttpController.acceptChange(@req, @res, @next) + + it "should call next with the error", -> + @next + .calledWith(new Error("oops")) + .should.equal true From 540d0f7ec7783bc74b99ea832bd038b3e6c6f778 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Jan 2017 11:55:38 +0100 Subject: [PATCH 22/25] Make sure comment ids are unique --- .../app/coffee/RangesTracker.coffee | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index 09d471d476..f2794f2c07 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -35,7 +35,7 @@ load = (EventEmitter) -> # * Inserts by another user will not combine with inserts by the first user. If they are in the # middle of a previous insert by the first user, the original insert will be split into two. constructor: (@changes = [], @comments = []) -> - @setIdSeed("") + @setIdSeed(RangesTracker.generateIdSeed()) getIdSeed: () -> return @id_seed @@ -43,6 +43,19 @@ load = (EventEmitter) -> setIdSeed: (seed) -> @id_seed = seed @id_increment = 0 + + @generateIdSeed: () -> + # Generate a the first 18 characters of Mongo ObjectId, leaving 6 for the increment part + # Reference: https://github.com/dreampulse/ObjectId.js/blob/master/src/main/javascript/Objectid.js + pid = Math.floor(Math.random() * (32767)).toString(16) + machine = Math.floor(Math.random() * (16777216)).toString(16) + timestamp = Math.floor(new Date().valueOf() / 1000).toString(16) + return '00000000'.substr(0, 8 - timestamp.length) + timestamp + + '000000'.substr(0, 6 - machine.length) + machine + + '0000'.substr(0, 4 - pid.length) + pid + + @generateId: () -> + @generateId() + "000001" newId: () -> @id_increment++ From a3d5971d5462b0584cbb4243fa47faf9be88cdec Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Jan 2017 11:59:09 +0100 Subject: [PATCH 23/25] Update RangesTRacker --- services/document-updater/app/coffee/RangesTracker.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/RangesTracker.coffee b/services/document-updater/app/coffee/RangesTracker.coffee index f2794f2c07..722eab1aa5 100644 --- a/services/document-updater/app/coffee/RangesTracker.coffee +++ b/services/document-updater/app/coffee/RangesTracker.coffee @@ -55,7 +55,7 @@ load = (EventEmitter) -> '0000'.substr(0, 4 - pid.length) + pid @generateId: () -> - @generateId() + "000001" + @generateIdSeed() + "000001" newId: () -> @id_increment++ From 0706feb26bb6b182d25300d9f1c97ec5199dd67a Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Jan 2017 16:58:11 +0100 Subject: [PATCH 24/25] Add max limit on number of comments and changes per doc --- .../app/coffee/RangesManager.coffee | 6 +++++ .../app/coffee/ShareJsUpdateManager.coffee | 8 +------ .../app/coffee/UpdateManager.coffee | 7 +++++- .../coffee/ApplyingUpdatesToADocTests.coffee | 16 ++++++++++++- .../ShareJsUpdateManagerTests.coffee | 23 ------------------- .../UpdateManager/UpdateManagerTests.coffee | 20 ++++++++++++++++ 6 files changed, 48 insertions(+), 32 deletions(-) diff --git a/services/document-updater/app/coffee/RangesManager.coffee b/services/document-updater/app/coffee/RangesManager.coffee index 64f7059399..25da4ec9db 100644 --- a/services/document-updater/app/coffee/RangesManager.coffee +++ b/services/document-updater/app/coffee/RangesManager.coffee @@ -2,6 +2,9 @@ RangesTracker = require "./RangesTracker" logger = require "logger-sharelatex" module.exports = RangesManager = + MAX_COMMENTS: 500 + MAX_CHANGES: 500 + applyUpdate: (project_id, doc_id, entries = {}, updates = [], callback = (error, new_entries) ->) -> {changes, comments} = entries logger.log {changes, comments, updates}, "applying updates to ranges" @@ -12,6 +15,9 @@ module.exports = RangesManager = rangesTracker.setIdSeed(update.meta.tc) for op in update.op rangesTracker.applyOp(op, { user_id: update.meta?.user_id }) + + if rangesTracker.changes?.length > RangesManager.MAX_CHANGES or rangesTracker.comments?.length > RangesManager.MAX_COMMENTS + return callback new Error("too many comments or tracked changes") response = RangesManager._getRanges rangesTracker logger.log {response}, "applied updates to ranges" diff --git a/services/document-updater/app/coffee/ShareJsUpdateManager.coffee b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee index 876d56e71b..1d36776b9f 100644 --- a/services/document-updater/app/coffee/ShareJsUpdateManager.coffee +++ b/services/document-updater/app/coffee/ShareJsUpdateManager.coffee @@ -37,13 +37,10 @@ module.exports = ShareJsUpdateManager = update.dup = true ShareJsUpdateManager._sendOp(project_id, doc_id, update) else - ShareJsUpdateManager._sendError(project_id, doc_id, error) return callback(error) logger.log project_id: project_id, doc_id: doc_id, error: error, "applied update" model.getSnapshot doc_key, (error, data) => - if error? - ShareJsUpdateManager._sendError(project_id, doc_id, error) - return callback(error) + return callback(error) if error? docLines = data.snapshot.split(/\r\n|\n|\r/) callback(null, docLines, data.v, model.db.appliedOps[doc_key] or []) @@ -55,6 +52,3 @@ module.exports = ShareJsUpdateManager = _sendOp: (project_id, doc_id, op) -> WebRedisManager.sendData {project_id, doc_id, op} - _sendError: (project_id, doc_id, error) -> - WebRedisManager.sendData {project_id, doc_id, error: error.message || error} - diff --git a/services/document-updater/app/coffee/UpdateManager.coffee b/services/document-updater/app/coffee/UpdateManager.coffee index 1678c4d4c0..89f58bfd1f 100644 --- a/services/document-updater/app/coffee/UpdateManager.coffee +++ b/services/document-updater/app/coffee/UpdateManager.coffee @@ -46,7 +46,12 @@ module.exports = UpdateManager = (update, cb) -> UpdateManager.applyUpdate project_id, doc_id, update, cb callback - applyUpdate: (project_id, doc_id, update, callback = (error) ->) -> + applyUpdate: (project_id, doc_id, update, _callback = (error) ->) -> + callback = (error) -> + if error? + WebRedisManager.sendData {project_id, doc_id, error: error.message || error} + _callback(error) + UpdateManager._sanitizeUpdate update DocumentManager.getDoc project_id, doc_id, (error, lines, version, ranges) -> return callback(error) if error? diff --git a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee index 4166f8499e..bdfe89b990 100644 --- a/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee +++ b/services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee @@ -176,8 +176,12 @@ describe "Applying updates to a doc", -> describe "with a broken update", -> before (done) -> [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] + @broken_update = { doc_id: @doc_id, v: @version, op: [d: "not the correct content", p: 0 ] } MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.sendUpdate @project_id, @doc_id, @undefined, (error) -> + + DocUpdaterClient.subscribeToAppliedOps @messageCallback = sinon.stub() + + DocUpdaterClient.sendUpdate @project_id, @doc_id, @broken_update, (error) -> throw error if error? setTimeout done, 200 @@ -185,6 +189,16 @@ describe "Applying updates to a doc", -> DocUpdaterClient.getDoc @project_id, @doc_id, (error, res, doc) => doc.lines.should.deep.equal @lines done() + + it "should send a message with an error", -> + @messageCallback.called.should.equal true + [channel, message] = @messageCallback.args[0] + channel.should.equal "applied-ops" + JSON.parse(message).should.deep.equal { + project_id: @project_id, + doc_id: @doc_id, + error:'Delete component \'not the correct content\' does not match deleted text \'one\ntwo\nthree\'' + } describe "with enough updates to flush to the track changes api", -> before (done) -> diff --git a/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee index f3b871149d..42ba3f331b 100644 --- a/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/ShareJsUpdateManager/ShareJsUpdateManagerTests.coffee @@ -70,34 +70,22 @@ describe "ShareJsUpdateManager", -> describe "when applyOp fails", -> beforeEach (done) -> @error = new Error("Something went wrong") - @ShareJsUpdateManager._sendError = sinon.stub() @model.applyOp = sinon.stub().callsArgWith(2, @error) @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, @lines, @version, (err, docLines, version) => @callback(err, docLines, version) done() - it "should call sendError with the error", -> - @ShareJsUpdateManager._sendError - .calledWith(@project_id, @doc_id, @error) - .should.equal true - it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true describe "when getSnapshot fails", -> beforeEach (done) -> @error = new Error("Something went wrong") - @ShareJsUpdateManager._sendError = sinon.stub() @model.getSnapshot.callsArgWith(1, @error) @ShareJsUpdateManager.applyUpdate @project_id, @doc_id, @update, @lines, @version, (err, docLines, version) => @callback(err, docLines, version) done() - it "should call sendError with the error", -> - @ShareJsUpdateManager._sendError - .calledWith(@project_id, @doc_id, @error) - .should.equal true - it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true @@ -125,14 +113,3 @@ describe "ShareJsUpdateManager", -> .calledWith({project_id: @project_id, doc_id: @doc_id, op: @opData}) .should.equal true - describe "_sendError", -> - beforeEach -> - @error_text = "Something went wrong" - @WebRedisManager.sendData = sinon.stub() - @ShareJsUpdateManager._sendError(@project_id, @doc_id, new Error(@error_text)) - - it "should publish the error to the redis stream", -> - @WebRedisManager.sendData - .calledWith({project_id: @project_id, doc_id: @doc_id, error: @error_text}) - .should.equal true - diff --git a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee index e87391af44..33578cb6f0 100644 --- a/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/UpdateManager/UpdateManagerTests.coffee @@ -165,6 +165,7 @@ describe "UpdateManager", -> @RangesManager.applyUpdate = sinon.stub().yields(null, @updated_ranges) @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(null, @updatedDocLines, @version, @appliedOps) @RedisManager.updateDocument = sinon.stub().yields() + @WebRedisManager.sendData = sinon.stub() @HistoryManager.pushUncompressedHistoryOps = sinon.stub().callsArg(3) describe "normally", -> @@ -206,6 +207,25 @@ describe "UpdateManager", -> # \uFFFD is 'replacement character' @update.op[0].i.should.equal "\uFFFD\uFFFD" + + describe "with an error", -> + beforeEach -> + @error = new Error("something went wrong") + @ShareJsUpdateManager.applyUpdate = sinon.stub().yields(@error) + @UpdateManager.applyUpdate @project_id, @doc_id, @update, @callback + + it "should call WebRedisManager.sendData with the error", -> + @WebRedisManager.sendData + .calledWith({ + project_id: @project_id, + doc_id: @doc_id, + error: @error.message + }) + .should.equal true + + it "should call the callback with the error", -> + @callback.calledWith(@error).should.equal true + describe "lockUpdatesAndDo", -> beforeEach -> From 5fed2424d08a947e49721270fa299f856db18efb Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 16 Jan 2017 13:05:05 +0100 Subject: [PATCH 25/25] Remove unused redis package reference --- .../document-updater/test/acceptance/coffee/RangesTests.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/services/document-updater/test/acceptance/coffee/RangesTests.coffee b/services/document-updater/test/acceptance/coffee/RangesTests.coffee index 7498a8087b..0849d4551f 100644 --- a/services/document-updater/test/acceptance/coffee/RangesTests.coffee +++ b/services/document-updater/test/acceptance/coffee/RangesTests.coffee @@ -3,7 +3,6 @@ chai = require("chai") chai.should() expect = chai.expect async = require "async" -rclient = require("redis").createClient() MockWebApi = require "./helpers/MockWebApi" DocUpdaterClient = require "./helpers/DocUpdaterClient"