alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Explicitly separate calls to web and docupdater redis instances

Browse Source
This commit is contained in:
James Allen
2016-06-17 12:17:22 +01:00
parent 9ff3026807
commit bc00aab7b1
12 changed files with 175 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee
+7 -1
View File
@@ -1,4 +1,5 @@
rclient = require("redis").createClient()
Settings = require('settings-sharelatex')
rclient = require("redis").createClient(Settings.redis.web)
request = require("request").defaults(jar: false)
async = require "async"
@@ -7,6 +8,11 @@ module.exports = DocUpdaterClient =
chars = for i in [1..24]
Math.random().toString(16)[2]
return chars.join("")
subscribeToAppliedOps: (callback = (message) ->) ->
rclient_sub = require("redis").createClient()
rclient_sub.subscribe "applied-ops"
rclient_sub.on "message", callback
sendUpdate: (project_id, doc_id, update, callback = (error) ->) ->
rclient.rpush "PendingUpdates:#{doc_id}", JSON.stringify(update), (error)->
services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee
-51
View File
@@ -54,46 +54,6 @@ describe "RedisManager", ->
@callback
.calledWith(null, @lines, @version)
.should.equal true
describe "getPendingUpdatesForDoc", ->
beforeEach ->
@rclient.lrange = sinon.stub()
@rclient.del = sinon.stub()
describe "successfully", ->
beforeEach ->
@updates = [
{ op: [{ i: "foo", p: 4 }] }
{ op: [{ i: "foo", p: 4 }] }
]
@jsonUpdates = @updates.map (update) -> JSON.stringify update
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
@RedisManager.getPendingUpdatesForDoc @doc_id, @callback
it "should get the pending updates", ->
@rclient.lrange
.calledWith("PendingUpdates:#{@doc_id}", 0, -1)
.should.equal true
it "should delete the pending updates", ->
@rclient.del
.calledWith("PendingUpdates:#{@doc_id}")
.should.equal true
it "should call the callback with the updates", ->
@callback.calledWith(null, @updates).should.equal true
describe "when the JSON doesn't parse", ->
beforeEach ->
@jsonUpdates = [
JSON.stringify { op: [{ i: "foo", p: 4 }] }
"broken json"
]
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
@RedisManager.getPendingUpdatesForDoc @doc_id, @callback
it "should return an error to the callback", ->
@callback.calledWith(new Error("JSON parse error")).should.equal true
describe "getPreviousDocOpsTests", ->
describe "with a start and an end value", ->
@@ -179,17 +139,6 @@ describe "RedisManager", ->
it "should log out the problem", ->
@logger.warn.called.should.equal true
describe "getUpdatesLength", ->
beforeEach ->
@rclient.llen = sinon.stub().yields(null, @length = 3)
@RedisManager.getUpdatesLength @doc_id, @callback
it "should look up the length", ->
@rclient.llen.calledWith("PendingUpdates:#{@doc_id}").should.equal true
it "should return the length", ->
@callback.calledWith(null, @length).should.equal true
describe "pushDocOp", ->
beforeEach ->
@rclient.rpush = sinon.stub()
services/document-updater/test/unit/coffee/TrackChangesManager/TrackChangesManagerTests.coffee
+10 -14
View File
@@ -7,9 +7,9 @@ describe "TrackChangesManager", ->
beforeEach ->
@TrackChangesManager = SandboxedModule.require modulePath, requires:
"request": @request = {}
"settings-sharelatex": @Settings = { redis: web: {} }
"settings-sharelatex": @Settings = {}
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
"redis-sharelatex": createClient: () => @rclient = {}
"./WebRedisManager": @WebRedisManager = {}
@project_id = "mock-project-id"
@doc_id = "mock-doc-id"
@callback = sinon.stub()
@@ -42,23 +42,17 @@ describe "TrackChangesManager", ->
describe "pushUncompressedHistoryOp", ->
beforeEach ->
@op = { op: [{ i: "foo", p: 4 }] }
@rclient.multi = sinon.stub().returns(@multi = {})
@multi.rpush = sinon.stub()
@multi.sadd = sinon.stub()
@multi.exec = sinon.stub().yields(null, [@length = 42, "foo"])
@op = "mock-op"
@TrackChangesManager.flushDocChanges = sinon.stub().callsArg(2)
describe "pushing the op", ->
beforeEach ->
@WebRedisManager.pushUncompressedHistoryOp = sinon.stub().callsArgWith(3, null, 1)
@TrackChangesManager.pushUncompressedHistoryOp @project_id, @doc_id, @op, @callback
it "should push the op into redis", ->
@multi.rpush
.calledWith("UncompressedHistoryOps:#{@doc_id}", JSON.stringify @op)
.should.equal true
@multi.sadd
.calledWith("DocsWithHistoryOps:#{@project_id}", @doc_id)
@WebRedisManager.pushUncompressedHistoryOp
.calledWith(@project_id, @doc_id, @op)
.should.equal true
it "should call the callback", ->
@@ -69,7 +63,8 @@ describe "TrackChangesManager", ->
describe "when there are a multiple of FLUSH_EVERY_N_OPS ops", ->
beforeEach ->
@multi.exec = sinon.stub().yields(null, [2 * @TrackChangesManager.FLUSH_EVERY_N_OPS, "foo"])
@WebRedisManager.pushUncompressedHistoryOp =
sinon.stub().callsArgWith(3, null, 2 * @TrackChangesManager.FLUSH_EVERY_N_OPS)
@TrackChangesManager.pushUncompressedHistoryOp @project_id, @doc_id, @op, @callback
it "should tell the track changes api to flush", ->
@@ -79,7 +74,8 @@ describe "TrackChangesManager", ->
describe "when TrackChangesManager errors", ->
beforeEach ->
@multi.exec = sinon.stub().yields(null, [2 * @TrackChangesManager.FLUSH_EVERY_N_OPS, "foo"])
@WebRedisManager.pushUncompressedHistoryOp =
sinon.stub().callsArgWith(3, null, 2 * @TrackChangesManager.FLUSH_EVERY_N_OPS)
@TrackChangesManager.flushDocChanges = sinon.stub().callsArgWith(2, @error = new Error("oops"))
@TrackChangesManager.pushUncompressedHistoryOp @project_id, @doc_id, @op, @callback
services/document-updater/test/unit/coffee/UpdateManager/ApplyingUpdates.coffee
+6 -5
View File
@@ -12,6 +12,7 @@ describe "UpdateManager", ->
@UpdateManager = SandboxedModule.require modulePath, requires:
"./LockManager" : @LockManager = {}
"./RedisManager" : @RedisManager = {}
"./WebRedisManager" : @WebRedisManager = {}
"./ShareJsUpdateManager" : @ShareJsUpdateManager = {}
"logger-sharelatex": @logger = { log: sinon.stub() }
"./Metrics": @Metrics =
@@ -89,7 +90,7 @@ describe "UpdateManager", ->
describe "continueProcessingUpdatesWithLock", ->
describe "when there are outstanding updates", ->
beforeEach ->
@RedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 3)
@WebRedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 3)
@UpdateManager.processOutstandingUpdatesWithLock = sinon.stub().callsArg(2)
@UpdateManager.continueProcessingUpdatesWithLock @project_id, @doc_id, @callback
@@ -101,7 +102,7 @@ describe "UpdateManager", ->
describe "when there are no outstanding updates", ->
beforeEach ->
@RedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 0)
@WebRedisManager.getUpdatesLength = sinon.stub().callsArgWith(1, null, 0)
@UpdateManager.processOutstandingUpdatesWithLock = sinon.stub().callsArg(2)
@UpdateManager.continueProcessingUpdatesWithLock @project_id, @doc_id, @callback
@@ -117,12 +118,12 @@ describe "UpdateManager", ->
@updates = [{p: 1, t: "foo"}]
@updatedDocLines = ["updated", "lines"]
@version = 34
@RedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates)
@WebRedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates)
@UpdateManager.applyUpdates = sinon.stub().callsArgWith(3, null, @updatedDocLines, @version)
@UpdateManager.fetchAndApplyUpdates @project_id, @doc_id, @callback
it "should get the pending updates", ->
@RedisManager.getPendingUpdatesForDoc.calledWith(@doc_id).should.equal true
@WebRedisManager.getPendingUpdatesForDoc.calledWith(@doc_id).should.equal true
it "should apply the updates", ->
@UpdateManager.applyUpdates
@@ -135,7 +136,7 @@ describe "UpdateManager", ->
describe "when there are no updates", ->
beforeEach ->
@updates = []
@RedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates)
@WebRedisManager.getPendingUpdatesForDoc = sinon.stub().callsArgWith(1, null, @updates)
@UpdateManager.applyUpdates = sinon.stub()
@RedisManager.setDocument = sinon.stub()
@UpdateManager.fetchAndApplyUpdates @project_id, @doc_id, @callback
services/document-updater/test/unit/coffee/UpdateManager/lockUpdatesAndDoTests.coffee
+1
View File
@@ -9,6 +9,7 @@ describe 'UpdateManager - lockUpdatesAndDo', ->
@UpdateManager = SandboxedModule.require modulePath, requires:
"./LockManager" : @LockManager = {}
"./RedisManager" : @RedisManager = {}
"./WebRedisManager" : @WebRedisManager = {}
"./ShareJsUpdateManager" : @ShareJsUpdateManager = {}
"logger-sharelatex": @logger = { log: sinon.stub() }
@project_id = "project-id-123"
services/document-updater/test/unit/coffee/WebRedisManager/WebRedisManagerTests.coffee
+93
View File
@@ -0,0 +1,93 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/WebRedisManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
describe "WebRedisManager", ->
beforeEach ->
@rclient =
auth: () ->
exec: sinon.stub()
@rclient.multi = () => @rclient
@WebRedisManager = SandboxedModule.require modulePath, requires:
"redis-sharelatex": createClient: () => @rclient
"settings-sharelatex": redis: web: @settings = {"mock": "settings"}
@doc_id = "doc-id-123"
@project_id = "project-id-123"
@callback = sinon.stub()
describe "getPendingUpdatesForDoc", ->
beforeEach ->
@rclient.lrange = sinon.stub()
@rclient.del = sinon.stub()
describe "successfully", ->
beforeEach ->
@updates = [
{ op: [{ i: "foo", p: 4 }] }
{ op: [{ i: "foo", p: 4 }] }
]
@jsonUpdates = @updates.map (update) -> JSON.stringify update
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
@WebRedisManager.getPendingUpdatesForDoc @doc_id, @callback
it "should get the pending updates", ->
@rclient.lrange
.calledWith("PendingUpdates:#{@doc_id}", 0, -1)
.should.equal true
it "should delete the pending updates", ->
@rclient.del
.calledWith("PendingUpdates:#{@doc_id}")
.should.equal true
it "should call the callback with the updates", ->
@callback.calledWith(null, @updates).should.equal true
describe "when the JSON doesn't parse", ->
beforeEach ->
@jsonUpdates = [
JSON.stringify { op: [{ i: "foo", p: 4 }] }
"broken json"
]
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonUpdates])
@WebRedisManager.getPendingUpdatesForDoc @doc_id, @callback
it "should return an error to the callback", ->
@callback.calledWith(new Error("JSON parse error")).should.equal true
describe "getUpdatesLength", ->
beforeEach ->
@rclient.llen = sinon.stub().yields(null, @length = 3)
@WebRedisManager.getUpdatesLength @doc_id, @callback
it "should look up the length", ->
@rclient.llen.calledWith("PendingUpdates:#{@doc_id}").should.equal true
it "should return the length", ->
@callback.calledWith(null, @length).should.equal true
describe "pushUncompressedHistoryOp", ->
beforeEach (done) ->
@op = { op: [{ i: "foo", p: 4 }] }
@rclient.rpush = sinon.stub().yields(null, @length = 42)
@rclient.sadd = sinon.stub().yields()
@WebRedisManager.pushUncompressedHistoryOp @project_id, @doc_id, @op, (args...) =>
@callback(args...)
done()
it "should push the doc op into the doc ops list", ->
@rclient.rpush
.calledWith("UncompressedHistoryOps:#{@doc_id}", JSON.stringify(@op))
.should.equal true
it "should add the doc_id to the set of which records the project docs", ->
@rclient.sadd
.calledWith("DocsWithHistoryOps:#{@project_id}", @doc_id)
.should.equal true
it "should call the callback with the length", ->
@callback.calledWith(null, @length).should.equal true