alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove pack worker

Browse Source 
remove the op-specific code

remove tests for ops, now only packing

remove unused packing code

work in progress

store index for completed packs only

support archiving and unarchiving of individual packs

remove support for archiving whole document history

split out ArchiveManager, IndexManager

remove old DocArchive code

remove docHistoryStats collection

comment about archiving

added method to look at index when last pack has been archived

added start of iterator for project results

use a proper iterator

added heap module

getting it working

increase pack size since bulk operations no longer needed

remove unused MongoAWSexternal

cleanup

added doc iterator

remove old query code

added missing files

cleanup

clean upclean up

started adding pack worker for archiving

work in progress

work in progress

getting pack worker working

updating worker

getting packworker working

added lock

use correct key name for track changes aws access

use correct key name for track changes aws access

always send back users array

fix up comparison of retrieved objects

handle op ids inside packs

log when s3 download completes

comments

cleanup, remove finalisation ideacleanup, remove finalisation idea

remove logging
This commit is contained in:
Brian Gough
2016-03-01 10:10:02 +00:00
parent a23ddf31c0
commit 3d9dfeccc3
18 changed files with 627 additions and 1321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee
+2 -2
View File
@@ -9,12 +9,12 @@ describe "MongoAWS", ->
beforeEach ->
@MongoAWS = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings =
filestore:
trackchanges:
s3:
secret: "s3-secret"
key: "s3-key"
stores:
user_files: "s3-bucket"
doc_history: "s3-bucket"
"child_process": @child_process = {}
"mongo-uri": @mongouri = {}
"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub(), err:->}
services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee
-60
View File
@@ -101,66 +101,6 @@ describe "MongoManager", ->
@callback.calledWith(null, null, @update.v).should.equal true
describe "insertCompressedUpdate", ->
beforeEach ->
@update = { op: "op", meta: "meta", v: "v"}
@db.docHistory =
insert: sinon.stub().callsArg(1)
describe "temporarly", ->
beforeEach ->
@MongoManager.insertCompressedUpdate @project_id, @doc_id, @update, true, @callback
it "should insert the update with a expiresAt field one week away", ->
@db.docHistory.insert
.calledWith({
project_id: ObjectId(@project_id),
doc_id: ObjectId(@doc_id),
op: @update.op,
meta: @update.meta,
v: @update.v
expiresAt: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000)
})
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "permanenty", ->
beforeEach ->
@MongoManager.insertCompressedUpdate @project_id, @doc_id, @update, false, @callback
it "should insert the update with no expiresAt field", ->
@db.docHistory.insert
.calledWith({
project_id: ObjectId(@project_id),
doc_id: ObjectId(@doc_id),
op: @update.op,
meta: @update.meta,
v: @update.v
})
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "insertCompressedUpdates", ->
beforeEach (done) ->
@updates = [ "mock-update-1", "mock-update-2" ]
@MongoManager.insertCompressedUpdate = sinon.stub().callsArg(4)
@MongoManager.insertCompressedUpdates @project_id, @doc_id, @updates, @temporary = true, (args...) =>
@callback(args...)
done()
it "should insert each update", ->
for update in @updates
@MongoManager.insertCompressedUpdate
.calledWith(@project_id, @doc_id, update, @temporary)
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "getDocUpdates", ->
beforeEach ->
@results = [
services/track-changes/test/unit/coffee/PackManager/PackManagerTests.coffee
-181
View File
@@ -233,184 +233,3 @@ describe "PackManager", ->
it "should call the callback", ->
@callback.called.should.equal true
describe "convertDocsToPacks", ->
describe "with several small ops", ->
beforeEach ->
@ops = [
{ _id: "3", op: "op-3", meta: {start_ts: "ts3_s", end_ts: "ts3_e", user_id: "u3"}, v: 3},
{ _id: "4", op: "op-4", meta: {start_ts: "ts4_s", end_ts: "ts4_e", user_id: "u4"}, v: 4},
]
@PackManager.convertDocsToPacks @ops, @callback
it "should create a single pack", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: [
{ _id: "3", op: "op-3", meta: {start_ts: "ts3_s", end_ts: "ts3_e", user_id: "u3"}, v: 3},
{ _id: "4", op: "op-4", meta: {start_ts: "ts4_s", end_ts: "ts4_e", user_id: "u4"}, v: 4},
]
v: 3
v_end: 4
meta: {start_ts: "ts3_s", end_ts: "ts4_e"}
n: 2
sz: 202
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
describe "with many small ops", ->
beforeEach ->
@ops = ({ _id: "#{n}", op: "op-#{n}", meta: {start_ts: "ts#{n}_s", end_ts: "ts#{n}_e", user_id: "u#{n}"}, v: n} for n in [0...1024])
@PackManager.convertDocsToPacks @ops, @callback
it "should create two packs", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: @ops[0...512]
v: 0
v_end: 511
meta: {start_ts: "ts0_s", end_ts: "ts511_e"}
n: 512
sz: 56282
}, {
pack: @ops[512...1024]
v: 512
v_end: 1023
meta: {start_ts: "ts512_s", end_ts: "ts1023_e"}
n: 512
sz: 56952
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
describe "with many temporary ops", ->
beforeEach ->
@ops = ({ _id: "#{n}", op: "op-#{n}", meta: {start_ts: n, end_ts: n+1, user_id: "u#{n}"}, v: n, expiresAt: n+24*3600*1000 } for n in [0...1024])
@PackManager.convertDocsToPacks @ops, @callback
it "should create two packs", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: (_.omit(op, "expiresAt") for op in @ops[0...512])
v: 0
v_end: 511
meta: {start_ts: 0 , end_ts: 512}
n: 512
sz: 55990
expiresAt: @ops[511].expiresAt
}, {
pack: (_.omit(op, "expiresAt") for op in @ops[512...1024])
v: 512
v_end: 1023
meta: {start_ts: 512, end_ts: 1024}
n: 512
sz: 56392
expiresAt: @ops[1023].expiresAt
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
describe "with temporary ops spanning more than 1 day", ->
beforeEach ->
@ops = ({ _id: "#{n}", op: "op-#{n}", meta: {start_ts: n*3600*1000, end_ts: n*3600*1000+1, user_id: "u#{n}"}, v: n, expiresAt: n+24*3600*1000 } for n in [0...48])
@PackManager.convertDocsToPacks @ops, @callback
it "should create two packs", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: (_.omit(op, "expiresAt") for op in @ops[0...24])
v: 0
v_end: 23
meta: {start_ts: 0 , end_ts: 23*3600*1000+1}
n: 24
sz: 2538
expiresAt: @ops[23].expiresAt
}, {
pack: (_.omit(op, "expiresAt") for op in @ops[24...48])
v: 24
v_end: 47
meta: {start_ts: 24*3600*1000, end_ts: 47*3600*1000+1}
n: 24
sz: 2568
expiresAt: @ops[47].expiresAt
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
describe "with mixture of temporary and permanent ops", ->
beforeEach ->
@ops = ({ _id: "#{n}", op: "op-#{n}", meta: {start_ts: n*3600*1000, end_ts: n*3600*1000+1, user_id: "u#{n}"}, v: n, expiresAt: n+24*3600*1000 } for n in [0...48])
for n in [10...48]
delete @ops[n].expiresAt
@PackManager.convertDocsToPacks @ops, @callback
it "should create two packs", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: (_.omit(op, "expiresAt") for op in @ops[0...10])
v: 0
v_end: 9
meta: {start_ts: 0 , end_ts: 9*3600*1000+1}
n: 10
sz: 1040
expiresAt: @ops[9].expiresAt
}, {
pack: (_.omit(op, "expiresAt") for op in @ops[10...48])
v: 10
v_end: 47
meta: {start_ts: 10*3600*1000, end_ts: 47*3600*1000+1}
n: 38
sz: 3496
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
describe "with mixture of temporary and permanent ops and an existing pack", ->
beforeEach ->
@ops = ({ _id: "#{n}", op: "op-#{n}", meta: {start_ts: n*3600*1000, end_ts: n*3600*1000+1, user_id: "u#{n}"}, v: n, expiresAt: n+24*3600*1000 } for n in [0...48])
for n in [10...48]
delete @ops[n].expiresAt
# convert op 16 into a pack
@ops[16].pack = [ @ops[16].op ]
delete @ops[16].op
@PackManager.convertDocsToPacks @ops, @callback
it "should create three packs", (done) ->
@PackManager.convertDocsToPacks @ops, (err, packs) =>
assert.deepEqual packs, [ {
pack: (_.omit(op, "expiresAt") for op in @ops[0...10])
v: 0
v_end: 9
meta: {start_ts: 0 , end_ts: 9*3600*1000+1}
n: 10
sz: 1040
expiresAt: @ops[9].expiresAt
}, {
pack: @ops[10...16]
v: 10
v_end: 15
meta: {start_ts: 10*3600*1000, end_ts: 15*3600*1000+1}
n: 6
sz: 552
}, {
pack: @ops[17...48]
v: 17
v_end: 47
meta: {start_ts: 17*3600*1000, end_ts: 47*3600*1000+1}
n: 31
sz: 2852
}]
done()
it "should call the callback", ->
@callback.called.should.equal true
services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee
+7 -19
View File
@@ -26,12 +26,10 @@ describe "UpdatesManager", ->
describe "when there are no raw ops", ->
beforeEach ->
@MongoManager.peekLastCompressedUpdate = sinon.stub()
@MongoManager.insertCompressedUpdates = sinon.stub()
@UpdatesManager.compressAndSaveRawUpdates @project_id, @doc_id, [], @temporary, @callback
it "should not need to access the database", ->
@MongoManager.peekLastCompressedUpdate.called.should.equal false
@MongoManager.insertCompressedUpdates.called.should.equal false
it "should call the callback", ->
@callback.called.should.equal true
@@ -65,8 +63,6 @@ describe "UpdatesManager", ->
@compressedUpdates = [ { v: 12, op: "compressed-op-11+12" }, { v: 13, op: "compressed-op-12" } ]
@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate, @lastCompressedUpdate.v)
@MongoManager.modifyCompressedUpdate = sinon.stub().callsArg(2)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(4)
@PackManager.insertCompressedUpdates = sinon.stub().callsArg(5)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)
@@ -80,19 +76,14 @@ describe "UpdatesManager", ->
.calledWith(@doc_id)
.should.equal true
it "should compress the last compressed op and the raw ops", ->
it "should compress the raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates)
.calledWith(null, @rawUpdates)
.should.equal true
it "should update the existing op", ->
@MongoManager.modifyCompressedUpdate
.calledWith(@lastCompressedUpdate, @compressedUpdates[0])
.should.equal true
it "should save the new compressed ops", ->
@MongoManager.insertCompressedUpdates
.calledWith(@project_id, @doc_id, @compressedUpdates[1..], @temporary)
it "should save the new compressed ops into a pack", ->
@PackManager.insertCompressedUpdates
.calledWith(@project_id, @doc_id, @lastCompressedUpdate, @compressedUpdates, @temporary)
.should.equal true
it "should call the callback", ->
@@ -130,7 +121,7 @@ describe "UpdatesManager", ->
it "should only compress the more recent raw ops", ->
@UpdateCompressor.compressRawUpdates
.calledWith(@lastCompressedUpdate, @rawUpdates.slice(-2))
.calledWith(null, @rawUpdates.slice(-2))
.should.equal true
describe "when the raw ops do not follow from the last compressed op version", ->
@@ -143,11 +134,8 @@ describe "UpdatesManager", ->
.calledWith(new Error("Tried to apply raw op at version 13 to last compressed update with version 11"))
.should.equal true
it "should not modify any update in mongo", ->
@MongoManager.modifyCompressedUpdate.called.should.equal false
it "should not insert any update into mongo", ->
@MongoManager.insertCompressedUpdates.called.should.equal false
@PackManager.insertCompressedUpdates.called.should.equal false
describe "when the raw ops need appending to existing history which is in S3", ->
beforeEach ->