From ec8a12d445bcd32b38e7a03a244f70e3f9539683 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 27 Jan 2016 13:33:42 +0000 Subject: [PATCH] Flush changes to mongo before sending request to references service --- .../ReferencesSearchController.coffee | 2 +- .../ReferencesSearchHandler.coffee | 49 ++++++++++++------- .../ReferencesSearchManager.coffee | 14 ++++-- .../ReferencesSearchHandlerTests.coffee | 36 ++++++++++++++ 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee index db85585724..ae7be5f73e 100644 --- a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee +++ b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee @@ -12,7 +12,7 @@ module.exports = ReferencesSearchController = if (!docIds or (!(docIds instanceof Array) and (docIds != 'ALL'))) logger.err {projectId, docIds}, "docIds is not valid, should be either Array or String 'ALL'" return res.send 400 - logger.log {projectId, docIds}, "index references for project" + logger.log {projectId, docIds: docIds}, "index references for project" ReferencesSearchHandler.index projectId, docIds, (err, data) -> if err logger.err {err, projectId}, "error indexing references" diff --git a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee index 12f31bccbf..7722c9b241 100644 --- a/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee +++ b/services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee @@ -2,6 +2,7 @@ logger = require("logger-sharelatex") request = require("request") settings = require("settings-sharelatex") Project = require("../../models/Project").Project +DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler') U = require('underscore') Async = require('async') @@ -46,22 +47,32 @@ module.exports = ReferencesSearchHandler = if err logger.err {err, projectId}, "error checking whether to do full index" return callback(err) - bibDocUrls = docIds.map (docId) -> - ReferencesSearchHandler._buildDocUrl projectId, docId - logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service" - request.post { - url: "#{settings.apis.references.url}/project/#{projectId}/index" - json: - docUrls: bibDocUrls - fullIndex: isFullIndex - }, (err, res, data) -> - if err - logger.err {err, projectId}, "error communicating with references api" - return callback(err) - if 200 <= res.statusCode < 300 - logger.log {projectId}, "got keys from references api" - return callback(null, data) - else - err = new Error("references api responded with non-success code: #{res.statusCode}") - logger.log {err, projectId}, "error updating references" - return callback(err) + # TODO: flush documents to mongo + logger.log {projectId, docIds}, 'flushing docs to mongo before calling references service' + Async.series( + docIds.map((docId) -> (cb) -> DocumentUpdaterHandler.flushDocToMongo(projectId, docId, cb)), + (err) -> + # continue + if err + logger.err {err, projectId, docIds}, "error flushing docs to mongo" + return callback(err) + bibDocUrls = docIds.map (docId) -> + ReferencesSearchHandler._buildDocUrl projectId, docId + logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service" + request.post { + url: "#{settings.apis.references.url}/project/#{projectId}/index" + json: + docUrls: bibDocUrls + fullIndex: isFullIndex + }, (err, res, data) -> + if err + logger.err {err, projectId}, "error communicating with references api" + return callback(err) + if 200 <= res.statusCode < 300 + logger.log {projectId}, "got keys from references api" + return callback(null, data) + else + err = new Error("references api responded with non-success code: #{res.statusCode}") + logger.log {err, projectId}, "error updating references" + return callback(err) + ) diff --git a/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee b/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee index a90acbcbd7..d553fa84ab 100644 --- a/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee +++ b/services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee @@ -11,13 +11,16 @@ define [ if entity?.name?.match /.*\.bib$/ @indexReferences([doc.doc_id], true) + # When we join the project: + # index all references files + # and don't broadcast to all clients @$scope.$on 'project:joined', (e) => @indexReferences("ALL", false) setTimeout( (self) -> self.ide.socket.on 'references:keys:updated', (keys) -> - console.log '>> got keys from socket' + # console.log '>> got keys from socket' self._storeReferencesKeys(keys) , 100 , this @@ -26,8 +29,11 @@ define [ _storeReferencesKeys: (newKeys) -> if window._ENABLE_REFERENCES_AUTOCOMPLETE != true return - console.log '>> storing references keys' - @$scope.$root._references.keys = newKeys + # console.log '>> storing references keys' + oldKeys = @$scope.$root._references.keys + console.log "#{oldKeys.length} + #{newKeys.length}" + @$scope.$root._references.keys = _.union(oldKeys, newKeys) + console.log "end>> #{@$scope.$root._references.keys.length}" # docIds: List[String]|String('ALL'), shouldBroadcast: Bool indexReferences: (docIds, shouldBroadcast) -> @@ -39,6 +45,6 @@ define [ "/project/#{@$scope.project_id}/references/index", opts, (data) => - console.log ">> got keys ", data + # console.log ">> got keys ", data @_storeReferencesKeys(data.keys) ) diff --git a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee index 4e63779096..faa6e49a62 100644 --- a/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee @@ -44,6 +44,9 @@ describe 'ReferencesSearchHandler', -> findPopulatedById: sinon.stub().callsArgWith(1, null, @fakeProject) } } + '../DocumentUpdater/DocumentUpdaterHandler': @DocumentUpdaterHandler = { + flushDocToMongo: sinon.stub().callsArgWith(2, null) + } @fakeResponseData = projectId: @projectId keys: ['k1', 'k2'] @@ -73,6 +76,13 @@ describe 'ReferencesSearchHandler', -> @Project.findPopulatedById.calledWith(@projectId).should.equal true done() + it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> + @call (err, data) => + @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 + @docIds.forEach (docId) => + @DocumentUpdaterHandler.flushDocToMongo.calledWith(@projectId, docId).should.equal true + done() + it 'should make a request to references service', (done) -> @call (err, data) => @request.post.callCount.should.equal 1 @@ -106,6 +116,11 @@ describe 'ReferencesSearchHandler', -> @handler._findBibDocIds.calledWith(@fakeProject).should.equal true done() + it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) -> + @call (err, data) => + @DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2 + done() + it 'should not produce an error', (done) -> @call (err, data) => expect(err).to.equal null @@ -153,11 +168,32 @@ describe 'ReferencesSearchHandler', -> @request.post.callCount.should.equal 0 done() + describe 'when flushDocToMongo produces an error', -> + + beforeEach -> + @Project.findPopulatedById.callsArgWith(1, null, @fakeProject) + @handler._isFullIndex.callsArgWith(1, false) + @DocumentUpdaterHandler.flushDocToMongo.callsArgWith(2, new Error('woops')) + + it 'should produce an error', (done) -> + @call (err, data) => + expect(err).to.not.equal null + expect(err).to.be.instanceof Error + expect(data).to.equal undefined + done() + + it 'should not send request', (done) -> + @call (err, data) => + @request.post.callCount.should.equal 0 + done() + + describe 'when request produces an error', -> beforeEach -> @Project.findPopulatedById.callsArgWith(1, null, @fakeProject) @handler._isFullIndex.callsArgWith(1, null, false) + @DocumentUpdaterHandler.flushDocToMongo.callsArgWith(2, null) @request.post.callsArgWith(1, new Error('woops')) it 'should produce an error', (done) ->