From e5bd7ce55153d1c56d6c186b4e5341a8c648a518 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 13 Jun 2018 17:22:34 +0100 Subject: [PATCH 01/63] Add message explaining Collaborator subscription to v1 users --- services/web/app/views/subscriptions/new.pug | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/web/app/views/subscriptions/new.pug b/services/web/app/views/subscriptions/new.pug index 4eae442f1e..c8228f4802 100644 --- a/services/web/app/views/subscriptions/new.pug +++ b/services/web/app/views/subscriptions/new.pug @@ -10,6 +10,7 @@ block scripts block content .content.content-alt + .container(ng-controller="NewSubscriptionController" ng-cloak) .row.card-group .col-md-5.col-md-push-4 @@ -35,6 +36,15 @@ block content .row(ng-if="plansVariant == 'more-details' && planCode == 'student-annual' || plansVariant == 'more-details' && planCode == 'student-monthly'") .col-xs-12 p.student-disclaimer #{translate('student_disclaimer')} + + if plan_code == 'collaborator_free_trial_7_days' && !!settings.overleaf + .row + .col-xs-12 + p.small.row-spaced-small + | The Collaborator plan is a new Overleaf v2 plan which includes track-changes, references search, and Dropbox & GitHub integration. While Overleaf v2 is in beta, you can also still subscribe to a + | + a(href=settings.overleaf.host + "/plans") legacy plan in Overleaf v1 + | . hr.thin .row .col-md-12.text-center From eda031023cc6893d40b1c6796ab0bc4bdd2e18e7 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 14 Jun 2018 10:15:20 +0100 Subject: [PATCH 02/63] Update subscription thank you page to remove ShareLaTeX references --- .../views/subscriptions/successful_subscription.pug | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/services/web/app/views/subscriptions/successful_subscription.pug b/services/web/app/views/subscriptions/successful_subscription.pug index 184c89c0d2..6fb4fc7958 100644 --- a/services/web/app/views/subscriptions/successful_subscription.pug +++ b/services/web/app/views/subscriptions/successful_subscription.pug @@ -18,17 +18,11 @@ block content p.letter-from-founders p #{translate("thanks_for_subscribing_you_help_sl", {planName:subscription.name})} p #{translate("need_anything_contact_us_at")} - a(href='mailto:support@sharelatex.com') support@sharelatex.com - | . #{translate("goes_straight_to_our_inboxes")}. + a(href='mailto:support@sharelatex.com') #{settings.adminEmail} + | . p #{translate("regards")}, br - | Henry and James - .portraits - span.img-circle - img(src=buildImgPath("about/henry_oswald.jpg")) - |   - span.img-circle - img(src=buildImgPath("about/james_allen.jpg")) + | The #{settings.appName} Team p a.btn.btn-primary(href="/project") < #{translate("back_to_your_projects")} From 32e14527c6fed9b6be642af83f8769f18a481583 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 14 Jun 2018 10:31:06 +0100 Subject: [PATCH 03/63] Make links readable in v2 alerts --- services/web/public/stylesheets/components/alerts.less | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/web/public/stylesheets/components/alerts.less b/services/web/public/stylesheets/components/alerts.less index 6ed93d29d4..ecdcba07a8 100755 --- a/services/web/public/stylesheets/components/alerts.less +++ b/services/web/public/stylesheets/components/alerts.less @@ -65,3 +65,10 @@ .alert-danger { .alert-variant(@alert-danger-bg; @alert-danger-border; @alert-danger-text); } + +.alert when (@is-overleaf = true) { + a { + color: white; + text-decoration: underline; + } +} \ No newline at end of file From b8787e6e68359be60f9791043870d2bbea5be2df Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 15 Jun 2018 08:56:03 -0400 Subject: [PATCH 04/63] modify dropdown to trigger on command completion if possible --- .../aceEditor/auto-complete/AutoCompleteManager.coffee | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee index a10dcf618b..a3f7da17a9 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee @@ -192,6 +192,13 @@ define [ setTimeout () => @editor.execCommand("startAutocomplete") , 0 + if ( + change.action == "insert" and + change.lines[0] in ["\\begin{}", "\\ref{}", "\\usepackage{}", "\\cite{}"] + ) + setTimeout () => + @editor.execCommand("startAutocomplete") + , 0 monkeyPatchAutocomplete: () -> Autocomplete = ace.require("ace/autocomplete").Autocomplete From 4b1bc291cb1738c88608b6f83f06bdbf32423bc2 Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 15 Jun 2018 10:31:51 -0400 Subject: [PATCH 05/63] misspelled variable completionAfterCursor --- .../aceEditor/auto-complete/CommandManager.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/CommandManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/CommandManager.coffee index d194c7fb4a..d786dbb9ec 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/CommandManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/CommandManager.coffee @@ -154,14 +154,14 @@ define [ totalArgs = squareArgsNo + curlyArgsNo if totalArgs == 0 completionBeforeCursor = completionBase - completionAfterCurspr = "" + completionAfterCursor = "" else completionBeforeCursor = completionBase + args[0] completionAfterCursor = args.slice(1) return { - base: base, - completion: completionBase + args, + base: base + completion: completionBase + args completionBeforeCursor: completionBeforeCursor completionAfterCursor: completionAfterCursor } From b1c0552214e435d906b75169115953c48de73d3f Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 15 Jun 2018 16:59:35 -0400 Subject: [PATCH 06/63] add ref command to snippets --- .../aceEditor/auto-complete/top_hundred_snippets.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee index 5c3de75af5..0a945c9e4d 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee @@ -74,6 +74,11 @@ define -> [{ "meta": "cmd", "score": 1.4425339817971206 }, { + "caption": "\\ref{}", + "snippet": "\\ref{$1}", + "meta": "cross-reference", + "score": 0.014379554883991673 + }{ "caption": "\\frac{}{}", "snippet": "\\frac{$1}{$2}", "meta": "cmd", From 3de1721aa25fa6b23676f9d0496c8146329cedc1 Mon Sep 17 00:00:00 2001 From: Jessica Lawshe Date: Fri, 15 Jun 2018 16:11:45 -0500 Subject: [PATCH 07/63] Send event to Google Analytics when variant selected --- services/web/public/coffee/main/plans.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/web/public/coffee/main/plans.coffee b/services/web/public/coffee/main/plans.coffee index 2915425dc0..1637d79374 100644 --- a/services/web/public/coffee/main/plans.coffee +++ b/services/web/public/coffee/main/plans.coffee @@ -153,6 +153,7 @@ define [ if $scope.shouldABTestPlans sixpack.participate 'plans-details', ['default', 'more-details'], (chosenVariation, rawResponse)-> $scope.plansVariant = chosenVariation + event_tracking.send 'subscription-funnel', 'plans-page-loaded', chosenVariation $scope.showPlans = true From fe225fbbd3eab7506bcf276fa978d28a8860ac2b Mon Sep 17 00:00:00 2001 From: Jessica Lawshe Date: Fri, 15 Jun 2018 16:29:42 -0500 Subject: [PATCH 08/63] Include event label when features table viewed --- .../web/app/views/subscriptions/_plans_page_details_more.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/views/subscriptions/_plans_page_details_more.pug b/services/web/app/views/subscriptions/_plans_page_details_more.pug index 17c683201e..d4867c2811 100644 --- a/services/web/app/views/subscriptions/_plans_page_details_more.pug +++ b/services/web/app/views/subscriptions/_plans_page_details_more.pug @@ -87,7 +87,7 @@ div +plan_switch('table') .col-md-3.text-right +currency_dropdown - .row(event-tracking="features-table-viewed" event-tracking-ga="subscription-funnel" event-tracking-trigger="scroll" event-tracking-send-once="true") + .row(event-tracking="features-table-viewed" event-tracking-ga="subscription-funnel" event-tracking-trigger="scroll" event-tracking-send-once="true" event-tracking-label=`exp-{{plansVariant}}`) .col-sm-12(ng-if="ui.view != 'student'") +table_premium .col-sm-12(ng-if="ui.view == 'student'") From f9c074a31efd04196ccc7b57a9c0604368d7244a Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 15 Jun 2018 17:29:08 +0100 Subject: [PATCH 09/63] Simplify check for when we are editing lines with metadata commands --- .../aceEditor/metadata/MetadataManager.coffee | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee index c9308626bc..1ff12efc38 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee @@ -18,41 +18,15 @@ define [ return if change.action not in ['remove', 'insert'] return - cursorPosition = @editor.getCursorPosition() + end = change.end - range = new Range(end.row, 0, end.row, end.column) - lineUpToCursor = @editor.getSession().getTextRange range - if lineUpToCursor.trim() == '%' or lineUpToCursor.slice(0, 1) == '\\' - range = new Range(end.row, 0, end.row, end.column + 80) - lineUpToCursor = @editor.getSession().getTextRange range - commandFragment = getLastCommandFragment lineUpToCursor + lineText = @editor.getSession().getLine end.row - linesContainPackage = _.any( - change.lines, - (line) -> line.match(/^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/) - ) - linesContainReqPackage = _.any( - change.lines, - (line) -> line.match(/^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/) - ) - linesContainLabel = _.any( - change.lines, - (line) -> line.match(/\\label{(.{0,80}?)}/) - ) - linesContainMeta = - linesContainPackage or - linesContainLabel or - linesContainReqPackage + # Defensive check against extremely long lines + return if lineText.length > 10000 - lastCommandFragmentIsLabel = commandFragment?.slice(0, 7) == '\\label{' - lastCommandFragmentIsPackage = commandFragment?.slice(0, 11) == '\\usepackage' - lastCommandFragmentIsReqPack = commandFragment?.slice(0, 15) == '\\RequirePackage' - lastCommandFragmentIsMeta = - lastCommandFragmentIsPackage or - lastCommandFragmentIsLabel or - lastCommandFragmentIsReqPack - - if linesContainMeta or lastCommandFragmentIsMeta + # Check if edited line contains metadata commands + if /\\(usepackage|RequirePackage|label)(\[.*])?({.*})?/.test(lineText) @scheduleLoadCurrentDocMetaFromServer() @editor.on "changeSession", (e) => From 81c102b5010944ca11f4ff2ae00f11afe1d71d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Mon, 18 Jun 2018 14:18:23 +0100 Subject: [PATCH 10/63] Add method to import invites --- .../Subscription/TeamInvitesHandler.coffee | 15 ++++++++++++++- .../Subscription/TeamInvitesHandlerTests.coffee | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Subscription/TeamInvitesHandler.coffee b/services/web/app/coffee/Features/Subscription/TeamInvitesHandler.coffee index e09e99c94a..c3b3b11089 100644 --- a/services/web/app/coffee/Features/Subscription/TeamInvitesHandler.coffee +++ b/services/web/app/coffee/Features/Subscription/TeamInvitesHandler.coffee @@ -56,6 +56,20 @@ module.exports = TeamInvitesHandler = return callback(error) if error? createInvite(subscription, email, inviterName, callback) + importInvite: (subscription, inviterName, email, token, sentAt, callback) -> + checkIfInviteIsPossible subscription, email, (error, possible, reason) -> + return callback(error) if error? + return callback(reason) unless possible + + subscription.teamInvites.push({ + email: email + inviterName: inviterName + token: token + sentAt: sentAt + }) + + subscription.save callback + acceptInvite: (token, userId, callback) -> logger.log {userId}, "Accepting invite" TeamInvitesHandler.getInvite token, (err, invite, subscription) -> @@ -93,7 +107,6 @@ createInvite = (subscription, email, inviterName, callback) -> return callback(error) if error? return callback(reason) unless possible - invite = subscription.teamInvites.find (invite) -> invite.email == email if !invite? diff --git a/services/web/test/unit/coffee/Subscription/TeamInvitesHandlerTests.coffee b/services/web/test/unit/coffee/Subscription/TeamInvitesHandlerTests.coffee index 527ba8a143..d23d1fc3a6 100644 --- a/services/web/test/unit/coffee/Subscription/TeamInvitesHandlerTests.coffee +++ b/services/web/test/unit/coffee/Subscription/TeamInvitesHandlerTests.coffee @@ -175,6 +175,22 @@ describe "TeamInvitesHandler", -> ).should.equal true done() + describe "importInvite", -> + beforeEach -> + @sentAt = new Date() + + it "can imports an invite from v1", -> + @TeamInvitesHandler.importInvite @subscription, "A-Team", "hannibal@a-team.org", + "secret", @sentAt, (error) => + expect(error).not.to.exist + + @subscription.save.calledOnce.should.eq true + + invite = @subscription.teamInvites.find (i) -> i.email == "hannibal@a-team.org" + expect(invite.token).to.eq("secret") + expect(invite.sentAt).to.eq(@sentAt) + + describe "acceptInvite", -> beforeEach -> @user = { From 9f309e22a7c0a5621af54677372c567383d43305 Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Mon, 18 Jun 2018 12:26:45 -0400 Subject: [PATCH 11/63] refactor AutoCompleteManager --- .../auto-complete/AutoCompleteManager.coffee | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee index a3f7da17a9..160c28bd1e 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee @@ -44,13 +44,11 @@ define [ GraphicsCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - context = Helpers.getContext(editor, pos) - {commandFragment, closingBrace} = context + {commandFragment, closingBrace} = Helpers.getContext(editor, pos) if commandFragment match = commandFragment.match(/^~?\\(includegraphics(?:\[.*])?){([^}]*, *)?(\w*)/) if match - commandName = match[1] - currentArg = match[3] + [commandName, _, currentArg] = match graphicsPaths = Preamble.getGraphicsPaths() result = [] for graphic in Graphics.getGraphicsFiles() @@ -70,13 +68,11 @@ define [ metadataManager = @metadataManager FilesCompleter = getCompletions: (editor, session, pos, prefix, callback) => - context = Helpers.getContext(editor, pos) - {commandFragment, closingBrace} = context + {commandFragment, closingBrace} = Helpers.getContext(editor, pos) if commandFragment match = commandFragment.match(/^\\(input|include){(\w*)/) if match - commandName = match[1] - currentArg = match[2] + [commandName, currentArg] = match result = [] for file in Files.getTeXFiles() if file.id != @$scope.docId @@ -91,13 +87,11 @@ define [ LabelsCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - context = Helpers.getContext(editor, pos) - {commandFragment, closingBrace} = context + {commandFragment, closingBrace} = Helpers.getContext(editor, pos) if commandFragment refMatch = commandFragment.match(/^~?\\([a-z]*ref){([^}]*, *)?(\w*)/) if refMatch - commandName = refMatch[1] - currentArg = refMatch[2] + [commandName, currentArg] = refMatch result = [] if commandName != 'ref' # ref is in top 100 commands result.push { @@ -118,17 +112,14 @@ define [ references = @$scope.$root._references ReferencesCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - context = Helpers.getContext(editor, pos) - {commandFragment, closingBrace} = context + {commandFragment, closingBrace} = Helpers.getContext(editor, pos) if commandFragment citeMatch = commandFragment.match( /^~?\\([a-z]*cite[a-z]*(?:\[.*])?){([^}]*, *)?(\w*)/ ) if citeMatch - commandName = citeMatch[1] - previousArgs = citeMatch[2] - currentArg = citeMatch[3] - if previousArgs == undefined + [commandName, previousArgs, currentArg] = citeMatch + if !previousArgs? previousArgs = "" previousArgsCaption = if previousArgs.length > 8 then "…," else previousArgs result = [] @@ -140,7 +131,7 @@ define [ } if references.keys and references.keys.length > 0 references.keys.forEach (key) -> - if !(key in [null, undefined]) + if key? result.push({ caption: "\\#{commandName}{#{previousArgsCaption}#{key}#{closingBrace}" value: "\\#{commandName}{#{previousArgs}#{key}#{closingBrace}" @@ -170,8 +161,7 @@ define [ onChange: (change) -> cursorPosition = @editor.getCursorPosition() end = change.end - context = Helpers.getContext(@editor, end) - {lineUpToCursor, commandFragment} = context + {lineUpToCursor, commandFragment} = Helpers.getContext(@editor, end) if lineUpToCursor.match(/.*%.*/) return lastCharIsBackslash = lineUpToCursor.slice(-1) == "\\" @@ -194,7 +184,7 @@ define [ , 0 if ( change.action == "insert" and - change.lines[0] in ["\\begin{}", "\\ref{}", "\\usepackage{}", "\\cite{}"] + change.lines[0].match(/\\(\w+){}/)?[1].match(/(begin|[a-z]*ref|usepackage|[a-z]*cite[a-z]*)/) ) setTimeout () => @editor.execCommand("startAutocomplete") From 3d272ca297075cf2473697bf862a25e9573bd163 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Mon, 18 Jun 2018 18:37:58 +0200 Subject: [PATCH 12/63] replace OldAssetProxy --- .../infrastructure/OldAssetProxy.coffee | 16 ----- .../coffee/infrastructure/ProxyManager.coffee | 43 +++++++++++++ .../app/coffee/infrastructure/Server.coffee | 4 +- .../infrastructure/ProxyManagerTests.coffee | 64 +++++++++++++++++++ 4 files changed, 109 insertions(+), 18 deletions(-) delete mode 100644 services/web/app/coffee/infrastructure/OldAssetProxy.coffee create mode 100644 services/web/app/coffee/infrastructure/ProxyManager.coffee create mode 100644 services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee diff --git a/services/web/app/coffee/infrastructure/OldAssetProxy.coffee b/services/web/app/coffee/infrastructure/OldAssetProxy.coffee deleted file mode 100644 index 7912f0ed14..0000000000 --- a/services/web/app/coffee/infrastructure/OldAssetProxy.coffee +++ /dev/null @@ -1,16 +0,0 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -request = require("request") - -module.exports = (req, res, next)-> - requestedUrl = req.url - - redirectUrl = settings.proxyUrls[requestedUrl] - if redirectUrl? - logger.log redirectUrl:redirectUrl, reqUrl:req.url, "proxying url" - upstream = request(redirectUrl) - upstream.on "error", (error) -> - logger.error err: error, "error in OldAssetProxy" - upstream.pipe(res) - else - next() diff --git a/services/web/app/coffee/infrastructure/ProxyManager.coffee b/services/web/app/coffee/infrastructure/ProxyManager.coffee new file mode 100644 index 0000000000..45ca6b7a48 --- /dev/null +++ b/services/web/app/coffee/infrastructure/ProxyManager.coffee @@ -0,0 +1,43 @@ +settings = require("settings-sharelatex") +logger = require("logger-sharelatex") +httpProxy = require 'express-http-proxy' + +module.exports = + # add proxy for all paths listed in `settings.proxyUrls`and log errors + apply: (app) -> + for requestUrl, target of settings.proxyUrls + targetUrl = @makeTargetUrl(target) + if targetUrl? + app.use requestUrl, httpProxy(targetUrl) + else + logger.error "Cannot make proxy target from #{target}" + + # takes a 'target' and return an URL to proxy to. + # 'target' can be: + # - a String, representing the URL + # - an Object with: + # - a path attribute (String) + # - a baseURL attribute (String) + # - a baseURL attribute (Object) with: + # - a setting attribute pointing to a value in the settings + makeTargetUrl: (target) -> + return target if typeof target is 'string' + return target.path unless target.baseUrl? + + if typeof target.baseUrl is 'string' + baseUrl = target.baseUrl + else if target.baseUrl.setting? + baseUrl = digSettingValue target.baseUrl.setting + + return null unless baseUrl? + "#{baseUrl}#{target.path}" + +# given a setting path (e.g. 'apis.v1.url') recursively find the corresponding +# settings value +digSettingValue = (attributesPath, dig = null) -> + dig ||= settings + [nextAttribute, leftAttributes...] = attributesPath.split('.') + dig = dig[nextAttribute] + return null unless dig? + return dig if leftAttributes.length == 0 + digSettingValue(leftAttributes.join('.'), dig) diff --git a/services/web/app/coffee/infrastructure/Server.coffee b/services/web/app/coffee/infrastructure/Server.coffee index 39fc3c131b..522599c659 100644 --- a/services/web/app/coffee/infrastructure/Server.coffee +++ b/services/web/app/coffee/infrastructure/Server.coffee @@ -32,7 +32,7 @@ Mongoose = require("./Mongoose") oneDayInMilliseconds = 86400000 ReferalConnect = require('../Features/Referal/ReferalConnect') RedirectManager = require("./RedirectManager") -OldAssetProxy = require("./OldAssetProxy") +ProxyManager = require("./ProxyManager") translations = require("translations-sharelatex").setup(Settings.i18n) Modules = require "./Modules" @@ -74,7 +74,7 @@ app.use methodOverride() app.use metrics.http.monitor(logger) app.use RedirectManager -app.use OldAssetProxy +ProxyManager.apply(app) webRouter.use cookieParser(Settings.security.sessionSecret) diff --git a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee new file mode 100644 index 0000000000..3c3c4c7ccd --- /dev/null +++ b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee @@ -0,0 +1,64 @@ +sinon = require('sinon') +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = '../../../../app/js/infrastructure/ProxyManager' +SandboxedModule = require('sandboxed-module') +MockRequest = require "../helpers/MockRequest" +MockResponse = require "../helpers/MockResponse" + +describe "ProxyManager", -> + before -> + @settings = proxyUrls: {} + @httpProxy = sinon.stub() + @proxyManager = SandboxedModule.require modulePath, requires: + 'settings-sharelatex': @settings + 'logger-sharelatex': + error: -> + log: -> console.log(arguments) + 'httpProxy': @httpProxy + + describe 'apply', -> + before -> + @sandbox = sinon.sandbox.create() + @app = use: sinon.stub() + @sandbox.stub(@proxyManager, 'makeTargetUrl') + + after -> + @sandbox.restore() + + beforeEach -> + @app.use.reset() + @requestUrl = '/foo/bar' + @settings.proxyUrls[@requestUrl] = 'http://whatever' + + it 'sets routes', -> + @proxyManager.makeTargetUrl.returns('http://whatever') + @proxyManager.apply(@app) + @app.use.calledWith(@requestUrl).should.equal true + + it 'logs errors', -> + @proxyManager.makeTargetUrl.returns(null) + @proxyManager.apply(@app) + @app.use.called.should.equal false + + describe 'makeTargetUrl', -> + it 'returns Strings', -> + target = 'http://whatever' + @proxyManager.makeTargetUrl(target).should.equal target + + it 'makes from object', -> + target = path: 'baz' + @proxyManager.makeTargetUrl(target).should.equal 'baz' + + it 'makes with baseUrl', -> + target = path: 'baz', baseUrl: 'foo.bar/' + @proxyManager.makeTargetUrl(target).should.equal 'foo.bar/baz' + + it 'makes with settingsUrl', -> + @settings.apis = v1: url: 'foo.bar/' + target = path: 'baz', baseUrl: { setting: 'apis.v1.url' } + @proxyManager.makeTargetUrl(target).should.equal 'foo.bar/baz' + + target = path: 'baz', baseUrl: { setting: 'incorrect.setting' } + expect(@proxyManager.makeTargetUrl(target)).to.equal null From e7038f0a579c969162c1184e36a4be55e495c6bf Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Mon, 18 Jun 2018 15:08:25 -0400 Subject: [PATCH 13/63] remove trailing } for all commands rather than relying on Helpers module --- .../auto-complete/AutoCompleteManager.coffee | 40 ++++++++++--------- .../auto-complete/EnvironmentManager.coffee | 4 -- .../aceEditor/auto-complete/Helpers.coffee | 5 +-- .../auto-complete/PackageManager.coffee | 9 ++--- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee index 160c28bd1e..638ab20b30 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee @@ -44,11 +44,11 @@ define [ GraphicsCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - {commandFragment, closingBrace} = Helpers.getContext(editor, pos) + {commandFragment} = Helpers.getContext(editor, pos) if commandFragment match = commandFragment.match(/^~?\\(includegraphics(?:\[.*])?){([^}]*, *)?(\w*)/) if match - [commandName, _, currentArg] = match + [_, commandName, _, currentArg] = match graphicsPaths = Preamble.getGraphicsPaths() result = [] for graphic in Graphics.getGraphicsFiles() @@ -58,8 +58,8 @@ define [ path = path.slice(graphicsPath.length) break result.push { - caption: "\\#{commandName}{#{path}#{closingBrace}" - value: "\\#{commandName}{#{path}#{closingBrace}" + caption: "\\#{commandName}{#{path}}" + value: "\\#{commandName}{#{path}}" meta: "graphic" score: 50 } @@ -68,18 +68,18 @@ define [ metadataManager = @metadataManager FilesCompleter = getCompletions: (editor, session, pos, prefix, callback) => - {commandFragment, closingBrace} = Helpers.getContext(editor, pos) + {commandFragment} = Helpers.getContext(editor, pos) if commandFragment match = commandFragment.match(/^\\(input|include){(\w*)/) if match - [commandName, currentArg] = match + [_, commandName, currentArg] = match result = [] for file in Files.getTeXFiles() if file.id != @$scope.docId path = file.path result.push { - caption: "\\#{commandName}{#{path}#{closingBrace}" - value: "\\#{commandName}{#{path}#{closingBrace}" + caption: "\\#{commandName}{#{path}}" + value: "\\#{commandName}{#{path}}" meta: "file" score: 50 } @@ -87,11 +87,11 @@ define [ LabelsCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - {commandFragment, closingBrace} = Helpers.getContext(editor, pos) + {commandFragment} = Helpers.getContext(editor, pos) if commandFragment refMatch = commandFragment.match(/^~?\\([a-z]*ref){([^}]*, *)?(\w*)/) if refMatch - [commandName, currentArg] = refMatch + [_, commandName, currentArg] = refMatch result = [] if commandName != 'ref' # ref is in top 100 commands result.push { @@ -102,8 +102,8 @@ define [ } for label in metadataManager.getAllLabels() result.push { - caption: "\\#{commandName}{#{label}#{closingBrace}" - value: "\\#{commandName}{#{label}#{closingBrace}" + caption: "\\#{commandName}{#{label}}" + value: "\\#{commandName}{#{label}}" meta: "cross-reference" score: 50 } @@ -112,13 +112,13 @@ define [ references = @$scope.$root._references ReferencesCompleter = getCompletions: (editor, session, pos, prefix, callback) -> - {commandFragment, closingBrace} = Helpers.getContext(editor, pos) + {commandFragment} = Helpers.getContext(editor, pos) if commandFragment citeMatch = commandFragment.match( /^~?\\([a-z]*cite[a-z]*(?:\[.*])?){([^}]*, *)?(\w*)/ ) if citeMatch - [commandName, previousArgs, currentArg] = citeMatch + [_, commandName, previousArgs, currentArg] = citeMatch if !previousArgs? previousArgs = "" previousArgsCaption = if previousArgs.length > 8 then "…," else previousArgs @@ -133,8 +133,8 @@ define [ references.keys.forEach (key) -> if key? result.push({ - caption: "\\#{commandName}{#{previousArgsCaption}#{key}#{closingBrace}" - value: "\\#{commandName}{#{previousArgs}#{key}#{closingBrace}" + caption: "\\#{commandName}{#{previousArgsCaption}#{key}}" + value: "\\#{commandName}{#{previousArgs}#{key}}" meta: "reference" score: 50 }) @@ -178,13 +178,15 @@ define [ end.row == cursorPosition.row and end.column == cursorPosition.column + 1 ) - if (commandFragment? and commandFragment.length > 2) or lastCharIsBackslash + if commandFragment?.length > 2 or lastCharIsBackslash setTimeout () => @editor.execCommand("startAutocomplete") , 0 if ( change.action == "insert" and - change.lines[0].match(/\\(\w+){}/)?[1].match(/(begin|[a-z]*ref|usepackage|[a-z]*cite[a-z]*)/) + change.lines[0].match(/\\(\w+){}/)?[1].match( + /(begin|end|[a-z]*ref|usepackage|[a-z]*cite[a-z]*|input|include)/ + ) ) setTimeout () => @editor.execCommand("startAutocomplete") @@ -206,7 +208,7 @@ define [ # If we are in \begin{it|}, then we need to remove the trailing } # since it will be adding in with the autocomplete of \begin{item}... - if this.completions.filterText.match(/^\\begin\{/) and nextChar == "}" + if this.completions.filterText.match(/^\\\w+{/) and nextChar == "}" editor.session.remove(range) # Provide our own `insertMatch` implementation. diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/EnvironmentManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/EnvironmentManager.coffee index 383bf1bd2e..156389e73d 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/EnvironmentManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/EnvironmentManager.coffee @@ -160,14 +160,10 @@ define () -> hasDocumentEnvironment = (text) -> re = /^\\begin{document}/m - envs = [] - iterations = 0 return re.exec(text) != null hasBibliographyEnvironment = (text) -> re = /^\\begin{thebibliography}/m - envs = [] - iterations = 0 return re.exec(text) != null class EnvironmentManager diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Helpers.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Helpers.coffee index 87a41f06d3..c8bcbdb6ab 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Helpers.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Helpers.coffee @@ -33,14 +33,11 @@ define [ commandName = Helpers.getCommandNameFromFragment(commandFragment) beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999) lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange) - needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/) - closingBrace = if needsClosingBrace then '}' else '' return { lineUpToCursor, commandFragment, commandName, - lineBeyondCursor, - closingBrace + lineBeyondCursor } return Helpers diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/PackageManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/PackageManager.coffee index 4ab1b6933c..12bcc26a8f 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/PackageManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/PackageManager.coffee @@ -1,6 +1,4 @@ -define [ - "./Helpers" -], (Helpers) -> +define [], () -> packages = [ 'inputenc', 'graphicx', 'amsmath', 'geometry', 'amssymb', 'hyperref', 'babel', 'color', 'xcolor', 'url', 'natbib', 'fontenc', 'fancyhdr', @@ -26,14 +24,13 @@ define [ constructor: (@metadataManager) -> getCompletions: (editor, session, pos, prefix, callback) -> - {closingBrace} = Helpers.getContext(editor, pos) usedPackages = Object.keys(@metadataManager.getAllPackages()) packageSnippets = [] for pkg in packages if pkg not in usedPackages packageSnippets.push { - caption: "\\usepackage{#{pkg}#{closingBrace}" - snippet: "\\usepackage{#{pkg}#{closingBrace}" + caption: "\\usepackage{#{pkg}}" + snippet: "\\usepackage{#{pkg}}" meta: "pkg" } From 0c86a7dc9b89773858b823c3196ae7beb9809fa2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 19 Jun 2018 08:43:27 +0100 Subject: [PATCH 14/63] Revert "Simplify package metadata check" --- .../aceEditor/metadata/MetadataManager.coffee | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee index 1ff12efc38..c9308626bc 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/metadata/MetadataManager.coffee @@ -18,15 +18,41 @@ define [ return if change.action not in ['remove', 'insert'] return - + cursorPosition = @editor.getCursorPosition() end = change.end - lineText = @editor.getSession().getLine end.row + range = new Range(end.row, 0, end.row, end.column) + lineUpToCursor = @editor.getSession().getTextRange range + if lineUpToCursor.trim() == '%' or lineUpToCursor.slice(0, 1) == '\\' + range = new Range(end.row, 0, end.row, end.column + 80) + lineUpToCursor = @editor.getSession().getTextRange range + commandFragment = getLastCommandFragment lineUpToCursor - # Defensive check against extremely long lines - return if lineText.length > 10000 + linesContainPackage = _.any( + change.lines, + (line) -> line.match(/^\\usepackage(?:\[.{0,80}?])?{(.{0,80}?)}/) + ) + linesContainReqPackage = _.any( + change.lines, + (line) -> line.match(/^\\RequirePackage(?:\[.{0,80}?])?{(.{0,80}?)}/) + ) + linesContainLabel = _.any( + change.lines, + (line) -> line.match(/\\label{(.{0,80}?)}/) + ) + linesContainMeta = + linesContainPackage or + linesContainLabel or + linesContainReqPackage - # Check if edited line contains metadata commands - if /\\(usepackage|RequirePackage|label)(\[.*])?({.*})?/.test(lineText) + lastCommandFragmentIsLabel = commandFragment?.slice(0, 7) == '\\label{' + lastCommandFragmentIsPackage = commandFragment?.slice(0, 11) == '\\usepackage' + lastCommandFragmentIsReqPack = commandFragment?.slice(0, 15) == '\\RequirePackage' + lastCommandFragmentIsMeta = + lastCommandFragmentIsPackage or + lastCommandFragmentIsLabel or + lastCommandFragmentIsReqPack + + if linesContainMeta or lastCommandFragmentIsMeta @scheduleLoadCurrentDocMetaFromServer() @editor.on "changeSession", (e) => From 024663144317e7351a0f7f793cbe5b2045e4aa5a Mon Sep 17 00:00:00 2001 From: Michael Mazour Date: Wed, 13 Jun 2018 15:30:46 +0100 Subject: [PATCH 15/63] Add public API endpoints to reach CLSIs - `/api/clsi/compile/:submission_id` - `/api/clsi/compile/:submission_id/build/:build_id/output/:file` Also per review: - DRY up ClsiManager.sendRequestOnce and ClsiManager.sendExternalRequest - Include submission_id in a log message - Don't include timeout in limits when getting file --- .../Features/Compile/ClsiManager.coffee | 51 +++++++---- .../Features/Compile/CompileController.coffee | 33 +++++++ services/web/app/coffee/router.coffee | 16 ++++ .../acceptance/coffee/ApiClsiTests.coffee | 88 ++++++++++++++++++ .../coffee/helpers/MockClsiApi.coffee | 39 ++++++++ .../coffee/Compile/ClsiManagerTests.coffee | 87 ++++++++++++++++-- .../Compile/CompileControllerTests.coffee | 91 +++++++++++++++++++ 7 files changed, 378 insertions(+), 27 deletions(-) create mode 100644 services/web/test/acceptance/coffee/ApiClsiTests.coffee create mode 100644 services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index c27bc2fe90..e5158cdb9b 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -14,6 +14,7 @@ async = require("async") ClsiFormatChecker = require("./ClsiFormatChecker") DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler" Metrics = require('metrics-sharelatex') +Errors = require ('../Errors/Errors') module.exports = ClsiManager = @@ -35,24 +36,16 @@ module.exports = ClsiManager = else return callback(error) logger.log project_id: project_id, "sending compile to CLSI" - ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)-> - if err? - logger.err err, project_id, "could not check resources for potential problems before sending to clsi" - return callback(err) - if validationProblems? - logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted" - return callback(null, "validation-problems", null, null, validationProblems) - ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) -> - if error? - logger.err err:error, project_id:project_id, "error sending request to clsi" - return callback(error) - logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI" - ClsiCookieManager._getServerId project_id, (err, clsiServerId)-> - if err? - logger.err err:err, project_id:project_id, "error getting server id" - return callback(err) - outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles) - callback(null, response?.compile?.status, outputFiles, clsiServerId) + ClsiManager._sendBuiltRequest project_id, user_id, req, options, (error, status, result...) -> + return callback(error) if error? + callback(error, status, result...) + + # for public API requests where there is no project id + sendExternalRequest: (submission_id, clsi_request, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) -> + logger.log submission_id: submission_id, "sending external compile to CLSI", clsi_request + ClsiManager._sendBuiltRequest submission_id, null, clsi_request, options, (error, status, result...) -> + return callback(error) if error? + callback(error, status, result...) stopCompile: (project_id, user_id, options, callback = (error) ->) -> compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id, "compile/stop") @@ -74,6 +67,26 @@ module.exports = ClsiManager = return callback(error) if error? callback() + _sendBuiltRequest: (project_id, user_id, req, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) -> + ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)-> + if err? + logger.err err, project_id, "could not check resources for potential problems before sending to clsi" + return callback(err) + if validationProblems? + logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted" + return callback(null, "validation-problems", null, null, validationProblems) + ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) -> + if error? + logger.err err:error, project_id:project_id, "error sending request to clsi" + return callback(error) + logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI" + ClsiCookieManager._getServerId project_id, (err, clsiServerId)-> + if err? + logger.err err:err, project_id:project_id, "error getting server id" + return callback(err) + outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles) + callback(null, response?.compile?.status, outputFiles, clsiServerId) + _makeRequest: (project_id, opts, callback)-> ClsiCookieManager.getCookieJar project_id, (err, jar)-> if err? @@ -87,7 +100,7 @@ module.exports = ClsiManager = ClsiCookieManager.setServerId project_id, response, (err)-> if err? logger.warn err:err, project_id:project_id, "error setting server id" - + return callback err, response, body _getCompilerUrl: (compileGroup, project_id, user_id, action) -> diff --git a/services/web/app/coffee/Features/Compile/CompileController.coffee b/services/web/app/coffee/Features/Compile/CompileController.coffee index decfac515b..ee3e20fdc9 100755 --- a/services/web/app/coffee/Features/Compile/CompileController.coffee +++ b/services/web/app/coffee/Features/Compile/CompileController.coffee @@ -55,6 +55,33 @@ module.exports = CompileController = return next(error) if error? res.status(200).send() + # Used for submissions through the public API + compileSubmission: (req, res, next = (error) ->) -> + res.setTimeout(5 * 60 * 1000) + submission_id = req.params.submission_id + options = {} + if req.body?.rootResourcePath? + options.rootResourcePath = req.body.rootResourcePath + if req.body?.compiler + options.compiler = req.body.compiler + if req.body?.draft + options.draft = req.body.draft + if req.body?.check in ['validate', 'error', 'silent'] + options.check = req.body.check + options.compileGroup = req.body?.compileGroup || Settings.defaultFeatures.compileGroup + options.timeout = req.body?.timeout || Settings.defaultFeatures.compileTimeout + logger.log {options:options, submission_id:submission_id}, "got compileSubmission request" + ClsiManager.sendExternalRequest submission_id, req.body, options, (error, status, outputFiles, clsiServerId, validationProblems) -> + return next(error) if error? + logger.log {submission_id:submission_id, files:outputFiles}, "compileSubmission output files" + res.contentType("application/json") + res.status(200).send JSON.stringify { + status: status + outputFiles: outputFiles + clsiServerId: clsiServerId + validationProblems: validationProblems + } + _compileAsUser: (req, callback) -> # callback with user_id if per-user, undefined otherwise if not Settings.disablePerUserCompiles @@ -139,6 +166,12 @@ module.exports = CompileController = url = CompileController._getFileUrl project_id, user_id, req.params.build_id, req.params.file CompileController.proxyToClsi(project_id, url, req, res, next) + getFileFromClsiWithoutUser: (req, res, next = (error) ->) -> + submission_id = req.params.submission_id + url = CompileController._getFileUrl submission_id, null, req.params.build_id, req.params.file + limits = { compileGroup: req.body?.compileGroup || Settings.defaultFeatures.compileGroup } + CompileController.proxyToClsiWithLimits(submission_id, url, limits, req, res, next) + # compute a GET file url for a given project, user (optional), build (optional) and file _getFileUrl: (project_id, user_id, build_id, file) -> if user_id? and build_id? diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index 339b4abe1c..0abc170589 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -295,6 +295,22 @@ module.exports = class Router webRouter.post "/confirm-password", AuthenticationController.requireLogin(), SudoModeController.submitPassword + # New "api" endpoints. Started as a way for v1 to call over to v2 (for + # long-term features, as opposed to the nominally temporary ones in the + # overleaf-integration module), but may expand beyond that role. + publicApiRouter.post '/api/clsi/compile/:submission_id', AuthenticationController.httpAuth, CompileController.compileSubmission + publicApiRouter.get /^\/api\/clsi\/compile\/([^\/]*)\/build\/([0-9a-f-]+)\/output\/(.*)$/, + ((req, res, next) -> + params = + "submission_id": req.params[0] + "build_id": req.params[1] + "file": req.params[2] + req.params = params + next() + ), + AuthenticationController.httpAuth, + CompileController.getFileFromClsiWithoutUser + #Admin Stuff webRouter.get '/admin', AuthorizationMiddlewear.ensureUserIsSiteAdmin, AdminController.index webRouter.get '/admin/user', AuthorizationMiddlewear.ensureUserIsSiteAdmin, (req, res)-> res.redirect("/admin/register") #this gets removed by admin-panel addon diff --git a/services/web/test/acceptance/coffee/ApiClsiTests.coffee b/services/web/test/acceptance/coffee/ApiClsiTests.coffee new file mode 100644 index 0000000000..d19e11f933 --- /dev/null +++ b/services/web/test/acceptance/coffee/ApiClsiTests.coffee @@ -0,0 +1,88 @@ +expect = require("chai").expect +request = require './helpers/request' +Settings = require "settings-sharelatex" + +auth = new Buffer('sharelatex:password').toString("base64") +authed_request = request.defaults + headers: + Authorization: "Basic #{auth}" + + +describe 'ApiClsiTests', -> + describe 'compile', -> + before (done) -> + @compileSpec = + compile: + options: + compiler: 'pdflatex' + timeout: 60 + rootResourcePath: 'main.tex' + resources: [ + path: 'main/tex' + content: "\\documentclass{article}\n\\begin{document}\nHello World\n\\end{document}" + , + path: 'image.png' + url: 'www.example.com/image.png' + modified: 123456789 + ] + done() + + describe 'valid request', -> + it 'returns success and a list of output files', (done) -> + authed_request.post { + uri: '/api/clsi/compile/abcd' + json: @compileSpec + }, (error, response, body) -> + throw error if error? + expect(response.statusCode).to.equal 200 + expect(response.body).to.deep.equal { + status: 'success' + outputFiles: [ + path: 'project.pdf' + url: '/project/abcd/build/1234/output/project.pdf' + type: 'pdf' + build: 1234 + , + path: 'project.log' + url: '/project/abcd/build/1234/output/project.log' + type: 'log' + build: 1234 + ] + } + done() + + describe 'unauthorized', -> + it 'returns 401', (done) -> + request.post { + uri: '/api/clsi/compile/abcd' + json: @compileSpec + }, (error, response, body) -> + throw error if error? + expect(response.statusCode).to.equal 401 + expect(response.body).to.equal 'Unauthorized' + done() + + describe 'get output', -> + describe 'valid file', -> + it 'returns the file', (done) -> + authed_request.get '/api/clsi/compile/abcd/build/1234/output/project.pdf', (error, response, body) -> + throw error if error? + expect(response.statusCode).to.equal 200 + expect(response.body).to.equal 'mock-pdf' + done() + + describe 'invalid file', -> + it 'returns 404', (done) -> + authed_request.get '/api/clsi/compile/abcd/build/1234/output/project.aux', (error, response, body) -> + throw error if error? + expect(response.statusCode).to.equal 404 + expect(response.body).to.not.equal 'mock-pdf' + done() + + describe 'unauthorized', -> + it 'returns 401', (done) -> + request.get '/api/clsi/compile/abcd/build/1234/output/project.pdf', (error, response, body) -> + throw error if error? + expect(response.statusCode).to.equal 401 + expect(response.body).to.not.equal 'mock-pdf' + done() diff --git a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee new file mode 100644 index 0000000000..2f9180960d --- /dev/null +++ b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee @@ -0,0 +1,39 @@ +express = require("express") +app = express() + +module.exports = MockClsiApi = + run: () -> + app.post "/project/:project_id/compile", (req, res, next) => + res.status(200).send { + compile: + status: 'success' + error: null + outputFiles: [ + url: "/project/#{req.params.project_id}/build/1234/output/project.pdf" + path: 'project.pdf' + type: 'pdf' + build: 1234 + , + url: "/project/#{req.params.project_id}/build/1234/output/project.log" + path: 'project.log' + type: 'log' + build: 1234 + ] + } + + app.get "/project/:project_id/build/:build_id/output/*", (req, res, next) -> + filename = req.params[0] + if filename == 'project.pdf' + res.status(200).send 'mock-pdf' + else if filename == 'project.log' + res.status(200).send 'mock-log' + else + res.sendStatus(404) + + app.listen 3013, (error) -> + throw error if error? + .on "error", (error) -> + console.error "error starting MockClsiApi:", error.message + process.exit(1) + +MockClsiApi.run() diff --git a/services/web/test/unit/coffee/Compile/ClsiManagerTests.coffee b/services/web/test/unit/coffee/Compile/ClsiManagerTests.coffee index bd02af2f28..c8c39d9952 100644 --- a/services/web/test/unit/coffee/Compile/ClsiManagerTests.coffee +++ b/services/web/test/unit/coffee/Compile/ClsiManagerTests.coffee @@ -8,7 +8,7 @@ SandboxedModule = require('sandboxed-module') describe "ClsiManager", -> beforeEach -> @jar = {cookie:"stuff"} - @ClsiCookieManager = + @ClsiCookieManager = getCookieJar: sinon.stub().callsArgWith(1, null, @jar) setServerId: sinon.stub().callsArgWith(2) _getServerId:sinon.stub() @@ -99,8 +99,8 @@ describe "ClsiManager", -> status: @status = "failure" }) @ClsiManager.sendRequest @project_id, @user_id, {}, @callback - - it "should call the callback with a failure statue", -> + + it "should call the callback with a failure status", -> @callback.calledWith(null, @status).should.equal true describe "with a sync conflict", -> @@ -137,11 +137,82 @@ describe "ClsiManager", -> it "should call the callback with an error", -> @callback.calledWithExactly(new Error("failed")).should.equal true + describe "sendExternalRequest", -> + beforeEach -> + @submission_id = "submission-id" + @clsi_request = "mock-request" + @ClsiCookieManager._getServerId.callsArgWith(1, null, "clsi3") + + describe "with a successful compile", -> + beforeEach -> + @ClsiManager._postToClsi = sinon.stub().callsArgWith(4, null, { + compile: + status: @status = "success" + outputFiles: [{ + url: "#{@settings.apis.clsi.url}/project/#{@submission_id}/build/1234/output/output.pdf" + path: "output.pdf" + type: "pdf" + build: 1234 + },{ + url: "#{@settings.apis.clsi.url}/project/#{@submission_id}/build/1234/output/output.log" + path: "output.log" + type: "log" + build: 1234 + }] + }) + @ClsiManager.sendExternalRequest @submission_id, @clsi_request, {compileGroup:"standard"}, @callback + + it "should send the request to the CLSI", -> + @ClsiManager._postToClsi + .calledWith(@submission_id, null, @clsi_request, "standard") + .should.equal true + + it "should call the callback with the status and output files", -> + outputFiles = [{ + url: "/project/#{@submission_id}/build/1234/output/output.pdf" + path: "output.pdf" + type: "pdf" + build: 1234 + },{ + url: "/project/#{@submission_id}/build/1234/output/output.log" + path: "output.log" + type: "log" + build: 1234 + }] + @callback.calledWith(null, @status, outputFiles).should.equal true + + describe "with a failed compile", -> + beforeEach -> + @ClsiManager._postToClsi = sinon.stub().callsArgWith(4, null, { + compile: + status: @status = "failure" + }) + @ClsiManager.sendExternalRequest @submission_id, @clsi_request, {}, @callback + + it "should call the callback with a failure status", -> + @callback.calledWith(null, @status).should.equal true + + describe "when the resources fail the precompile check", -> + beforeEach -> + @ClsiFormatChecker.checkRecoursesForProblems = sinon.stub().callsArgWith(1, new Error("failed")) + @ClsiManager._postToClsi = sinon.stub().callsArgWith(4, null, { + compile: + status: @status = "failure" + }) + @ClsiManager.sendExternalRequest @submission_id, @clsi_request, {}, @callback + + it "should call the callback only once", -> + @callback.calledOnce.should.equal true + + it "should call the callback with an error", -> + @callback.calledWithExactly(new Error("failed")).should.equal true + + describe "deleteAuxFiles", -> beforeEach -> @ClsiManager._makeRequest = sinon.stub().callsArg(2) @DocumentUpdaterHandler.clearProjectState = sinon.stub().callsArg(1) - + describe "with the standard compileGroup", -> beforeEach -> @ClsiManager.deleteAuxFiles @project_id, @user_id, {compileGroup: "standard"}, @callback @@ -158,7 +229,7 @@ describe "ClsiManager", -> it "should call the callback", -> @callback.called.should.equal true - + describe "_buildRequest", -> beforeEach -> @@ -441,7 +512,7 @@ describe "ClsiManager", -> it "should call the callback", -> @callback.called.should.equal true - + describe "with param file", -> beforeEach -> @ClsiManager.wordCount @project_id, @user_id, "main.tex", {}, @callback @@ -450,7 +521,7 @@ describe "ClsiManager", -> @ClsiManager._makeRequest .calledWith(@project_id, { method: "GET", url: "http://clsi.example.com/project/#{@project_id}/user/#{@user_id}/wordcount", qs:{file:"main.tex",image:undefined}}) .should.equal true - + describe "with image", -> beforeEach -> @req.compile.options.imageName = @image = "example.com/mock/image" @@ -468,7 +539,7 @@ describe "ClsiManager", -> beforeEach -> @response = {there:"something"} @request.callsArgWith(1, null, @response) - @opts = + @opts = method: "SOMETHIGN" url: "http://a place on the web" diff --git a/services/web/test/unit/coffee/Compile/CompileControllerTests.coffee b/services/web/test/unit/coffee/Compile/CompileControllerTests.coffee index 8a09ac866e..ced3260412 100644 --- a/services/web/test/unit/coffee/Compile/CompileControllerTests.coffee +++ b/services/web/test/unit/coffee/Compile/CompileControllerTests.coffee @@ -29,6 +29,9 @@ describe "CompileController", -> url: "clsi.example.com" clsi_priority: url: "clsi-priority.example.com" + defaultFeatures: + compileGroup: 'standard' + compileTimeout: 60 @jar = {cookie:"stuff"} @ClsiCookieManager = getCookieJar:sinon.stub().callsArgWith(1, null, @jar) @@ -109,6 +112,62 @@ describe "CompileController", -> .calledWith(@project_id, @user_id, { isAutoCompile: false, draft: true }) .should.equal true + describe "compileSubmission", -> + beforeEach -> + @submission_id = 'sub-1234' + @req.params = + submission_id: @submission_id + @req.body = {} + @ClsiManager.sendExternalRequest = sinon.stub() + .callsArgWith(3, null, @status = "success", @outputFiles = ["mock-output-files"], \ + @clsiServerId = "mock-server-id", @validationProblems = null) + + it "should set the content-type of the response to application/json", -> + @CompileController.compileSubmission @req, @res, @next + @res.contentType + .calledWith("application/json") + .should.equal true + + it "should send a successful response reporting the status and files", -> + @CompileController.compileSubmission @req, @res, @next + @res.statusCode.should.equal 200 + @res.body.should.equal JSON.stringify({ + status: @status + outputFiles: @outputFiles + clsiServerId: 'mock-server-id' + validationProblems: null + }) + + describe "with compileGroup and timeout", -> + beforeEach -> + @req.body = + compileGroup: 'special' + timeout: 600 + @CompileController.compileSubmission @req, @res, @next + + it "should use the supplied values", -> + @ClsiManager.sendExternalRequest + .calledWith(@submission_id, { compileGroup: 'special', timeout: 600 }, \ + { compileGroup: 'special', timeout: 600 }) + .should.equal true + + describe "with other supported options but not compileGroup and timeout", -> + beforeEach -> + @req.body = + rootResourcePath: 'main.tex' + compiler: 'lualatex' + draft: true + check: 'validate' + @CompileController.compileSubmission @req, @res, @next + + it "should use the other options but default values for compileGroup and timeout", -> + @ClsiManager.sendExternalRequest + .calledWith(@submission_id, \ + {rootResourcePath: 'main.tex', compiler: 'lualatex', draft: true, check: 'validate'}, \ + {rootResourcePath: 'main.tex', compiler: 'lualatex', draft: true, check: 'validate', \ + compileGroup: 'standard', timeout: 60}) + .should.equal true + describe "downloadPdf", -> beforeEach -> @req.params = @@ -167,6 +226,38 @@ describe "CompileController", -> done() @CompileController.downloadPdf @req, @res + describe "getFileFromClsiWithoutUser", -> + beforeEach -> + @submission_id = 'sub-1234' + @build_id = 123456 + @file = 'project.pdf' + @req.params = + submission_id: @submission_id + build_id: @build_id + file: @file + @req.body = {} + @expected_url = "/project/#{@submission_id}/build/#{@build_id}/output/#{@file}" + @CompileController.proxyToClsiWithLimits = sinon.stub() + + describe "without limits specified", -> + beforeEach -> + @CompileController.getFileFromClsiWithoutUser @req, @res, @next + + it "should proxy to CLSI with correct URL and default limits", -> + @CompileController.proxyToClsiWithLimits + .calledWith(@submission_id, @expected_url, {compileGroup: 'standard'}) + .should.equal true + + describe "with limits specified", -> + beforeEach -> + @req.body = {compileTimeout: 600, compileGroup: 'special'} + @CompileController.getFileFromClsiWithoutUser @req, @res, @next + + it "should proxy to CLSI with correct URL and specified limits", -> + @CompileController.proxyToClsiWithLimits + .calledWith(@submission_id, @expected_url, {compileGroup: 'special'}) + .should.equal true + describe "proxyToClsi", -> beforeEach -> @request.returns(@proxy = { From 99728d8f18022f9a9d14abea61656093ccd3d50b Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 19 Jun 2018 13:55:01 +0100 Subject: [PATCH 16/63] Bail out with exit code on acceptance test failure --- services/web/Makefile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/services/web/Makefile b/services/web/Makefile index 474155a478..26534b8612 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -196,8 +196,11 @@ test_frontend: test_clean # stop service test_acceptance: test_acceptance_app test_acceptance_modules -test_acceptance_app: test_acceptance_app_start_service test_acceptance_app_run - $(MAKE) test_acceptance_app_stop_service +test_acceptance_app: + @set -e; \ + $(MAKE) test_acceptance_app_start_service; \ + $(MAKE) test_acceptance_app_run; \ + $(MAKE) test_acceptance_app_stop_service; test_acceptance_app_start_service: test_clean # stop service and clear dbs $(MAKE) compile @@ -208,10 +211,12 @@ test_acceptance_app_stop_service: test_acceptance_app_run: @docker-compose ${DOCKER_COMPOSE_FLAGS} exec -T test_acceptance npm -q run test:acceptance -- ${MOCHA_ARGS}; \ - if [ $$? -eq 137 ]; then \ - echo "\nOh dear, it looks like the web process crashed! To see why, run:\n\n\tdocker-compose logs test_acceptance\n"; \ - exit 1; \ - fi + result=$$?; \ + if [ $$result -eq 137 ]; then \ + docker-compose logs --tail=50 test_acceptance; \ + echo "\nOh dear, it looks like the web process crashed! Some logs are above, but to see them all, run:\n\n\tdocker-compose logs test_acceptance\n"; \ + fi; \ + exit $$result test_acceptance_modules: @set -e; \ From 8492373a14d8a1aa015989697bf4e9df03cb9c94 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Tue, 19 Jun 2018 16:24:27 +0100 Subject: [PATCH 17/63] Add V1ConnectionError --- services/web/app/coffee/Features/Errors/Errors.coffee | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/web/app/coffee/Features/Errors/Errors.coffee b/services/web/app/coffee/Features/Errors/Errors.coffee index af1e437d4b..25b3f3e06d 100644 --- a/services/web/app/coffee/Features/Errors/Errors.coffee +++ b/services/web/app/coffee/Features/Errors/Errors.coffee @@ -61,6 +61,13 @@ ProjectHistoryDisabledError = (message) -> return error ProjectHistoryDisabledError.prototype.__proto___ = Error.prototype +V1ConnectionError = (message) -> + error = new Error(message) + error.name = "V1ConnectionError" + error.__proto__ = V1ConnectionError.prototype + return error +V1ConnectionError.prototype.__proto___ = Error.prototype + module.exports = Errors = NotFoundError: NotFoundError ServiceNotConfiguredError: ServiceNotConfiguredError @@ -71,3 +78,4 @@ module.exports = Errors = UnsupportedExportRecordsError: UnsupportedExportRecordsError V1HistoryNotSyncedError: V1HistoryNotSyncedError ProjectHistoryDisabledError: ProjectHistoryDisabledError + V1ConnectionError: V1ConnectionError From 82a8e370710724b5239af4dc62b4ee513d7cd917 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Tue, 19 Jun 2018 16:25:31 +0100 Subject: [PATCH 18/63] Handle V1 connection refused when getting v1 subscription --- .../app/coffee/Features/Project/ProjectController.coffee | 6 +++++- .../Features/Subscription/V1SubscriptionManager.coffee | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/Features/Project/ProjectController.coffee b/services/web/app/coffee/Features/Project/ProjectController.coffee index 8ea2dc3189..05ebb9f108 100644 --- a/services/web/app/coffee/Features/Project/ProjectController.coffee +++ b/services/web/app/coffee/Features/Project/ProjectController.coffee @@ -27,6 +27,7 @@ CollaboratorsHandler = require '../Collaborators/CollaboratorsHandler' Modules = require '../../infrastructure/Modules' ProjectEntityHandler = require './ProjectEntityHandler' crypto = require 'crypto' +{ V1ConnectionError } = require '../Errors/Errors' module.exports = ProjectController = @@ -183,7 +184,10 @@ module.exports = ProjectController = return cb(null, projects: [], tags: [], noConnection: true) return cb(error, projects[0]) # hooks.fire returns an array of results, only need first hasSubscription: (cb)-> - LimitationsManager.userHasSubscriptionOrIsGroupMember currentUser, cb + LimitationsManager.userHasSubscriptionOrIsGroupMember currentUser, (error, hasSub) -> + if error? and error instanceof V1ConnectionError + return cb(null, true) + return cb(error, hasSub) user: (cb) -> User.findById user_id, "featureSwitches overleaf awareOfV2 features", cb }, (err, results)-> diff --git a/services/web/app/coffee/Features/Subscription/V1SubscriptionManager.coffee b/services/web/app/coffee/Features/Subscription/V1SubscriptionManager.coffee index dcf23a3453..857881df4d 100644 --- a/services/web/app/coffee/Features/Subscription/V1SubscriptionManager.coffee +++ b/services/web/app/coffee/Features/Subscription/V1SubscriptionManager.coffee @@ -2,6 +2,7 @@ UserGetter = require "../User/UserGetter" request = require "request" settings = require "settings-sharelatex" logger = require "logger-sharelatex" +{ V1ConnectionError } = require "../Errors/Errors" module.exports = V1SubscriptionManager = # Returned planCode = 'v1_pro' | 'v1_pro_plus' | 'v1_student' | 'v1_free' | null @@ -58,7 +59,10 @@ module.exports = V1SubscriptionManager = json: true, timeout: 5 * 1000 }, (error, response, body) -> - return callback(error) if error? + if error? + # Specially handle no connection err, so warning can be shown + error = new V1ConnectionError('No V1 connection') if error.code == 'ECONNREFUSED' + return callback(error) if 200 <= response.statusCode < 300 return callback null, body else From 8de9e9fae43f234d45e51e9f2720e835de028630 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Tue, 19 Jun 2018 16:25:56 +0100 Subject: [PATCH 19/63] Use error type instead of error message to check v1 connection error --- .../web/app/coffee/Features/Project/ProjectController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Project/ProjectController.coffee b/services/web/app/coffee/Features/Project/ProjectController.coffee index 05ebb9f108..1b7051e39a 100644 --- a/services/web/app/coffee/Features/Project/ProjectController.coffee +++ b/services/web/app/coffee/Features/Project/ProjectController.coffee @@ -180,7 +180,7 @@ module.exports = ProjectController = ProjectGetter.findAllUsersProjects user_id, 'name lastUpdated publicAccesLevel archived owner_ref tokens', cb v1Projects: (cb) -> Modules.hooks.fire "findAllV1Projects", user_id, (error, projects = []) -> - if error? and error.message == 'No V1 connection' + if error? and error instanceof V1ConnectionError return cb(null, projects: [], tags: [], noConnection: true) return cb(error, projects[0]) # hooks.fire returns an array of results, only need first hasSubscription: (cb)-> From bbed5fca9ab6f140b339562df56132426e226ad3 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Wed, 20 Jun 2018 10:58:19 +0200 Subject: [PATCH 20/63] simplify proxy --- .../coffee/infrastructure/ProxyManager.coffee | 20 +------------------ .../infrastructure/ProxyManagerTests.coffee | 14 +++++-------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/services/web/app/coffee/infrastructure/ProxyManager.coffee b/services/web/app/coffee/infrastructure/ProxyManager.coffee index 45ca6b7a48..d93c696ab1 100644 --- a/services/web/app/coffee/infrastructure/ProxyManager.coffee +++ b/services/web/app/coffee/infrastructure/ProxyManager.coffee @@ -18,26 +18,8 @@ module.exports = # - an Object with: # - a path attribute (String) # - a baseURL attribute (String) - # - a baseURL attribute (Object) with: - # - a setting attribute pointing to a value in the settings makeTargetUrl: (target) -> return target if typeof target is 'string' return target.path unless target.baseUrl? + "#{target.baseUrl}#{target.path or ''}" - if typeof target.baseUrl is 'string' - baseUrl = target.baseUrl - else if target.baseUrl.setting? - baseUrl = digSettingValue target.baseUrl.setting - - return null unless baseUrl? - "#{baseUrl}#{target.path}" - -# given a setting path (e.g. 'apis.v1.url') recursively find the corresponding -# settings value -digSettingValue = (attributesPath, dig = null) -> - dig ||= settings - [nextAttribute, leftAttributes...] = attributesPath.split('.') - dig = dig[nextAttribute] - return null unless dig? - return dig if leftAttributes.length == 0 - digSettingValue(leftAttributes.join('.'), dig) diff --git a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee index 3c3c4c7ccd..5782a4e2fc 100644 --- a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee +++ b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee @@ -47,18 +47,14 @@ describe "ProxyManager", -> target = 'http://whatever' @proxyManager.makeTargetUrl(target).should.equal target - it 'makes from object', -> + it 'makes with path', -> target = path: 'baz' @proxyManager.makeTargetUrl(target).should.equal 'baz' it 'makes with baseUrl', -> + target = baseUrl: 'foo.bar' + @proxyManager.makeTargetUrl(target).should.equal 'foo.bar' + + it 'makes with baseUrl and path', -> target = path: 'baz', baseUrl: 'foo.bar/' @proxyManager.makeTargetUrl(target).should.equal 'foo.bar/baz' - - it 'makes with settingsUrl', -> - @settings.apis = v1: url: 'foo.bar/' - target = path: 'baz', baseUrl: { setting: 'apis.v1.url' } - @proxyManager.makeTargetUrl(target).should.equal 'foo.bar/baz' - - target = path: 'baz', baseUrl: { setting: 'incorrect.setting' } - expect(@proxyManager.makeTargetUrl(target)).to.equal null From 6c1994e25b815a1970e6c64d292eb8289703d14d Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Wed, 20 Jun 2018 11:19:23 +0100 Subject: [PATCH 21/63] Stub errors in tests to cache --- .../web/test/unit/coffee/Project/ProjectControllerTests.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee b/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee index 1b66ba9bf9..8de7c6aed1 100644 --- a/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee +++ b/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee @@ -5,6 +5,7 @@ path = require('path') sinon = require('sinon') modulePath = path.join __dirname, "../../../../app/js/Features/Project/ProjectController" expect = require("chai").expect +Errors = require "../../../../app/js/Features/Errors/Errors" describe "ProjectController", -> @@ -100,6 +101,7 @@ describe "ProjectController", -> "../Collaborators/CollaboratorsHandler": @CollaboratorsHandler "../../infrastructure/Modules": @Modules "./ProjectEntityHandler": @ProjectEntityHandler + "../Errors/Errors": Errors @projectName = "£12321jkj9ujkljds" @req = From 74ca0c4220ed241f16a711bcba12188185bda8b0 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 19 Jun 2018 13:52:56 +0100 Subject: [PATCH 22/63] Refactor email template system --- .../Email/Bodies/SingleCTAEmailBody.coffee | 16 +- .../Email/Bodies/ol-SingleCTAEmailBody.coffee | 16 +- .../coffee/Features/Email/EmailBuilder.coffee | 236 +++++++----------- 3 files changed, 110 insertions(+), 158 deletions(-) diff --git a/services/web/app/coffee/Features/Email/Bodies/SingleCTAEmailBody.coffee b/services/web/app/coffee/Features/Email/Bodies/SingleCTAEmailBody.coffee index cf4adf8cd7..bffa485f07 100644 --- a/services/web/app/coffee/Features/Email/Bodies/SingleCTAEmailBody.coffee +++ b/services/web/app/coffee/Features/Email/Bodies/SingleCTAEmailBody.coffee @@ -4,13 +4,17 @@ settings = require "settings-sharelatex" module.exports = _.template """
-

- <%= title %> -

+ <% if (title) { %> +

+ <%= title %> +

+ <% } %>
 
-

- <%= greeting %> -

+ <% if (greeting) { %> +

+ <%= greeting %> +

+ <% } %>

<%= message %>

diff --git a/services/web/app/coffee/Features/Email/Bodies/ol-SingleCTAEmailBody.coffee b/services/web/app/coffee/Features/Email/Bodies/ol-SingleCTAEmailBody.coffee index 45bc00383c..ee9907c019 100644 --- a/services/web/app/coffee/Features/Email/Bodies/ol-SingleCTAEmailBody.coffee +++ b/services/web/app/coffee/Features/Email/Bodies/ol-SingleCTAEmailBody.coffee @@ -4,13 +4,17 @@ settings = require "settings-sharelatex" module.exports = _.template """
-

- <%= title %> -

+ <% if (title) { %> +

+ <%= title %> +

+ <% } %>
 
-

- <%= greeting %> -

+ <% if (greeting) { %> +

+ <%= greeting %> +

+ <% } %>

<%= message %>

diff --git a/services/web/app/coffee/Features/Email/EmailBuilder.coffee b/services/web/app/coffee/Features/Email/EmailBuilder.coffee index 8952e7b864..a33f02e695 100644 --- a/services/web/app/coffee/Features/Email/EmailBuilder.coffee +++ b/services/web/app/coffee/Features/Email/EmailBuilder.coffee @@ -1,5 +1,6 @@ _ = require('underscore') settings = require("settings-sharelatex") +marked = require('marked') PersonalEmailLayout = require("./Layouts/PersonalEmailLayout") NotificationEmailLayout = require("./Layouts/NotificationEmailLayout") @@ -7,169 +8,113 @@ BaseWithHeaderEmailLayout = require("./Layouts/" + settings.brandPrefix + "BaseW SingleCTAEmailBody = require("./Bodies/" + settings.brandPrefix + "SingleCTAEmailBody") +CTAEmailTemplate = (content) -> + content.greeting ?= () -> 'Hi,' + content.secondaryMessage ?= () -> "" + return { + subject: (opts) -> content.subject(opts), + layout: BaseWithHeaderEmailLayout, + plainTextTemplate: (opts) -> """ +#{content.greeting(opts)} + +#{content.message(opts).trim()} + +#{content.ctaText(opts)}: #{content.ctaURL(opts)} + +#{content.secondaryMessage?(opts).trim() or ""} + +Regards, +The #{settings.appName} Team - #{settings.siteUrl} + """ + compiledTemplate: (opts) -> + SingleCTAEmailBody({ + title: content.title?(opts) + greeting: content.greeting(opts) + message: marked(content.message(opts).trim()) + secondaryMessage: marked(content.secondaryMessage(opts).trim()) + ctaText: content.ctaText(opts) + ctaURL: content.ctaURL(opts) + gmailGoToAction: content.gmailGoToAction?(opts) + }) + } + templates = {} -templates.registered = - subject: _.template "Activate your #{settings.appName} Account" - layout: PersonalEmailLayout - type: "notification" - plainTextTemplate: _.template """ -Congratulations, you've just had an account created for you on #{settings.appName} with the email address "<%= to %>". +templates.registered = CTAEmailTemplate({ + subject: () -> "Activate your #{settings.appName} Account" + message: (opts) -> """ +Congratulations, you've just had an account created for you on #{settings.appName} with the email address '#{opts.to}'. -Click here to set your password and log in: <%= setNewPasswordUrl %> - -If you have any questions or problems, please contact #{settings.adminEmail} +Click here to set your password and log in: """ - compiledTemplate: _.template """ -

Congratulations, you've just had an account created for you on #{settings.appName} with the email address "<%= to %>".

+ secondaryMessage: () -> "If you have any questions or problems, please contact #{settings.adminEmail}" + ctaText: () -> "Set password" + ctaURL: (opts) -> opts.setNewPasswordUrl +}) -

Click here to set your password and log in.

- -

If you have any questions or problems, please contact #{settings.adminEmail}.

+templates.canceledSubscription = CTAEmailTemplate({ + subject: () -> "#{settings.appName} thoughts" + message: () -> """ +I'm sorry to see you cancelled your #{settings.appName} premium account. Would you mind giving us some feedback on what the site is lacking at the moment via this quick survey? """ + secondaryMessage: () -> "Thank you in advance!" + ctaText: () -> "Leave Feedback" + ctaURL: (opts) -> "https://sharelatex.typeform.com/to/f5lBiZ" +}) - -templates.canceledSubscription = - subject: _.template "ShareLaTeX thoughts" - layout: PersonalEmailLayout - type:"lifecycle" - plainTextTemplate: _.template """ -Hi <%= first_name %>, - -I'm sorry to see you cancelled your ShareLaTeX premium account. Would you mind giving me some advice on what the site is lacking at the moment via this survey?: - -https://sharelatex.typeform.com/to/f5lBiZ - -Thank you in advance. - -Henry - -ShareLaTeX Co-founder -""" - compiledTemplate: _.template ''' -

Hi <%= first_name %>,

- -

I'm sorry to see you cancelled your ShareLaTeX premium account. Would you mind giving me some advice on what the site is lacking at the moment via this survey?

- -

Thank you in advance.

- -

-Henry
-ShareLaTeX Co-founder -

-''' - - -templates.passwordResetRequested = - subject: _.template "Password Reset - #{settings.appName}" - layout: BaseWithHeaderEmailLayout - type:"notification" - plainTextTemplate: _.template """ -Password Reset - -We got a request to reset your #{settings.appName} password. - -Click this link to reset your password: <%= setNewPasswordUrl %> - +templates.passwordResetRequested = CTAEmailTemplate({ + subject: () -> "Password Reset - #{settings.appName}" + title: () -> "Password Reset" + message: () -> "We got a request to reset your #{settings.appName} password." + secondaryMessage: () -> """ If you ignore this message, your password won't be changed. If you didn't request a password reset, let us know. - -Thank you - -#{settings.appName} - <%= siteUrl %> """ - compiledTemplate: (opts) -> - SingleCTAEmailBody({ - title: "Password Reset" - greeting: "Hi," - message: "We got a request to reset your #{settings.appName} password." - secondaryMessage: "If you ignore this message, your password won't be changed.
If you didn't request a password reset, let us know." - ctaText: "Reset password" - ctaURL: opts.setNewPasswordUrl - gmailGoToAction: null - }) + ctaText: () -> "Reset password" + ctaURL: (opts) -> opts.setNewPasswordUrl +}) +templates.confirmEmail = CTAEmailTemplate({ + subject: () -> "Confirm Email - #{settings.appName}" + title: () -> "Confirm Email" + message: () -> "Please confirm your email on #{settings.appName}." + ctaText: () -> "Confirm Email" + ctaURL: (opts) -> opts.confirmEmailUrl +}) -templates.projectInvite = - subject: _.template "<%= project.name %> - shared by <%= owner.email %>" - layout: BaseWithHeaderEmailLayout - type:"notification" - plainTextTemplate: _.template """ -Hi, <%= owner.email %> wants to share '<%= project.name %>' with you. +templates.projectInvite = CTAEmailTemplate({ + subject: (opts) -> "#{opts.project.name} - shared by #{opts.owner.email}" + title: (opts) -> "#{ opts.project.name } - shared by #{ opts.owner.email }" + message: (opts) -> "#{ opts.owner.email } wants to share '#{ opts.project.name }' with you." + ctaText: () -> "View project" + ctaURL: (opts) -> opts.inviteUrl + gmailGoToAction: (opts) -> + target: opts.inviteUrl + name: "View project" + description: "Join #{ opts.project.name } at #{ settings.appName }" +}) -Follow this link to view the project: <%= inviteUrl %> - -Thank you - -#{settings.appName} - <%= siteUrl %> -""" - compiledTemplate: (opts) -> - SingleCTAEmailBody({ - title: "#{ opts.project.name } – shared by #{ opts.owner.email }" - greeting: "Hi," - message: "#{ opts.owner.email } wants to share “#{ opts.project.name }” with you." - secondaryMessage: null - ctaText: "View project" - ctaURL: opts.inviteUrl - gmailGoToAction: - target: opts.inviteUrl - name: "View project" - description: "Join #{ opts.project.name } at ShareLaTeX" - }) - - -templates.verifyEmailToJoinTeam = - subject: _.template "<%= inviterName %> has invited you to join a team on #{settings.appName}" - layout: BaseWithHeaderEmailLayout - type:"notification" - plainTextTemplate: _.template """ - -Please click the button below to join the team and enjoy the benefits of an upgraded <%= appName %> account. - -<%= acceptInviteUrl %> - -Thank You - -#{settings.appName} - <%= siteUrl %> -""" - compiledTemplate: (opts) -> - SingleCTAEmailBody({ - title: "#{opts.inviterName} has invited you to join a team on #{settings.appName}" - greeting: "Hi," - message: "Please click the button below to join the team and enjoy the benefits of an upgraded #{ opts.appName } account." - secondaryMessage: null - ctaText: "Verify now" - ctaURL: opts.acceptInviteUrl - gmailGoToAction: null - }) - -templates.testEmail = - subject: _.template "A Test Email from ShareLaTeX" - layout: BaseWithHeaderEmailLayout - type:"notification" - plainTextTemplate: _.template """ -Hi, - -This is a test email sent from ShareLaTeX. - -#{settings.appName} - <%= siteUrl %> -""" - compiledTemplate: (opts) -> - SingleCTAEmailBody({ - title: "A Test Email from ShareLaTeX" - greeting: "Hi," - message: "This is a test email sent from ShareLaTeX" - secondaryMessage: null - ctaText: "Open ShareLaTeX" - ctaURL: "/" - gmailGoToAction: null - }) +templates.verifyEmailToJoinTeam = CTAEmailTemplate({ + subject: (opts) -> "#{ opts.inviterName } has invited you to join a team on #{settings.appName}" + title: (opts) -> "#{opts.inviterName} has invited you to join a team on #{settings.appName}" + message: (opts) -> "Please click the button below to join the team and enjoy the benefits of an upgraded #{ settings.appName } account." + ctaText: (opts) -> "Join now" + ctaURL: (opts) -> opts.acceptInviteUrl +}) +templates.testEmail = CTAEmailTemplate({ + subject: () -> "A Test Email from #{settings.appName}" + title: () -> "A Test Email from #{settings.appName}" + greeting: () -> "Hi," + message: () -> "This is a test Email from #{settings.appName}" + ctaText: () -> "Open #{settings.appName}" + ctaURL: () -> settings.siteUrl +}) module.exports = templates: templates - + CTAEmailTemplate: CTAEmailTemplate buildEmail: (templateName, opts)-> template = templates[templateName] opts.siteUrl = settings.siteUrl @@ -180,5 +125,4 @@ module.exports = subject : template.subject(opts) html: template.layout(opts) text: template?.plainTextTemplate?(opts) - type:template.type } From 0dcbc5facb934907ab212eebaa556e4e191091b1 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 19 Jun 2018 13:55:34 +0100 Subject: [PATCH 23/63] Send out confirmation emails on register and record confirmedAt date --- .../PasswordReset/PasswordResetHandler.coffee | 4 +- .../Security/OneTimeTokenHandler.coffee | 18 +-- .../User/UserEmailsConfirmationHandler.coffee | 42 ++++++ .../Features/User/UserEmailsController.coffee | 43 ++++-- .../User/UserRegistrationHandler.coffee | 2 +- .../coffee/Features/User/UserUpdater.coffee | 25 +++- services/web/app/coffee/models/User.coffee | 3 +- services/web/app/coffee/router.coffee | 4 + services/web/app/views/user/confirm_email.pug | 28 ++++ .../public/coffee/directives/asyncForm.coffee | 16 ++- .../acceptance/coffee/UserEmailsTests.coffee | 132 ++++++++++++++++++ .../PasswordResetHandlerTests.coffee | 10 +- .../Security/OneTimeTokenHandlerTests.coffee | 8 +- .../UserEmailsConfirmationHandlerTests.coffee | 125 +++++++++++++++++ .../User/UserEmailsControllerTests.coffee | 92 +++++++----- .../User/UserRegistrationHandlerTests.coffee | 4 +- .../unit/coffee/User/UserUpdaterTests.coffee | 39 +++++- 17 files changed, 516 insertions(+), 79 deletions(-) create mode 100644 services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee create mode 100644 services/web/app/views/user/confirm_email.pug create mode 100644 services/web/test/acceptance/coffee/UserEmailsTests.coffee create mode 100644 services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee diff --git a/services/web/app/coffee/Features/PasswordReset/PasswordResetHandler.coffee b/services/web/app/coffee/Features/PasswordReset/PasswordResetHandler.coffee index 3947b63004..d383e8b669 100644 --- a/services/web/app/coffee/Features/PasswordReset/PasswordResetHandler.coffee +++ b/services/web/app/coffee/Features/PasswordReset/PasswordResetHandler.coffee @@ -14,7 +14,7 @@ module.exports = if !user? or user.holdingAccount logger.err email:email, "user could not be found for password reset" return callback(null, false) - OneTimeTokenHandler.getNewToken user._id, (err, token)-> + OneTimeTokenHandler.getNewToken 'password', user._id, (err, token)-> if err then return callback(err) emailOptions = to : email @@ -24,7 +24,7 @@ module.exports = callback null, true setNewUserPassword: (token, password, callback = (error, found, user_id) ->)-> - OneTimeTokenHandler.getValueFromTokenAndExpire token, (err, user_id)-> + OneTimeTokenHandler.getValueFromTokenAndExpire 'password', token, (err, user_id)-> if err then return callback(err) if !user_id? return callback null, false, null diff --git a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee index 3824703efb..c989ef9505 100644 --- a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee +++ b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee @@ -6,29 +6,29 @@ logger = require("logger-sharelatex") ONE_HOUR_IN_S = 60 * 60 -buildKey = (token)-> return "password_token:#{token}" +buildKey = (use, token)-> return "#{use}_token:#{token}" module.exports = - getNewToken: (value, options = {}, callback)-> + getNewToken: (use, value, options = {}, callback)-> # options is optional if typeof options == "function" callback = options options = {} expiresIn = options.expiresIn or ONE_HOUR_IN_S - logger.log value:value, "generating token for password reset" token = crypto.randomBytes(32).toString("hex") + logger.log {value, expiresIn, token_start: token.slice(0,8)}, "generating token for #{use}" multi = rclient.multi() - multi.set buildKey(token), value - multi.expire buildKey(token), expiresIn + multi.set buildKey(use, token), value + multi.expire buildKey(use, token), expiresIn multi.exec (err)-> callback(err, token) - getValueFromTokenAndExpire: (token, callback)-> - logger.log token:token, "getting user id from password token" + getValueFromTokenAndExpire: (use, token, callback)-> + logger.log token_start: token.slice(0,8), "getting value from #{use} token" multi = rclient.multi() - multi.get buildKey(token) - multi.del buildKey(token) + multi.get buildKey(use, token) + multi.del buildKey(use, token) multi.exec (err, results)-> callback err, results?[0] diff --git a/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee b/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee new file mode 100644 index 0000000000..1fbad4a294 --- /dev/null +++ b/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee @@ -0,0 +1,42 @@ +EmailHelper = require "../Helpers/EmailHelper" +EmailHandler = require "../Email/EmailHandler" +OneTimeTokenHandler = require "../Security/OneTimeTokenHandler" +settings = require 'settings-sharelatex' +Errors = require "../Errors/Errors" +logger = require "logger-sharelatex" +UserUpdater = require "./UserUpdater" + +ONE_YEAR_IN_S = 365 * 24 * 60 * 60 + +module.exports = UserEmailsConfirmationHandler = + serializeData: (user_id, email) -> + JSON.stringify({user_id, email}) + + deserializeData: (data) -> + JSON.parse(data) + + sendConfirmationEmail: (user_id, email, emailTemplate, callback = (error) ->) -> + if arguments.length == 3 + callback = emailTemplate + emailTemplate = 'confirmEmail' + email = EmailHelper.parseEmail(email) + return callback(new Error('invalid email')) if !email? + value = UserEmailsConfirmationHandler.serializeData(user_id, email) + OneTimeTokenHandler.getNewToken 'email_confirmation', value, {expiresIn: ONE_YEAR_IN_S}, (err, token)-> + return callback(err) if err? + emailOptions = + to: email + confirmEmailUrl: "#{settings.siteUrl}/user/emails/confirm?token=#{token}" + EmailHandler.sendEmail emailTemplate, emailOptions, callback + + confirmEmailFromToken: (token, callback = (error) ->) -> + logger.log {token_start: token.slice(0,8)}, 'confirming email from token' + OneTimeTokenHandler.getValueFromTokenAndExpire 'email_confirmation', token, (error, data) -> + return callback(error) if error? + if !data? + return callback(new Errors.NotFoundError('no token found')) + {user_id, email} = UserEmailsConfirmationHandler.deserializeData(data) + logger.log {data, user_id, email, token_start: token.slice(0,8)}, 'found data for email confirmation' + if !user_id? or email != EmailHelper.parseEmail(email) + return callback(new Errors.NotFoundError('invalid data')) + UserUpdater.confirmEmail user_id, email, callback diff --git a/services/web/app/coffee/Features/User/UserEmailsController.coffee b/services/web/app/coffee/Features/User/UserEmailsController.coffee index 07433d2dde..f877cbe345 100644 --- a/services/web/app/coffee/Features/User/UserEmailsController.coffee +++ b/services/web/app/coffee/Features/User/UserEmailsController.coffee @@ -2,42 +2,67 @@ AuthenticationController = require('../Authentication/AuthenticationController') UserGetter = require("./UserGetter") UserUpdater = require("./UserUpdater") EmailHelper = require("../Helpers/EmailHelper") +UserEmailsConfirmationHandler = require "./UserEmailsConfirmationHandler" logger = require("logger-sharelatex") +Errors = require "../Errors/Errors" module.exports = UserEmailsController = - list: (req, res) -> + list: (req, res, next) -> userId = AuthenticationController.getLoggedInUserId(req) UserGetter.getUserFullEmails userId, (error, fullEmails) -> - return res.sendStatus 500 if error? + return next(error) if error? res.json fullEmails - add: (req, res) -> + add: (req, res, next) -> userId = AuthenticationController.getLoggedInUserId(req) email = EmailHelper.parseEmail(req.body.email) return res.sendStatus 422 unless email? UserUpdater.addEmailAddress userId, email, (error)-> - return res.sendStatus 500 if error? - res.sendStatus 200 + return next(error) if error? + UserEmailsConfirmationHandler.sendConfirmationEmail userId, email, (err) -> + return next(error) if error? + res.sendStatus 204 - remove: (req, res) -> + remove: (req, res, next) -> userId = AuthenticationController.getLoggedInUserId(req) email = EmailHelper.parseEmail(req.body.email) return res.sendStatus 422 unless email? UserUpdater.removeEmailAddress userId, email, (error)-> - return res.sendStatus 500 if error? + return next(error) if error? res.sendStatus 200 - setDefault: (req, res) -> + setDefault: (req, res, next) -> userId = AuthenticationController.getLoggedInUserId(req) email = EmailHelper.parseEmail(req.body.email) return res.sendStatus 422 unless email? UserUpdater.setDefaultEmailAddress userId, email, (error)-> - return res.sendStatus 500 if error? + return next(error) if error? res.sendStatus 200 + + showConfirm: (req, res, next) -> + res.render 'user/confirm_email', { + token: req.query.token, + title: 'confirm_email' + } + + confirm: (req, res, next) -> + token = req.body.token + if !token? + return res.sendStatus 422 + UserEmailsConfirmationHandler.confirmEmailFromToken token, (error) -> + if error? + if error instanceof Errors.NotFoundError + res.status(404).json({ + message: 'Sorry, your confirmation token is invalid or has expired. Please request a new email confirmation link.' + }) + else + next(error) + else + res.sendStatus 200 \ No newline at end of file diff --git a/services/web/app/coffee/Features/User/UserRegistrationHandler.coffee b/services/web/app/coffee/Features/User/UserRegistrationHandler.coffee index 85cdabdbdf..df7fe93218 100644 --- a/services/web/app/coffee/Features/User/UserRegistrationHandler.coffee +++ b/services/web/app/coffee/Features/User/UserRegistrationHandler.coffee @@ -74,7 +74,7 @@ module.exports = UserRegistrationHandler = logger.log {email}, "user already exists, resending welcome email" ONE_WEEK = 7 * 24 * 60 * 60 # seconds - OneTimeTokenHandler.getNewToken user._id, { expiresIn: ONE_WEEK }, (err, token)-> + OneTimeTokenHandler.getNewToken 'password', user._id, { expiresIn: ONE_WEEK }, (err, token)-> return callback(err) if err? setNewPasswordUrl = "#{settings.siteUrl}/user/activate?token=#{token}&user_id=#{user._id}" diff --git a/services/web/app/coffee/Features/User/UserUpdater.coffee b/services/web/app/coffee/Features/User/UserUpdater.coffee index 22b31239bd..4de994b5c2 100644 --- a/services/web/app/coffee/Features/User/UserUpdater.coffee +++ b/services/web/app/coffee/Features/User/UserUpdater.coffee @@ -5,6 +5,8 @@ db = mongojs.db async = require("async") ObjectId = mongojs.ObjectId UserGetter = require("./UserGetter") +EmailHelper = require "../Helpers/EmailHelper" +Errors = require "../Errors/Errors" module.exports = UserUpdater = updateUser: (query, update, callback = (error) ->) -> @@ -53,7 +55,6 @@ module.exports = UserUpdater = return callback(error) callback() - # remove one of the user's email addresses. The email cannot be the user's # default email address removeEmailAddress: (userId, email, callback) -> @@ -63,7 +64,7 @@ module.exports = UserUpdater = if error? logger.err error:error, 'problem removing users email' return callback(error) - if res.nMatched == 0 + if res.n == 0 return callback(new Error('Cannot remove default email')) callback() @@ -77,10 +78,28 @@ module.exports = UserUpdater = if error? logger.err error:error, 'problem setting default emails' return callback(error) - if res.nMatched == 0 + if res.n == 0 # TODO: Check n or nMatched? return callback(new Error('Default email does not belong to user')) callback() + confirmEmail: (userId, email, callback) -> + email = EmailHelper.parseEmail(email) + return callback(new Error('invalid email')) if !email? + logger.log {userId, email}, 'confirming user email' + query = + _id: userId + 'emails.email': email + update = + $set: + 'emails.$.confirmedAt': new Date() + @updateUser query, update, (error, res) -> + return callback(error) if error? + logger.log {res, userId, email}, "tried to confirm email" + if res.n == 0 + return callback(new Errors.NotFoundError('user id and email do no match')) + callback() + + [ 'updateUser' 'changeEmailAddress' diff --git a/services/web/app/coffee/models/User.coffee b/services/web/app/coffee/models/User.coffee index 71982ba40b..730b3668c6 100644 --- a/services/web/app/coffee/models/User.coffee +++ b/services/web/app/coffee/models/User.coffee @@ -10,7 +10,8 @@ UserSchema = new Schema email : {type : String, default : ''} emails: [{ email: { type : String, default : '' }, - createdAt: { type : Date, default: () -> new Date() } + createdAt: { type : Date, default: () -> new Date() }, + confirmedAt: { type: Date } }], first_name : {type : String, default : ''} last_name : {type : String, default : ''} diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index 339b4abe1c..6014fea661 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -120,6 +120,10 @@ module.exports = class Router webRouter.post '/user/emails/default', AuthenticationController.requireLogin(), UserEmailsController.setDefault + webRouter.get '/user/emails/confirm', + UserEmailsController.showConfirm + webRouter.post '/user/emails/confirm', + UserEmailsController.confirm webRouter.get '/user/sessions', AuthenticationController.requireLogin(), diff --git a/services/web/app/views/user/confirm_email.pug b/services/web/app/views/user/confirm_email.pug new file mode 100644 index 0000000000..388a0a0252 --- /dev/null +++ b/services/web/app/views/user/confirm_email.pug @@ -0,0 +1,28 @@ +extends ../layout + +block content + .content.content-alt + .container + .row + .col-md-6.col-md-offset-3.col-lg-4.col-lg-offset-4 + .card + .page-header + h1 #{translate("confirm_email")} + form( + async-form="confirm-email", + name="confirmEmailForm" + action="/user/emails/confirm", + method="POST", + id="confirmEmailForm", + auto-submit="true", + ng-cloak + ) + input(type="hidden", name="_csrf", value=csrfToken) + input(type="hidden", name="token", value=token) + form-messages(for="confirmEmailForm") + .alert.alert-success(ng-show="confirmEmailForm.response.success") + | Thank you, your email is now confirmed + p.text-center(ng-show="!confirmEmailForm.response.success && !confirmEmailForm.response.error") + i.fa.fa-fw.fa-spin.fa-spinner + | + | Confirming your email... diff --git a/services/web/public/coffee/directives/asyncForm.coffee b/services/web/public/coffee/directives/asyncForm.coffee index 15e3c9b3fe..acafec563d 100644 --- a/services/web/public/coffee/directives/asyncForm.coffee +++ b/services/web/public/coffee/directives/asyncForm.coffee @@ -15,11 +15,6 @@ define [ scope[attrs.name].response = response = {} scope[attrs.name].inflight = false - element.on "submit", (e) -> - e.preventDefault() - validateCaptchaIfEnabled (response) -> - submitRequest response - validateCaptchaIfEnabled = (callback = (response) ->) -> if attrs.captcha? validateCaptcha callback @@ -84,6 +79,17 @@ define [ text: data.message?.text or data.message or "Something went wrong talking to the server :(. Please try again." type: 'error' ga('send', 'event', formName, 'failure', data.message) + + submit = () -> + validateCaptchaIfEnabled (response) -> + submitRequest response + + element.on "submit", (e) -> + e.preventDefault() + submit() + + if attrs.autoSubmit + submit() } App.directive "formMessages", () -> diff --git a/services/web/test/acceptance/coffee/UserEmailsTests.coffee b/services/web/test/acceptance/coffee/UserEmailsTests.coffee new file mode 100644 index 0000000000..8cdbea75fa --- /dev/null +++ b/services/web/test/acceptance/coffee/UserEmailsTests.coffee @@ -0,0 +1,132 @@ +expect = require("chai").expect +async = require("async") +User = require "./helpers/User" +request = require "./helpers/request" +settings = require "settings-sharelatex" +rclient = require("redis-sharelatex").createClient(settings.redis.web) + +describe "UserEmails", -> + beforeEach (done) -> + @timeout(20000) + @user = new User() + @user.login done + + describe 'confirming an email', -> + it 'should confirm the email', (done) -> + token = null + async.series [ + (cb) => + @user.request { + method: 'POST', + url: '/user/emails', + json: + email: 'newly-added-email@example.com' + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 204 + cb() + (cb) => + @user.request { url: '/user/emails', json: true }, (error, response, body) -> + expect(response.statusCode).to.equal 200 + expect(body[0].confirmedAt).to.not.exist + expect(body[1].confirmedAt).to.not.exist + cb() + (cb) => + rclient.keys 'email_confirmation_token:*', (error, keys) -> + # There should only be one confirmation token at the moment + expect(keys.length).to.equal 1 + token = keys[0].split(':')[1] + cb() + (cb) => + @user.request { + method: 'POST', + url: '/user/emails/confirm', + json: + token: token + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 200 + cb() + (cb) => + @user.request { url: '/user/emails', json: true }, (error, response, body) -> + expect(response.statusCode).to.equal 200 + expect(body[0].confirmedAt).to.not.exist + expect(body[1].confirmedAt).to.exist + cb() + (cb) => + rclient.keys 'email_confirmation_token:*', (error, keys) -> + # Token should be deleted after use + expect(keys.length).to.equal 0 + cb() + ], done + + it 'should not allow confirmation of the email if the user has changed', (done) -> + token1 = null + token2 = null + @user2 = new User() + @email = 'duplicate-email@example.com' + async.series [ + (cb) => @user2.login cb + (cb) => + # Create email for first user + @user.request { + method: 'POST', + url: '/user/emails', + json: {@email} + }, cb + (cb) => + rclient.keys 'email_confirmation_token:*', (error, keys) -> + # There should only be one confirmation token at the moment, + # for the first user + expect(keys.length).to.equal 1 + token1 = keys[0].split(':')[1] + cb() + (cb) => + # Delete the email from the first user + @user.request { + method: 'DELETE', + url: '/user/emails', + json: {@email} + }, cb + (cb) => + # Create email for second user + @user2.request { + method: 'POST', + url: '/user/emails', + json: {@email} + }, cb + (cb) => + # Original confirmation token should no longer work + @user.request { + method: 'POST', + url: '/user/emails/confirm', + json: + token: token1 + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 404 + cb() + (cb) => + rclient.keys 'email_confirmation_token:*', (error, keys) -> + # The first token has been used, so this should be token2 now + expect(keys.length).to.equal 1 + token2 = keys[0].split(':')[1] + cb() + (cb) => + # Second user should be able to confirm the email + @user2.request { + method: 'POST', + url: '/user/emails/confirm', + json: + token: token2 + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 200 + cb() + (cb) => + @user2.request { url: '/user/emails', json: true }, (error, response, body) -> + expect(response.statusCode).to.equal 200 + expect(body[0].confirmedAt).to.not.exist + expect(body[1].confirmedAt).to.exist + cb() + ], done diff --git a/services/web/test/unit/coffee/PasswordReset/PasswordResetHandlerTests.coffee b/services/web/test/unit/coffee/PasswordReset/PasswordResetHandlerTests.coffee index 261f5582dd..7ee82fbd7e 100644 --- a/services/web/test/unit/coffee/PasswordReset/PasswordResetHandlerTests.coffee +++ b/services/web/test/unit/coffee/PasswordReset/PasswordResetHandlerTests.coffee @@ -41,7 +41,7 @@ describe "PasswordResetHandler", -> it "should check the user exists", (done)-> @UserGetter.getUserByMainEmail.callsArgWith(1) - @OneTimeTokenHandler.getNewToken.callsArgWith(1) + @OneTimeTokenHandler.getNewToken.yields() @PasswordResetHandler.generateAndEmailResetToken @user.email, (err, exists)=> exists.should.equal false done() @@ -50,7 +50,7 @@ describe "PasswordResetHandler", -> it "should send the email with the token", (done)-> @UserGetter.getUserByMainEmail.callsArgWith(1, null, @user) - @OneTimeTokenHandler.getNewToken.callsArgWith(1, null, @token) + @OneTimeTokenHandler.getNewToken.yields(null, @token) @EmailHandler.sendEmail.callsArgWith(2) @PasswordResetHandler.generateAndEmailResetToken @user.email, (err, exists)=> @EmailHandler.sendEmail.called.should.equal true @@ -63,7 +63,7 @@ describe "PasswordResetHandler", -> it "should return exists = false for a holdingAccount", (done) -> @user.holdingAccount = true @UserGetter.getUserByMainEmail.callsArgWith(1, null, @user) - @OneTimeTokenHandler.getNewToken.callsArgWith(1) + @OneTimeTokenHandler.getNewToken.yields() @PasswordResetHandler.generateAndEmailResetToken @user.email, (err, exists)=> exists.should.equal false done() @@ -71,14 +71,14 @@ describe "PasswordResetHandler", -> describe "setNewUserPassword", -> it "should return false if no user id can be found", (done)-> - @OneTimeTokenHandler.getValueFromTokenAndExpire.callsArgWith(1) + @OneTimeTokenHandler.getValueFromTokenAndExpire.yields() @PasswordResetHandler.setNewUserPassword @token, @password, (err, found) => found.should.equal false @AuthenticationManager.setUserPassword.called.should.equal false done() it "should set the user password", (done)-> - @OneTimeTokenHandler.getValueFromTokenAndExpire.callsArgWith(1, null, @user_id) + @OneTimeTokenHandler.getValueFromTokenAndExpire.yields(null, @user_id) @AuthenticationManager.setUserPassword.callsArgWith(2) @PasswordResetHandler.setNewUserPassword @token, @password, (err, found, user_id) => found.should.equal true diff --git a/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee b/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee index 1940f34d07..21e8d9f288 100644 --- a/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee +++ b/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee @@ -37,21 +37,21 @@ describe "OneTimeTokenHandler", -> it "should set a new token into redis with a ttl", (done)-> @redisMulti.exec.callsArgWith(0) - @OneTimeTokenHandler.getNewToken @value, (err, token) => + @OneTimeTokenHandler.getNewToken 'password', @value, (err, token) => @redisMulti.set.calledWith("password_token:#{@stubbedToken.toString("hex")}", @value).should.equal true @redisMulti.expire.calledWith("password_token:#{@stubbedToken.toString("hex")}", 60 * 60).should.equal true done() it "should return if there was an error", (done)-> @redisMulti.exec.callsArgWith(0, "error") - @OneTimeTokenHandler.getNewToken @value, (err, token)=> + @OneTimeTokenHandler.getNewToken 'password', @value, (err, token)=> err.should.exist done() it "should allow the expiry time to be overridden", (done) -> @redisMulti.exec.callsArgWith(0) @ttl = 42 - @OneTimeTokenHandler.getNewToken @value, {expiresIn: @ttl}, (err, token) => + @OneTimeTokenHandler.getNewToken 'password', @value, {expiresIn: @ttl}, (err, token) => @redisMulti.expire.calledWith("password_token:#{@stubbedToken.toString("hex")}", @ttl).should.equal true done() @@ -59,7 +59,7 @@ describe "OneTimeTokenHandler", -> it "should get and delete the token", (done)-> @redisMulti.exec.callsArgWith(0, null, [@value]) - @OneTimeTokenHandler.getValueFromTokenAndExpire @stubbedToken, (err, value)=> + @OneTimeTokenHandler.getValueFromTokenAndExpire 'password', @stubbedToken, (err, value)=> value.should.equal @value @redisMulti.get.calledWith("password_token:#{@stubbedToken}").should.equal true @redisMulti.del.calledWith("password_token:#{@stubbedToken}").should.equal true diff --git a/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee b/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee new file mode 100644 index 0000000000..22cd70c4bc --- /dev/null +++ b/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee @@ -0,0 +1,125 @@ +should = require('chai').should() +SandboxedModule = require('sandboxed-module') +assert = require('assert') +path = require('path') +sinon = require('sinon') +modulePath = path.join __dirname, "../../../../app/js/Features/User/UserEmailsConfirmationHandler" +expect = require("chai").expect +Errors = require "../../../../app/js/Features/Errors/Errors" +EmailHelper = require "../../../../app/js/Features/Helpers/EmailHelper" + +describe "UserEmailsConfirmationHandler", -> + beforeEach -> + @UserEmailsConfirmationHandler = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings = + siteUrl: "emails.example.com" + "logger-sharelatex": @logger = { log: sinon.stub() } + "../Security/OneTimeTokenHandler": @OneTimeTokenHandler = {} + "../Errors/Errors": Errors + "./UserUpdater": @UserUpdater = {} + "../Email/EmailHandler": @EmailHandler = {} + "../Helpers/EmailHelper": EmailHelper + @user_id = "mock-user-id" + @email = "mock@example.com" + @callback = sinon.stub() + + describe "sendConfirmationEmail", -> + beforeEach -> + @OneTimeTokenHandler.getNewToken = sinon.stub().yields(null, @token = "new-token") + @EmailHandler.sendEmail = sinon.stub().yields() + + describe 'successfully', -> + beforeEach -> + @UserEmailsConfirmationHandler.sendConfirmationEmail @user_id, @email, @callback + + it "should generate a token for the user which references their id and email", -> + @OneTimeTokenHandler.getNewToken + .calledWith( + 'email_confirmation', + JSON.stringify({@user_id, @email}), + { expiresIn: 365 * 24 * 60 * 60 } + ) + .should.equal true + + it 'should send an email to the user', -> + @EmailHandler.sendEmail + .calledWith('confirmEmail', { + to: @email, + confirmEmailUrl: 'emails.example.com/user/emails/confirm?token=new-token' + }) + .should.equal true + + it 'should call the callback', -> + @callback.called.should.equal true + + describe 'with invalid email', -> + beforeEach -> + @UserEmailsConfirmationHandler.sendConfirmationEmail @user_id, '!"£$%^&*()', @callback + + it 'should return an error', -> + @callback.calledWith(sinon.match.instanceOf(Error)).should.equal true + + describe 'a custom template', -> + beforeEach -> + @UserEmailsConfirmationHandler.sendConfirmationEmail @user_id, @email, 'myCustomTemplate', @callback + + it 'should send an email with the given template', -> + @EmailHandler.sendEmail + .calledWith('myCustomTemplate') + .should.equal true + + describe "confirmEmailFromToken", -> + beforeEach -> + @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( + null, + JSON.stringify({@user_id, @email}) + ) + @UserUpdater.confirmEmail = sinon.stub().yields() + + describe "successfully", -> + beforeEach -> + @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback + + it "should call getValueFromTokenAndExpire", -> + @OneTimeTokenHandler.getValueFromTokenAndExpire + .calledWith('email_confirmation', @token) + .should.equal true + + it "should confirm the email of the user_id", -> + @UserUpdater.confirmEmail + .calledWith(@user_id, @email) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe 'with an expired token', -> + beforeEach -> + @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields(null, null) + @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback + + it "should call the callback with a NotFoundError", -> + @callback.calledWith(sinon.match.instanceOf(Errors.NotFoundError)).should.equal true + + describe 'with no user_id in the token', -> + beforeEach -> + @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( + null, + JSON.stringify({@email}) + ) + @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback + + it "should call the callback with a NotFoundError", -> + @callback.calledWith(sinon.match.instanceOf(Errors.NotFoundError)).should.equal true + + describe 'with no email in the token', -> + beforeEach -> + @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( + null, + JSON.stringify({@user_id}) + ) + @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback + + it "should call the callback with a NotFoundError", -> + @callback.calledWith(sinon.match.instanceOf(Errors.NotFoundError)).should.equal true + diff --git a/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee b/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee index 5b4a32fb38..d0e8076276 100644 --- a/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee +++ b/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee @@ -8,6 +8,7 @@ modulePath = "../../../../app/js/Features/User/UserEmailsController.js" SandboxedModule = require('sandboxed-module') MockRequest = require "../helpers/MockRequest" MockResponse = require "../helpers/MockResponse" +Errors = require("../../../../app/js/Features/Errors/Errors") describe "UserEmailsController", -> beforeEach -> @@ -30,6 +31,8 @@ describe "UserEmailsController", -> "./UserGetter": @UserGetter "./UserUpdater": @UserUpdater "../Helpers/EmailHelper": @EmailHelper + "./UserEmailsConfirmationHandler": @UserEmailsConfirmationHandler = {} + "../Errors/Errors": Errors "logger-sharelatex": log: -> console.log(arguments) err: -> @@ -47,30 +50,29 @@ describe "UserEmailsController", -> assertCalledWith @UserGetter.getUserFullEmails, @user._id done() - it 'handles error', (done) -> - @UserGetter.getUserFullEmails.callsArgWith 1, new Error('Oups') - - @UserEmailsController.list @req, - sendStatus: (code) => - code.should.equal 500 - done() - describe 'Add', -> beforeEach -> @newEmail = 'new_email@baz.com' @req.body.email = @newEmail @EmailHelper.parseEmail.returns @newEmail + @UserUpdater.addEmailAddress.callsArgWith 2, null + @UserEmailsConfirmationHandler.sendConfirmationEmail = sinon.stub().yields() it 'adds new email', (done) -> - @UserUpdater.addEmailAddress.callsArgWith 2, null - @UserEmailsController.add @req, sendStatus: (code) => - code.should.equal 200 + code.should.equal 204 assertCalledWith @EmailHelper.parseEmail, @newEmail assertCalledWith @UserUpdater.addEmailAddress, @user._id, @newEmail done() + it 'sends an email confirmation', (done) -> + @UserEmailsController.add @req, + sendStatus: (code) => + code.should.equal 204 + assertCalledWith @UserEmailsConfirmationHandler.sendConfirmationEmail, @user._id, @newEmail + done() + it 'handles email parse error', (done) -> @EmailHelper.parseEmail.returns null @@ -80,14 +82,6 @@ describe "UserEmailsController", -> assertNotCalled @UserUpdater.addEmailAddress done() - it 'handles error', (done) -> - @UserUpdater.addEmailAddress.callsArgWith 2, new Error('Oups') - - @UserEmailsController.add @req, - sendStatus: (code) => - code.should.equal 500 - done() - describe 'remove', -> beforeEach -> @email = 'email_to_remove@bar.com' @@ -113,15 +107,6 @@ describe "UserEmailsController", -> assertNotCalled @UserUpdater.removeEmailAddress done() - it 'handles error', (done) -> - @UserUpdater.removeEmailAddress.callsArgWith 2, new Error('Oups') - - @UserEmailsController.remove @req, - sendStatus: (code) => - code.should.equal 500 - done() - - describe 'setDefault', -> beforeEach -> @email = "email_to_set_default@bar.com" @@ -147,11 +132,50 @@ describe "UserEmailsController", -> assertNotCalled @UserUpdater.setDefaultEmailAddress done() - it 'handles error', (done) -> - @UserUpdater.setDefaultEmailAddress.callsArgWith 2, new Error('Oups') + describe 'confirm', -> + beforeEach -> + @UserEmailsConfirmationHandler.confirmEmailFromToken = sinon.stub().yields() + @res = + sendStatus: sinon.stub() + json: sinon.stub() + @res.status = sinon.stub().returns(@res) + @next = sinon.stub() + @token = 'mock-token' + @req.body = token: @token + + describe 'successfully', -> + beforeEach -> + @UserEmailsController.confirm @req, @res, @next + + it 'should confirm the email from the token', -> + @UserEmailsConfirmationHandler.confirmEmailFromToken + .calledWith(@token) + .should.equal true + + it 'should return a 200 status', -> + @res.sendStatus.calledWith(200).should.equal true + + describe 'without a token', -> + beforeEach -> + @req.body.token = null + @UserEmailsController.confirm @req, @res, @next + + it 'should return a 422 status', -> + @res.sendStatus.calledWith(422).should.equal true + + describe 'when confirming fails', -> + beforeEach -> + @UserEmailsConfirmationHandler.confirmEmailFromToken = sinon.stub().yields( + new Errors.NotFoundError('not found') + ) + @UserEmailsController.confirm @req, @res, @next + + it 'should return a 404 error code with a message', -> + @res.status.calledWith(404).should.equal true + @res.json.calledWith({ + message: 'Sorry, your confirmation token is invalid or has expired. Please request a new email confirmation link.' + }).should.equal true + + - @UserEmailsController.setDefault @req, - sendStatus: (code) => - code.should.equal 500 - done() diff --git a/services/web/test/unit/coffee/User/UserRegistrationHandlerTests.coffee b/services/web/test/unit/coffee/User/UserRegistrationHandlerTests.coffee index 3a8801bc08..e9a6c568dd 100644 --- a/services/web/test/unit/coffee/User/UserRegistrationHandlerTests.coffee +++ b/services/web/test/unit/coffee/User/UserRegistrationHandlerTests.coffee @@ -152,7 +152,7 @@ describe "UserRegistrationHandler", -> beforeEach -> @email = "email@example.com" @crypto.randomBytes = sinon.stub().returns({toString: () => @password = "mock-password"}) - @OneTimeTokenHandler.getNewToken.callsArgWith(2, null, @token = "mock-token") + @OneTimeTokenHandler.getNewToken.yields(null, @token = "mock-token") @handler.registerNewUser = sinon.stub() @callback = sinon.stub() @@ -171,7 +171,7 @@ describe "UserRegistrationHandler", -> it "should generate a new password reset token", -> @OneTimeTokenHandler.getNewToken - .calledWith(@user_id, expiresIn: 7 * 24 * 60 * 60) + .calledWith('password', @user_id, expiresIn: 7 * 24 * 60 * 60) .should.equal true it "should send a registered email", -> diff --git a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee index 7b3be3df2b..6b9a1ecc85 100644 --- a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee +++ b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee @@ -5,11 +5,12 @@ path = require('path') sinon = require('sinon') modulePath = path.join __dirname, "../../../../app/js/Features/User/UserUpdater" expect = require("chai").expect +tk = require('timekeeper') describe "UserUpdater", -> beforeEach -> - + tk.freeze(Date.now()) @settings = {} @mongojs = db:{} @@ -32,6 +33,9 @@ describe "UserUpdater", -> email:"hello@world.com" @newEmail = "bob@bob.com" + afterEach -> + tk.reset() + describe 'changeEmailAddress', -> beforeEach -> @UserGetter.getUserEmail.callsArgWith(1, null, @stubbedUser.email) @@ -103,7 +107,7 @@ describe "UserUpdater", -> done() it 'handle missed update', (done)-> - @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, nMatched: 0) + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 0) @UserUpdater.removeEmailAddress @stubbedUser._id, @newEmail, (err)=> should.exist(err) @@ -111,7 +115,7 @@ describe "UserUpdater", -> describe 'setDefaultEmailAddress', -> it 'set default', (done)-> - @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, nMatched: 1) + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 1) @UserUpdater.setDefaultEmailAddress @stubbedUser._id, @newEmail, (err)=> should.not.exist(err) @@ -129,10 +133,37 @@ describe "UserUpdater", -> done() it 'handle missed update', (done)-> - @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, nMatched: 0) + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 0) @UserUpdater.setDefaultEmailAddress @stubbedUser._id, @newEmail, (err)=> should.exist(err) done() + describe 'confirmEmail', -> + it 'should update the email record', (done)-> + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 1) + + @UserUpdater.confirmEmail @stubbedUser._id, @newEmail, (err)=> + should.not.exist(err) + @UserUpdater.updateUser.calledWith( + { _id: @stubbedUser._id, 'emails.email': @newEmail }, + $set: { 'emails.$.confirmedAt': new Date() } + ).should.equal true + done() + + it 'handle error', (done)-> + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, new Error('nope')) + + @UserUpdater.confirmEmail @stubbedUser._id, @newEmail, (err)=> + should.exist(err) + done() + + it 'handle missed update', (done)-> + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 0) + + @UserUpdater.confirmEmail @stubbedUser._id, @newEmail, (err)=> + should.exist(err) + done() + + From 4608a59e3db340deb489e2872da5e70bc2e56c62 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 20 Jun 2018 17:27:22 +0100 Subject: [PATCH 24/63] Store OneTimeTokens in mongo rather than redis --- .../Security/OneTimeTokenHandler.coffee | 54 +++++--- .../User/UserEmailsConfirmationHandler.coffee | 12 +- .../app/coffee/infrastructure/mongojs.coffee | 2 +- .../acceptance/coffee/UserEmailsTests.coffee | 92 ++++++++++--- .../Security/OneTimeTokenHandlerTests.coffee | 121 +++++++++++------- .../UserEmailsConfirmationHandlerTests.coffee | 8 +- 6 files changed, 198 insertions(+), 91 deletions(-) diff --git a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee index c989ef9505..69c9f5b0e9 100644 --- a/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee +++ b/services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee @@ -1,34 +1,50 @@ Settings = require('settings-sharelatex') -RedisWrapper = require("../../infrastructure/RedisWrapper") -rclient = RedisWrapper.client("one_time_token") crypto = require("crypto") logger = require("logger-sharelatex") +{db} = require "../../infrastructure/mongojs" +Errors = require "../Errors/Errors" ONE_HOUR_IN_S = 60 * 60 -buildKey = (use, token)-> return "#{use}_token:#{token}" - module.exports = - - getNewToken: (use, value, options = {}, callback)-> + getNewToken: (use, data, options = {}, callback = (error, data) ->)-> # options is optional if typeof options == "function" callback = options options = {} expiresIn = options.expiresIn or ONE_HOUR_IN_S + createdAt = new Date() + expiresAt = new Date(createdAt.getTime() + expiresIn * 1000) token = crypto.randomBytes(32).toString("hex") - logger.log {value, expiresIn, token_start: token.slice(0,8)}, "generating token for #{use}" - multi = rclient.multi() - multi.set buildKey(use, token), value - multi.expire buildKey(use, token), expiresIn - multi.exec (err)-> - callback(err, token) + logger.log {data, expiresIn, token_start: token.slice(0,8)}, "generating token for #{use}" + db.tokens.insert { + use: use + token: token, + data: data, + createdAt: createdAt, + expiresAt: expiresAt + }, (error) -> + return callback(error) if error? + callback null, token - getValueFromTokenAndExpire: (use, token, callback)-> - logger.log token_start: token.slice(0,8), "getting value from #{use} token" - multi = rclient.multi() - multi.get buildKey(use, token) - multi.del buildKey(use, token) - multi.exec (err, results)-> - callback err, results?[0] + getValueFromTokenAndExpire: (use, token, callback = (error, data) ->)-> + logger.log token_start: token.slice(0,8), "getting data from #{use} token" + now = new Date() + db.tokens.findAndModify { + query: { + use: use, + token: token, + expiresAt: { $gt: now }, + usedAt: { $exists: false } + }, + update: { + $set: { + usedAt: now + } + } + }, (error, token) -> + return callback(error) if error? + if !token? + return callback(new Errors.NotFoundError('no token found')) + return callback null, token.data diff --git a/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee b/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee index 1fbad4a294..dd87570450 100644 --- a/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee +++ b/services/web/app/coffee/Features/User/UserEmailsConfirmationHandler.coffee @@ -9,20 +9,14 @@ UserUpdater = require "./UserUpdater" ONE_YEAR_IN_S = 365 * 24 * 60 * 60 module.exports = UserEmailsConfirmationHandler = - serializeData: (user_id, email) -> - JSON.stringify({user_id, email}) - - deserializeData: (data) -> - JSON.parse(data) - sendConfirmationEmail: (user_id, email, emailTemplate, callback = (error) ->) -> if arguments.length == 3 callback = emailTemplate emailTemplate = 'confirmEmail' email = EmailHelper.parseEmail(email) return callback(new Error('invalid email')) if !email? - value = UserEmailsConfirmationHandler.serializeData(user_id, email) - OneTimeTokenHandler.getNewToken 'email_confirmation', value, {expiresIn: ONE_YEAR_IN_S}, (err, token)-> + data = {user_id, email} + OneTimeTokenHandler.getNewToken 'email_confirmation', data, {expiresIn: ONE_YEAR_IN_S}, (err, token)-> return callback(err) if err? emailOptions = to: email @@ -35,7 +29,7 @@ module.exports = UserEmailsConfirmationHandler = return callback(error) if error? if !data? return callback(new Errors.NotFoundError('no token found')) - {user_id, email} = UserEmailsConfirmationHandler.deserializeData(data) + {user_id, email} = data logger.log {data, user_id, email, token_start: token.slice(0,8)}, 'found data for email confirmation' if !user_id? or email != EmailHelper.parseEmail(email) return callback(new Errors.NotFoundError('invalid data')) diff --git a/services/web/app/coffee/infrastructure/mongojs.coffee b/services/web/app/coffee/infrastructure/mongojs.coffee index f1ed213435..8492b9023c 100644 --- a/services/web/app/coffee/infrastructure/mongojs.coffee +++ b/services/web/app/coffee/infrastructure/mongojs.coffee @@ -1,6 +1,6 @@ Settings = require "settings-sharelatex" mongojs = require "mongojs" -db = mongojs(Settings.mongo.url, ["projects", "users", "userstubs"]) +db = mongojs(Settings.mongo.url, ["projects", "users", "userstubs", "tokens"]) module.exports = db: db ObjectId: mongojs.ObjectId diff --git a/services/web/test/acceptance/coffee/UserEmailsTests.coffee b/services/web/test/acceptance/coffee/UserEmailsTests.coffee index 8cdbea75fa..248c0331ae 100644 --- a/services/web/test/acceptance/coffee/UserEmailsTests.coffee +++ b/services/web/test/acceptance/coffee/UserEmailsTests.coffee @@ -3,7 +3,7 @@ async = require("async") User = require "./helpers/User" request = require "./helpers/request" settings = require "settings-sharelatex" -rclient = require("redis-sharelatex").createClient(settings.redis.web) +{db, ObjectId} = require("../../../app/js/infrastructure/mongojs") describe "UserEmails", -> beforeEach (done) -> @@ -32,10 +32,15 @@ describe "UserEmails", -> expect(body[1].confirmedAt).to.not.exist cb() (cb) => - rclient.keys 'email_confirmation_token:*', (error, keys) -> + db.tokens.find { + use: 'email_confirmation', + usedAt: { $exists: false } + }, (error, tokens) => # There should only be one confirmation token at the moment - expect(keys.length).to.equal 1 - token = keys[0].split(':')[1] + expect(tokens.length).to.equal 1 + expect(tokens[0].data.email).to.equal 'newly-added-email@example.com' + expect(tokens[0].data.user_id).to.equal @user._id + token = tokens[0].token cb() (cb) => @user.request { @@ -54,9 +59,12 @@ describe "UserEmails", -> expect(body[1].confirmedAt).to.exist cb() (cb) => - rclient.keys 'email_confirmation_token:*', (error, keys) -> + db.tokens.find { + use: 'email_confirmation', + usedAt: { $exists: false } + }, (error, tokens) => # Token should be deleted after use - expect(keys.length).to.equal 0 + expect(tokens.length).to.equal 0 cb() ], done @@ -75,11 +83,15 @@ describe "UserEmails", -> json: {@email} }, cb (cb) => - rclient.keys 'email_confirmation_token:*', (error, keys) -> - # There should only be one confirmation token at the moment, - # for the first user - expect(keys.length).to.equal 1 - token1 = keys[0].split(':')[1] + db.tokens.find { + use: 'email_confirmation', + usedAt: { $exists: false } + }, (error, tokens) => + # There should only be one confirmation token at the moment + expect(tokens.length).to.equal 1 + expect(tokens[0].data.email).to.equal @email + expect(tokens[0].data.user_id).to.equal @user._id + token1 = tokens[0].token cb() (cb) => # Delete the email from the first user @@ -107,14 +119,19 @@ describe "UserEmails", -> expect(response.statusCode).to.equal 404 cb() (cb) => - rclient.keys 'email_confirmation_token:*', (error, keys) -> + db.tokens.find { + use: 'email_confirmation', + usedAt: { $exists: false } + }, (error, tokens) => # The first token has been used, so this should be token2 now - expect(keys.length).to.equal 1 - token2 = keys[0].split(':')[1] + expect(tokens.length).to.equal 1 + expect(tokens[0].data.email).to.equal @email + expect(tokens[0].data.user_id).to.equal @user2._id + token2 = tokens[0].token cb() (cb) => # Second user should be able to confirm the email - @user2.request { + @user2.request { method: 'POST', url: '/user/emails/confirm', json: @@ -130,3 +147,48 @@ describe "UserEmails", -> expect(body[1].confirmedAt).to.exist cb() ], done + + describe "with an expired token", -> + it 'should not confirm the email', (done) -> + token = null + async.series [ + (cb) => + @user.request { + method: 'POST', + url: '/user/emails', + json: + email: @email = 'expired-token-email@example.com' + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 204 + cb() + (cb) => + db.tokens.find { + use: 'email_confirmation', + usedAt: { $exists: false } + }, (error, tokens) => + # There should only be one confirmation token at the moment + expect(tokens.length).to.equal 1 + expect(tokens[0].data.email).to.equal @email + expect(tokens[0].data.user_id).to.equal @user._id + token = tokens[0].token + cb() + (cb) => + db.tokens.update { + token: token + }, { + $set: { + expiresAt: new Date(Date.now() - 1000000) + } + }, cb + (cb) => + @user.request { + method: 'POST', + url: '/user/emails/confirm', + json: + token: token + }, (error, response, body) => + return done(error) if error? + expect(response.statusCode).to.equal 404 + cb() + ], done diff --git a/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee b/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee index 21e8d9f288..c0e6d9cc13 100644 --- a/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee +++ b/services/web/test/unit/coffee/Security/OneTimeTokenHandlerTests.coffee @@ -5,64 +5,99 @@ path = require('path') sinon = require('sinon') modulePath = path.join __dirname, "../../../../app/js/Features/Security/OneTimeTokenHandler" expect = require("chai").expect +Errors = require "../../../../app/js/Features/Errors/Errors" +tk = require("timekeeper") describe "OneTimeTokenHandler", -> - beforeEach -> - @value = "user id here" - @stubbedToken = require("crypto").randomBytes(32) - - @settings = - redis: - web:{} - @redisMulti = - set:sinon.stub() - get:sinon.stub() - del:sinon.stub() - expire:sinon.stub() - exec:sinon.stub() - self = @ + tk.freeze Date.now() # freeze the time for these tests + @stubbedToken = "mock-token" + @callback = sinon.stub() @OneTimeTokenHandler = SandboxedModule.require modulePath, requires: - "../../infrastructure/RedisWrapper" : - client: => - auth:-> - multi: -> return self.redisMulti - "settings-sharelatex":@settings "logger-sharelatex": log:-> "crypto": randomBytes: () => @stubbedToken + "../Errors/Errors": Errors + "../../infrastructure/mongojs": db: @db = tokens: {} + afterEach -> + tk.reset() describe "getNewToken", -> + beforeEach -> + @db.tokens.insert = sinon.stub().yields() - it "should set a new token into redis with a ttl", (done)-> - @redisMulti.exec.callsArgWith(0) - @OneTimeTokenHandler.getNewToken 'password', @value, (err, token) => - @redisMulti.set.calledWith("password_token:#{@stubbedToken.toString("hex")}", @value).should.equal true - @redisMulti.expire.calledWith("password_token:#{@stubbedToken.toString("hex")}", 60 * 60).should.equal true - done() + describe 'normally', -> + beforeEach -> + @OneTimeTokenHandler.getNewToken 'password', 'mock-data-to-store', @callback - it "should return if there was an error", (done)-> - @redisMulti.exec.callsArgWith(0, "error") - @OneTimeTokenHandler.getNewToken 'password', @value, (err, token)=> - err.should.exist - done() + it "should insert a generated token with a 1 hour expiry", -> + @db.tokens.insert + .calledWith({ + use: 'password' + token: @stubbedToken, + createdAt: new Date(), + expiresAt: new Date(Date.now() + 60 * 60 * 1000) + data: 'mock-data-to-store' + }) + .should.equal true - it "should allow the expiry time to be overridden", (done) -> - @redisMulti.exec.callsArgWith(0) - @ttl = 42 - @OneTimeTokenHandler.getNewToken 'password', @value, {expiresIn: @ttl}, (err, token) => - @redisMulti.expire.calledWith("password_token:#{@stubbedToken.toString("hex")}", @ttl).should.equal true - done() + it 'should call the callback with the token', -> + @callback.calledWith(null, @stubbedToken).should.equal true + + describe 'with an optional expiresIn parameter', -> + beforeEach -> + @OneTimeTokenHandler.getNewToken 'password', 'mock-data-to-store', { expiresIn: 42 }, @callback + + it "should insert a generated token with a custom expiry", -> + @db.tokens.insert + .calledWith({ + use: 'password' + token: @stubbedToken, + createdAt: new Date(), + expiresAt: new Date(Date.now() + 42 * 1000) + data: 'mock-data-to-store' + }) + .should.equal true + + it 'should call the callback with the token', -> + @callback.calledWith(null, @stubbedToken).should.equal true describe "getValueFromTokenAndExpire", -> + describe 'successfully', -> + beforeEach -> + @db.tokens.findAndModify = sinon.stub().yields(null, { data: 'mock-data' }) + @OneTimeTokenHandler.getValueFromTokenAndExpire 'password', 'mock-token', @callback + + it 'should expire the token', -> + @db.tokens.findAndModify + .calledWith({ + query: { + use: 'password' + token: 'mock-token', + expiresAt: { $gt: new Date() }, + usedAt: { $exists: false } + }, + update: { + $set: { usedAt: new Date() } + } + }) + .should.equal true + + it 'should return the data', -> + @callback.calledWith(null, 'mock-data').should.equal true + + describe 'when a valid token is not found', -> + beforeEach -> + @db.tokens.findAndModify = sinon.stub().yields(null, null) + @OneTimeTokenHandler.getValueFromTokenAndExpire 'password', 'mock-token', @callback + + it 'should return a NotFoundError', -> + @callback + .calledWith(sinon.match.instanceOf(Errors.NotFoundError)) + .should.equal true + + - it "should get and delete the token", (done)-> - @redisMulti.exec.callsArgWith(0, null, [@value]) - @OneTimeTokenHandler.getValueFromTokenAndExpire 'password', @stubbedToken, (err, value)=> - value.should.equal @value - @redisMulti.get.calledWith("password_token:#{@stubbedToken}").should.equal true - @redisMulti.del.calledWith("password_token:#{@stubbedToken}").should.equal true - done() diff --git a/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee b/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee index 22cd70c4bc..95f72e0907 100644 --- a/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee +++ b/services/web/test/unit/coffee/User/UserEmailsConfirmationHandlerTests.coffee @@ -36,7 +36,7 @@ describe "UserEmailsConfirmationHandler", -> @OneTimeTokenHandler.getNewToken .calledWith( 'email_confirmation', - JSON.stringify({@user_id, @email}), + {@user_id, @email}, { expiresIn: 365 * 24 * 60 * 60 } ) .should.equal true @@ -72,7 +72,7 @@ describe "UserEmailsConfirmationHandler", -> beforeEach -> @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( null, - JSON.stringify({@user_id, @email}) + {@user_id, @email} ) @UserUpdater.confirmEmail = sinon.stub().yields() @@ -105,7 +105,7 @@ describe "UserEmailsConfirmationHandler", -> beforeEach -> @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( null, - JSON.stringify({@email}) + {@email} ) @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback @@ -116,7 +116,7 @@ describe "UserEmailsConfirmationHandler", -> beforeEach -> @OneTimeTokenHandler.getValueFromTokenAndExpire = sinon.stub().yields( null, - JSON.stringify({@user_id}) + {@user_id} ) @UserEmailsConfirmationHandler.confirmEmailFromToken @token = 'mock-token', @callback From de45c08585bc93d124518336a4b6763e82c81c8c Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 21 Jun 2018 11:00:25 +0100 Subject: [PATCH 25/63] Restrict token look ups by id to not conflict with other tests --- services/web/test/acceptance/coffee/UserEmailsTests.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/web/test/acceptance/coffee/UserEmailsTests.coffee b/services/web/test/acceptance/coffee/UserEmailsTests.coffee index 248c0331ae..01f1dde531 100644 --- a/services/web/test/acceptance/coffee/UserEmailsTests.coffee +++ b/services/web/test/acceptance/coffee/UserEmailsTests.coffee @@ -34,6 +34,7 @@ describe "UserEmails", -> (cb) => db.tokens.find { use: 'email_confirmation', + 'data.user_id': @user._id, usedAt: { $exists: false } }, (error, tokens) => # There should only be one confirmation token at the moment @@ -61,6 +62,7 @@ describe "UserEmails", -> (cb) => db.tokens.find { use: 'email_confirmation', + 'data.user_id': @user._id, usedAt: { $exists: false } }, (error, tokens) => # Token should be deleted after use @@ -85,6 +87,7 @@ describe "UserEmails", -> (cb) => db.tokens.find { use: 'email_confirmation', + 'data.user_id': @user._id, usedAt: { $exists: false } }, (error, tokens) => # There should only be one confirmation token at the moment @@ -121,6 +124,7 @@ describe "UserEmails", -> (cb) => db.tokens.find { use: 'email_confirmation', + 'data.user_id': @user2._id, usedAt: { $exists: false } }, (error, tokens) => # The first token has been used, so this should be token2 now @@ -165,6 +169,7 @@ describe "UserEmails", -> (cb) => db.tokens.find { use: 'email_confirmation', + 'data.user_id': @user._id, usedAt: { $exists: false } }, (error, tokens) => # There should only be one confirmation token at the moment From caee25d85d3b6aa778f3bb04015a3a5efc9b7107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Alby?= Date: Thu, 21 Jun 2018 15:52:20 +0200 Subject: [PATCH 26/63] Revert "Replace OldAssetsProxy" --- .../infrastructure/OldAssetProxy.coffee | 16 +++++ .../coffee/infrastructure/ProxyManager.coffee | 25 -------- .../app/coffee/infrastructure/Server.coffee | 4 +- .../infrastructure/ProxyManagerTests.coffee | 60 ------------------- 4 files changed, 18 insertions(+), 87 deletions(-) create mode 100644 services/web/app/coffee/infrastructure/OldAssetProxy.coffee delete mode 100644 services/web/app/coffee/infrastructure/ProxyManager.coffee delete mode 100644 services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee diff --git a/services/web/app/coffee/infrastructure/OldAssetProxy.coffee b/services/web/app/coffee/infrastructure/OldAssetProxy.coffee new file mode 100644 index 0000000000..7912f0ed14 --- /dev/null +++ b/services/web/app/coffee/infrastructure/OldAssetProxy.coffee @@ -0,0 +1,16 @@ +settings = require("settings-sharelatex") +logger = require("logger-sharelatex") +request = require("request") + +module.exports = (req, res, next)-> + requestedUrl = req.url + + redirectUrl = settings.proxyUrls[requestedUrl] + if redirectUrl? + logger.log redirectUrl:redirectUrl, reqUrl:req.url, "proxying url" + upstream = request(redirectUrl) + upstream.on "error", (error) -> + logger.error err: error, "error in OldAssetProxy" + upstream.pipe(res) + else + next() diff --git a/services/web/app/coffee/infrastructure/ProxyManager.coffee b/services/web/app/coffee/infrastructure/ProxyManager.coffee deleted file mode 100644 index d93c696ab1..0000000000 --- a/services/web/app/coffee/infrastructure/ProxyManager.coffee +++ /dev/null @@ -1,25 +0,0 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -httpProxy = require 'express-http-proxy' - -module.exports = - # add proxy for all paths listed in `settings.proxyUrls`and log errors - apply: (app) -> - for requestUrl, target of settings.proxyUrls - targetUrl = @makeTargetUrl(target) - if targetUrl? - app.use requestUrl, httpProxy(targetUrl) - else - logger.error "Cannot make proxy target from #{target}" - - # takes a 'target' and return an URL to proxy to. - # 'target' can be: - # - a String, representing the URL - # - an Object with: - # - a path attribute (String) - # - a baseURL attribute (String) - makeTargetUrl: (target) -> - return target if typeof target is 'string' - return target.path unless target.baseUrl? - "#{target.baseUrl}#{target.path or ''}" - diff --git a/services/web/app/coffee/infrastructure/Server.coffee b/services/web/app/coffee/infrastructure/Server.coffee index 522599c659..39fc3c131b 100644 --- a/services/web/app/coffee/infrastructure/Server.coffee +++ b/services/web/app/coffee/infrastructure/Server.coffee @@ -32,7 +32,7 @@ Mongoose = require("./Mongoose") oneDayInMilliseconds = 86400000 ReferalConnect = require('../Features/Referal/ReferalConnect') RedirectManager = require("./RedirectManager") -ProxyManager = require("./ProxyManager") +OldAssetProxy = require("./OldAssetProxy") translations = require("translations-sharelatex").setup(Settings.i18n) Modules = require "./Modules" @@ -74,7 +74,7 @@ app.use methodOverride() app.use metrics.http.monitor(logger) app.use RedirectManager -ProxyManager.apply(app) +app.use OldAssetProxy webRouter.use cookieParser(Settings.security.sessionSecret) diff --git a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee deleted file mode 100644 index 5782a4e2fc..0000000000 --- a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee +++ /dev/null @@ -1,60 +0,0 @@ -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = '../../../../app/js/infrastructure/ProxyManager' -SandboxedModule = require('sandboxed-module') -MockRequest = require "../helpers/MockRequest" -MockResponse = require "../helpers/MockResponse" - -describe "ProxyManager", -> - before -> - @settings = proxyUrls: {} - @httpProxy = sinon.stub() - @proxyManager = SandboxedModule.require modulePath, requires: - 'settings-sharelatex': @settings - 'logger-sharelatex': - error: -> - log: -> console.log(arguments) - 'httpProxy': @httpProxy - - describe 'apply', -> - before -> - @sandbox = sinon.sandbox.create() - @app = use: sinon.stub() - @sandbox.stub(@proxyManager, 'makeTargetUrl') - - after -> - @sandbox.restore() - - beforeEach -> - @app.use.reset() - @requestUrl = '/foo/bar' - @settings.proxyUrls[@requestUrl] = 'http://whatever' - - it 'sets routes', -> - @proxyManager.makeTargetUrl.returns('http://whatever') - @proxyManager.apply(@app) - @app.use.calledWith(@requestUrl).should.equal true - - it 'logs errors', -> - @proxyManager.makeTargetUrl.returns(null) - @proxyManager.apply(@app) - @app.use.called.should.equal false - - describe 'makeTargetUrl', -> - it 'returns Strings', -> - target = 'http://whatever' - @proxyManager.makeTargetUrl(target).should.equal target - - it 'makes with path', -> - target = path: 'baz' - @proxyManager.makeTargetUrl(target).should.equal 'baz' - - it 'makes with baseUrl', -> - target = baseUrl: 'foo.bar' - @proxyManager.makeTargetUrl(target).should.equal 'foo.bar' - - it 'makes with baseUrl and path', -> - target = path: 'baz', baseUrl: 'foo.bar/' - @proxyManager.makeTargetUrl(target).should.equal 'foo.bar/baz' From 5abb745e05131a0d7ab5a88e15d7fe24bdb00ea4 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 22 Jun 2018 10:00:39 +0100 Subject: [PATCH 27/63] Copy linkedFileData when cloning a project --- .../Features/Project/ProjectEntityUpdateHandler.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee b/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee index 22034200f5..b7be80cb47 100644 --- a/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee +++ b/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee @@ -55,7 +55,10 @@ module.exports = ProjectEntityUpdateHandler = self = logger.err { project_id, folder_id, originalProject_id, origonalFileRef }, "file trying to copy is null" return callback() # convert any invalid characters in original file to '_' - fileRef = new File name : SafePath.clean(origonalFileRef.name) + fileProperties = name : SafePath.clean(origonalFileRef.name) + if origonalFileRef.linkedFileData? + fileProperties.linkedFileData = origonalFileRef.linkedFileData + fileRef = new File(fileProperties) FileStoreHandler.copyFile originalProject_id, origonalFileRef._id, project._id, fileRef._id, (err, fileStoreUrl)-> if err? logger.err { err, project_id, folder_id, originalProject_id, origonalFileRef }, "error coping file in s3" From fadbd72837997eeb7fd5eb63fbf26d7f933d2493 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 22 Jun 2018 10:52:49 +0100 Subject: [PATCH 28/63] Add unit test for copying project with linked files --- .../ProjectEntityUpdateHandlerTests.coffee | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee b/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee index 3a5b7a58bd..3de4546e6d 100644 --- a/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee +++ b/services/web/test/unit/coffee/Project/ProjectEntityUpdateHandlerTests.coffee @@ -44,6 +44,8 @@ describe 'ProjectEntityUpdateHandler', -> else @._id = file_id @rev = 0 + if options.linkedFileData? + @linkedFileData = options.linkedFileData @docName = "doc-name" @docLines = ['1234','abc'] @@ -121,6 +123,35 @@ describe 'ProjectEntityUpdateHandler', -> .calledWithMatch(project_id, projectHistoryId, userId, changesMatcher) .should.equal true + describe 'copyFileFromExistingProjectWithProject, with linkedFileData', -> + + beforeEach -> + @oldProject_id = "123kljadas" + @oldFileRef = { + _id:"oldFileRef", + name:@fileName, + linkedFileData: @linkedFileData + } + @ProjectEntityMongoUpdateHandler._confirmFolder = sinon.stub().yields(folder_id) + @ProjectEntityMongoUpdateHandler._putElement = sinon.stub().yields(null, {path:{fileSystem: @fileSystemPath}}) + + @ProjectEntityUpdateHandler.copyFileFromExistingProjectWithProject @project, folder_id, @oldProject_id, @oldFileRef, userId, @callback + + it 'should copy the file in FileStoreHandler', -> + @FileStoreHandler.copyFile + .calledWith(@oldProject_id, @oldFileRef._id, project_id, file_id) + .should.equal true + + it 'should put file into folder by calling put element, with the linkedFileData', -> + @ProjectEntityMongoUpdateHandler._putElement + .calledWithMatch( + @project, + folder_id, + { _id: file_id, name: @fileName, linkedFileData: @linkedFileData}, + "file" + ) + .should.equal true + describe 'updateDocLines', -> beforeEach -> @path = "/somewhere/something.tex" @@ -285,7 +316,7 @@ describe 'ProjectEntityUpdateHandler', -> beforeEach -> @path = "/path/to/file" - @newFile = {_id: file_id, rev: 0, name: @fileName} + @newFile = {_id: file_id, rev: 0, name: @fileName, linkedFileData: @linkedFileData} @TpdsUpdateSender.addFile = sinon.stub().yields() @ProjectEntityMongoUpdateHandler.addFile = sinon.stub().yields(null, {path: fileSystem: @path}, @project) @ProjectEntityUpdateHandler.addFile project_id, folder_id, @fileName, @fileSystemPath, @linkedFileData, userId, @callback @@ -330,7 +361,7 @@ describe 'ProjectEntityUpdateHandler', -> @newFileUrl = "new-file-url" @FileStoreHandler.uploadFileFromDisk = sinon.stub().yields(null, @newFileUrl) - @newFile = _id: new_file_id, name: "dummy-upload-filename", rev: 0 + @newFile = _id: new_file_id, name: "dummy-upload-filename", rev: 0, linkedFileData: @linkedFileData @oldFile = _id: file_id @path = "/path/to/file" @ProjectEntityMongoUpdateHandler._insertDeletedFileReference = sinon.stub().yields() From 60ca298db3e179d52de5afb44577b5ee9e615e33 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 09:35:52 +0100 Subject: [PATCH 29/63] WIP --- .../app/views/project/editor/file-tree.pug | 2 +- .../app/views/project/editor/left-menu.pug | 4 + .../controllers/FileTreeController.coffee | 85 ++++++++++++++++--- .../ide/pdf/controllers/PdfController.coffee | 1 + .../controllers/TestControlsController.coffee | 3 + 5 files changed, 80 insertions(+), 15 deletions(-) diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug index d16258d1c9..18d313f5cb 100644 --- a/services/web/app/views/project/editor/file-tree.pug +++ b/services/web/app/views/project/editor/file-tree.pug @@ -345,7 +345,7 @@ script(type='text/ng-template', id='newDocModalTemplate') // Project Linked Files Modal script(type='text/ng-template', id='projectLinkedFileModalTemplate') .modal-header - h3 New file from Project + h3 New file from Project ({{ isOutputFiles }}) .modal-body div diff --git a/services/web/app/views/project/editor/left-menu.pug b/services/web/app/views/project/editor/left-menu.pug index d85a45723b..c1ff891844 100644 --- a/services/web/app/views/project/editor/left-menu.pug +++ b/services/web/app/views/project/editor/left-menu.pug @@ -73,6 +73,10 @@ aside#left-menu.full-size( a(href="#" ng-click="openProjectLinkedFileModal()") i.fa.fa-exclamation.fa-fw | Project-Linked-File Modal + li + a(href="#" ng-click="openProjectOutputLinkedFileModal()") + i.fa.fa-exclamation.fa-fw + | Project-Output-Linked-File Modal li a(href="#" ng-click="openLinkedFileModal()") i.fa.fa-exclamation.fa-fw diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index 010e00476f..422e294129 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -53,6 +53,21 @@ define [ scope: $scope resolve: { parent_folder: () -> ide.fileTreeManager.getCurrentFolder() + isOutputFiles: () -> false + } + ) + + $scope.openProjectOutputLinkedFileModal = window.openProjectOutputLinkedFileModal = () -> + unless 'project_output_file' in window.data.enabledLinkedFileTypes + console.warn("Project linked output files are not enabled") + return + $modal.open( + templateUrl: "projectLinkedFileModalTemplate" + controller: "ProjectLinkedFileModalController" + scope: $scope + resolve: { + parent_folder: () -> ide.fileTreeManager.getCurrentFolder() + isOutputFiles: () -> true } ) @@ -215,25 +230,33 @@ define [ ] App.controller "ProjectLinkedFileModalController", [ - "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", - ($scope, ide, $modalInstance, $timeout, parent_folder) -> + "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", "isOutputFiles", + ($scope, ide, $modalInstance, $timeout, parent_folder, isOutputFiles) -> + $scope.isOutputFiles = isOutputFiles + console.log ">>>> LINKED", isOutputFiles + $scope.data = projects: null # or [] selectedProjectId: null projectEntities: null # or [] + projectOutputFiles: null # or [] selectedProjectEntity: null name: null $scope.state = inFlight: projects: false entities: false + compile: false create: false error: false $scope.$watch 'data.selectedProjectId', (newVal, oldVal) -> return if !newVal $scope.data.selectedProjectEntity = null - $scope.getProjectEntities($scope.data.selectedProjectId) + if isOutputFiles + $scope.compileProjectAndGetOutputFiles($scope.data.selectedProjectId) + else + $scope.getProjectEntities($scope.data.selectedProjectId) # auto-set filename based on selected file $scope.$watch 'data.selectedProjectEntity', (newVal, oldVal) -> @@ -248,7 +271,7 @@ define [ _reset = (opts) -> isError = opts.err == true inFlight = $scope.state.inFlight - inFlight.projects = inFlight.entities = inFlight.create = false + inFlight.projects = inFlight.entities = inFlight.compile = inFlight.create = false $scope.state.error = isError $scope.shouldEnableProjectSelect = () -> @@ -259,6 +282,10 @@ define [ { state, data } = $scope return !state.inFlight.projects && !state.inFlight.entities && data.projects && data.selectedProjectId + $scope.shouldEnableProjectOutputFileSelect = () -> + { state, data } = $scope + return !state.inFlight.projects && !state.inFlight.compile && data.projects && data.selectedProjectId + $scope.shouldEnableCreateButton = () -> state = $scope.state data = $scope.data @@ -266,8 +293,18 @@ define [ !state.inFlight.entities && data.projects && data.selectedProjectId && - data.projectEntities && - data.selectedProjectEntity && + ( + ( + !isOutputFiles && + data.projectEntities && + data.selectedProjectEntity + ) || + ( + isOutputFiles && + data.projectOutputFiles && + data.selectedProjectOutputFile + ) + ) && data.name $scope.getUserProjects = () -> @@ -295,23 +332,43 @@ define [ .catch (err) -> _reset(err: true) + window._C = $scope.compileProjectAndGetOutputFiles = (project_id) => + _setInFlight('compile') + $http.post("/project/${project_id}/compile", { + check: "silent", + draft: false, + incrementalCompilesEnabled: false + _csrf: window.csrfToken + }) + .then (data) -> + console.log ">> COMPILE", data + _reset(err: false) + .catch (err) -> + console.error(err) + _reset(err: true) + $scope.init = () -> $scope.getUserProjects() $timeout($scope.init, 0) $scope.create = () -> projectId = $scope.data.selectedProjectId - path = $scope.data.selectedProjectEntity name = $scope.data.name - if !name || !path || !projectId - _reset(err: true) - return + if isOutputFiles + provider = 'project_output_file' + payload = { + source_project_id: projectId, + source_output_file_path: $scope.data.selectedProjectOutputFile + } + else + provider = 'project_file' + payload = { + source_project_id: projectId, + source_entity_path: $scope.data.selectedProjectEntity + } _setInFlight('create') ide.fileTreeManager - .createLinkedFile(name, parent_folder, 'project_file', { - source_project_id: projectId, - source_entity_path: path - }) + .createLinkedFile(name, parent_folder, provider, payload) .then () -> _reset(err: false) $modalInstance.close() diff --git a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee index 77a94a564b..098b436e86 100644 --- a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee +++ b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee @@ -524,6 +524,7 @@ define [ { data } = response $scope.pdf.view = "pdf" $scope.pdf.compiling = false + console.log ">>", data parseCompileResponse(data) .catch (response) -> { data, status } = response diff --git a/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee b/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee index ae7db45905..0dca64dc5b 100644 --- a/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee +++ b/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee @@ -7,6 +7,9 @@ define [ $scope.openProjectLinkedFileModal = () -> window.openProjectLinkedFileModal() + $scope.openProjectOutputLinkedFileModal = () -> + window.openProjectOutputLinkedFileModal() + $scope.openLinkedFileModal = () -> window.openLinkedFileModal() From ead245721be6cc4d0c8b6ab4446f0437e5e7f47a Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 12:32:22 +0100 Subject: [PATCH 30/63] Mostly working selection of output files from another project --- .../app/views/project/editor/file-tree.pug | 20 ++++++++- .../controllers/FileTreeController.coffee | 43 ++++++++++++------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug index 18d313f5cb..70fa5badc9 100644 --- a/services/web/app/views/project/editor/file-tree.pug +++ b/services/web/app/views/project/editor/file-tree.pug @@ -345,7 +345,7 @@ script(type='text/ng-template', id='newDocModalTemplate') // Project Linked Files Modal script(type='text/ng-template', id='projectLinkedFileModalTemplate') .modal-header - h3 New file from Project ({{ isOutputFiles }}) + h3 New file from Project ({{ isOutputFilesMode }}) .modal-body div @@ -369,7 +369,7 @@ script(type='text/ng-template', id='projectLinkedFileModalTemplate') ) {{ project.name }} br - .form-controls + .form-controls(ng-if="!isOutputFilesMode") label(for="project-entity-select") Select a File span(ng-show="state.inFlight.entities") |   @@ -384,6 +384,22 @@ script(type='text/ng-template', id='projectLinkedFileModalTemplate') ng-repeat="projectEntity in data.projectEntities" value="{{ projectEntity.path }}" ) {{ projectEntity.path.slice(1) }} + + .form-controls(ng-if="isOutputFilesMode") + label(for="project-entity-select") Select an Output File + span(ng-show="state.inFlight.compile") + |   + i.fa.fa-spinner.fa-spin + select.form-control( + name="project-output-file-select" + ng-model="data.selectedProjectOutputFile" + ng-disabled="!shouldEnableProjectOutputFileSelect()" + ) + option(value="" disabled selected) - Please Select an Output File + option( + ng-repeat="outputFile in data.projectOutputFiles" + value="{{ outputFile.path }}" + ) {{ outputFile.path }} br .form-controls diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index 422e294129..eec7d0627f 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -53,7 +53,7 @@ define [ scope: $scope resolve: { parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - isOutputFiles: () -> false + isOutputFilesMode: () -> false } ) @@ -67,7 +67,7 @@ define [ scope: $scope resolve: { parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - isOutputFiles: () -> true + isOutputFilesMode: () -> true } ) @@ -230,10 +230,10 @@ define [ ] App.controller "ProjectLinkedFileModalController", [ - "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", "isOutputFiles", - ($scope, ide, $modalInstance, $timeout, parent_folder, isOutputFiles) -> - $scope.isOutputFiles = isOutputFiles - console.log ">>>> LINKED", isOutputFiles + "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", "isOutputFilesMode", + ($scope, ide, $modalInstance, $timeout, parent_folder, isOutputFilesMode) -> + $scope.isOutputFilesMode = isOutputFilesMode + console.log ">>>> LINKED", isOutputFilesMode $scope.data = projects: null # or [] @@ -241,6 +241,7 @@ define [ projectEntities: null # or [] projectOutputFiles: null # or [] selectedProjectEntity: null + selectedProjectOutputFile: null name: null $scope.state = inFlight: @@ -253,7 +254,7 @@ define [ $scope.$watch 'data.selectedProjectId', (newVal, oldVal) -> return if !newVal $scope.data.selectedProjectEntity = null - if isOutputFiles + if isOutputFilesMode $scope.compileProjectAndGetOutputFiles($scope.data.selectedProjectId) else $scope.getProjectEntities($scope.data.selectedProjectId) @@ -265,6 +266,13 @@ define [ if fileName $scope.data.name = fileName + # auto-set filename based on selected file + $scope.$watch 'data.selectedProjectOutputFile', (newVal, oldVal) -> + return if !newVal + fileName = newVal.split('/').reverse()[0] + if fileName + $scope.data.name = fileName + _setInFlight = (type) -> $scope.state.inFlight[type] = true @@ -295,12 +303,12 @@ define [ data.selectedProjectId && ( ( - !isOutputFiles && + !isOutputFilesMode && data.projectEntities && data.selectedProjectEntity ) || ( - isOutputFiles && + isOutputFilesMode && data.projectOutputFiles && data.selectedProjectOutputFile ) @@ -332,17 +340,22 @@ define [ .catch (err) -> _reset(err: true) - window._C = $scope.compileProjectAndGetOutputFiles = (project_id) => + $scope.compileProjectAndGetOutputFiles = (project_id) => _setInFlight('compile') - $http.post("/project/${project_id}/compile", { + ide.$http.post("/project/#{project_id}/compile", { check: "silent", draft: false, incrementalCompilesEnabled: false _csrf: window.csrfToken }) - .then (data) -> - console.log ">> COMPILE", data - _reset(err: false) + .then (resp) -> + console.log ">> COMPILE", resp.data + if resp.data.status == 'success' + $scope.data.projectOutputFiles = resp.data.outputFiles + _reset(err: false) + else + $scope.data.projectOutputFiles = null + _reset(err: true) .catch (err) -> console.error(err) _reset(err: true) @@ -354,7 +367,7 @@ define [ $scope.create = () -> projectId = $scope.data.selectedProjectId name = $scope.data.name - if isOutputFiles + if isOutputFilesMode provider = 'project_output_file' payload = { source_project_id: projectId, From e916d967929cfa1b09d1965e1921534afbb83628 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 15:33:02 +0100 Subject: [PATCH 31/63] WIP: basic backend for project-output-file agent --- .../coffee/Features/LinkedFiles/LinkedFilesController.coffee | 3 ++- services/web/app/views/project/editor/file-tree.pug | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee index 13de54e947..00c061778c 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee @@ -8,7 +8,8 @@ _ = require 'underscore' module.exports = LinkedFilesController = { Agents: { url: require('./UrlAgent'), - project_file: require('./ProjectFileAgent') + project_file: require('./ProjectFileAgent'), + project_output_file: require('./ProjectOutputFileAgent') } _getAgent: (provider) -> diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug index 70fa5badc9..89e6db9f17 100644 --- a/services/web/app/views/project/editor/file-tree.pug +++ b/services/web/app/views/project/editor/file-tree.pug @@ -345,7 +345,9 @@ script(type='text/ng-template', id='newDocModalTemplate') // Project Linked Files Modal script(type='text/ng-template', id='projectLinkedFileModalTemplate') .modal-header - h3 New file from Project ({{ isOutputFilesMode }}) + h3 + span(ng-if="!isOutputFilesMode") New file from Project + span(ng-if="isOutputFilesMode") New file from Project output .modal-body div From d4beba24b62626bd411dea0c3734b953d3263a91 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 16:06:47 +0100 Subject: [PATCH 32/63] Backend for project output file agent --- .../LinkedFiles/ProjectOutputFileAgent.coffee | 102 ++++++++++++++++++ .../app/views/project/editor/binary-file.pug | 20 +++- 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee new file mode 100644 index 0000000000..4850abd0e0 --- /dev/null +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -0,0 +1,102 @@ +FileWriter = require('../../infrastructure/FileWriter') +AuthorizationManager = require('../Authorization/AuthorizationManager') +ProjectGetter = require('../Project/ProjectGetter') +FileWriter = require('../../infrastructure/FileWriter') +Settings = require 'settings-sharelatex' +CompileManager = require '../Compile/CompileManager' +CompileController = require '../Compile/CompileController' +ClsiCookieManager = require '../Compile/ClsiCookieManager' +_ = require "underscore" +request = require "request" + + +BadDataError = (message) -> + error = new Error(message) + error.name = 'BadData' + error.__proto__ = BadDataError.prototype + return error +BadDataError.prototype.__proto__ = Error.prototype + + +ProjectNotFoundError = (message) -> + error = new Error(message) + error.name = 'ProjectNotFound' + error.__proto__ = ProjectNotFoundError.prototype + return error +ProjectNotFoundError.prototype.__proto__ = Error.prototype + + +OutputFileFetchFailedError = (message) -> + error = new Error(message) + error.name = 'OutputFileFetchFailedError' + error.__proto__ = OutputFileFetchFailedError.prototype + return error +OutputFileFetchFailedError.prototype.__proto__ = Error.prototype + + +module.exports = ProjectOutputFileAgent = { + + sanitizeData: (data) -> + return { + source_project_id: data.source_project_id, + source_output_file_path: data.source_output_file_path + } + + canCreate: (data) -> true + + decorateLinkedFileData: (data, callback = (err, newData) ->) -> + callback = _.once(callback) + ProjectGetter.getProject data.source_project_id, {name: 1}, (err, project) -> + return callback(err) if err? + if !project? + return callback(new ProjectNotFoundError()) + callback(err, _.extend(data, {source_project_display_name: project.name})) + + checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> + callback = _.once(callback) + { source_project_id } = data + AuthorizationManager.canUserReadProject current_user_id, source_project_id, null, (err, canRead) -> + return callback(err) if err? + callback(null, canRead) + + _validate: (data) -> + data.source_project_id? && data.source_output_file_path? + + writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> + callback = _.once(callback) + # TODO: + # - Compile project + # - Get output file content + # - Write to disk + # - callback with fs-path + if !ProjectOutputFileAgent._validate(data) + return callback(new BadDataError()) + { source_project_id, source_output_file_path } = data + CompileManager.compile source_project_id, null, {}, (err) -> + return callback(err) if err? + url = "#{Settings.apis.clsi.url}/project/#{source_project_id}/output/#{source_output_file_path}" + ClsiCookieManager.getCookieJar source_project_id, (err, jar)-> + return callback(err) if err? + oneMinute = 60 * 1000 + # the base request + options = { url: url, method: "GET", timeout: oneMinute, jar : jar } + readStream = request(options) + readStream.on "error", callback + readStream.on "response", (response) -> + if 200 <= response.statusCode < 300 + FileWriter.writeStreamToDisk project_id, readStream, callback + else + error = new OutputFileFetchFailedError("Output file fetch failed: #{url}") + error.statusCode = response.statusCode + callback(error) + + handleError: (error, req, res, next) -> + if error instanceof BadDataError + res.status(400).send("The submitted data is not valid") + else if error instanceof SourceFileNotFoundError + res.status(404).send("Source file not found") + else if error instanceof ProjectNotFoundError + res.status(404).send("Project not found") + else + next(error) +} diff --git a/services/web/app/views/project/editor/binary-file.pug b/services/web/app/views/project/editor/binary-file.pug index f4ccf8c9dc..bd0e4b9732 100644 --- a/services/web/app/views/project/editor/binary-file.pug +++ b/services/web/app/views/project/editor/binary-file.pug @@ -38,6 +38,7 @@ div.binary-file.full-size( ) #{translate("no_preview_available")} div.binary-file-footer + // Linked Files: URL div(ng-if="openFile.linkedFileData.provider == 'url'") p i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon @@ -47,6 +48,7 @@ div.binary-file.full-size( | | at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }} + // Linked Files: Project File div(ng-if="openFile.linkedFileData.provider == 'project_file'") p i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon @@ -61,7 +63,23 @@ div.binary-file.full-size( | | at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }} - span(ng-if="openFile.linkedFileData.provider == 'url' || openFile.linkedFileData.provider == 'project_file'") + // Linked Files: Project Output File + div(ng-if="openFile.linkedFileData.provider == 'project_output_file'") + p + i.fa.fa-fw.fa-external-link-square.fa-rotate-180.linked-file-icon + | Imported from the output of + | + a(ng-if='!openFile.linkedFileData.v1_source_doc_id' + ng-href='/project/{{openFile.linkedFileData.source_project_id}}' target="_blank") + | {{ openFile.linkedFileData.source_project_display_name }} + span(ng-if='openFile.linkedFileData.v1_source_doc_id') + | {{ openFile.linkedFileData.source_project_display_name }} + | : {{ openFile.linkedFileData.source_output_file_path }}, + | + | at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }} + + // Bottom Controls + span(ng-if="openFile.linkedFileData.provider") button.btn.btn-success( href, ng-click="refreshFile(openFile)", ng-disabled="refreshing" From 87474ce060470d2206c3bdc6cb0f3cff48e95391 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 16:13:18 +0100 Subject: [PATCH 33/63] Remove commentary --- .../Features/LinkedFiles/ProjectOutputFileAgent.coffee | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 4850abd0e0..8af8b5bf63 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -64,11 +64,6 @@ module.exports = ProjectOutputFileAgent = { writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> callback = _.once(callback) - # TODO: - # - Compile project - # - Get output file content - # - Write to disk - # - callback with fs-path if !ProjectOutputFileAgent._validate(data) return callback(new BadDataError()) { source_project_id, source_output_file_path } = data From 2da1d57948cea414efd1dd123a9a228f8ca44dc6 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 8 Jun 2018 16:22:34 +0100 Subject: [PATCH 34/63] Clean up logging --- .../coffee/ide/file-tree/controllers/FileTreeController.coffee | 2 -- .../web/public/coffee/ide/pdf/controllers/PdfController.coffee | 1 - 2 files changed, 3 deletions(-) diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index eec7d0627f..f929474952 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -233,7 +233,6 @@ define [ "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", "isOutputFilesMode", ($scope, ide, $modalInstance, $timeout, parent_folder, isOutputFilesMode) -> $scope.isOutputFilesMode = isOutputFilesMode - console.log ">>>> LINKED", isOutputFilesMode $scope.data = projects: null # or [] @@ -349,7 +348,6 @@ define [ _csrf: window.csrfToken }) .then (resp) -> - console.log ">> COMPILE", resp.data if resp.data.status == 'success' $scope.data.projectOutputFiles = resp.data.outputFiles _reset(err: false) diff --git a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee index 098b436e86..77a94a564b 100644 --- a/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee +++ b/services/web/public/coffee/ide/pdf/controllers/PdfController.coffee @@ -524,7 +524,6 @@ define [ { data } = response $scope.pdf.view = "pdf" $scope.pdf.compiling = false - console.log ">>", data parseCompileResponse(data) .catch (response) -> { data, status } = response From 5717496685caee9144e7dba5840626c0f8cd49d8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Sun, 10 Jun 2018 15:44:00 +0100 Subject: [PATCH 35/63] Create unified new file modal with linked files --- .../app/views/project/editor/file-tree.pug | 246 +----------------- .../views/project/editor/new-file-modal.pug | 209 +++++++++++++++ .../controllers/FileTreeController.coffee | 222 ++++++++-------- .../stylesheets/app/editor/file-tree.less | 37 +++ .../stylesheets/components/fineupload.less | 12 +- 5 files changed, 363 insertions(+), 363 deletions(-) create mode 100644 services/web/app/views/project/editor/new-file-modal.pug diff --git a/services/web/app/views/project/editor/file-tree.pug b/services/web/app/views/project/editor/file-tree.pug index 89e6db9f17..d877421a9d 100644 --- a/services/web/app/views/project/editor/file-tree.pug +++ b/services/web/app/views/project/editor/file-tree.pug @@ -308,183 +308,10 @@ script(type='text/ng-template', id='entityListItemTemplate') ng-repeat="child in entity.children | orderBy:[orderByFoldersFirst, 'name']" ) -script(type='text/ng-template', id='newDocModalTemplate') - .modal-header - h3 #{translate("new_file")} - .modal-body - form(novalidate, name="newDocForm") - div.alert.alert-danger(ng-if="error") - div(ng-switch="error") - span(ng-switch-when="already exists") #{translate("file_already_exists")} - span(ng-switch-default) {{error}} - input.form-control( - type="text", - placeholder="File Name", - required, - ng-model="inputs.name", - on-enter="create()", - select-name-on="open", - valid-file, - name="name" - ) - .text-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile") - | #{translate('files_cannot_include_invalid_characters')} - .modal-footer - button.btn.btn-default( - ng-disabled="state.inflight" - ng-click="cancel()" - ) #{translate("cancel")} - button.btn.btn-primary( - ng-disabled="newDocForm.$invalid || state.inflight" - ng-click="create()" - ) - span(ng-hide="state.inflight") #{translate("create")} - span(ng-show="state.inflight") #{translate("creating")}... - - -// Project Linked Files Modal -script(type='text/ng-template', id='projectLinkedFileModalTemplate') - .modal-header - h3 - span(ng-if="!isOutputFilesMode") New file from Project - span(ng-if="isOutputFilesMode") New file from Project output - - .modal-body - div - div.alert.alert-danger(ng-if="state.error") Error, something went wrong! - div - form - .form-controls - label(for="project-select") Select a Project - span(ng-show="state.inFlight.projects") - |   - i.fa.fa-spinner.fa-spin - select.form-control( - name="project-select" - ng-model="data.selectedProjectId" - ng-disabled="!shouldEnableProjectSelect()" - ) - option(value="" disabled selected) - Please Select a Project - option( - ng-repeat="project in data.projects" - value="{{ project._id }}" - ) {{ project.name }} - - br - .form-controls(ng-if="!isOutputFilesMode") - label(for="project-entity-select") Select a File - span(ng-show="state.inFlight.entities") - |   - i.fa.fa-spinner.fa-spin - select.form-control( - name="project-entity-select" - ng-model="data.selectedProjectEntity" - ng-disabled="!shouldEnableProjectEntitySelect()" - ) - option(value="" disabled selected) - Please Select a File - option( - ng-repeat="projectEntity in data.projectEntities" - value="{{ projectEntity.path }}" - ) {{ projectEntity.path.slice(1) }} - - .form-controls(ng-if="isOutputFilesMode") - label(for="project-entity-select") Select an Output File - span(ng-show="state.inFlight.compile") - |   - i.fa.fa-spinner.fa-spin - select.form-control( - name="project-output-file-select" - ng-model="data.selectedProjectOutputFile" - ng-disabled="!shouldEnableProjectOutputFileSelect()" - ) - option(value="" disabled selected) - Please Select an Output File - option( - ng-repeat="outputFile in data.projectOutputFiles" - value="{{ outputFile.path }}" - ) {{ outputFile.path }} - br - - .form-controls - label(for="name") File Name In This Project - input.form-control( - type="text" - placeholder="example.tex" - required - ng-model="data.name" - name="name" - ) - br - - .modal-footer - span(ng-show="state.inFlight.create") - i.fa.fa-spinner.fa-spin - |   - button.btn.btn-default( - ng-disabled="state.inflight" - ng-click="cancel()" - ) #{translate("cancel")} - button.btn.btn-primary( - ng-disabled="!shouldEnableCreateButton()" - ng-click="create()" - ) - span(ng-hide="state.inflight") #{translate("create")} - span(ng-show="state.inflight") #{translate("creating")}... - - -script(type='text/ng-template', id='linkedFileModalTemplate') - .modal-header - h3 New file from URL - .modal-body - form(novalidate, name="newLinkedFileForm") - div.alert.alert-danger(ng-if="error") - div(ng-switch="error") - span(ng-switch-when="already exists") #{translate("file_already_exists")} - span(ng-switch-default) {{error}} - label(for="url") URL to fetch the file from - input.form-control( - type="text", - placeholder="www.example.com/my_file", - required, - ng-model="inputs.url", - focus-on="open", - on-enter="create()", - name="url" - ) - .row-spaced - label(for="name") File name in this project - input.form-control( - type="text", - placeholder="my_file", - required, - ng-model="inputs.name", - ng-change="nameChangedByUser = true" - valid-file, - on-enter="create()", - name="name" - ) - .text-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile") - | #{translate('files_cannot_include_invalid_characters')} - .modal-footer - button.btn.btn-default( - ng-disabled="state.inflight" - ng-click="cancel()" - ) #{translate("cancel")} - button.btn.btn-primary( - ng-disabled="newLinkedFileForm.$invalid || state.inflight" - ng-click="create()" - ) - span(ng-hide="state.inflight") #{translate("create")} - span(ng-show="state.inflight") #{translate("creating")}... - - script(type='text/ng-template', id='newFolderModalTemplate') .modal-header h3 #{translate("new_folder")} .modal-body - div.alert.alert-danger(ng-if="error") - div(ng-switch="error") - span(ng-switch-when="already exists") #{translate("file_already_exists")} - span(ng-switch-default) {{error}} form(novalidate, name="newFolderForm") input.form-control( type="text", @@ -496,8 +323,12 @@ script(type='text/ng-template', id='newFolderModalTemplate') valid-file, name="name" ) - .text-danger.row-spaced-small(ng-show="newFolderForm.name.$error.validFile") - | #{translate('files_cannot_include_invalid_characters')} + div.alert.alert-danger.row-spaced-small(ng-show="newFolderForm.name.$error.validFile") + | #{translate('files_cannot_include_invalid_characters')} + div.alert.alert-danger.row-spaced-small(ng-if="error") + div(ng-switch="error") + span(ng-switch-when="already exists") #{translate("file_already_exists")} + span(ng-switch-default) {{error}} .modal-footer button.btn.btn-default( ng-disabled="state.inflight" @@ -510,70 +341,7 @@ script(type='text/ng-template', id='newFolderModalTemplate') span(ng-hide="state.inflight") #{translate("create")} span(ng-show="state.inflight") #{translate("creating")}... - -script(type="text/template", id="qq-file-uploader-template") - div.qq-uploader-selector - div(qq-hide-dropzone="").qq-upload-drop-area-selector.qq-upload-drop-area - span.qq-upload-drop-area-text-selector #{translate('drop_files_here_to_upload')} - div.qq-upload-button-selector.btn.btn-primary.btn-lg - div #{translate('upload')} - span.or.btn-lg #{translate('or')} - span.drag-here.btn-lg #{translate('drag_here')} - span.qq-drop-processing-selector - span #{translate('processing')} - span.qq-drop-processing-spinner-selector - ul.qq-upload-list-selector - li - div.qq-progress-bar-container-selector - div( - role="progressbar" - aria-valuenow="0" - aria-valuemin="0" - aria-valuemax="100" - class="qq-progress-bar-selector qq-progress-bar" - ) - span.qq-upload-file-selector.qq-upload-file - span.qq-upload-size-selector.qq-upload-size - a(type="button").qq-btn.qq-upload-cancel-selector.qq-upload-cancel #{translate('cancel')} - button(type="button").qq-btn.qq-upload-retry-selector.qq-upload-retry #{translate('retry')} - span(role="status").qq-upload-status-text-selector.qq-upload-status-text - -script(type="text/ng-template", id="uploadFileModalTemplate") - .modal-header - h3 #{translate("upload_files")} - .alert.alert-warning.small.modal-alert(ng-if="tooManyFiles") #{translate("maximum_files_uploaded_together", {max:"{{max_files}}"})} - .alert.alert-warning.small.modal-alert(ng-if="rateLimitHit") #{translate("too_many_files_uploaded_throttled_short_period")} - .alert.alert-warning.small.modal-alert(ng-if="notLoggedIn") #{translate("session_expired_redirecting_to_login", {seconds:"{{secondsToRedirect}}"})} - .alert.alert-warning.small.modal-alert(ng-if="conflicts.length > 0") - p.text-center - | The following files already exist in this project: - ul.text-center.list-unstyled.row-spaced-small - li(ng-repeat="conflict in conflicts"): strong {{ conflict }} - p.text-center.row-spaced-small - | Do you want to overwrite them? - p.text-center - a(href, ng-click="doUpload()").btn.btn-primary Overwrite - |   - a(href, ng-click="cancel()").btn.btn-default Cancel - - .modal-body( - fine-upload - endpoint="/project/{{ project_id }}/upload" - template-id="qq-file-uploader-template" - multiple="true" - auto-upload="false" - on-complete-callback="onComplete" - on-upload-callback="onUpload" - on-validate-batch="onValidateBatch" - on-error-callback="onError" - on-submit-callback="onSubmit" - on-cancel-callback="onCancel" - control="control" - params="{'folder_id': parent_folder_id}" - ) - .modal-footer - button.btn.btn-default(ng-click="cancel()") #{translate("cancel")} - +include ./new-file-modal script(type='text/ng-template', id='deleteEntityModalTemplate') .modal-header diff --git a/services/web/app/views/project/editor/new-file-modal.pug b/services/web/app/views/project/editor/new-file-modal.pug new file mode 100644 index 0000000000..48c8863e5b --- /dev/null +++ b/services/web/app/views/project/editor/new-file-modal.pug @@ -0,0 +1,209 @@ +script(type='text/ng-template', id='newFileModalTemplate') + .modal-header + h3 Add Files + .modal-body.modal-new-file + table + tr + td.modal-new-file--list + ul.list-unstyled + li(ng-class="type == 'doc' ? 'active' : null") + a(href, ng-click="type = 'doc'") + i.fa.fa-fw.fa-file + | + | New File + li(ng-class="type == 'upload' ? 'active' : null") + a(href, ng-click="type = 'upload'") + i.fa.fa-fw.fa-upload + | + | Upload + li(ng-class="type == 'project' ? 'active' : null") + a(href, ng-click="type = 'project'") + i.fa.fa-fw.fa-folder-open + | + | From Another Project + li(ng-class="type == 'url' ? 'active' : null") + a(href, ng-click="type = 'url'") + i.fa.fa-fw.fa-globe + | + | From External URL + td(class="modal-new-file--body modal-new-file--body-{{type}}") + div(ng-if="type == 'doc'", ng-controller="NewDocModalController") + form(novalidate, name="newDocForm") + label(for="name") File Name + input.form-control( + type="text", + placeholder="File Name", + required, + ng-model="inputs.name", + on-enter="create()", + select-name-on="open", + valid-file, + name="name" + ) + div.alert.alert-danger.row-spaced-small(ng-if="error") + div(ng-switch="error") + span(ng-switch-when="already exists") #{translate("file_already_exists")} + span(ng-switch-default) {{error}} + div.alert.alert-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile") + | #{translate('files_cannot_include_invalid_characters')} + div(ng-if="type == 'upload'", ng-controller="UploadFileModalController") + .alert.alert-warning.small.modal(ng-if="tooManyFiles") #{translate("maximum_files_uploaded_together", {max:"{{max_files}}"})} + .alert.alert-warning.small.modal(ng-if="rateLimitHit") #{translate("too_many_files_uploaded_throttled_short_period")} + .alert.alert-warning.small.modal(ng-if="notLoggedIn") #{translate("session_expired_redirecting_to_login", {seconds:"{{secondsToRedirect}}"})} + .alert.alert-warning.small.modal(ng-if="conflicts.length > 0") + p.text-center + | The following files already exist in this project: + ul.text-center.list-unstyled.row-spaced-small + li(ng-repeat="conflict in conflicts"): strong {{ conflict }} + p.text-center.row-spaced-small + | Do you want to overwrite them? + p.text-center + a(href, ng-click="doUpload()").btn.btn-primary Overwrite + |   + a(href, ng-click="cancel()").btn.btn-default Cancel + div( + fine-upload + endpoint="/project/{{ project_id }}/upload" + template-id="qq-file-uploader-template" + multiple="true" + auto-upload="false" + on-complete-callback="onComplete" + on-upload-callback="onUpload" + on-validate-batch="onValidateBatch" + on-error-callback="onError" + on-submit-callback="onSubmit" + on-cancel-callback="onCancel" + control="control" + params="{'folder_id': parent_folder_id}" + ) + div(ng-if="type == 'project'", ng-controller="ProjectLinkedFileModalController") + div + form + .form-controls + label(for="project-select") Select a Project + span(ng-show="state.inFlight.projects") + |   + i.fa.fa-spinner.fa-spin + select.form-control( + name="project-select" + ng-model="data.selectedProjectId" + ng-disabled="!shouldEnableProjectSelect()" + ) + option(value="" disabled selected) - Please Select a Project + option( + ng-repeat="project in data.projects" + value="{{ project._id }}" + ) {{ project.name }} + + .form-controls.row-spaced-small(ng-if="!isOutputFilesMode") + label(for="project-entity-select") Select a File + span(ng-show="state.inFlight.entities") + |   + i.fa.fa-spinner.fa-spin + select.form-control( + name="project-entity-select" + ng-model="data.selectedProjectEntity" + ng-disabled="!shouldEnableProjectEntitySelect()" + ) + option(value="" disabled selected) - Please Select a File + option( + ng-repeat="projectEntity in data.projectEntities" + value="{{ projectEntity.path }}" + ) {{ projectEntity.path.slice(1) }} + + .form-controls.row-spaced-small(ng-if="isOutputFilesMode") + label(for="project-entity-select") Select an Output File + span(ng-show="state.inFlight.compile") + |   + i.fa.fa-spinner.fa-spin + select.form-control( + name="project-output-file-select" + ng-model="data.selectedProjectOutputFile" + ng-disabled="!shouldEnableProjectOutputFileSelect()" + ) + option(value="" disabled selected) - Please Select an Output File + option( + ng-repeat="outputFile in data.projectOutputFiles" + value="{{ outputFile.path }}" + ) {{ outputFile.path }} + + .form-controls.row-spaced-small + label(for="name") File Name In This Project + input.form-control( + type="text" + placeholder="example.tex" + required + ng-model="data.name" + name="name" + ) + div.alert.alert-danger.row-spaced-small(ng-if="state.error") Error, something went wrong! + div(ng-if="type == 'url'", ng-controller="UrlLinkedFileModalController") + form(novalidate, name="newLinkedFileForm") + label(for="url") URL to fetch the file from + input.form-control( + type="text", + placeholder="www.example.com/my_file", + required, + ng-model="inputs.url", + focus-on="open", + on-enter="create()", + name="url" + ) + .row-spaced.small + label(for="name") File name in this project + input.form-control( + type="text", + placeholder="my_file", + required, + ng-model="inputs.name", + ng-change="nameChangedByUser = true" + valid-file, + on-enter="create()", + name="name" + ) + .text-danger.row-spaced-small(ng-show="newDocForm.name.$error.validFile") + | #{translate('files_cannot_include_invalid_characters')} + div.alert.alert-danger.row-spaced-small(ng-if="error") + div(ng-switch="error") + span(ng-switch-when="already exists") #{translate("file_already_exists")} + span(ng-switch-default) {{error}} + .modal-footer + button.btn.btn-default( + ng-disabled="state.inflight" + ng-click="cancel()" + ) #{translate("cancel")} + button.btn.btn-primary( + ng-disabled="state.inflight || !state.valid" + ng-click="create()" + ng-hide="type == 'upload'" + ) + span(ng-hide="state.inflight") #{translate("create")} + span(ng-show="state.inflight") #{translate("creating")}... + +script(type="text/template", id="qq-file-uploader-template") + div.qq-uploader-selector + div(qq-hide-dropzone="").qq-upload-drop-area-selector.qq-upload-drop-area + span.qq-upload-drop-area-text-selector #{translate('drop_files_here_to_upload')} + div Drag here + div.row-spaced-small.small #{translate('or')} + div.row-spaced-small + div.qq-upload-button-selector.btn.btn-primary + | Select from your computer + span.qq-drop-processing-selector + span #{translate('processing')} + span.qq-drop-processing-spinner-selector + ul.qq-upload-list-selector + li + div.qq-progress-bar-container-selector + div( + role="progressbar" + aria-valuenow="0" + aria-valuemin="0" + aria-valuemax="100" + class="qq-progress-bar-selector qq-progress-bar" + ) + span.qq-upload-file-selector.qq-upload-file + span.qq-upload-size-selector.qq-upload-size + a(type="button").qq-btn.qq-upload-cancel-selector.qq-upload-cancel #{translate('cancel')} + button(type="button").qq-btn.qq-upload-retry-selector.qq-upload-retry #{translate('retry')} + span(role="status").qq-upload-status-text-selector.qq-upload-status-text diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index f929474952..6d92675b4e 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -4,10 +4,12 @@ define [ App.controller "FileTreeController", ["$scope", "$modal", "ide", "$rootScope", ($scope, $modal, ide, $rootScope) -> $scope.openNewDocModal = () -> $modal.open( - templateUrl: "newDocModalTemplate" - controller: "NewDocModalController" + templateUrl: "newFileModalTemplate" + controller: "NewFileModalController" + size: 'lg' resolve: { parent_folder: () -> ide.fileTreeManager.getCurrentFolder() + type: () -> 'doc' } ) @@ -22,52 +24,12 @@ define [ $scope.openUploadFileModal = () -> $modal.open( - templateUrl: "uploadFileModalTemplate" - controller: "UploadFileModalController" - scope: $scope + templateUrl: "newFileModalTemplate" + controller: "NewFileModalController" + size: 'lg' resolve: { parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - } - ) - - $scope.openLinkedFileModal = window.openLinkedFileModal = () -> - unless 'url' in window.data.enabledLinkedFileTypes - console.warn("Url linked files are not enabled") - return - $modal.open( - templateUrl: "linkedFileModalTemplate" - controller: "LinkedFileModalController" - scope: $scope - resolve: { - parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - } - ) - - $scope.openProjectLinkedFileModal = window.openProjectLinkedFileModal = () -> - unless 'project_file' in window.data.enabledLinkedFileTypes - console.warn("Project linked files are not enabled") - return - $modal.open( - templateUrl: "projectLinkedFileModalTemplate" - controller: "ProjectLinkedFileModalController" - scope: $scope - resolve: { - parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - isOutputFilesMode: () -> false - } - ) - - $scope.openProjectOutputLinkedFileModal = window.openProjectOutputLinkedFileModal = () -> - unless 'project_output_file' in window.data.enabledLinkedFileTypes - console.warn("Project linked output files are not enabled") - return - $modal.open( - templateUrl: "projectLinkedFileModalTemplate" - controller: "ProjectLinkedFileModalController" - scope: $scope - resolve: { - parent_folder: () -> ide.fileTreeManager.getCurrentFolder() - isOutputFilesMode: () -> true + type: () -> 'upload' } ) @@ -82,42 +44,10 @@ define [ $scope.$broadcast "delete:selected" ] - App.controller "NewDocModalController", [ - "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", - ($scope, ide, $modalInstance, $timeout, parent_folder) -> - $scope.inputs = - name: "name.tex" - $scope.state = - inflight: false - - $modalInstance.opened.then () -> - $timeout () -> - $scope.$broadcast "open" - , 200 - - $scope.create = () -> - name = $scope.inputs.name - if !name? or name.length == 0 - return - $scope.state.inflight = true - ide.fileTreeManager - .createDoc(name, parent_folder) - .then () -> - $scope.state.inflight = false - $modalInstance.close() - .catch (response)-> - { data } = response - $scope.error = data - $scope.state.inflight = false - - $scope.cancel = () -> - $modalInstance.dismiss('cancel') - ] - App.controller "NewFolderModalController", [ "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", ($scope, ide, $modalInstance, $timeout, parent_folder) -> - $scope.inputs = + $scope.inputs = name: "name" $scope.state = inflight: false @@ -133,10 +63,10 @@ define [ return $scope.state.inflight = true ide.fileTreeManager - .createFolder(name, parent_folder) + .createFolder(name, $scope.parent_folder) .then () -> $scope.state.inflight = false - $modalInstance.close() + $modalInstance.dismiss('done') .catch (response)-> { data } = response $scope.error = data @@ -146,10 +76,60 @@ define [ $modalInstance.dismiss('cancel') ] + App.controller "NewFileModalController", [ + "$scope", "type", "parent_folder", "$modalInstance" + ($scope, type, parent_folder, $modalInstance) -> + $scope.type = type + $scope.parent_folder = parent_folder + $scope.state = { + inflight: false + valid: true + } + $scope.cancel = () -> + $modalInstance.dismiss('cancel') + $scope.create = () -> + $scope.$broadcast 'create' + $scope.$on 'done', () -> + $modalInstance.dismiss('done') + ] + + App.controller "NewDocModalController", [ + "$scope", "ide", "$timeout" + ($scope, ide, $timeout) -> + $scope.inputs = + name: "name.tex" + + validate = () -> + name = $scope.inputs.name + $scope.state.valid = (name? and name.length > 0) + $scope.$watch 'inputs.name', validate + + $timeout () -> + $scope.$broadcast "open" + , 200 + + $scope.$on 'create', () -> + name = $scope.inputs.name + if !name? or name.length == 0 + return + $scope.state.inflight = true + ide.fileTreeManager + .createDoc(name, $scope.parent_folder) + .then () -> + $scope.state.inflight = false + $scope.$emit 'done' + .catch (response)-> + { data } = response + $scope.error = data + $scope.state.inflight = false + + ] + App.controller "UploadFileModalController", [ - "$scope", "$rootScope", "ide", "$modalInstance", "$timeout", "parent_folder", "$window" - ($scope, $rootScope, ide, $modalInstance, $timeout, parent_folder, $window) -> - $scope.parent_folder_id = parent_folder?.id + "$scope", "$rootScope", "ide", "$timeout", "$window" + ($scope, $rootScope, ide, $timeout, $window) -> + $scope.parent_folder_id = $scope.parent_folder?.id + $scope.project_id = ide.project_id $scope.tooManyFiles = false $scope.rateLimitHit = false $scope.secondsToRedirect = 10 @@ -177,7 +157,7 @@ define [ if response.success $rootScope.$broadcast 'file:upload:complete', response if uploadCount == 0 and response? and response.success - $modalInstance.close("done") + $scope.$emit 'done' ), 250 $scope.onValidateBatch = (files)-> @@ -225,14 +205,11 @@ define [ $scope.doUpload = () -> $scope.control?.q?.uploadStoredFiles() - $scope.cancel = () -> - $modalInstance.dismiss('cancel') ] App.controller "ProjectLinkedFileModalController", [ - "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", "isOutputFilesMode", - ($scope, ide, $modalInstance, $timeout, parent_folder, isOutputFilesMode) -> - $scope.isOutputFilesMode = isOutputFilesMode + "$scope", "ide", "$timeout", + ($scope, ide, $timeout) -> $scope.data = projects: null # or [] @@ -242,13 +219,11 @@ define [ selectedProjectEntity: null selectedProjectOutputFile: null name: null - $scope.state = - inFlight: - projects: false - entities: false - compile: false - create: false - error: false + $scope.state.inFlight = + projects: false + entities: false + compile: false + $scope.state.error = false $scope.$watch 'data.selectedProjectId', (newVal, oldVal) -> return if !newVal @@ -278,7 +253,8 @@ define [ _reset = (opts) -> isError = opts.err == true inFlight = $scope.state.inFlight - inFlight.projects = inFlight.entities = inFlight.compile = inFlight.create = false + inFlight.projects = inFlight.entities = inFlight.compile = false + $scope.state.inflight = false $scope.state.error = isError $scope.shouldEnableProjectSelect = () -> @@ -293,10 +269,11 @@ define [ { state, data } = $scope return !state.inFlight.projects && !state.inFlight.compile && data.projects && data.selectedProjectId - $scope.shouldEnableCreateButton = () -> + + validate = () -> state = $scope.state data = $scope.data - return !state.inFlight.projects && + $scope.state.valid = !state.inFlight.projects && !state.inFlight.entities && data.projects && data.selectedProjectId && @@ -313,6 +290,8 @@ define [ ) ) && data.name + $scope.$watch 'state', validate, true + $scope.$watch 'data', validate, true $scope.getUserProjects = () -> _setInFlight('projects') @@ -362,7 +341,7 @@ define [ $scope.getUserProjects() $timeout($scope.init, 0) - $scope.create = () -> + $scope.$on 'create', () -> projectId = $scope.data.selectedProjectId name = $scope.data.name if isOutputFilesMode @@ -379,34 +358,39 @@ define [ } _setInFlight('create') ide.fileTreeManager - .createLinkedFile(name, parent_folder, provider, payload) + .createLinkedFile(name, $scope.parent_folder, provider, payload) .then () -> _reset(err: false) - $modalInstance.close() + $scope.$emit 'done' .catch (response)-> { data } = response _reset(err: true) - $scope.cancel = () -> - $modalInstance.dismiss('cancel') ] - # TODO: rename all this to UrlLinkedFilModalController - App.controller "LinkedFileModalController", [ - "$scope", "ide", "$modalInstance", "$timeout", "parent_folder", - ($scope, ide, $modalInstance, $timeout, parent_folder) -> + App.controller "UrlLinkedFileModalController", [ + "$scope", "ide", "$timeout" + ($scope, ide, $timeout) -> $scope.inputs = name: "" url: "" $scope.nameChangedByUser = false - $scope.state = - inflight: false - $modalInstance.opened.then () -> - $timeout () -> - $scope.$broadcast "open" - , 200 + $timeout () -> + $scope.$broadcast "open" + , 200 + + validate = () -> + {name, url} = $scope.inputs + if !name? or name.length == 0 + $scope.state.valid = false + else if !url? or url.length == 0 + $scope.state.valid = false + else + $scope.state.valid = true + $scope.$watch 'inputs.name', validate + $scope.$watch 'inputs.url', validate $scope.$watch "inputs.url", (url) -> if url? and url != "" and !$scope.nameChangedByUser @@ -415,7 +399,7 @@ define [ if parts.length > 1 # Wait for at one / $scope.inputs.name = parts[0] - $scope.create = () -> + $scope.$on 'create', () -> {name, url} = $scope.inputs if !name? or name.length == 0 return @@ -423,15 +407,13 @@ define [ return $scope.state.inflight = true ide.fileTreeManager - .createLinkedFile(name, parent_folder, 'url', {url}) + .createLinkedFile(name, $scope.parent_folder, 'url', {url}) .then () -> $scope.state.inflight = false - $modalInstance.close() + $scope.$emit 'done' .catch (response)-> { data } = response $scope.error = data $scope.state.inflight = false - $scope.cancel = () -> - $modalInstance.dismiss('cancel') ] diff --git a/services/web/public/stylesheets/app/editor/file-tree.less b/services/web/public/stylesheets/app/editor/file-tree.less index 4e26e4751d..2cf73115d7 100644 --- a/services/web/public/stylesheets/app/editor/file-tree.less +++ b/services/web/public/stylesheets/app/editor/file-tree.less @@ -261,3 +261,40 @@ } } } + +.modal-new-file { + padding: 0; + table { + width: 100%; + td { + vertical-align: top; + } + } +} + .modal-new-file--list { + background-color: @modal-footer-background-color; + width: 220px; + ul { + li { + padding: (@line-height-computed / 4); + a { + color: @text-color; + } + } + li.active { + background-color: white; + a { + color: @link-color; + } + } + } + } + + .modal-new-file--body { + padding: 20px; + padding-top: (@line-height-computed / 4); + } + + .modal-new-file--body-upload { + padding-top: 20px; + } diff --git a/services/web/public/stylesheets/components/fineupload.less b/services/web/public/stylesheets/components/fineupload.less index ee1e96164f..bb037b4087 100644 --- a/services/web/public/stylesheets/components/fineupload.less +++ b/services/web/public/stylesheets/components/fineupload.less @@ -10,13 +10,17 @@ } .qq-uploader-selector { text-align: center; - .drag-here { - border: 1px dashed #666; - vertical-align: middle; - } + border: 1px dashed #666; + border-radius: 6px; + vertical-align: middle; .help { margin-top: 6px; } + min-height: 200px; + padding: 20px; + display: flex; + flex-direction: column; + justify-content: center; } /*.qq-upload-button-selector { display: block; From 64ec90f34f76f608fce6e6c0a9c922dbf4491f98 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 11 Jun 2018 10:29:40 +0100 Subject: [PATCH 36/63] Move the isOutputFilesMode flag onto the scope --- services/web/app/views/project/editor/new-file-modal.pug | 4 ++-- .../ide/file-tree/controllers/FileTreeController.coffee | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/services/web/app/views/project/editor/new-file-modal.pug b/services/web/app/views/project/editor/new-file-modal.pug index 48c8863e5b..021f822eab 100644 --- a/services/web/app/views/project/editor/new-file-modal.pug +++ b/services/web/app/views/project/editor/new-file-modal.pug @@ -95,7 +95,7 @@ script(type='text/ng-template', id='newFileModalTemplate') value="{{ project._id }}" ) {{ project.name }} - .form-controls.row-spaced-small(ng-if="!isOutputFilesMode") + .form-controls.row-spaced-small(ng-if="!state.isOutputFilesMode") label(for="project-entity-select") Select a File span(ng-show="state.inFlight.entities") |   @@ -111,7 +111,7 @@ script(type='text/ng-template', id='newFileModalTemplate') value="{{ projectEntity.path }}" ) {{ projectEntity.path.slice(1) }} - .form-controls.row-spaced-small(ng-if="isOutputFilesMode") + .form-controls.row-spaced-small(ng-if="state.isOutputFilesMode") label(for="project-entity-select") Select an Output File span(ng-show="state.inFlight.compile") |   diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index 6d92675b4e..fd7b297de3 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -223,12 +223,13 @@ define [ projects: false entities: false compile: false + $scope.state.isOutputFilesMode = false $scope.state.error = false $scope.$watch 'data.selectedProjectId', (newVal, oldVal) -> return if !newVal $scope.data.selectedProjectEntity = null - if isOutputFilesMode + if $scope.state.isOutputFilesMode $scope.compileProjectAndGetOutputFiles($scope.data.selectedProjectId) else $scope.getProjectEntities($scope.data.selectedProjectId) @@ -279,12 +280,12 @@ define [ data.selectedProjectId && ( ( - !isOutputFilesMode && + !$scope.state.isOutputFilesMode && data.projectEntities && data.selectedProjectEntity ) || ( - isOutputFilesMode && + $scope.state.isOutputFilesMode && data.projectOutputFiles && data.selectedProjectOutputFile ) @@ -344,7 +345,7 @@ define [ $scope.$on 'create', () -> projectId = $scope.data.selectedProjectId name = $scope.data.name - if isOutputFilesMode + if $scope.state.isOutputFilesMode provider = 'project_output_file' payload = { source_project_id: projectId, From 6672a20c2bc1bb1985a7906be210fac736b09956 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 11 Jun 2018 11:27:57 +0100 Subject: [PATCH 37/63] Enable switching between source and output files --- .../web/app/views/project/editor/new-file-modal.pug | 7 +++++++ .../file-tree/controllers/FileTreeController.coffee | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/services/web/app/views/project/editor/new-file-modal.pug b/services/web/app/views/project/editor/new-file-modal.pug index 021f822eab..2b139a9d06 100644 --- a/services/web/app/views/project/editor/new-file-modal.pug +++ b/services/web/app/views/project/editor/new-file-modal.pug @@ -126,6 +126,13 @@ script(type='text/ng-template', id='newFileModalTemplate') ng-repeat="outputFile in data.projectOutputFiles" value="{{ outputFile.path }}" ) {{ outputFile.path }} + div + a( + href="#" + ng-click="toggleOutputFilesMode()" + ) + span(ng-show="state.isOutputFilesMode") Switch to source files + span(ng-show="!state.isOutputFilesMode") Switch to output files .form-controls.row-spaced-small label(for="name") File Name In This Project diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index fd7b297de3..da9c9d37e0 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -229,11 +229,20 @@ define [ $scope.$watch 'data.selectedProjectId', (newVal, oldVal) -> return if !newVal $scope.data.selectedProjectEntity = null + $scope.data.selectedProjectOutputFile = null if $scope.state.isOutputFilesMode $scope.compileProjectAndGetOutputFiles($scope.data.selectedProjectId) else $scope.getProjectEntities($scope.data.selectedProjectId) + $scope.$watch 'state.isOutputFilesMode', (newVal, oldVal) -> + return if !newVal and !oldVal + $scope.data.selectedProjectOutputFile = null + if newVal == true + $scope.compileProjectAndGetOutputFiles($scope.data.selectedProjectId) + else + $scope.getProjectEntities($scope.data.selectedProjectId) + # auto-set filename based on selected file $scope.$watch 'data.selectedProjectEntity', (newVal, oldVal) -> return if !newVal @@ -258,6 +267,10 @@ define [ $scope.state.inflight = false $scope.state.error = isError + $scope.toggleOutputFilesMode = () -> + return if !$scope.data.selectedProjectId + $scope.state.isOutputFilesMode = !$scope.state.isOutputFilesMode + $scope.shouldEnableProjectSelect = () -> { state, data } = $scope return !state.inFlight.projects && data.projects From 7fc99a38f8a98b2a27292b728bf078065c261859 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 11 Jun 2018 11:53:30 +0100 Subject: [PATCH 38/63] Better styling on the output-files toggle --- services/web/app/views/project/editor/new-file-modal.pug | 7 ++++--- services/web/public/stylesheets/app/editor/file-tree.less | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/web/app/views/project/editor/new-file-modal.pug b/services/web/app/views/project/editor/new-file-modal.pug index 2b139a9d06..3dcd55bfb9 100644 --- a/services/web/app/views/project/editor/new-file-modal.pug +++ b/services/web/app/views/project/editor/new-file-modal.pug @@ -126,13 +126,14 @@ script(type='text/ng-template', id='newFileModalTemplate') ng-repeat="outputFile in data.projectOutputFiles" value="{{ outputFile.path }}" ) {{ outputFile.path }} - div + div.toggle-output-files-button + | or  a( href="#" ng-click="toggleOutputFilesMode()" ) - span(ng-show="state.isOutputFilesMode") Switch to source files - span(ng-show="!state.isOutputFilesMode") Switch to output files + span(ng-show="state.isOutputFilesMode") select from source files + span(ng-show="!state.isOutputFilesMode") select from output files .form-controls.row-spaced-small label(for="name") File Name In This Project diff --git a/services/web/public/stylesheets/app/editor/file-tree.less b/services/web/public/stylesheets/app/editor/file-tree.less index 2cf73115d7..845cc74a93 100644 --- a/services/web/public/stylesheets/app/editor/file-tree.less +++ b/services/web/public/stylesheets/app/editor/file-tree.less @@ -270,6 +270,9 @@ vertical-align: top; } } + .toggle-output-files-button { + font-size: 80%; + } } .modal-new-file--list { background-color: @modal-footer-background-color; From 2cfc2b473803d46bffc0d939b7cfd61a08df4a73 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 13 Jun 2018 14:20:10 +0100 Subject: [PATCH 39/63] Only show output files which are images or pdfs --- .../ide/file-tree/controllers/FileTreeController.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index da9c9d37e0..71f3956118 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -342,7 +342,9 @@ define [ }) .then (resp) -> if resp.data.status == 'success' - $scope.data.projectOutputFiles = resp.data.outputFiles + filteredFiles = resp.data.outputFiles.filter (f) -> + f.path.match(/.*\.(pdf|png|jpeg|jpg|gif)/) + $scope.data.projectOutputFiles = filteredFiles _reset(err: false) else $scope.data.projectOutputFiles = null From 48a4f6c4c4aa75be0d01f4d505153ffc85142bcd Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 13 Jun 2018 14:49:16 +0100 Subject: [PATCH 40/63] Fix error handling for failing to get output file --- .../coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 8af8b5bf63..fff464945a 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -88,8 +88,8 @@ module.exports = ProjectOutputFileAgent = { handleError: (error, req, res, next) -> if error instanceof BadDataError res.status(400).send("The submitted data is not valid") - else if error instanceof SourceFileNotFoundError - res.status(404).send("Source file not found") + else if error instanceof OutputFileFetchFailedError + res.status(404).send("Could not get output file") else if error instanceof ProjectNotFoundError res.status(404).send("Project not found") else From 28257462aefe767f83838716284a95cf03ada47e Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jun 2018 10:03:29 +0100 Subject: [PATCH 41/63] Acceptance tests for project-output-file --- services/web/docker-compose.yml | 2 +- .../acceptance/coffee/LinkedFilesTests.coffee | 68 +++++++++++++++++++ .../coffee/helpers/MockClsiApi.coffee | 9 +++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index 9596fe5126..f2966da141 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -17,7 +17,7 @@ services: PROJECT_HISTORY_ENABLED: 'true' ENABLED_LINKED_FILE_TYPES: 'url' LINKED_URL_PROXY: 'http://localhost:6543' - ENABLED_LINKED_FILE_TYPES: 'url,project_file' + ENABLED_LINKED_FILE_TYPES: 'url,project_file,project_output_file' SHARELATEX_CONFIG: /app/test/acceptance/config/settings.test.coffee depends_on: - redis diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee index 875e887d81..af7573a151 100644 --- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee +++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee @@ -8,6 +8,8 @@ MockFileStoreApi = require './helpers/MockFileStoreApi' request = require "./helpers/request" User = require "./helpers/User" +MockClsiApi = require "./helpers/MockClsiApi" + express = require("express") LinkedUrlProxy = express() @@ -344,3 +346,69 @@ describe "LinkedFiles", -> # TODO: Add test for asking for host that return ENOTFOUND # (This will probably end up handled by the proxy) + + describe "creating a linked output file", -> + before (done) -> + async.series [ + (cb) => + @owner.createProject 'output-test-one', {template: 'blank'}, (error, project_id) => + @project_one_id = project_id + cb(error) + (cb) => + @owner.getProject @project_one_id, (error, project) => + @project_one = project + @project_one_root_folder_id = project.rootFolder[0]._id.toString() + cb(error) + (cb) => + @owner.createProject 'output-test-two', {template: 'blank'}, (error, project_id) => + @project_two_id = project_id + cb(error) + (cb) => + @owner.getProject @project_two_id, (error, project) => + @project_two = project + @project_two_root_folder_id = project.rootFolder[0]._id.toString() + cb(error) + ], done + + it 'should import the output.pdf file from the source project', (done) -> + @owner.request.post { + url: "/project/#{@project_one_id}/linked_file", + json: + name: 'test.pdf', + parent_folder_id: @project_one_root_folder_id, + provider: 'project_output_file', + data: + source_project_id: @project_two_id, + source_output_file_path: "output.pdf", + }, (error, response, body) => + new_file_id = body.new_file_id + @existing_file_id = new_file_id + expect(new_file_id).to.exist + @owner.getProject @project_one_id, (error, project) => + return done(error) if error? + firstFile = project.rootFolder[0].fileRefs[0] + expect(firstFile._id.toString()).to.equal(new_file_id.toString()) + expect(firstFile.linkedFileData).to.deep.equal { + provider: 'project_output_file', + source_project_id: @project_two_id, + source_output_file_path: "output.pdf", + source_project_display_name: "output-test-two" + } + expect(firstFile.name).to.equal('test.pdf') + done() + + it 'should refresh the file', (done) -> + @owner.request.post { + url: "/project/#{@project_one_id}/linked_file/#{@existing_file_id}/refresh", + json: true + }, (error, response, body) => + new_file_id = body.new_file_id + expect(new_file_id).to.exist + expect(new_file_id).to.not.equal @existing_file_id + @refreshed_file_id = new_file_id + @owner.getProject @project_one_id, (error, project) => + return done(error) if error? + firstFile = project.rootFolder[0].fileRefs[0] + expect(firstFile._id.toString()).to.equal(new_file_id.toString()) + expect(firstFile.name).to.equal('test.pdf') + done() diff --git a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee index 2f9180960d..5b01b7a81b 100644 --- a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee @@ -1,4 +1,5 @@ express = require("express") +bodyParser = require "body-parser" app = express() module.exports = MockClsiApi = @@ -30,6 +31,14 @@ module.exports = MockClsiApi = else res.sendStatus(404) + app.post "/project/:project_id/compile", (req, res, next) => + res.json { + outputFiles: [{path: 'output.pdf'}] + } + + app.get "/project/:project_id/output/:output_path", (req, res, next) => + res.status(200).send("hello") + app.listen 3013, (error) -> throw error if error? .on "error", (error) -> From a313184c712758df73871dad21506a6e6491fa30 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jun 2018 12:27:20 +0100 Subject: [PATCH 42/63] Handle linked-output-files from v1 imports --- .../LinkedFiles/ProjectFileAgent.coffee | 6 +- .../LinkedFiles/ProjectOutputFileAgent.coffee | 68 +++++++++++-------- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee index 9ba8b5f1e7..78fc31ee03 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee @@ -60,6 +60,10 @@ SourceFileNotFoundError.prototype.__proto__ = Error.prototype module.exports = ProjectFileAgent = + V1ProjectNotFoundError: V1ProjectNotFoundError + + _v1ProjectNotFoundMessage: "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file" + sanitizeData: (data) -> return _.pick( data, @@ -155,7 +159,7 @@ module.exports = ProjectFileAgent = else if error instanceof ProjectNotFoundError res.status(404).send("Project not found") else if error instanceof V1ProjectNotFoundError - res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file") + res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage) else next(error) next() diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index fff464945a..5d81420cd8 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -6,6 +6,7 @@ Settings = require 'settings-sharelatex' CompileManager = require '../Compile/CompileManager' CompileController = require '../Compile/CompileController' ClsiCookieManager = require '../Compile/ClsiCookieManager' +ProjectFileAgent = require './ProjectFileAgent' _ = require "underscore" request = require "request" @@ -42,48 +43,53 @@ module.exports = ProjectOutputFileAgent = { source_output_file_path: data.source_output_file_path } - canCreate: (data) -> true + canCreate: ProjectFileAgent.canCreate - decorateLinkedFileData: (data, callback = (err, newData) ->) -> - callback = _.once(callback) - ProjectGetter.getProject data.source_project_id, {name: 1}, (err, project) -> - return callback(err) if err? - if !project? - return callback(new ProjectNotFoundError()) - callback(err, _.extend(data, {source_project_display_name: project.name})) + _getSourceProject: ProjectFileAgent._getSourceProject + + decorateLinkedFileData: ProjectFileAgent.decorateLinkedFileData + + _validate: (data) -> + return ( + (data.source_project_id? || data.v1_source_doc_id?) && + data.source_output_file_path? + ) checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> callback = _.once(callback) - { source_project_id } = data - AuthorizationManager.canUserReadProject current_user_id, source_project_id, null, (err, canRead) -> + if !ProjectOutputFileAgent._validate(data) + return callback(new BadDataError()) + @_getSourceProject data, (err, project) -> return callback(err) if err? - callback(null, canRead) - - _validate: (data) -> - data.source_project_id? && data.source_output_file_path? + AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) -> + return callback(err) if err? + callback(null, canRead) writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> callback = _.once(callback) if !ProjectOutputFileAgent._validate(data) return callback(new BadDataError()) - { source_project_id, source_output_file_path } = data - CompileManager.compile source_project_id, null, {}, (err) -> + { source_output_file_path } = data + @_getSourceProject data, (err, project) -> return callback(err) if err? - url = "#{Settings.apis.clsi.url}/project/#{source_project_id}/output/#{source_output_file_path}" - ClsiCookieManager.getCookieJar source_project_id, (err, jar)-> + source_project_id = project._id + CompileManager.compile source_project_id, null, {}, (err) -> return callback(err) if err? - oneMinute = 60 * 1000 - # the base request - options = { url: url, method: "GET", timeout: oneMinute, jar : jar } - readStream = request(options) - readStream.on "error", callback - readStream.on "response", (response) -> - if 200 <= response.statusCode < 300 - FileWriter.writeStreamToDisk project_id, readStream, callback - else - error = new OutputFileFetchFailedError("Output file fetch failed: #{url}") - error.statusCode = response.statusCode - callback(error) + url = "#{Settings.apis.clsi.url}/project/#{source_project_id}/output/#{source_output_file_path}" + ClsiCookieManager.getCookieJar source_project_id, (err, jar)-> + return callback(err) if err? + oneMinute = 60 * 1000 + # the base request + options = { url: url, method: "GET", timeout: oneMinute, jar : jar } + readStream = request(options) + readStream.on "error", callback + readStream.on "response", (response) -> + if 200 <= response.statusCode < 300 + FileWriter.writeStreamToDisk project_id, readStream, callback + else + error = new OutputFileFetchFailedError("Output file fetch failed: #{url}") + error.statusCode = response.statusCode + callback(error) handleError: (error, req, res, next) -> if error instanceof BadDataError @@ -92,6 +98,8 @@ module.exports = ProjectOutputFileAgent = { res.status(404).send("Could not get output file") else if error instanceof ProjectNotFoundError res.status(404).send("Project not found") + else if error instanceof ProjectFileAgent.V1ProjectNotFoundError + res.status(404).send(ProjectFileAgent._v1ProjectNotFoundMessage) else next(error) } From 2ade78783bf14b7477592f2c1c2d587989187dee Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 14 Jun 2018 14:25:44 +0100 Subject: [PATCH 43/63] Add acceptance test for refreshing output file from v1 project --- .../LinkedFiles/ProjectOutputFileAgent.coffee | 2 +- .../acceptance/coffee/LinkedFilesTests.coffee | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 5d81420cd8..5969b25c5b 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -99,7 +99,7 @@ module.exports = ProjectOutputFileAgent = { else if error instanceof ProjectNotFoundError res.status(404).send("Project not found") else if error instanceof ProjectFileAgent.V1ProjectNotFoundError - res.status(404).send(ProjectFileAgent._v1ProjectNotFoundMessage) + res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage) else next(error) } diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee index af7573a151..173aefa154 100644 --- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee +++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee @@ -412,3 +412,37 @@ describe "LinkedFiles", -> expect(firstFile._id.toString()).to.equal(new_file_id.toString()) expect(firstFile.name).to.equal('test.pdf') done() + + describe "with a linked project_output_file from a v1 project that has not been imported", -> + before (done) -> + async.series [ + (cb) => + @owner.createProject 'output-v1-test-one', {template: 'blank'}, (error, project_id) => + @project_one_id = project_id + cb(error) + (cb) => + @owner.getProject @project_one_id, (error, project) => + @project_one = project + @project_one_root_folder_id = project.rootFolder[0]._id.toString() + @project_one.rootFolder[0].fileRefs.push { + linkedFileData: { + provider: "project_output_file", + v1_source_doc_id: 9999999, # We won't find this id in the database + source_output_file_path: "output.pdf" + }, + _id: "abcdef", + rev: 0, + created: new Date(), + name: "whatever.pdf" + } + @owner.saveProject @project_one, cb + ], done + + it 'should refuse to refresh', (done) -> + @owner.request.post { + url: "/project/#{@project_one_id}/linked_file/abcdef/refresh", + json: true + }, (error, response, body) => + expect(response.statusCode).to.equal 409 + expect(body).to.equal "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file" + done() From 708e809df677aae91b1e0858564cfe22354d491b Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 15 Jun 2018 11:17:20 +0100 Subject: [PATCH 44/63] Use errors from ProjectFileAgent --- .../LinkedFiles/ProjectFileAgent.coffee | 8 ++++++-- .../LinkedFiles/ProjectOutputFileAgent.coffee | 20 ++----------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee index 78fc31ee03..1b9f964733 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee @@ -58,9 +58,12 @@ SourceFileNotFoundError = (message) -> SourceFileNotFoundError.prototype.__proto__ = Error.prototype -module.exports = ProjectFileAgent = +module.exports = ProjectFileAgent = { - V1ProjectNotFoundError: V1ProjectNotFoundError + V1ProjectNotFoundError + BadDataError + ProjectNotFoundError + V1ProjectNotFoundError _v1ProjectNotFoundMessage: "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file" @@ -163,3 +166,4 @@ module.exports = ProjectFileAgent = else next(error) next() +} diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 5969b25c5b..d194032273 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -11,22 +11,6 @@ _ = require "underscore" request = require "request" -BadDataError = (message) -> - error = new Error(message) - error.name = 'BadData' - error.__proto__ = BadDataError.prototype - return error -BadDataError.prototype.__proto__ = Error.prototype - - -ProjectNotFoundError = (message) -> - error = new Error(message) - error.name = 'ProjectNotFound' - error.__proto__ = ProjectNotFoundError.prototype - return error -ProjectNotFoundError.prototype.__proto__ = Error.prototype - - OutputFileFetchFailedError = (message) -> error = new Error(message) error.name = 'OutputFileFetchFailedError' @@ -92,11 +76,11 @@ module.exports = ProjectOutputFileAgent = { callback(error) handleError: (error, req, res, next) -> - if error instanceof BadDataError + if error instanceof ProjectFileAgent.BadDataError res.status(400).send("The submitted data is not valid") else if error instanceof OutputFileFetchFailedError res.status(404).send("Could not get output file") - else if error instanceof ProjectNotFoundError + else if error instanceof ProjectFileAgent.ProjectNotFoundError res.status(404).send("Project not found") else if error instanceof ProjectFileAgent.V1ProjectNotFoundError res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage) From 67dcbff450c608f0473205c7910f2683952692ef Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 15 Jun 2018 15:34:37 +0100 Subject: [PATCH 45/63] Remove linked-files items from the left-menu test controls --- services/web/app/views/project/editor/left-menu.pug | 12 ------------ .../controllers/TestControlsController.coffee | 9 --------- 2 files changed, 21 deletions(-) diff --git a/services/web/app/views/project/editor/left-menu.pug b/services/web/app/views/project/editor/left-menu.pug index c1ff891844..2cedc0b37e 100644 --- a/services/web/app/views/project/editor/left-menu.pug +++ b/services/web/app/views/project/editor/left-menu.pug @@ -69,18 +69,6 @@ aside#left-menu.full-size( a(href="#" ng-click="richText()") i.fa.fa-exclamation.fa-fw | Rich Text - li - a(href="#" ng-click="openProjectLinkedFileModal()") - i.fa.fa-exclamation.fa-fw - | Project-Linked-File Modal - li - a(href="#" ng-click="openProjectOutputLinkedFileModal()") - i.fa.fa-exclamation.fa-fw - | Project-Output-Linked-File Modal - li - a(href="#" ng-click="openLinkedFileModal()") - i.fa.fa-exclamation.fa-fw - | URL-Linked-File Modal h4(ng-show="!anonymous") #{translate("settings")} diff --git a/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee b/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee index 0dca64dc5b..881d03d3cb 100644 --- a/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee +++ b/services/web/public/coffee/ide/test-controls/controllers/TestControlsController.coffee @@ -4,15 +4,6 @@ define [ ], (App) -> App.controller "TestControlsController", ($scope) -> - $scope.openProjectLinkedFileModal = () -> - window.openProjectLinkedFileModal() - - $scope.openProjectOutputLinkedFileModal = () -> - window.openProjectOutputLinkedFileModal() - - $scope.openLinkedFileModal = () -> - window.openLinkedFileModal() - $scope.richText = () -> current = window.location.toString() target = "#{current}#{if window.location.search then '&' else '?'}rt=true" From 54cdbd738cbc2f37ee2521732f3aeddb119b17e9 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 15 Jun 2018 16:04:42 +0100 Subject: [PATCH 46/63] If selecting 'output.pdf', set the filename to project-name.pdf --- .../file-tree/controllers/FileTreeController.coffee | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index 71f3956118..cd3c35b866 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -253,9 +253,13 @@ define [ # auto-set filename based on selected file $scope.$watch 'data.selectedProjectOutputFile', (newVal, oldVal) -> return if !newVal - fileName = newVal.split('/').reverse()[0] - if fileName - $scope.data.name = fileName + if newVal == 'output.pdf' + project = _.find($scope.data.projects, (p) -> p._id == $scope.data.selectedProjectId) + $scope.data.name = if project?.name? then "#{project.name}.pdf" else 'output.pdf' + else + fileName = newVal.split('/').reverse()[0] + if fileName + $scope.data.name = fileName _setInFlight = (type) -> $scope.state.inFlight[type] = true From d93eb448e3d987d2783db98b0c1c5892764802cd Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 15 Jun 2018 16:20:55 +0100 Subject: [PATCH 47/63] Move Clsi logic to ClsiManager --- .../app/coffee/Features/Compile/ClsiManager.coffee | 8 ++++++++ .../LinkedFiles/ProjectOutputFileAgent.coffee | 11 ++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index e5158cdb9b..232b416e66 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -182,6 +182,14 @@ module.exports = ClsiManager = return callback(error) if error? callback(null, projectStateHash, docs) + getOutputFileStream: (project_id, output_file_path, callback=(err, readStream)->) -> + url = "#{Settings.apis.clsi.url}/project/#{project_id}/output/#{output_file_path}" + ClsiCookieManager.getCookieJar project_id, (err, jar)-> + return callback(err) if err? + options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar } + readStream = request(options) + callback(null, readStream) + _buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) -> ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) -> return callback(error) if error? diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index d194032273..0c84d3f7dc 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -4,8 +4,8 @@ ProjectGetter = require('../Project/ProjectGetter') FileWriter = require('../../infrastructure/FileWriter') Settings = require 'settings-sharelatex' CompileManager = require '../Compile/CompileManager' -CompileController = require '../Compile/CompileController' ClsiCookieManager = require '../Compile/ClsiCookieManager' +ClsiManager = require '../Compile/ClsiManager' ProjectFileAgent = require './ProjectFileAgent' _ = require "underscore" request = require "request" @@ -59,16 +59,13 @@ module.exports = ProjectOutputFileAgent = { source_project_id = project._id CompileManager.compile source_project_id, null, {}, (err) -> return callback(err) if err? - url = "#{Settings.apis.clsi.url}/project/#{source_project_id}/output/#{source_output_file_path}" - ClsiCookieManager.getCookieJar source_project_id, (err, jar)-> + ClsiManager.getOutputFileStream source_project_id, source_output_file_path, (err, readStream) -> return callback(err) if err? - oneMinute = 60 * 1000 - # the base request - options = { url: url, method: "GET", timeout: oneMinute, jar : jar } - readStream = request(options) + readStream.pause() readStream.on "error", callback readStream.on "response", (response) -> if 200 <= response.statusCode < 300 + readStream.resume() FileWriter.writeStreamToDisk project_id, readStream, callback else error = new OutputFileFetchFailedError("Output file fetch failed: #{url}") From 6058f3ef9bb5f7e5d9b1b4f6c1a438d0d459a8b6 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 20 Jun 2018 10:01:03 +0100 Subject: [PATCH 48/63] Refactor the LinkedFiles/Agent system, and track build_id for output files --- .../Features/Compile/ClsiManager.coffee | 4 +- .../LinkedFiles/LinkedFilesController.coffee | 60 ++---- .../LinkedFiles/LinkedFilesErrors.coffee | 122 +++++++++++ .../LinkedFiles/LinkedFilesHandler.coffee | 53 +++++ .../LinkedFiles/ProjectFileAgent.coffee | 191 ++++++++---------- .../LinkedFiles/ProjectOutputFileAgent.coffee | 173 +++++++++++----- .../Features/LinkedFiles/UrlAgent.coffee | 80 +++----- .../coffee/infrastructure/FileWriter.coffee | 5 +- .../controllers/FileTreeController.coffee | 6 +- .../acceptance/coffee/LinkedFilesTests.coffee | 4 +- .../coffee/helpers/MockClsiApi.coffee | 18 +- 11 files changed, 462 insertions(+), 254 deletions(-) create mode 100644 services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee create mode 100644 services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee diff --git a/services/web/app/coffee/Features/Compile/ClsiManager.coffee b/services/web/app/coffee/Features/Compile/ClsiManager.coffee index 232b416e66..d1dfe21dad 100755 --- a/services/web/app/coffee/Features/Compile/ClsiManager.coffee +++ b/services/web/app/coffee/Features/Compile/ClsiManager.coffee @@ -182,8 +182,8 @@ module.exports = ClsiManager = return callback(error) if error? callback(null, projectStateHash, docs) - getOutputFileStream: (project_id, output_file_path, callback=(err, readStream)->) -> - url = "#{Settings.apis.clsi.url}/project/#{project_id}/output/#{output_file_path}" + getOutputFileStream: (project_id, user_id, build_id, output_file_path, callback=(err, readStream)->) -> + url = "#{Settings.apis.clsi.url}/project/#{project_id}/user/#{user_id}/build/#{build_id}/output/#{output_file_path}" ClsiCookieManager.getCookieJar project_id, (err, jar)-> return callback(err) if err? options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar } diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee index 00c061778c..2d42c2e19c 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee @@ -4,8 +4,11 @@ ProjectLocator = require '../Project/ProjectLocator' Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' _ = require 'underscore' +LinkedFilesErrors = require './LinkedFilesErrors' + module.exports = LinkedFilesController = { + Agents: { url: require('./UrlAgent'), project_file: require('./ProjectFileAgent'), @@ -38,16 +41,16 @@ module.exports = LinkedFilesController = { if !Agent? return res.sendStatus(400) - linkedFileData = Agent.sanitizeData(data) - linkedFileData.provider = provider + data.provider = provider - if !Agent.canCreate(linkedFileData) - return res.status(403).send('Cannot create linked file') - - LinkedFilesController._doImport( - req, res, next, Agent, project_id, user_id, - parent_folder_id, name, linkedFileData - ) + Agent.createLinkedFile project_id, + data, + name, + parent_folder_id, + user_id, + (err, newFileId) -> + return LinkedFilesErrors.handleError(err, req, res, next) if err? + res.json(new_file_id: newFileId) refreshLinkedFile: (req, res, next) -> {project_id, file_id} = req.params @@ -66,37 +69,14 @@ module.exports = LinkedFilesController = { Agent = LinkedFilesController._getAgent(provider) if !Agent? return res.sendStatus(400) - LinkedFilesController._doImport( - req, res, next, Agent, project_id, user_id, - parent_folder_id, name, linkedFileData - ) - _doImport: (req, res, next, Agent, project_id, user_id, parent_folder_id, name, linkedFileData) -> - Agent.checkAuth project_id, linkedFileData, user_id, (err, allowed) -> - return Agent.handleError(err, req, res, next) if err? - return res.sendStatus(403) if !allowed - Agent.decorateLinkedFileData linkedFileData, (err, newLinkedFileData) -> - return Agent.handleError(err) if err? - linkedFileData = newLinkedFileData - Agent.writeIncomingFileToDisk project_id, - linkedFileData, - user_id, - (error, fsPath) -> - if error? - logger.error( - {err: error, project_id, name, linkedFileData, parent_folder_id, user_id}, - 'error writing linked file to disk' - ) - return Agent.handleError(error, req, res, next) - EditorController.upsertFile project_id, - parent_folder_id, - name, - fsPath, - linkedFileData, - "upload", - user_id, - (error, file) -> - return next(error) if error? - res.json(new_file_id: file._id) # created + Agent.refreshLinkedFile project_id, + linkedFileData, + name, + parent_folder_id, + user_id, + (err, newFileId) -> + return LinkedFilesErrors.handleError(err, req, res, next) if err? + res.json(new_file_id: newFileId) } diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee new file mode 100644 index 0000000000..69c66e11ea --- /dev/null +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee @@ -0,0 +1,122 @@ +UrlFetchFailedError = (message) -> + error = new Error(message) + error.name = 'UrlFetchFailedError' + error.__proto__ = UrlFetchFailedError.prototype + return error +UrlFetchFailedError.prototype.__proto__ = Error.prototype + + +InvalidUrlError = (message) -> + error = new Error(message) + error.name = 'InvalidUrlError' + error.__proto__ = InvalidUrlError.prototype + return error +InvalidUrlError.prototype.__proto__ = Error.prototype + + +OutputFileFetchFailedError = (message) -> + error = new Error(message) + error.name = 'OutputFileFetchFailedError' + error.__proto__ = OutputFileFetchFailedError.prototype + return error +OutputFileFetchFailedError.prototype.__proto__ = Error.prototype + + +AccessDeniedError = (message) -> + error = new Error(message) + error.name = 'AccessDenied' + error.__proto__ = AccessDeniedError.prototype + return error +AccessDeniedError.prototype.__proto__ = Error.prototype + + +BadEntityTypeError = (message) -> + error = new Error(message) + error.name = 'BadEntityType' + error.__proto__ = BadEntityTypeError.prototype + return error +BadEntityTypeError.prototype.__proto__ = Error.prototype + + +BadDataError = (message) -> + error = new Error(message) + error.name = 'BadData' + error.__proto__ = BadDataError.prototype + return error +BadDataError.prototype.__proto__ = Error.prototype + + +ProjectNotFoundError = (message) -> + error = new Error(message) + error.name = 'ProjectNotFound' + error.__proto__ = ProjectNotFoundError.prototype + return error +ProjectNotFoundError.prototype.__proto__ = Error.prototype + + +V1ProjectNotFoundError = (message) -> + error = new Error(message) + error.name = 'V1ProjectNotFound' + error.__proto__ = V1ProjectNotFoundError.prototype + return error +V1ProjectNotFoundError.prototype.__proto__ = Error.prototype + + +SourceFileNotFoundError = (message) -> + error = new Error(message) + error.name = 'SourceFileNotFound' + error.__proto__ = SourceFileNotFoundError.prototype + return error +SourceFileNotFoundError.prototype.__proto__ = Error.prototype + + +module.exports = { + + UrlFetchFailedError, + InvalidUrlError, + OutputFileFetchFailedError, + AccessDeniedError, + BadEntityTypeError, + BadDataError, + ProjectNotFoundError, + V1ProjectNotFoundError, + SourceFileNotFoundError, + + handleError: (error, req, res, next) -> + if error instanceof BadDataError + res.status(400).send("The submitted data is not valid") + + else if error instanceof AccessDeniedError + res.status(403).send("You do not have access to this project") + + else if error instanceof BadDataError + res.status(400).send("The submitted data is not valid") + + else if error instanceof BadEntityTypeError + res.status(400).send("The file is the wrong type") + + else if error instanceof SourceFileNotFoundError + res.status(404).send("Source file not found") + + else if error instanceof ProjectNotFoundError + res.status(404).send("Project not found") + + else if error instanceof V1ProjectNotFoundError + res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file") + + else if error instanceof OutputFileFetchFailedError + res.status(404).send("Could not get output file") + + else if error instanceof UrlFetchFailedError + res.status(422).send( + "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again." + ) + + else if error instanceof InvalidUrlError + res.status(422).send( + "Your URL is not valid. Please check it and try again." + ) + + else + next(error) +} diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee new file mode 100644 index 0000000000..4eb3391035 --- /dev/null +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee @@ -0,0 +1,53 @@ +LinkedFilesErrors = require './LinkedFilesErrors' +FileWriter = require '../../infrastructure/FileWriter' +EditorController = require '../Editor/EditorController' +_ = require 'underscore' + + +module.exports = LinkedFilesHandler = + + importFromStream: ( + project_id, + readStream, + linkedFileData, + name, + parent_folder_id, + user_id, + callback=(err, file)-> + ) -> + callback = _.once(callback) + FileWriter.writeStreamToDisk project_id, readStream, (err, fsPath) -> + return callback(err) if err? + EditorController.upsertFile project_id, + parent_folder_id, + name, + fsPath, + linkedFileData, + "upload", + user_id, + (err, file) => + return callback(err) if err? + callback(null, file) + + importContent: ( + project_id, + content, + linkedFileData, + name, + parent_folder_id, + user_id, + callback=(err, file)-> + ) -> + callback = _.once(callback) + FileWriter.writeContentToDisk project_id, content, (err, fsPath) -> + return callback(err) if err? + EditorController.upsertFile project_id, + parent_folder_id, + name, + fsPath, + linkedFileData, + "upload", + user_id, + (err, file) => + return callback(err) if err? + callback(null, file) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee index 1b9f964733..dab91a5162 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee @@ -1,75 +1,97 @@ -FileWriter = require('../../infrastructure/FileWriter') AuthorizationManager = require('../Authorization/AuthorizationManager') ProjectLocator = require('../Project/ProjectLocator') ProjectGetter = require('../Project/ProjectGetter') Project = require("../../models/Project").Project DocstoreManager = require('../Docstore/DocstoreManager') FileStoreHandler = require('../FileStore/FileStoreHandler') -FileWriter = require('../../infrastructure/FileWriter') _ = require "underscore" Settings = require 'settings-sharelatex' - - -AccessDeniedError = (message) -> - error = new Error(message) - error.name = 'AccessDenied' - error.__proto__ = AccessDeniedError.prototype - return error -AccessDeniedError.prototype.__proto__ = Error.prototype - - -BadEntityTypeError = (message) -> - error = new Error(message) - error.name = 'BadEntityType' - error.__proto__ = BadEntityTypeError.prototype - return error -BadEntityTypeError.prototype.__proto__ = Error.prototype - - -BadDataError = (message) -> - error = new Error(message) - error.name = 'BadData' - error.__proto__ = BadDataError.prototype - return error -BadDataError.prototype.__proto__ = Error.prototype - - -ProjectNotFoundError = (message) -> - error = new Error(message) - error.name = 'ProjectNotFound' - error.__proto__ = ProjectNotFoundError.prototype - return error -ProjectNotFoundError.prototype.__proto__ = Error.prototype - - -V1ProjectNotFoundError = (message) -> - error = new Error(message) - error.name = 'V1ProjectNotFound' - error.__proto__ = V1ProjectNotFoundError.prototype - return error -V1ProjectNotFoundError.prototype.__proto__ = Error.prototype - - -SourceFileNotFoundError = (message) -> - error = new Error(message) - error.name = 'SourceFileNotFound' - error.__proto__ = SourceFileNotFoundError.prototype - return error -SourceFileNotFoundError.prototype.__proto__ = Error.prototype - +LinkedFilesHandler = require './LinkedFilesHandler' +{ + BadDataError, + AccessDeniedError, + BadEntityTypeError, + SourceFileNotFoundError, + ProjectNotFoundError, + V1ProjectNotFoundError +} = require './LinkedFilesErrors' module.exports = ProjectFileAgent = { - V1ProjectNotFoundError - BadDataError - ProjectNotFoundError - V1ProjectNotFoundError + createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + if !@_canCreate(linkedFileData) + return callback(new AccessDeniedError()) + @_go(project_id, linkedFileData, name, parent_folder_id, user_id, callback) - _v1ProjectNotFoundMessage: "Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file" + refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + @_go project_id, linkedFileData, name, parent_folder_id, user_id, callback - sanitizeData: (data) -> + _prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) -> + @_checkAuth project_id, linkedFileData, user_id, (err, allowed) => + return callback(err) if err? + return callback(new AccessDeniedError()) if !allowed + @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) => + return callback(err) if err? + if !@_validate(newLinkedFileData) + return callback(new BadDataError()) + callback(null, newLinkedFileData) + + _go: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + linkedFileData = @_sanitizeData(linkedFileData) + @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) => + return callback(err) if err? + if !@_validate(linkedFileData) + return callback(new BadDataError()) + @_getEntity linkedFileData, user_id, (err, source_project, entity, type) => + return callback(err) if err? + if type == 'doc' + DocstoreManager.getDoc source_project._id, entity._id, (err, lines) -> + return callback(err) if err? + LinkedFilesHandler.importContent project_id, + lines.join('\n'), + linkedFileData, + name, + parent_folder_id, + user_id, + (err, file) -> + return callback(err) if err? + callback(null, file._id) # Created + else if type == 'file' + FileStoreHandler.getFileStream source_project._id, entity._id, null, (err, fileStream) -> + return callback(err) if err? + LinkedFilesHandler.importFromStream project_id, + fileStream, + linkedFileData, + name, + parent_folder_id, + user_id, + (err, file) -> + return callback(err) if err? + callback(null, file._id) # Created + else + callback(new BadEntityTypeError()) + + _getEntity: + (linkedFileData, current_user_id, callback = (err, entity, type) ->) -> + callback = _.once(callback) + { source_entity_path } = linkedFileData + @_getSourceProject linkedFileData, (err, project) -> + return callback(err) if err? + source_project_id = project._id + ProjectLocator.findElementByPath { + project_id: source_project_id, + path: source_entity_path + }, (err, entity, type) -> + if err? + if err.toString().match(/^not found.*/) + err = new SourceFileNotFoundError() + return callback(err) + callback(null, project, entity, type) + + _sanitizeData: (data) -> return _.pick( data, + 'provider', 'source_project_id', 'v1_source_doc_id', 'source_entity_path' @@ -81,7 +103,7 @@ module.exports = ProjectFileAgent = { data.source_entity_path? ) - canCreate: (data) -> + _canCreate: (data) -> # Don't allow creation of linked-files with v1 doc ids !data.v1_source_doc_id? @@ -102,13 +124,13 @@ module.exports = ProjectFileAgent = { else callback(new BadDataError('neither v1 nor v2 id present')) - decorateLinkedFileData: (data, callback = (err, newData) ->) -> + _decorateLinkedFileData: (data, callback = (err, newData) ->) -> callback = _.once(callback) @_getSourceProject data, (err, project) -> return callback(err) if err? callback(err, _.extend(data, {source_project_display_name: project.name})) - checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> + _checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> callback = _.once(callback) if !ProjectFileAgent._validate(data) return callback(new BadDataError()) @@ -117,53 +139,4 @@ module.exports = ProjectFileAgent = { AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) -> return callback(err) if err? callback(null, canRead) - - writeIncomingFileToDisk: - (project_id, data, current_user_id, callback = (error, fsPath) ->) -> - callback = _.once(callback) - if !ProjectFileAgent._validate(data) - return callback(new BadDataError()) - { source_entity_path } = data - @_getSourceProject data, (err, project) -> - return callback(err) if err? - source_project_id = project._id - ProjectLocator.findElementByPath { - project_id: source_project_id, - path: source_entity_path - }, (err, entity, type) -> - if err? - if err.toString().match(/^not found.*/) - err = new SourceFileNotFoundError() - return callback(err) - ProjectFileAgent._writeEntityToDisk source_project_id, entity._id, type, callback - - _writeEntityToDisk: (project_id, entity_id, type, callback=(err, location)->) -> - callback = _.once(callback) - if type == 'doc' - DocstoreManager.getDoc project_id, entity_id, (err, lines) -> - return callback(err) if err? - FileWriter.writeLinesToDisk entity_id, lines, callback - else if type == 'file' - FileStoreHandler.getFileStream project_id, entity_id, null, (err, fileStream) -> - return callback(err) if err? - FileWriter.writeStreamToDisk entity_id, fileStream, callback - else - callback(new BadEntityTypeError()) - - handleError: (error, req, res, next) -> - if error instanceof AccessDeniedError - res.status(403).send("You do not have access to this project") - else if error instanceof BadDataError - res.status(400).send("The submitted data is not valid") - else if error instanceof BadEntityTypeError - res.status(400).send("The file is the wrong type") - else if error instanceof SourceFileNotFoundError - res.status(404).send("Source file not found") - else if error instanceof ProjectNotFoundError - res.status(404).send("Project not found") - else if error instanceof V1ProjectNotFoundError - res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage) - else - next(error) - next() } diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 0c84d3f7dc..aff876f864 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -1,86 +1,157 @@ -FileWriter = require('../../infrastructure/FileWriter') AuthorizationManager = require('../Authorization/AuthorizationManager') ProjectGetter = require('../Project/ProjectGetter') -FileWriter = require('../../infrastructure/FileWriter') Settings = require 'settings-sharelatex' CompileManager = require '../Compile/CompileManager' -ClsiCookieManager = require '../Compile/ClsiCookieManager' ClsiManager = require '../Compile/ClsiManager' ProjectFileAgent = require './ProjectFileAgent' _ = require "underscore" -request = require "request" - - -OutputFileFetchFailedError = (message) -> - error = new Error(message) - error.name = 'OutputFileFetchFailedError' - error.__proto__ = OutputFileFetchFailedError.prototype - return error -OutputFileFetchFailedError.prototype.__proto__ = Error.prototype +LinkedFilesErrors = require './LinkedFilesErrors' +LinkedFilesHandler = require './LinkedFilesHandler' +logger = require 'logger-sharelatex' module.exports = ProjectOutputFileAgent = { - sanitizeData: (data) -> + _prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) -> + @_checkAuth project_id, linkedFileData, user_id, (err, allowed) => + return callback(err) if err? + return callback(new LinkedFilesErrors.AccessDeniedError()) if !allowed + @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) => + return callback(err) if err? + if !@_validate(newLinkedFileData) + return callback(new BadDataError()) + callback(null, newLinkedFileData) + + createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + if !@_canCreate(linkedFileData) + return callback(new LinkedFilesErrors.AccessDeniedError()) + linkedFileData = @_sanitizeData(linkedFileData) + @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) => + return callback(err) if err? + @_getFileStream linkedFileData, user_id, (err, readStream) => + return callback(err) if err? + readStream.on "error", callback + readStream.on "response", (response) => + if 200 <= response.statusCode < 300 + readStream.resume() + LinkedFilesHandler.importFromStream project_id, + readStream, + linkedFileData, + name, + parent_folder_id, + user_id, + (err, file) -> + return callback(err) if err? + callback(null, file._id) # Created + else + err = new LinkedFilesErrors.OutputFileFetchFailedError( + "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}" + ) + err.statusCode = response.statusCode + callback(err) + + refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) => + return callback(err) if err? + @_compileAndGetFileStream linkedFileData, user_id, (err, readStream, new_build_id) => + return callback(err) if err? + readStream.on "error", callback + readStream.on "response", (response) => + if 200 <= response.statusCode < 300 + readStream.resume() + linkedFileData.build_id = new_build_id + LinkedFilesHandler.importFromStream project_id, + readStream, + linkedFileData, + name, + parent_folder_id, + user_id, + (err, file) -> + return callback(err) if err? + callback(null, file._id) # Created + else + err = new LinkedFilesErrors.OutputFileFetchFailedError( + "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}" + ) + err.statusCode = response.statusCode + callback(err) + + + _sanitizeData: (data) -> return { + provider: data.provider, source_project_id: data.source_project_id, - source_output_file_path: data.source_output_file_path + source_output_file_path: data.source_output_file_path, + build_id: data.build_id } - canCreate: ProjectFileAgent.canCreate + _canCreate: ProjectFileAgent._canCreate _getSourceProject: ProjectFileAgent._getSourceProject - decorateLinkedFileData: ProjectFileAgent.decorateLinkedFileData + _decorateLinkedFileData: ProjectFileAgent._decorateLinkedFileData _validate: (data) -> return ( (data.source_project_id? || data.v1_source_doc_id?) && - data.source_output_file_path? + data.source_output_file_path? && + data.build_id? ) - checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> + _checkAuth: (project_id, data, current_user_id, callback = (err, allowed)->) -> callback = _.once(callback) - if !ProjectOutputFileAgent._validate(data) - return callback(new BadDataError()) + if !@_validate(data) + return callback(new LinkedFilesErrors.BadDataError()) @_getSourceProject data, (err, project) -> return callback(err) if err? - AuthorizationManager.canUserReadProject current_user_id, project._id, null, (err, canRead) -> - return callback(err) if err? - callback(null, canRead) + AuthorizationManager.canUserReadProject current_user_id, + project._id, + null, + (err, canRead) -> + return callback(err) if err? + callback(null, canRead) - writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> + _getFileStream: (linkedFileData, user_id, callback=(err, fileStream)->) -> callback = _.once(callback) - if !ProjectOutputFileAgent._validate(data) - return callback(new BadDataError()) - { source_output_file_path } = data - @_getSourceProject data, (err, project) -> + { source_output_file_path, build_id } = linkedFileData + @_getSourceProject linkedFileData, (err, project) -> return callback(err) if err? source_project_id = project._id - CompileManager.compile source_project_id, null, {}, (err) -> - return callback(err) if err? - ClsiManager.getOutputFileStream source_project_id, source_output_file_path, (err, readStream) -> + ClsiManager.getOutputFileStream source_project_id, + user_id, + build_id, + source_output_file_path, + (err, readStream) -> return callback(err) if err? readStream.pause() - readStream.on "error", callback - readStream.on "response", (response) -> - if 200 <= response.statusCode < 300 - readStream.resume() - FileWriter.writeStreamToDisk project_id, readStream, callback - else - error = new OutputFileFetchFailedError("Output file fetch failed: #{url}") - error.statusCode = response.statusCode - callback(error) + callback(null, readStream) - handleError: (error, req, res, next) -> - if error instanceof ProjectFileAgent.BadDataError - res.status(400).send("The submitted data is not valid") - else if error instanceof OutputFileFetchFailedError - res.status(404).send("Could not get output file") - else if error instanceof ProjectFileAgent.ProjectNotFoundError - res.status(404).send("Project not found") - else if error instanceof ProjectFileAgent.V1ProjectNotFoundError - res.status(409).send(ProjectFileAgent._v1ProjectNotFoundMessage) - else - next(error) + _compileAndGetFileStream: (linkedFileData, user_id, callback=(err, stream, build_id)->) -> + callback = _.once(callback) + { source_output_file_path } = linkedFileData + @_getSourceProject linkedFileData, (err, project) -> + return callback(err) if err? + source_project_id = project._id + CompileManager.compile source_project_id, + user_id, + {}, + (err, status, outputFiles) -> + return callback(err) if err? + if status != 'success' + return callback(new LinkedFilesErrors.OutputFileFetchFailedError()) + outputFile = _.find( + outputFiles, + (o) => o.path == source_output_file_path + ) + if !outputFile? + return callback(new LinkedFilesErrors.OutputFileFetchFailedError()) + build_id = outputFile.build + ClsiManager.getOutputFileStream source_project_id, + user_id, + build_id, + source_output_file_path, + (err, readStream) -> + return callback(err) if err? + readStream.pause() + callback(null, readStream, build_id) } diff --git a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee index d3748cf8d2..b5a6e49020 100644 --- a/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee @@ -1,67 +1,53 @@ request = require 'request' -FileWriter = require('../../infrastructure/FileWriter') _ = require "underscore" urlValidator = require 'valid-url' Settings = require 'settings-sharelatex' +{ InvalidUrlError, UrlFetchFailedError } = require './LinkedFilesErrors' +LinkedFilesHandler = require './LinkedFilesHandler' -UrlFetchFailedError = (message) -> - error = new Error(message) - error.name = 'UrlFetchFailedError' - error.__proto__ = UrlFetchFailedError.prototype - return error -UrlFetchFailedError.prototype.__proto__ = Error.prototype - -InvalidUrlError = (message) -> - error = new Error(message) - error.name = 'InvalidUrlError' - error.__proto__ = InvalidUrlError.prototype - return error -InvalidUrlError.prototype.__proto__ = Error.prototype module.exports = UrlAgent = { - UrlFetchFailedError: UrlFetchFailedError - InvalidUrlError: InvalidUrlError - sanitizeData: (data) -> + createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + linkedFileData = @._sanitizeData(linkedFileData) + @_getUrlStream project_id, linkedFileData, user_id, (err, readStream) -> + return callback(err) if err? + readStream.on "error", callback + readStream.on "response", (response) -> + if 200 <= response.statusCode < 300 + readStream.resume() + LinkedFilesHandler.importFromStream project_id, + readStream, + linkedFileData, + name, + parent_folder_id, + user_id, + (err, file) -> + return callback(err) if err? + callback(null, file._id) # Created + else + error = new UrlFetchFailedError("url fetch failed: #{linkedFileData.url}") + error.statusCode = response.statusCode + callback(error) + + refreshLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> + @createLinkedFile project_id, linkedFileData, name, parent_folder_id, user_id, callback + + _sanitizeData: (data) -> return { + provider: data.provider url: @._prependHttpIfNeeded(data.url) } - canCreate: (data) -> true - - decorateLinkedFileData: (data, callback = (err, newData) ->) -> - return callback(null, data) - - checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> - callback(null, true) - - writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> + _getUrlStream: (project_id, data, current_user_id, callback = (error, fsPath) ->) -> callback = _.once(callback) url = data.url if !urlValidator.isWebUri(url) return callback(new InvalidUrlError("invalid url: #{url}")) - url = UrlAgent._wrapWithProxy(url) + url = @_wrapWithProxy(url) readStream = request.get(url) - readStream.on "error", callback - readStream.on "response", (response) -> - if 200 <= response.statusCode < 300 - FileWriter.writeStreamToDisk project_id, readStream, callback - else - error = new UrlFetchFailedError("url fetch failed: #{url}") - error.statusCode = response.statusCode - callback(error) - - handleError: (error, req, res, next) -> - if error instanceof UrlFetchFailedError - res.status(422).send( - "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again." - ) - else if error instanceof InvalidUrlError - res.status(422).send( - "Your URL is not valid. Please check it and try again." - ) - else - next(error) + readStream.pause() + callback(null, readStream) _prependHttpIfNeeded: (url) -> if !url.match('://') diff --git a/services/web/app/coffee/infrastructure/FileWriter.coffee b/services/web/app/coffee/infrastructure/FileWriter.coffee index 27b1f16921..4e37273338 100644 --- a/services/web/app/coffee/infrastructure/FileWriter.coffee +++ b/services/web/app/coffee/infrastructure/FileWriter.coffee @@ -15,11 +15,14 @@ module.exports = FileWriter = callback(null) writeLinesToDisk: (identifier, lines, callback = (error, fsPath)->) -> + FileWriter.writeContentToDisk(identifier, lines.join('\n'), callback) + + writeContentToDisk: (identifier, content, callback = (error, fsPath)->) -> callback = _.once(callback) fsPath = "#{Settings.path.dumpFolder}/#{identifier}_#{uuid.v4()}" FileWriter._ensureDumpFolderExists (error) -> return callback(error) if error? - fs.writeFile fsPath, lines.join('\n'), (error) -> + fs.writeFile fsPath, content, (error) -> return callback(error) if error? callback(null, fsPath) diff --git a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee index cd3c35b866..8573b3742d 100644 --- a/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee +++ b/services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee @@ -218,6 +218,7 @@ define [ projectOutputFiles: null # or [] selectedProjectEntity: null selectedProjectOutputFile: null + buildId: null name: null $scope.state.inFlight = projects: false @@ -349,6 +350,8 @@ define [ filteredFiles = resp.data.outputFiles.filter (f) -> f.path.match(/.*\.(pdf|png|jpeg|jpg|gif)/) $scope.data.projectOutputFiles = filteredFiles + $scope.data.buildId = filteredFiles?[0]?.build + console.log ">> build_id", $scope.data.buildId _reset(err: false) else $scope.data.projectOutputFiles = null @@ -368,7 +371,8 @@ define [ provider = 'project_output_file' payload = { source_project_id: projectId, - source_output_file_path: $scope.data.selectedProjectOutputFile + source_output_file_path: $scope.data.selectedProjectOutputFile, + build_id: $scope.data.buildId } else provider = 'project_file' diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee index 173aefa154..c892848e08 100644 --- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee +++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee @@ -151,7 +151,7 @@ describe "LinkedFiles", -> source_entity_path: "/#{@source_doc_name}", }, (error, response, body) => expect(response.statusCode).to.equal 403 - expect(body).to.equal 'Cannot create linked file' + expect(body).to.equal 'You do not have access to this project' done() describe "with a linked project_file from a v1 project that has not been imported", -> @@ -380,6 +380,7 @@ describe "LinkedFiles", -> data: source_project_id: @project_two_id, source_output_file_path: "output.pdf", + build_id: '1234-abcd' }, (error, response, body) => new_file_id = body.new_file_id @existing_file_id = new_file_id @@ -393,6 +394,7 @@ describe "LinkedFiles", -> source_project_id: @project_two_id, source_output_file_path: "output.pdf", source_project_display_name: "output-test-two" + build_id: '1234-abcd' } expect(firstFile.name).to.equal('test.pdf') done() diff --git a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee index 5b01b7a81b..60424af2a8 100644 --- a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee @@ -33,12 +33,26 @@ module.exports = MockClsiApi = app.post "/project/:project_id/compile", (req, res, next) => res.json { - outputFiles: [{path: 'output.pdf'}] + compile: + status: 'success' + outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}] + } + app.post "/project/:project_id/user/:user_id/compile", (req, res, next) => + res.json { + compile: + status: 'success' + outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}] } - app.get "/project/:project_id/output/:output_path", (req, res, next) => + app.get "/project/:project_id/status", (req, res, next) => + res.status(200).send() + + app.get "/project/:project_id/user/:user_id/build/:build_id/output/:output_path", (req, res, next) => res.status(200).send("hello") + app.all "*", (req, res, next) => + next() + app.listen 3013, (error) -> throw error if error? .on "error", (error) -> From c8012f296889c331586f4117295787f7b1cedf60 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 21 Jun 2018 11:40:16 +0100 Subject: [PATCH 49/63] Cleaner import of errors --- .../LinkedFiles/ProjectOutputFileAgent.coffee | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index aff876f864..53d7652142 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -5,7 +5,12 @@ CompileManager = require '../Compile/CompileManager' ClsiManager = require '../Compile/ClsiManager' ProjectFileAgent = require './ProjectFileAgent' _ = require "underscore" -LinkedFilesErrors = require './LinkedFilesErrors' +{ + BadDataError, + AccessDeniedError, + BadEntityTypeError, + OutputFileFetchFailedError +} = require './LinkedFilesErrors' LinkedFilesHandler = require './LinkedFilesHandler' logger = require 'logger-sharelatex' @@ -15,7 +20,7 @@ module.exports = ProjectOutputFileAgent = { _prepare: (project_id, linkedFileData, user_id, callback=(err, linkedFileData)->) -> @_checkAuth project_id, linkedFileData, user_id, (err, allowed) => return callback(err) if err? - return callback(new LinkedFilesErrors.AccessDeniedError()) if !allowed + return callback(new AccessDeniedError()) if !allowed @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) => return callback(err) if err? if !@_validate(newLinkedFileData) @@ -24,7 +29,7 @@ module.exports = ProjectOutputFileAgent = { createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> if !@_canCreate(linkedFileData) - return callback(new LinkedFilesErrors.AccessDeniedError()) + return callback(new AccessDeniedError()) linkedFileData = @_sanitizeData(linkedFileData) @_prepare project_id, linkedFileData, user_id, (err, linkedFileData) => return callback(err) if err? @@ -44,7 +49,7 @@ module.exports = ProjectOutputFileAgent = { return callback(err) if err? callback(null, file._id) # Created else - err = new LinkedFilesErrors.OutputFileFetchFailedError( + err = new OutputFileFetchFailedError( "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}" ) err.statusCode = response.statusCode @@ -70,7 +75,7 @@ module.exports = ProjectOutputFileAgent = { return callback(err) if err? callback(null, file._id) # Created else - err = new LinkedFilesErrors.OutputFileFetchFailedError( + err = new OutputFileFetchFailedError( "Output file fetch failed: #{linkedFileData.build_id}, #{linkedFileData.source_output_file_path}" ) err.statusCode = response.statusCode @@ -101,7 +106,7 @@ module.exports = ProjectOutputFileAgent = { _checkAuth: (project_id, data, current_user_id, callback = (err, allowed)->) -> callback = _.once(callback) if !@_validate(data) - return callback(new LinkedFilesErrors.BadDataError()) + return callback(new BadDataError()) @_getSourceProject data, (err, project) -> return callback(err) if err? AuthorizationManager.canUserReadProject current_user_id, @@ -138,13 +143,13 @@ module.exports = ProjectOutputFileAgent = { (err, status, outputFiles) -> return callback(err) if err? if status != 'success' - return callback(new LinkedFilesErrors.OutputFileFetchFailedError()) + return callback(new OutputFileFetchFailedError()) outputFile = _.find( outputFiles, (o) => o.path == source_output_file_path ) if !outputFile? - return callback(new LinkedFilesErrors.OutputFileFetchFailedError()) + return callback(new OutputFileFetchFailedError()) build_id = outputFile.build ClsiManager.getOutputFileStream source_project_id, user_id, From dfb4898be52d030b32b88db5216bf46dfeaa2c00 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 21 Jun 2018 11:40:36 +0100 Subject: [PATCH 50/63] Move `getFileById` to the `LinkedFilesHandler` module --- .../Features/LinkedFiles/LinkedFilesController.coffee | 11 +---------- .../Features/LinkedFiles/LinkedFilesHandler.coffee | 10 ++++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee index 2d42c2e19c..ba624e7814 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee @@ -22,15 +22,6 @@ module.exports = LinkedFilesController = { return null LinkedFilesController.Agents[provider] - _getFileById: (project_id, file_id, callback=(err, file)->) -> - ProjectLocator.findElement { - project_id, - element_id: file_id, - type: 'file' - }, (err, file, path, parentFolder) -> - return callback(err) if err? - callback(null, file, path, parentFolder) - createLinkedFile: (req, res, next) -> {project_id} = req.params {name, provider, data, parent_folder_id} = req.body @@ -57,7 +48,7 @@ module.exports = LinkedFilesController = { user_id = AuthenticationController.getLoggedInUserId(req) logger.log {project_id, file_id, user_id}, 'refresh linked file request' - LinkedFilesController._getFileById project_id, file_id, (err, file, path, parentFolder) -> + LinkedFilesHandler.getFileById project_id, file_id, (err, file, path, parentFolder) -> return next(err) if err? return res.sendStatus(404) if !file? name = file.name diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee index 4eb3391035..76693cb525 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee @@ -1,11 +1,21 @@ LinkedFilesErrors = require './LinkedFilesErrors' FileWriter = require '../../infrastructure/FileWriter' EditorController = require '../Editor/EditorController' +ProjectLocator = require '../Project/ProjectLocator' _ = require 'underscore' module.exports = LinkedFilesHandler = + getFileById: (project_id, file_id, callback=(err, file)->) -> + ProjectLocator.findElement { + project_id, + element_id: file_id, + type: 'file' + }, (err, file, path, parentFolder) -> + return callback(err) if err? + callback(null, file, path, parentFolder) + importFromStream: ( project_id, readStream, From ebe828aa625c69a563542cde436928c29d5bd64d Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 21 Jun 2018 15:27:32 +0100 Subject: [PATCH 51/63] Refactor, and remove the `source_project_display_name` prop from linkedFileData --- .../LinkedFiles/LinkedFilesController.coffee | 1 + .../LinkedFiles/LinkedFilesHandler.coffee | 25 ++++++++++++++- .../LinkedFiles/ProjectFileAgent.coffee | 32 +++---------------- .../LinkedFiles/ProjectOutputFileAgent.coffee | 12 +++---- .../app/views/project/editor/binary-file.pug | 8 ++--- .../acceptance/coffee/LinkedFilesTests.coffee | 5 ++- 6 files changed, 39 insertions(+), 44 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee index ba624e7814..9e70f6e200 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee @@ -5,6 +5,7 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' _ = require 'underscore' LinkedFilesErrors = require './LinkedFilesErrors' +LinkedFilesHandler = require './LinkedFilesHandler' module.exports = LinkedFilesController = { diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee index 76693cb525..6262f0a5ab 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesHandler.coffee @@ -1,8 +1,14 @@ -LinkedFilesErrors = require './LinkedFilesErrors' FileWriter = require '../../infrastructure/FileWriter' EditorController = require '../Editor/EditorController' ProjectLocator = require '../Project/ProjectLocator' +Project = require("../../models/Project").Project +ProjectGetter = require("../Project/ProjectGetter") _ = require 'underscore' +{ + ProjectNotFoundError, + V1ProjectNotFoundError, + BadDataError +} = require './LinkedFilesErrors' module.exports = LinkedFilesHandler = @@ -16,6 +22,23 @@ module.exports = LinkedFilesHandler = return callback(err) if err? callback(null, file, path, parentFolder) + getSourceProject: (data, callback=(err, project)->) -> + projection = {_id: 1, name: 1} + if data.v1_source_doc_id? + Project.findOne {'overleaf.id': data.v1_source_doc_id}, projection, (err, project) -> + return callback(err) if err? + if !project? + return callback(new V1ProjectNotFoundError()) + callback(null, project) + else if data.source_project_id? + ProjectGetter.getProject data.source_project_id, projection, (err, project) -> + return callback(err) if err? + if !project? + return callback(new ProjectNotFoundError()) + callback(null, project) + else + callback(new BadDataError('neither v1 nor v2 id present')) + importFromStream: ( project_id, readStream, diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee index dab91a5162..77196ef1d4 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectFileAgent.coffee @@ -1,7 +1,6 @@ AuthorizationManager = require('../Authorization/AuthorizationManager') ProjectLocator = require('../Project/ProjectLocator') ProjectGetter = require('../Project/ProjectGetter') -Project = require("../../models/Project").Project DocstoreManager = require('../Docstore/DocstoreManager') FileStoreHandler = require('../FileStore/FileStoreHandler') _ = require "underscore" @@ -30,11 +29,9 @@ module.exports = ProjectFileAgent = { @_checkAuth project_id, linkedFileData, user_id, (err, allowed) => return callback(err) if err? return callback(new AccessDeniedError()) if !allowed - @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) => - return callback(err) if err? - if !@_validate(newLinkedFileData) - return callback(new BadDataError()) - callback(null, newLinkedFileData) + if !@_validate(linkedFileData) + return callback(new BadDataError()) + callback(null, linkedFileData) _go: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> linkedFileData = @_sanitizeData(linkedFileData) @@ -107,28 +104,7 @@ module.exports = ProjectFileAgent = { # Don't allow creation of linked-files with v1 doc ids !data.v1_source_doc_id? - _getSourceProject: (data, callback=(err, project)->) -> - projection = {_id: 1, name: 1} - if data.v1_source_doc_id? - Project.findOne {'overleaf.id': data.v1_source_doc_id}, projection, (err, project) -> - return callback(err) if err? - if !project? - return callback(new V1ProjectNotFoundError()) - callback(null, project) - else if data.source_project_id? - ProjectGetter.getProject data.source_project_id, projection, (err, project) -> - return callback(err) if err? - if !project? - return callback(new ProjectNotFoundError()) - callback(null, project) - else - callback(new BadDataError('neither v1 nor v2 id present')) - - _decorateLinkedFileData: (data, callback = (err, newData) ->) -> - callback = _.once(callback) - @_getSourceProject data, (err, project) -> - return callback(err) if err? - callback(err, _.extend(data, {source_project_display_name: project.name})) + _getSourceProject: LinkedFilesHandler.getSourceProject _checkAuth: (project_id, data, current_user_id, callback = (error, allowed)->) -> callback = _.once(callback) diff --git a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee index 53d7652142..5be0866ee2 100644 --- a/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/ProjectOutputFileAgent.coffee @@ -21,11 +21,9 @@ module.exports = ProjectOutputFileAgent = { @_checkAuth project_id, linkedFileData, user_id, (err, allowed) => return callback(err) if err? return callback(new AccessDeniedError()) if !allowed - @_decorateLinkedFileData linkedFileData, (err, newLinkedFileData) => - return callback(err) if err? - if !@_validate(newLinkedFileData) - return callback(new BadDataError()) - callback(null, newLinkedFileData) + if !@_validate(linkedFileData) + return callback(new BadDataError()) + callback(null, linkedFileData) createLinkedFile: (project_id, linkedFileData, name, parent_folder_id, user_id, callback) -> if !@_canCreate(linkedFileData) @@ -92,9 +90,7 @@ module.exports = ProjectOutputFileAgent = { _canCreate: ProjectFileAgent._canCreate - _getSourceProject: ProjectFileAgent._getSourceProject - - _decorateLinkedFileData: ProjectFileAgent._decorateLinkedFileData + _getSourceProject: LinkedFilesHandler.getSourceProject _validate: (data) -> return ( diff --git a/services/web/app/views/project/editor/binary-file.pug b/services/web/app/views/project/editor/binary-file.pug index bd0e4b9732..e7cd1e32ed 100644 --- a/services/web/app/views/project/editor/binary-file.pug +++ b/services/web/app/views/project/editor/binary-file.pug @@ -56,9 +56,9 @@ div.binary-file.full-size( | a(ng-if='!openFile.linkedFileData.v1_source_doc_id' ng-href='/project/{{openFile.linkedFileData.source_project_id}}' target="_blank") - | {{ openFile.linkedFileData.source_project_display_name }} + | Another project span(ng-if='openFile.linkedFileData.v1_source_doc_id') - | {{ openFile.linkedFileData.source_project_display_name }} + | Another project | /{{ openFile.linkedFileData.source_entity_path.slice(1) }}, | | at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }} @@ -71,9 +71,9 @@ div.binary-file.full-size( | a(ng-if='!openFile.linkedFileData.v1_source_doc_id' ng-href='/project/{{openFile.linkedFileData.source_project_id}}' target="_blank") - | {{ openFile.linkedFileData.source_project_display_name }} + | Another project span(ng-if='openFile.linkedFileData.v1_source_doc_id') - | {{ openFile.linkedFileData.source_project_display_name }} + | Another project | : {{ openFile.linkedFileData.source_output_file_path }}, | | at {{ openFile.created | formatDate:'h:mm a' }} {{ openFile.created | relativeDate }} diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee index c892848e08..72fbf5502e 100644 --- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee +++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee @@ -118,7 +118,6 @@ describe "LinkedFiles", -> provider: 'project_file', source_project_id: @project_two_id, source_entity_path: "/#{@source_doc_name}", - source_project_display_name: "plf-test-two" } expect(firstFile.name).to.equal('test-link.txt') done() @@ -393,7 +392,6 @@ describe "LinkedFiles", -> provider: 'project_output_file', source_project_id: @project_two_id, source_output_file_path: "output.pdf", - source_project_display_name: "output-test-two" build_id: '1234-abcd' } expect(firstFile.name).to.equal('test.pdf') @@ -430,7 +428,8 @@ describe "LinkedFiles", -> linkedFileData: { provider: "project_output_file", v1_source_doc_id: 9999999, # We won't find this id in the database - source_output_file_path: "output.pdf" + source_output_file_path: "output.pdf", + build_id: '123' }, _id: "abcdef", rev: 0, From 096d3f28a1402686d6f429d0591a0fee17baa34b Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 22 Jun 2018 11:14:04 +0100 Subject: [PATCH 52/63] Move the `handleError` function into the `LinkedFilesController` --- .../LinkedFiles/LinkedFilesController.coffee | 56 +++++++++++++++++-- .../LinkedFiles/LinkedFilesErrors.coffee | 38 ------------- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee index 9e70f6e200..34f77fde20 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesController.coffee @@ -4,8 +4,19 @@ ProjectLocator = require '../Project/ProjectLocator' Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' _ = require 'underscore' -LinkedFilesErrors = require './LinkedFilesErrors' LinkedFilesHandler = require './LinkedFilesHandler' +{ + + UrlFetchFailedError, + InvalidUrlError, + OutputFileFetchFailedError, + AccessDeniedError, + BadEntityTypeError, + BadDataError, + ProjectNotFoundError, + V1ProjectNotFoundError, + SourceFileNotFoundError, +} = require './LinkedFilesErrors' module.exports = LinkedFilesController = { @@ -41,7 +52,7 @@ module.exports = LinkedFilesController = { parent_folder_id, user_id, (err, newFileId) -> - return LinkedFilesErrors.handleError(err, req, res, next) if err? + return LinkedFilesController.handleError(err, req, res, next) if err? res.json(new_file_id: newFileId) refreshLinkedFile: (req, res, next) -> @@ -68,7 +79,44 @@ module.exports = LinkedFilesController = { parent_folder_id, user_id, (err, newFileId) -> - return LinkedFilesErrors.handleError(err, req, res, next) if err? + return LinkedFilesController.handleError(err, req, res, next) if err? res.json(new_file_id: newFileId) - } + handleError: (error, req, res, next) -> + if error instanceof BadDataError + res.status(400).send("The submitted data is not valid") + + else if error instanceof AccessDeniedError + res.status(403).send("You do not have access to this project") + + else if error instanceof BadDataError + res.status(400).send("The submitted data is not valid") + + else if error instanceof BadEntityTypeError + res.status(400).send("The file is the wrong type") + + else if error instanceof SourceFileNotFoundError + res.status(404).send("Source file not found") + + else if error instanceof ProjectNotFoundError + res.status(404).send("Project not found") + + else if error instanceof V1ProjectNotFoundError + res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file") + + else if error instanceof OutputFileFetchFailedError + res.status(404).send("Could not get output file") + + else if error instanceof UrlFetchFailedError + res.status(422).send( + "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again." + ) + + else if error instanceof InvalidUrlError + res.status(422).send( + "Your URL is not valid. Please check it and try again." + ) + + else + next(error) +} diff --git a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee index 69c66e11ea..8cf671702e 100644 --- a/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee +++ b/services/web/app/coffee/Features/LinkedFiles/LinkedFilesErrors.coffee @@ -81,42 +81,4 @@ module.exports = { ProjectNotFoundError, V1ProjectNotFoundError, SourceFileNotFoundError, - - handleError: (error, req, res, next) -> - if error instanceof BadDataError - res.status(400).send("The submitted data is not valid") - - else if error instanceof AccessDeniedError - res.status(403).send("You do not have access to this project") - - else if error instanceof BadDataError - res.status(400).send("The submitted data is not valid") - - else if error instanceof BadEntityTypeError - res.status(400).send("The file is the wrong type") - - else if error instanceof SourceFileNotFoundError - res.status(404).send("Source file not found") - - else if error instanceof ProjectNotFoundError - res.status(404).send("Project not found") - - else if error instanceof V1ProjectNotFoundError - res.status(409).send("Sorry, the source project is not yet imported to Overleaf v2. Please import it to Overleaf v2 to refresh this file") - - else if error instanceof OutputFileFetchFailedError - res.status(404).send("Could not get output file") - - else if error instanceof UrlFetchFailedError - res.status(422).send( - "Your URL could not be reached (#{error.statusCode} status code). Please check it and try again." - ) - - else if error instanceof InvalidUrlError - res.status(422).send( - "Your URL is not valid. Please check it and try again." - ) - - else - next(error) } From a8222f2e31fcebbbafa8ca3037507b4608639b9c Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 22 Jun 2018 13:20:13 +0100 Subject: [PATCH 53/63] Fix tests after sync with master --- .../acceptance/coffee/LinkedFilesTests.coffee | 8 +++--- .../coffee/helpers/MockClsiApi.coffee | 26 +++++-------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee index 72fbf5502e..0c369097cd 100644 --- a/services/web/test/acceptance/coffee/LinkedFilesTests.coffee +++ b/services/web/test/acceptance/coffee/LinkedFilesTests.coffee @@ -369,7 +369,7 @@ describe "LinkedFiles", -> cb(error) ], done - it 'should import the output.pdf file from the source project', (done) -> + it 'should import the project.pdf file from the source project', (done) -> @owner.request.post { url: "/project/#{@project_one_id}/linked_file", json: @@ -378,7 +378,7 @@ describe "LinkedFiles", -> provider: 'project_output_file', data: source_project_id: @project_two_id, - source_output_file_path: "output.pdf", + source_output_file_path: "project.pdf", build_id: '1234-abcd' }, (error, response, body) => new_file_id = body.new_file_id @@ -391,7 +391,7 @@ describe "LinkedFiles", -> expect(firstFile.linkedFileData).to.deep.equal { provider: 'project_output_file', source_project_id: @project_two_id, - source_output_file_path: "output.pdf", + source_output_file_path: "project.pdf", build_id: '1234-abcd' } expect(firstFile.name).to.equal('test.pdf') @@ -428,7 +428,7 @@ describe "LinkedFiles", -> linkedFileData: { provider: "project_output_file", v1_source_doc_id: 9999999, # We won't find this id in the database - source_output_file_path: "output.pdf", + source_output_file_path: "project.pdf", build_id: '123' }, _id: "abcdef", diff --git a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee index 60424af2a8..0711ac613f 100644 --- a/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockClsiApi.coffee @@ -4,7 +4,8 @@ app = express() module.exports = MockClsiApi = run: () -> - app.post "/project/:project_id/compile", (req, res, next) => + + compile = (req, res, next) => res.status(200).send { compile: status: 'success' @@ -22,6 +23,9 @@ module.exports = MockClsiApi = ] } + app.post "/project/:project_id/compile", compile + app.post "/project/:project_id/user/:user_id/compile", compile + app.get "/project/:project_id/build/:build_id/output/*", (req, res, next) -> filename = req.params[0] if filename == 'project.pdf' @@ -31,27 +35,11 @@ module.exports = MockClsiApi = else res.sendStatus(404) - app.post "/project/:project_id/compile", (req, res, next) => - res.json { - compile: - status: 'success' - outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}] - } - app.post "/project/:project_id/user/:user_id/compile", (req, res, next) => - res.json { - compile: - status: 'success' - outputFiles: [{path: 'output.pdf', build: 'abcd', url: 'http://example.com'}] - } - - app.get "/project/:project_id/status", (req, res, next) => - res.status(200).send() - app.get "/project/:project_id/user/:user_id/build/:build_id/output/:output_path", (req, res, next) => res.status(200).send("hello") - app.all "*", (req, res, next) => - next() + app.get "/project/:project_id/status", (req, res, next) => + res.status(200).send() app.listen 3013, (error) -> throw error if error? From 0edd2b5fe779f3cb0a85f63fb8cd20c73805787c Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 22 Jun 2018 14:05:48 +0100 Subject: [PATCH 54/63] Fix check for hasSubscription --- .../Features/Project/ProjectController.coffee | 2 +- .../coffee/Project/ProjectControllerTests.coffee | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Project/ProjectController.coffee b/services/web/app/coffee/Features/Project/ProjectController.coffee index 1b7051e39a..e2c71a956e 100644 --- a/services/web/app/coffee/Features/Project/ProjectController.coffee +++ b/services/web/app/coffee/Features/Project/ProjectController.coffee @@ -214,7 +214,7 @@ module.exports = ProjectController = tags: tags notifications: notifications or [] user: user - hasSubscription: results.hasSubscription[0] + hasSubscription: results.hasSubscription isShowingV1Projects: results.v1Projects? warnings: warnings } diff --git a/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee b/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee index 8de7c6aed1..bfd67e42d2 100644 --- a/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee +++ b/services/web/test/unit/coffee/Project/ProjectControllerTests.coffee @@ -305,6 +305,20 @@ describe "ProjectController", -> done() @ProjectController.projectListPage @req, @res + it 'should send hasSubscription == false when no subscription', (done) -> + @res.render = (pageName, opts)=> + opts.hasSubscription.should.equal false + done() + @ProjectController.projectListPage @req, @res + + it 'should send hasSubscription == true when there is a subscription', (done) -> + @LimitationsManager.userHasSubscriptionOrIsGroupMember = sinon.stub().callsArgWith(1, null, true) + @res.render = (pageName, opts)=> + opts.hasSubscription.should.equal true + done() + @ProjectController.projectListPage @req, @res + + describe 'front widget', (done) -> beforeEach -> @settings.overleaf = From 9194e25742694092763b43f82e6ae474fc6b76f3 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 22 Jun 2018 14:23:43 +0100 Subject: [PATCH 55/63] Set font colour at entire side bar level --- services/web/public/stylesheets/app/project-list.less | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/web/public/stylesheets/app/project-list.less b/services/web/public/stylesheets/app/project-list.less index 569035ad72..a1a0dd9cd6 100644 --- a/services/web/public/stylesheets/app/project-list.less +++ b/services/web/public/stylesheets/app/project-list.less @@ -58,6 +58,10 @@ background-color: @sidebar-bg; padding-top: @content-margin-vertical; padding-bottom: @content-margin-vertical; + color: @sidebar-color; + .small { + color: @sidebar-color; + } } .project-list-sidebar when (@is-overleaf) { @@ -109,9 +113,6 @@ } p { margin-bottom: @line-height-computed / 4; - &.small { - color: @sidebar-color; - } } } From c004cdb200414db4a61a64dddad7a4c20bf246f7 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 22 Jun 2018 14:24:41 +0100 Subject: [PATCH 56/63] Remove hardcoded ShareLaTeX reference in upgrade blurb --- services/web/app/views/subscriptions/new.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/views/subscriptions/new.pug b/services/web/app/views/subscriptions/new.pug index c8228f4802..d3d99168d1 100644 --- a/services/web/app/views/subscriptions/new.pug +++ b/services/web/app/views/subscriptions/new.pug @@ -249,7 +249,7 @@ block content hr - p.small.text-center We're confident that you'll love ShareLaTeX, but if not you can cancel anytime. We'll give you your money back, no questions asked, if you let us know within 30 days. + p.small.text-center We're confident that you'll love #{settings.appName}, but if not you can cancel anytime. We'll give you your money back, no questions asked, if you let us know within 30 days. hr span                   a(href="https://www.positivessl.com" style="font-family: arial; font-size: 10px; color: #212121; text-decoration: none;") From e9f8cceabbee64464e88a93dc17f459e14122c6a Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 22 Jun 2018 10:23:33 -0400 Subject: [PATCH 57/63] fix syntax error --- .../aceEditor/auto-complete/top_hundred_snippets.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee index 0a945c9e4d..56060448a5 100644 --- a/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee +++ b/services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/top_hundred_snippets.coffee @@ -78,7 +78,7 @@ define -> [{ "snippet": "\\ref{$1}", "meta": "cross-reference", "score": 0.014379554883991673 - }{ +}, { "caption": "\\frac{}{}", "snippet": "\\frac{$1}{$2}", "meta": "cmd", From b65ba388bab30b6f45fe0fbd5c256241ec7db88e Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Mon, 18 Jun 2018 18:37:58 +0200 Subject: [PATCH 58/63] replace OldAssetProxy --- .../infrastructure/OldAssetProxy.coffee | 16 ----- .../coffee/infrastructure/ProxyManager.coffee | 36 ++++++++++ .../app/coffee/infrastructure/Server.coffee | 4 +- .../infrastructure/ProxyManagerTests.coffee | 66 +++++++++++++++++++ 4 files changed, 104 insertions(+), 18 deletions(-) delete mode 100644 services/web/app/coffee/infrastructure/OldAssetProxy.coffee create mode 100644 services/web/app/coffee/infrastructure/ProxyManager.coffee create mode 100644 services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee diff --git a/services/web/app/coffee/infrastructure/OldAssetProxy.coffee b/services/web/app/coffee/infrastructure/OldAssetProxy.coffee deleted file mode 100644 index 7912f0ed14..0000000000 --- a/services/web/app/coffee/infrastructure/OldAssetProxy.coffee +++ /dev/null @@ -1,16 +0,0 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -request = require("request") - -module.exports = (req, res, next)-> - requestedUrl = req.url - - redirectUrl = settings.proxyUrls[requestedUrl] - if redirectUrl? - logger.log redirectUrl:redirectUrl, reqUrl:req.url, "proxying url" - upstream = request(redirectUrl) - upstream.on "error", (error) -> - logger.error err: error, "error in OldAssetProxy" - upstream.pipe(res) - else - next() diff --git a/services/web/app/coffee/infrastructure/ProxyManager.coffee b/services/web/app/coffee/infrastructure/ProxyManager.coffee new file mode 100644 index 0000000000..3895ef6e2f --- /dev/null +++ b/services/web/app/coffee/infrastructure/ProxyManager.coffee @@ -0,0 +1,36 @@ +settings = require 'settings-sharelatex' +logger = require 'logger-sharelatex' +request = require 'request' +URL = require 'url' + +module.exports = ProxyManager = + call: (req, res, next) -> + requestUrl = URL.parse(req.url) + requestPath = requestUrl.pathname # ignore the query part + + target = settings.proxyUrls[requestPath] + return next() unless target? # nothing to proxy + + targetUrl = makeTargetUrl(target, req.query) + logger.log targetUrl: targetUrl, reqUrl: req.url, "proxying url" + + upstream = request(targetUrl) + upstream.on "error", (error) -> + logger.error err: error, "error in ProxyManager" + + # TODO: better handling of status code + # see https://github.com/overleaf/write_latex/wiki/Streams-and-pipes-in-Node.js + upstream.pipe(res) + +# make a URL from a proxy target. +# if the query is specified, set/replace the target's query with the given query +makeTargetUrl = (target, query) -> + targetUrl = URL.parse(parseSettingUrl(target)) + if query? and Object.keys(query).length > 0 + targetUrl.query = query + targetUrl.search = null # clear `search` as it takes precedence over `query` + targetUrl.format() + +parseSettingUrl = (target) -> + return target if typeof target is 'string' + "#{target.baseUrl}#{target.path or ''}" diff --git a/services/web/app/coffee/infrastructure/Server.coffee b/services/web/app/coffee/infrastructure/Server.coffee index 39fc3c131b..37359ac797 100644 --- a/services/web/app/coffee/infrastructure/Server.coffee +++ b/services/web/app/coffee/infrastructure/Server.coffee @@ -32,7 +32,7 @@ Mongoose = require("./Mongoose") oneDayInMilliseconds = 86400000 ReferalConnect = require('../Features/Referal/ReferalConnect') RedirectManager = require("./RedirectManager") -OldAssetProxy = require("./OldAssetProxy") +ProxyManager = require("./ProxyManager") translations = require("translations-sharelatex").setup(Settings.i18n) Modules = require "./Modules" @@ -74,7 +74,7 @@ app.use methodOverride() app.use metrics.http.monitor(logger) app.use RedirectManager -app.use OldAssetProxy +app.use ProxyManager.call webRouter.use cookieParser(Settings.security.sessionSecret) diff --git a/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee new file mode 100644 index 0000000000..89a9c0c6f2 --- /dev/null +++ b/services/web/test/unit/coffee/infrastructure/ProxyManagerTests.coffee @@ -0,0 +1,66 @@ +sinon = require('sinon') +assertCalledWith = sinon.assert.calledWith +chai = require('chai') +should = chai.should() +expect = chai.expect +modulePath = '../../../../app/js/infrastructure/ProxyManager' +SandboxedModule = require('sandboxed-module') +MockRequest = require "../helpers/MockRequest" +MockResponse = require "../helpers/MockResponse" + +describe "ProxyManager", -> + before -> + @settings = proxyUrls: {} + @request = sinon.stub().returns( + on: ()-> + pipe: ()-> + ) + @proxyManager = SandboxedModule.require modulePath, requires: + 'settings-sharelatex': @settings + 'logger-sharelatex': log: -> + 'request': @request + @proxyPath = '/foo/bar' + @req = new MockRequest() + @res = new MockResponse() + @next = sinon.stub() + + describe 'proxyUrls', -> + beforeEach -> + @req.url = @proxyPath + @req.query = {} + @settings.proxyUrls = {} + + afterEach -> + @next.reset() + @request.reset() + + it 'calls next when no match', -> + @proxyManager.call(@req, @res, @next) + sinon.assert.called(@next) + sinon.assert.notCalled(@request) + + it 'does not calls next when match', -> + @settings.proxyUrls[@proxyPath] = '/' + @proxyManager.call(@req, @res, @next) + sinon.assert.notCalled(@next) + sinon.assert.called(@request) + + it 'proxy full URL', -> + targetUrl = 'https://user:pass@foo.bar:123/pa/th.ext?query#hash' + @settings.proxyUrls[@proxyPath] = targetUrl + @proxyManager.call(@req) + assertCalledWith(@request, targetUrl) + + it 'overwrite query', -> + targetUrl = 'foo.bar/baz?query' + @req.query = { requestQuery: 'important' } + @settings.proxyUrls[@proxyPath] = targetUrl + @proxyManager.call(@req) + newTargetUrl = 'foo.bar/baz?requestQuery=important' + assertCalledWith(@request, newTargetUrl) + + it 'handles target objects', -> + targetUrl = { baseUrl: 'api.v1', path: '/pa/th'} + @settings.proxyUrls[@proxyPath] = targetUrl + @proxyManager.call(@req, @res, @next) + assertCalledWith(@request, 'api.v1/pa/th') From d3b2a2650fe7c02f999366c36ff18b5212330e21 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Tue, 19 Jun 2018 13:46:15 +0200 Subject: [PATCH 59/63] add/remove affiliations when adding/removing emails --- .../Features/User/UserEmailsController.coffee | 8 +- .../coffee/Features/User/UserUpdater.coffee | 78 ++++++++++++++++--- .../User/UserEmailsControllerTests.coffee | 16 +++- .../unit/coffee/User/UserUpdaterTests.coffee | 66 ++++++++++++++-- 4 files changed, 144 insertions(+), 24 deletions(-) diff --git a/services/web/app/coffee/Features/User/UserEmailsController.coffee b/services/web/app/coffee/Features/User/UserEmailsController.coffee index f877cbe345..a6335521d6 100644 --- a/services/web/app/coffee/Features/User/UserEmailsController.coffee +++ b/services/web/app/coffee/Features/User/UserEmailsController.coffee @@ -20,7 +20,11 @@ module.exports = UserEmailsController = email = EmailHelper.parseEmail(req.body.email) return res.sendStatus 422 unless email? - UserUpdater.addEmailAddress userId, email, (error)-> + affiliationOptions = + university: req.body.university + role: req.body.role + department: req.body.department + UserUpdater.addEmailAddress userId, email, affiliationOptions, (error)-> return next(error) if error? UserEmailsConfirmationHandler.sendConfirmationEmail userId, email, (err) -> return next(error) if error? @@ -65,4 +69,4 @@ module.exports = UserEmailsController = else next(error) else - res.sendStatus 200 \ No newline at end of file + res.sendStatus 200 diff --git a/services/web/app/coffee/Features/User/UserUpdater.coffee b/services/web/app/coffee/Features/User/UserUpdater.coffee index 4de994b5c2..c73bef7b73 100644 --- a/services/web/app/coffee/Features/User/UserUpdater.coffee +++ b/services/web/app/coffee/Features/User/UserUpdater.coffee @@ -7,6 +7,8 @@ ObjectId = mongojs.ObjectId UserGetter = require("./UserGetter") EmailHelper = require "../Helpers/EmailHelper" Errors = require "../Errors/Errors" +settings = require "settings-sharelatex" +request = require "request" module.exports = UserUpdater = updateUser: (query, update, callback = (error) ->) -> @@ -44,29 +46,43 @@ module.exports = UserUpdater = # Add a new email address for the user. Email cannot be already used by this # or any other user - addEmailAddress: (userId, newEmail, callback) -> + addEmailAddress: (userId, newEmail, affiliationOptions, callback) -> + unless callback? # affiliationOptions is optional + callback = affiliationOptions + affiliationOptions = {} + UserGetter.ensureUniqueEmailAddress newEmail, (error) => return callback(error) if error? - update = $push: emails: email: newEmail, createdAt: new Date() - @updateUser userId, update, (error) -> + addAffiliation userId, newEmail, affiliationOptions, (error) => if error? - logger.err error: error, 'problem updating users emails' + logger.err error: error, 'problem adding affiliation' return callback(error) - callback() + + update = $push: emails: email: newEmail, createdAt: new Date() + @updateUser userId, update, (error) -> + if error? + logger.err error: error, 'problem updating users emails' + return callback(error) + callback() # remove one of the user's email addresses. The email cannot be the user's # default email address removeEmailAddress: (userId, email, callback) -> - query = _id: userId, email: $ne: email - update = $pull: emails: email: email - @updateUser query, update, (error, res) -> + removeAffiliation userId, email, (error) => if error? - logger.err error:error, 'problem removing users email' + logger.err error: error, 'problem removing affiliation' return callback(error) - if res.n == 0 - return callback(new Error('Cannot remove default email')) - callback() + + query = _id: userId, email: $ne: email + update = $pull: emails: email: email + @updateUser query, update, (error, res) -> + if error? + logger.err error:error, 'problem removing users email' + return callback(error) + if res.n == 0 + return callback(new Error('Cannot remove default email')) + callback() # set the default email address by setting the `email` attribute. The email @@ -99,6 +115,44 @@ module.exports = UserUpdater = return callback(new Errors.NotFoundError('user id and email do no match')) callback() +addAffiliation = (userId, email, { university, department, role }, callback = (error) ->) -> + makeAffiliationRequest { + method: 'POST' + path: "/api/v2/users/#{userId.toString()}/affiliations" + body: { email, university, department, role } + defaultErrorMessage: "Couldn't create affiliation" + }, callback + +removeAffiliation = (userId, email, callback = (error) ->) -> + email = encodeURIComponent(email) + makeAffiliationRequest { + method: 'DELETE' + path: "/api/v2/users/#{userId.toString()}/affiliations/#{email}" + extraSuccessStatusCodes: [404] # `Not Found` responses are considered successful + defaultErrorMessage: "Couldn't remove affiliation" + }, callback + +makeAffiliationRequest = (requestOptions, callback = (error) ->) -> + requestOptions.extraSuccessStatusCodes ||= [] + request { + method: requestOptions.method + url: "#{settings.apis.v1.url}#{requestOptions.path}" + body: requestOptions.body + auth: { user: settings.apis.v1.user, pass: settings.apis.v1.pass } + json: true, + timeout: 20 * 1000 + }, (error, response, body) -> + return callback(error) if error? + isSuccess = 200 <= response.statusCode < 300 + isSuccess ||= response.statusCode in requestOptions.extraSuccessStatusCodes + unless isSuccess + if body?.errors + errorMessage = "#{response.statusCode}: #{body.errors}" + else + errorMessage = "#{requestOptions.defaultErrorMessage}: #{response.statusCode}" + return callback(new Error(errorMessage)) + + callback(null) [ 'updateUser' diff --git a/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee b/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee index d0e8076276..d55b7fd963 100644 --- a/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee +++ b/services/web/test/unit/coffee/User/UserEmailsControllerTests.coffee @@ -53,10 +53,14 @@ describe "UserEmailsController", -> describe 'Add', -> beforeEach -> @newEmail = 'new_email@baz.com' - @req.body.email = @newEmail + @req.body = + email: @newEmail + university: { name: 'University Name' } + department: 'Department' + role: 'Role' @EmailHelper.parseEmail.returns @newEmail - @UserUpdater.addEmailAddress.callsArgWith 2, null @UserEmailsConfirmationHandler.sendConfirmationEmail = sinon.stub().yields() + @UserUpdater.addEmailAddress.callsArgWith 3, null it 'adds new email', (done) -> @UserEmailsController.add @req, @@ -64,6 +68,13 @@ describe "UserEmailsController", -> code.should.equal 204 assertCalledWith @EmailHelper.parseEmail, @newEmail assertCalledWith @UserUpdater.addEmailAddress, @user._id, @newEmail + + affiliationOptions = @UserUpdater.addEmailAddress.lastCall.args[2] + Object.keys(affiliationOptions).length.should.equal 3 + affiliationOptions.university.should.equal @req.body.university + affiliationOptions.department.should.equal @req.body.department + affiliationOptions.role.should.equal @req.body.role + done() it 'sends an email confirmation', (done) -> @@ -75,7 +86,6 @@ describe "UserEmailsController", -> it 'handles email parse error', (done) -> @EmailHelper.parseEmail.returns null - @UserEmailsController.add @req, sendStatus: (code) => code.should.equal 422 diff --git a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee index 6b9a1ecc85..77f2c38c85 100644 --- a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee +++ b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee @@ -11,7 +11,6 @@ describe "UserUpdater", -> beforeEach -> tk.freeze(Date.now()) - @settings = {} @mongojs = db:{} ObjectId:(id)-> return id @@ -20,12 +19,15 @@ describe "UserUpdater", -> getUserByAnyEmail: sinon.stub() ensureUniqueEmailAddress: sinon.stub() @logger = err: sinon.stub(), log: -> + settings = apis: { v1: { url: '', user: '', pass: '' } } + @request = sinon.stub() @UserUpdater = SandboxedModule.require modulePath, requires: - "settings-sharelatex":@settings "logger-sharelatex": @logger "./UserGetter": @UserGetter "../../infrastructure/mongojs":@mongojs "metrics-sharelatex": timeAsyncMethod: sinon.stub() + 'settings-sharelatex': settings + 'request': @request @stubbedUser = _id: "3131231" @@ -66,10 +68,10 @@ describe "UserUpdater", -> describe 'addEmailAddress', -> beforeEach -> @UserGetter.ensureUniqueEmailAddress = sinon.stub().callsArgWith(1) + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null) + @request.callsArgWith(1, null, { statusCode: 201 }) it 'add email', (done)-> - @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null) - @UserUpdater.addEmailAddress @stubbedUser._id, @newEmail, (err)=> @UserGetter.ensureUniqueEmailAddress.called.should.equal true should.not.exist(err) @@ -79,6 +81,27 @@ describe "UserUpdater", -> ).should.equal true done() + it 'add affiliation', (done)-> + affiliationOptions = + university: { id: 1 } + role: 'Prof' + department: 'Math' + @UserUpdater.addEmailAddress @stubbedUser._id, @newEmail, affiliationOptions, (err)=> + should.not.exist(err) + @request.calledOnce.should.equal true + requestOptions = @request.lastCall.args[0] + expectedUrl = "/api/v2/users/#{@stubbedUser._id}/affiliations" + requestOptions.url.should.equal expectedUrl + requestOptions.method.should.equal 'POST' + + body = requestOptions.body + Object.keys(body).length.should.equal 4 + body.email.should.equal @newEmail + body.university.should.equal affiliationOptions.university + body.department.should.equal affiliationOptions.department + body.role.should.equal affiliationOptions.role + done() + it 'handle error', (done)-> @UserUpdater.updateUser = sinon.stub().callsArgWith(2, new Error('nope')) @@ -87,10 +110,21 @@ describe "UserUpdater", -> should.exist(err) done() - describe 'removeEmailAddress', -> - it 'remove email', (done)-> - @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, nMatched: 1) + it 'handle affiliation error', (done)-> + body = errors: 'affiliation error message' + @request.callsArgWith(1, null, { statusCode: 422 }, body) + @UserUpdater.addEmailAddress @stubbedUser._id, @newEmail, (err)=> + err.message.should.have.string 422 + err.message.should.have.string body.errors + @UserUpdater.updateUser.called.should.equal false + done() + describe 'removeEmailAddress', -> + beforeEach -> + @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, nMatched: 1) + @request.callsArgWith(1, null, { statusCode: 404 }) + + it 'remove email', (done)-> @UserUpdater.removeEmailAddress @stubbedUser._id, @newEmail, (err)=> should.not.exist(err) @UserUpdater.updateUser.calledWith( @@ -99,6 +133,17 @@ describe "UserUpdater", -> ).should.equal true done() + it 'remove affiliation', (done)-> + @UserUpdater.removeEmailAddress @stubbedUser._id, @newEmail, (err)=> + should.not.exist(err) + @request.calledOnce.should.equal true + requestOptions = @request.lastCall.args[0] + expectedUrl = "/api/v2/users/#{@stubbedUser._id}/affiliations/" + expectedUrl += encodeURIComponent(@newEmail) + requestOptions.url.should.equal expectedUrl + requestOptions.method.should.equal 'DELETE' + done() + it 'handle error', (done)-> @UserUpdater.updateUser = sinon.stub().callsArgWith(2, new Error('nope')) @@ -113,6 +158,13 @@ describe "UserUpdater", -> should.exist(err) done() + it 'handle affiliation error', (done)-> + @request.callsArgWith(1, null, { statusCode: 500 }) + @UserUpdater.removeEmailAddress @stubbedUser._id, @newEmail, (err)=> + err.message.should.exist + @UserUpdater.updateUser.called.should.equal false + done() + describe 'setDefaultEmailAddress', -> it 'set default', (done)-> @UserUpdater.updateUser = sinon.stub().callsArgWith(2, null, n: 1) From e41391fb4a46a4b53957c985806314275cd0d230 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Thu, 21 Jun 2018 13:31:55 +0200 Subject: [PATCH 60/63] ignore affiliations if v1 is not configured --- services/web/app/coffee/Features/User/UserUpdater.coffee | 3 ++- services/web/test/unit/coffee/User/UserUpdaterTests.coffee | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/web/app/coffee/Features/User/UserUpdater.coffee b/services/web/app/coffee/Features/User/UserUpdater.coffee index c73bef7b73..eb983cb76b 100644 --- a/services/web/app/coffee/Features/User/UserUpdater.coffee +++ b/services/web/app/coffee/Features/User/UserUpdater.coffee @@ -132,7 +132,8 @@ removeAffiliation = (userId, email, callback = (error) ->) -> defaultErrorMessage: "Couldn't remove affiliation" }, callback -makeAffiliationRequest = (requestOptions, callback = (error) ->) -> +makeAffiliationRequest = (requestOptions, callback = (error) ->) -> + return callback(null) unless settings?.apis?.v1?.url # service is not configured requestOptions.extraSuccessStatusCodes ||= [] request { method: requestOptions.method diff --git a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee index 77f2c38c85..a2348b54e2 100644 --- a/services/web/test/unit/coffee/User/UserUpdaterTests.coffee +++ b/services/web/test/unit/coffee/User/UserUpdaterTests.coffee @@ -19,7 +19,7 @@ describe "UserUpdater", -> getUserByAnyEmail: sinon.stub() ensureUniqueEmailAddress: sinon.stub() @logger = err: sinon.stub(), log: -> - settings = apis: { v1: { url: '', user: '', pass: '' } } + settings = apis: { v1: { url: 'v1.url', user: '', pass: '' } } @request = sinon.stub() @UserUpdater = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger @@ -90,7 +90,7 @@ describe "UserUpdater", -> should.not.exist(err) @request.calledOnce.should.equal true requestOptions = @request.lastCall.args[0] - expectedUrl = "/api/v2/users/#{@stubbedUser._id}/affiliations" + expectedUrl = "v1.url/api/v2/users/#{@stubbedUser._id}/affiliations" requestOptions.url.should.equal expectedUrl requestOptions.method.should.equal 'POST' @@ -138,7 +138,7 @@ describe "UserUpdater", -> should.not.exist(err) @request.calledOnce.should.equal true requestOptions = @request.lastCall.args[0] - expectedUrl = "/api/v2/users/#{@stubbedUser._id}/affiliations/" + expectedUrl = "v1.url/api/v2/users/#{@stubbedUser._id}/affiliations/" expectedUrl += encodeURIComponent(@newEmail) requestOptions.url.should.equal expectedUrl requestOptions.method.should.equal 'DELETE' From c81f9c24ed71f908bb721d526b470b9f749e8f01 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Thu, 21 Jun 2018 17:59:35 +0200 Subject: [PATCH 61/63] mock affiliations API in acceptance tests --- services/web/test/acceptance/coffee/UserEmailsTests.coffee | 1 + services/web/test/acceptance/coffee/helpers/MockV1Api.coffee | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/services/web/test/acceptance/coffee/UserEmailsTests.coffee b/services/web/test/acceptance/coffee/UserEmailsTests.coffee index 01f1dde531..137932e878 100644 --- a/services/web/test/acceptance/coffee/UserEmailsTests.coffee +++ b/services/web/test/acceptance/coffee/UserEmailsTests.coffee @@ -4,6 +4,7 @@ User = require "./helpers/User" request = require "./helpers/request" settings = require "settings-sharelatex" {db, ObjectId} = require("../../../app/js/infrastructure/mongojs") +MockV1Api = require "./helpers/MockV1Api" describe "UserEmails", -> beforeEach (done) -> diff --git a/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee b/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee index 389ecd1762..65a77eb0e4 100644 --- a/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee @@ -42,6 +42,11 @@ module.exports = MockV1Api = @exportParams = Object.assign({}, req.body) res.json exportId: @exportId + app.post "/api/v2/users/:userId/affiliations", (req, res, next) => + res.sendStatus 201 + + app.delete "/api/v2/users/:userId/affiliations/:email", (req, res, next) => + res.sendStatus 204 app.listen 5000, (error) -> throw error if error? From c6de896bb5ef3b87b87c5c4adc44082180d7a48e Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Thu, 21 Jun 2018 12:30:12 +0200 Subject: [PATCH 62/63] decorate emails list with affiliation data --- .../coffee/Features/User/UserGetter.coffee | 39 ++++++++++++++++--- .../coffee/helpers/MockV1Api.coffee | 3 ++ .../unit/coffee/User/UserGetterTests.coffee | 34 ++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/services/web/app/coffee/Features/User/UserGetter.coffee b/services/web/app/coffee/Features/User/UserGetter.coffee index 59e45e1795..21969656a0 100644 --- a/services/web/app/coffee/Features/User/UserGetter.coffee +++ b/services/web/app/coffee/Features/User/UserGetter.coffee @@ -3,6 +3,8 @@ metrics = require('metrics-sharelatex') logger = require('logger-sharelatex') db = mongojs.db ObjectId = mongojs.ObjectId +settings = require "settings-sharelatex" +request = require "request" module.exports = UserGetter = getUser: (query, projection, callback = (error, user) ->) -> @@ -30,11 +32,9 @@ module.exports = UserGetter = return callback error if error? return callback new Error('User not Found') unless user - fullEmails = user.emails.map (emailData) -> - emailData.default = emailData.email == user.email - emailData - - callback null, fullEmails + getAffiliations userId, (error, affiliationsData) -> + return callback error if error? + callback null, decorateFullEmails(user.email, user.emails, affiliationsData) getUserByMainEmail: (email, projection, callback = (error, user) ->) -> email = email.trim() @@ -81,6 +81,35 @@ module.exports = UserGetter = return callback(message: 'alread_exists') if user? callback(error) +decorateFullEmails = (defaultEmail, emailsData, affiliationsData) -> + emailsData.map (emailData) -> + emailData.default = emailData.email == defaultEmail + + affiliation = affiliationsData.find (aff) -> aff.email == emailData.email + if affiliation? + { institution, inferred, role, department } = affiliation + emailData.affiliation = { institution, inferred, role, department } + else + emailsData.affiliation = null + + emailData + +getAffiliations = (userId, callback = (error) ->) -> + return callback(null, []) unless settings?.apis?.v1?.url # service is not configured + request { + method: 'GET' + url: "#{settings.apis.v1.url}/api/v2/users/#{userId.toString()}/affiliations" + auth: { user: settings.apis.v1.user, pass: settings.apis.v1.pass } + json: true, + timeout: 20 * 1000 + }, (error, response, body) -> + return callback(error) if error? + unless 200 <= response.statusCode < 300 + errorMessage = "Couldn't get affiliations: #{response.statusCode}" + return callback(new Error(errorMessage)) + + callback(null, body) + [ 'getUser', 'getUserEmail', diff --git a/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee b/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee index 65a77eb0e4..91e3f0b25c 100644 --- a/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee +++ b/services/web/test/acceptance/coffee/helpers/MockV1Api.coffee @@ -42,6 +42,9 @@ module.exports = MockV1Api = @exportParams = Object.assign({}, req.body) res.json exportId: @exportId + app.get "/api/v2/users/:userId/affiliations", (req, res, next) => + res.json [] + app.post "/api/v2/users/:userId/affiliations", (req, res, next) => res.sendStatus 201 diff --git a/services/web/test/unit/coffee/User/UserGetterTests.coffee b/services/web/test/unit/coffee/User/UserGetterTests.coffee index f2b7895b57..6b80b5a2a7 100644 --- a/services/web/test/unit/coffee/User/UserGetterTests.coffee +++ b/services/web/test/unit/coffee/User/UserGetterTests.coffee @@ -20,11 +20,15 @@ describe "UserGetter", -> @Mongo = db: users: findOne: @findOne ObjectId: (id) -> return id + settings = apis: { v1: { url: 'v1.url', user: '', pass: '' } } + @request = sinon.stub() @UserGetter = SandboxedModule.require modulePath, requires: "logger-sharelatex": log:-> "../../infrastructure/mongojs": @Mongo "metrics-sharelatex": timeAsyncMethod: sinon.stub() + 'settings-sharelatex': settings + 'request': @request describe "getUser", -> it "should get user", (done)-> @@ -42,6 +46,9 @@ describe "UserGetter", -> done() describe "getUserFullEmails", - + beforeEach -> + @request.callsArgWith(1, null, { statusCode: 200 }, []) + it "should get user", (done)-> @UserGetter.getUser = sinon.stub().callsArgWith(2, null, @fakeUser) projection = email: 1, emails: 1 @@ -59,6 +66,33 @@ describe "UserGetter", -> ] done() + it "should merge affiliation data", (done)-> + @UserGetter.getUser = sinon.stub().callsArgWith(2, null, @fakeUser) + affiliationsData = [ + { + email: 'email1@foo.bar' + role: 'Prof' + department: 'Maths' + inferred: false + institution: { name: 'University Name', isUniversity: true } + } + ] + @request.callsArgWith(1, null, { statusCode: 200 }, affiliationsData) + @UserGetter.getUserFullEmails @fakeUser._id, (error, fullEmails) => + assert.deepEqual fullEmails, [ + { + email: 'email1@foo.bar' + default: false + affiliation: + institution: affiliationsData[0].institution + inferred: affiliationsData[0].inferred + department: affiliationsData[0].department + role: affiliationsData[0].role + } + { email: 'email2@foo.bar', default: true } + ] + done() + describe "getUserbyMainEmail", -> it "query user by main email", (done)-> email = 'hello@world.com' From 7a3fcf6d2c840395c26f6f3acc1c93a11585652d Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Mon, 25 Jun 2018 13:46:46 +0200 Subject: [PATCH 63/63] change error message on email update failure --- services/web/app/coffee/Features/User/UserUpdater.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/User/UserUpdater.coffee b/services/web/app/coffee/Features/User/UserUpdater.coffee index eb983cb76b..8bb6015597 100644 --- a/services/web/app/coffee/Features/User/UserUpdater.coffee +++ b/services/web/app/coffee/Features/User/UserUpdater.coffee @@ -81,7 +81,7 @@ module.exports = UserUpdater = logger.err error:error, 'problem removing users email' return callback(error) if res.n == 0 - return callback(new Error('Cannot remove default email')) + return callback(new Error('Cannot remove email')) callback()