From ff96f37b2dcfd8be437770e64eb54610b8184e30 Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Wed, 26 Feb 2014 15:10:55 +0000 Subject: [PATCH 1/6] unit test passing version of FSPersistorManager --- .../app/coffee/FSPersistorManager.coffee | 59 +++++++ .../coffee/FSPersistorManagerTests.coffee | 156 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 services/filestore/app/coffee/FSPersistorManager.coffee create mode 100644 services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee new file mode 100644 index 0000000000..a0779e5b61 --- /dev/null +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -0,0 +1,59 @@ +logger = require("logger-sharelatex") +fs = require("fs") +LocalFileWriter = require("./LocalFileWriter") + +module.exports = + + sendFile: ( location, target, source, callback = (err)->) -> + logger.log location:location, target:target, source:source, "sending file" + fs.rename source, "#{location}/#{target}", (err) -> + logger.err err:err, location:location, target:target, source:source, "Error on put of file" + callback err + + sendStream: ( location, target, sourceStream, callback = (err)->) -> + logger.log location:location, target:target, source:sourceStream, "sending file stream" + sourceStream.on "error", (err)-> + logger.err location:location, target:target, source:sourceStream, err:err "error on stream to send" + LocalFileWriter.writeStream sourceStream, null, (err, fsPath)=> + if err? + logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" + return callback err + @sendFile location, target, fsPath, callback + + getFileStream: (location, name, callback = (err, res)->)-> + logger.log location:location, name:name, "getting file" + sourceStream = fs.createReadStream "#{location}/#{name}" + sourceStream.on 'error', (err) -> + logger.err err:err, location:location, name:name, "Error reading from file" + callback err + callback null,sourceStream + + + copyFile: (location, fromName, toName, callback = (err)->)-> + logger.log location:location, fromName:fromName, toName:toName, "copying file" + sourceStream = fs.createReadStream "#{location}/#{fromName}" + sourceStream.on 'error', (err) -> + logger.err err:err, location:location, key:fromName, "Error reading from file" + callback err + targetStream = fs.createWriteStream "#{location}/#{toName}" + targetStream.on 'error', (err) -> + logger.err err:err, location:location, key:targetKey, "Error writing to file" + callback err + sourceStream.pipe targetStream + + deleteFile: (location, name, callback)-> + logger.log location:location, name:name, "delete file" + fs.unlink "#{location}/#{name}", (err) -> + logger.err err:err, location:location, name:name, "Error on delete." + callback err + + deleteDirectory: (location, name, callback = (err)->)-> + fs.rmdir "#{location}/#{name}", (err) -> + logger.err err:err, location:location, name:name, "Error on rmdir." + callback err + + checkIfFileExists:(location, name, callback = (err,exists)->)-> + logger.log location:location, name:name, "checking if file exists" + fs.exists "#{location}/#{name}", (exists) -> + logger.log location:location, name:name, exists:exists, "checked if file exists" + callback null, exists diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee new file mode 100644 index 0000000000..44580d115a --- /dev/null +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -0,0 +1,156 @@ +assert = require("chai").assert +sinon = require('sinon') +chai = require('chai') +should = chai.should +expect = chai.expect +modulePath = "../../../app/js/FSPersistorManager.js" +SandboxedModule = require('sandboxed-module') +fs = require("fs") + +describe "FSPersistorManagerTests", -> + + beforeEach -> + @Fs = + rename:sinon.stub() + createReadStream:sinon.stub() + createWriteStream:sinon.stub() + unlink:sinon.stub() + rmdir:sinon.stub() + exists:sinon.stub() + @LocalFileWriter = + writeStream: sinon.stub() + @requires = + "./LocalFileWriter":@LocalFileWriter + "fs":@Fs + "logger-sharelatex": + log:-> + err:-> + @location = "/tmp" + @name1 = "first_file" + @name2 = "second_file" + @error = "error_message" + @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires + + describe "sendFile", -> + it "should put the file", (done) -> + @Fs.rename.callsArgWith(2,@error) + @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> + @Fs.rename.calledWith( @name2, "#{@location}/#{@name1}" ).should.equal true + err.should.equal @error + done() + + describe "sendStream", -> + beforeEach -> + @FSPersistorManager.sendFile = sinon.stub().callsArgWith(3) + @LocalFileWriter.writeStream.callsArgWith(2, null, @name1) + @SourceStream = + on:-> + + it "should sent stream to LocalFileWriter", (done)-> + @FSPersistorManager.sendStream @location, @name1, @SourceStream, => + @LocalFileWriter.writeStream.calledWith(@SourceStream).should.equal true + done() + + it "should return the error from LocalFileWriter", (done)-> + @LocalFileWriter.writeStream.callsArgWith(2, @error) + @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=> + err.should.equal @error + done() + + it "should send the file to the filestore", (done)-> + @LocalFileWriter.writeStream.callsArgWith(2) + @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=> + @FSPersistorManager.sendFile.called.should.equal true + done() + + describe "getFileStream", -> + it "should use correct file location", (done) -> + @Fs.createReadStream.returns( + on:-> + ) + @FSPersistorManager.getFileStream @location, @name1, (err,res)=> + @Fs.createReadStream.calledWith("#{@location}/#{@name1}").should.equal.true + done() + + describe "copyFile", -> + beforeEach -> + @ReadStream= + on:-> + pipe:sinon.stub() + @WriteStream= + on:-> + @Fs.createReadStream.returns(@ReadStream) + @Fs.createWriteStream.returns(@WriteStream) + + it "Should open the source for reading", (done) -> + @FSPersistorManager.copyFile @location, @name1, @name2, -> + @Fs.createReadStream.calledWith("#{@location}/#{@name1}").should.equal.true + done() + + it "Should open the target for writing", (done) -> + @FSPersistorManager.copyFile @location, @name1, @name2, -> + @Fs.createWriteStream.calledWith("#{@location}/#{@name2}").should.equal.true + done() + + it "Should pipe the source to the target", (done) -> + @FSPersistorManager.copyFile @location, @name1, @name2, -> + @ReadStream.pipe.calledWith(@WriteStream).should.equal.true + done() + + describe "deleteFile", -> + beforeEach -> + @Fs.unlink.callsArgWith(1,@error) + + it "Should call unlink with correct options", (done) -> + @FSPersistorManager.deleteFile @location, @name1, (err) => + @Fs.unlink.calledWith("#{@location}/#{@name1}").should.equal.true + done() + + it "Should propogate the error", (done) -> + @FSPersistorManager.deleteFile @location, @name1, (err) => + err.should.equal @error + done() + + + describe "deleteDirectory", -> + beforeEach -> + @Fs.rmdir.callsArgWith(1,@error) + + it "Should call rmdir with correct options", (done) -> + @FSPersistorManager.deleteDirectory @location, @name1, (err) => + @Fs.rmdir.calledWith("#{@location}/#{@name1}").should.equal.true + done() + + it "Should propogate the error", (done) -> + @FSPersistorManager.deleteDirectory @location, @name1, (err) => + err.should.equal @error + done() + + describe "checkIfFileExists", -> + beforeEach -> + @Fs.exists.callsArgWith(1,true) + + it "Should call exists with correct options", (done) -> + @FSPersistorManager.checkIfFileExists @location, @name1, (exists) => + @Fs.exists.calledWith("#{@location}/#{@name1}").should.equal.true + done() + + # fs.exists simply returns false on any error, so... + it "should not return an error", (done) -> + @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => + expect(err).to.be.null + done() + + it "Should return true for existing files", (done) -> + @Fs.exists.callsArgWith(1,true) + @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => + exists.should.be.true + done() + + it "Should return false for non-existing files", (done) -> + @Fs.exists.callsArgWith(1,false) + @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => + exists.should.be.false + done() + + From 7c5634044f546960b8d43d99e151b9449bdc8170 Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Wed, 26 Feb 2014 15:15:03 +0000 Subject: [PATCH 2/6] Allow selection of FS persistor manager. Only require the chosen persistor manager. --- services/filestore/app/coffee/PersistorManager.coffee | 5 +++-- services/filestore/config/settings.development.coffee | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/coffee/PersistorManager.coffee b/services/filestore/app/coffee/PersistorManager.coffee index d4922b9c3b..1b200c58d3 100644 --- a/services/filestore/app/coffee/PersistorManager.coffee +++ b/services/filestore/app/coffee/PersistorManager.coffee @@ -1,6 +1,5 @@ settings = require("settings-sharelatex") logger = require("logger-sharelatex") -S3PersistorManager = require("./S3PersistorManager") # assume s3 if none specified settings.filestoreBackend ||= "s3" @@ -9,6 +8,8 @@ settings.filestoreBackend ||= "s3" logger.log backend:settings.filestoreBackend, "Loading backend" module.exports = switch settings.filestoreBackend when "s3" - S3PersistorManager + require("./S3PersistorManager") + when "fs" + require("./FSPersistorManager") else throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" ) diff --git a/services/filestore/config/settings.development.coffee b/services/filestore/config/settings.development.coffee index a2a1b5cc26..db79f35ca6 100644 --- a/services/filestore/config/settings.development.coffee +++ b/services/filestore/config/settings.development.coffee @@ -7,6 +7,7 @@ module.exports = # which persistor to use for file storage # current options are: # "s3" - Amazon S3 + # "fs" - local filesystem # if no persistor is chosen, s3 will be used by default filestoreBackend: "s3" From 88cc89a0d1176fb77d8e947563c7d4ce23a7beab Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Sat, 1 Mar 2014 15:10:47 +0000 Subject: [PATCH 3/6] filter /s from key ids --- .../app/coffee/FSPersistorManager.coffee | 49 ++++++++++++------- .../coffee/FSPersistorManagerTests.coffee | 7 +-- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index a0779e5b61..1226c57d90 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -2,12 +2,17 @@ logger = require("logger-sharelatex") fs = require("fs") LocalFileWriter = require("./LocalFileWriter") -module.exports = +filterName = (key) -> + return key.replace /\//, "_" + +module.exports = sendFile: ( location, target, source, callback = (err)->) -> - logger.log location:location, target:target, source:source, "sending file" - fs.rename source, "#{location}/#{target}", (err) -> - logger.err err:err, location:location, target:target, source:source, "Error on put of file" + filteredTarget = filterName target + logger.log location:location, target:filteredTarget, source:source, "sending file" + fs.rename source, "#{location}/#{filteredTarget}", (err) -> + if err!=null + logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file" callback err sendStream: ( location, target, sourceStream, callback = (err)->) -> @@ -21,8 +26,9 @@ module.exports = @sendFile location, target, fsPath, callback getFileStream: (location, name, callback = (err, res)->)-> - logger.log location:location, name:name, "getting file" - sourceStream = fs.createReadStream "#{location}/#{name}" + filteredName = filterName name + logger.log location:location, name:filteredName, "getting file" + sourceStream = fs.createReadStream "#{location}/#{filteredName}" sourceStream.on 'error', (err) -> logger.err err:err, location:location, name:name, "Error reading from file" callback err @@ -30,30 +36,35 @@ module.exports = copyFile: (location, fromName, toName, callback = (err)->)-> - logger.log location:location, fromName:fromName, toName:toName, "copying file" - sourceStream = fs.createReadStream "#{location}/#{fromName}" + filteredFromName=filterName fromName + filteredToName=filterName toName + logger.log location:location, fromName:filteredFromName, toName:filteredToName, "copying file" + sourceStream = fs.createReadStream "#{location}/#{filteredFromName}" sourceStream.on 'error', (err) -> - logger.err err:err, location:location, key:fromName, "Error reading from file" + logger.err err:err, location:location, key:filteredFromName, "Error reading from file" callback err - targetStream = fs.createWriteStream "#{location}/#{toName}" + targetStream = fs.createWriteStream "#{location}/#{filteredToName}" targetStream.on 'error', (err) -> - logger.err err:err, location:location, key:targetKey, "Error writing to file" + logger.err err:err, location:location, key:filteredToName, "Error writing to file" callback err sourceStream.pipe targetStream deleteFile: (location, name, callback)-> - logger.log location:location, name:name, "delete file" - fs.unlink "#{location}/#{name}", (err) -> - logger.err err:err, location:location, name:name, "Error on delete." + filteredName = filterName name + logger.log location:location, name:filteredName, "delete file" + fs.unlink "#{location}/#{filteredName}", (err) -> + logger.err err:err, location:location, name:filteredName, "Error on delete." callback err deleteDirectory: (location, name, callback = (err)->)-> - fs.rmdir "#{location}/#{name}", (err) -> - logger.err err:err, location:location, name:name, "Error on rmdir." + filteredName = filterName name + fs.rmdir "#{location}/#{filteredName}", (err) -> + logger.err err:err, location:location, name:filteredName, "Error on rmdir." callback err checkIfFileExists:(location, name, callback = (err,exists)->)-> - logger.log location:location, name:name, "checking if file exists" - fs.exists "#{location}/#{name}", (exists) -> - logger.log location:location, name:name, exists:exists, "checked if file exists" + filteredName = filterName name + logger.log location:location, name:filteredName, "checking if file exists" + fs.exists "#{location}/#{filteredName}", (exists) -> + logger.log location:location, name:filteredName, exists:exists, "checked if file exists" callback null, exists diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee index 44580d115a..bf5f08ea9d 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee @@ -26,7 +26,8 @@ describe "FSPersistorManagerTests", -> log:-> err:-> @location = "/tmp" - @name1 = "first_file" + @name1 = "530f2407e7ef165704000007/530f838b46d9a9e859000008" + @name1Filtered ="530f2407e7ef165704000007_530f838b46d9a9e859000008" @name2 = "second_file" @error = "error_message" @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires @@ -35,7 +36,7 @@ describe "FSPersistorManagerTests", -> it "should put the file", (done) -> @Fs.rename.callsArgWith(2,@error) @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> - @Fs.rename.calledWith( @name2, "#{@location}/#{@name1}" ).should.equal true + @Fs.rename.calledWith( @name2, "#{@location}/#{@name1Filtered}" ).should.equal true err.should.equal @error done() @@ -69,7 +70,7 @@ describe "FSPersistorManagerTests", -> on:-> ) @FSPersistorManager.getFileStream @location, @name1, (err,res)=> - @Fs.createReadStream.calledWith("#{@location}/#{@name1}").should.equal.true + @Fs.createReadStream.calledWith("#{@location}/#{@name1Filtered}").should.equal.true done() describe "copyFile", -> From 40e2cb1c6d3f6e2ef08ec5545443ea1cd18371de Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Sat, 1 Mar 2014 15:23:11 +0000 Subject: [PATCH 4/6] fix tabbing --- services/filestore/app/coffee/FSPersistorManager.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 1226c57d90..09bbe91255 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -3,7 +3,7 @@ fs = require("fs") LocalFileWriter = require("./LocalFileWriter") filterName = (key) -> - return key.replace /\//, "_" + return key.replace /\//, "_" module.exports = @@ -11,8 +11,8 @@ module.exports = filteredTarget = filterName target logger.log location:location, target:filteredTarget, source:source, "sending file" fs.rename source, "#{location}/#{filteredTarget}", (err) -> - if err!=null - logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file" + if err!=null + logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file" callback err sendStream: ( location, target, sourceStream, callback = (err)->) -> From 957df0eb04f1bbd7073d36ab3b9fcaade865e30d Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Tue, 4 Mar 2014 14:45:32 +0000 Subject: [PATCH 5/6] Don't dump streams to log files. --- services/filestore/app/coffee/FSPersistorManager.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 09bbe91255..dd4e39d981 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -16,9 +16,9 @@ module.exports = callback err sendStream: ( location, target, sourceStream, callback = (err)->) -> - logger.log location:location, target:target, source:sourceStream, "sending file stream" + logger.log location:location, target:target, "sending file stream" sourceStream.on "error", (err)-> - logger.err location:location, target:target, source:sourceStream, err:err "error on stream to send" + logger.err location:location, target:target, err:err "error on stream to send" LocalFileWriter.writeStream sourceStream, null, (err, fsPath)=> if err? logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" From f920fd0b16bd2b7d0664731927c974fb007109d1 Mon Sep 17 00:00:00 2001 From: Oliver Matthews Date: Tue, 4 Mar 2014 15:01:13 +0000 Subject: [PATCH 6/6] match refactor_config on cwoac/sharelatex --- .../filestore/app/coffee/KeyBuilder.coffee | 6 ++-- .../app/coffee/PersistorManager.coffee | 8 ++--- .../app/coffee/S3PersistorManager.coffee | 20 ++++++------- .../config/settings.development.coffee | 28 ++++++++++++----- .../unit/coffee/PersistorManagerTests.coffee | 6 ++-- .../coffee/S3PersistorManagerTests.coffee | 30 ++++++++++--------- 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.coffee index 113c0eac57..45aa351487 100644 --- a/services/filestore/app/coffee/KeyBuilder.coffee +++ b/services/filestore/app/coffee/KeyBuilder.coffee @@ -20,15 +20,15 @@ module.exports = userFileKey: (req, res, next)-> {project_id, file_id} = req.params req.key = "#{project_id}/#{file_id}" - req.bucket = settings.s3.buckets.user_files + req.bucket = settings.filestore.stores.user_files next() templateFileKey: (req, res, next)-> {template_id, format, version} = req.params req.key = "#{template_id}/#{version}/#{format}" - req.bucket = settings.s3.buckets.template_files + req.bucket = settings.filestore.stores.template_files req.version = version opts = req.query next() - \ No newline at end of file + diff --git a/services/filestore/app/coffee/PersistorManager.coffee b/services/filestore/app/coffee/PersistorManager.coffee index 1b200c58d3..1dad923098 100644 --- a/services/filestore/app/coffee/PersistorManager.coffee +++ b/services/filestore/app/coffee/PersistorManager.coffee @@ -2,14 +2,14 @@ settings = require("settings-sharelatex") logger = require("logger-sharelatex") # assume s3 if none specified -settings.filestoreBackend ||= "s3" +settings.filestore.backend ||= "s3" -logger.log backend:settings.filestoreBackend, "Loading backend" -module.exports = switch settings.filestoreBackend +logger.log backend:settings.filestore.backend, "Loading backend" +module.exports = switch settings.filestore.backend when "s3" require("./S3PersistorManager") when "fs" require("./FSPersistorManager") else - throw new Error( "Unknown filestore backend: #{settings.filestoreBackend}" ) + throw new Error( "Unknown filestore backend: #{settings.filestore.backend}" ) diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee index dd65b79abc..0b35ea7b52 100644 --- a/services/filestore/app/coffee/S3PersistorManager.coffee +++ b/services/filestore/app/coffee/S3PersistorManager.coffee @@ -24,8 +24,8 @@ printSockets() buildDefaultOptions = (bucketName, method, key)-> return { aws: - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName method: method timeout: thirtySeconds @@ -36,8 +36,8 @@ module.exports = sendFile: (bucketName, key, fsPath, callback)-> s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName putEventEmiter = s3Client.putFile fsPath, key, (err, res)-> if err? @@ -70,8 +70,8 @@ module.exports = getFileStream: (bucketName, key, callback = (err, res)->)-> logger.log bucketName:bucketName, key:key, "getting file from s3" s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Stream = s3Client.get(key) s3Stream.end() @@ -84,8 +84,8 @@ module.exports = copyFile: (bucketName, sourceKey, destKey, callback)-> logger.log bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "copying file in s3" s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Client.copyFile sourceKey, destKey, (err)-> if err? @@ -102,8 +102,8 @@ module.exports = deleteDirectory: (bucketName, key, callback)-> s3Client = knox.createClient - key: settings.s3.key - secret: settings.s3.secret + key: settings.filestore.s3.key + secret: settings.filestore.s3.secret bucket: bucketName s3Client.list prefix:key, (err, data)-> keys = _.map data.Contents, (entry)-> diff --git a/services/filestore/config/settings.development.coffee b/services/filestore/config/settings.development.coffee index db79f35ca6..19088d3c76 100644 --- a/services/filestore/config/settings.development.coffee +++ b/services/filestore/config/settings.development.coffee @@ -3,13 +3,27 @@ module.exports = filestore: port: 3009 host: "localhost" - - # which persistor to use for file storage - # current options are: - # "s3" - Amazon S3 - # "fs" - local filesystem - # if no persistor is chosen, s3 will be used by default - filestoreBackend: "s3" + + filestore: + # which backend persistor to use. + # choices are + # s3 - Amazon S3 + # fs - local filesystem + backend: "s3" + stores: + # where to store user and template binary files + # + # For Amazon S3 this is the bucket name to store binary files in + # Must contain full url like: .s3.amazonaws.com + # + # For local filesystem this is the directory to store the files in. + # Must contain full path, e.g. "/var/lib/sharelatex/data" + # This path must exist, not be tmpfs and be writable to by the user sharelatex is run as. + user_files: "" + s3: + # if you are using S3, then fill in your S3 details below + key: "" + secret: "" # ShareLaTeX stores binary files like images in S3. # Fill in your Amazon S3 credentials below. diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/PersistorManagerTests.coffee index af11fa7408..ca7a82cbaa 100644 --- a/services/filestore/test/unit/coffee/PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/PersistorManagerTests.coffee @@ -22,7 +22,8 @@ describe "PersistorManagerTests", -> describe "test s3 mixin", -> beforeEach -> @settings = - filestoreBackend: "s3" + filestore: + backend: "s3" @requires = "./S3PersistorManager": @S3PersistorManager "settings-sharelatex": @settings @@ -81,7 +82,8 @@ describe "PersistorManagerTests", -> describe "test invalid mixins", -> it "should not load an invalid wrapper", (done) -> @settings = - filestoreBackend:"magic" + filestore: + backend:"magic" @requires = "./S3PersistorManager": @S3PersistorManager "settings-sharelatex": @settings diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee index 76872fb140..fe70f1008d 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee @@ -9,24 +9,26 @@ SandboxedModule = require('sandboxed-module') describe "S3PersistorManagerTests", -> beforeEach -> - @settings = - s3: - secret: "secret" - key: "this_key" - buckets: - user_files:"sl_user_files" - @stubbedKnoxClient = + @settings = + filestore: + backend: "s3" + s3: + secret: "secret" + key: "this_key" + stores: + user_files:"sl_user_files" + @stubbedKnoxClient = putFile:sinon.stub() copyFile:sinon.stub() list: sinon.stub() deleteMultiple: sinon.stub() get: sinon.stub() - @knox = + @knox = createClient: sinon.stub().returns(@stubbedKnoxClient) - @LocalFileWriter = + @LocalFileWriter = writeStream: sinon.stub() deleteFile: sinon.stub() - @requires = + @requires = "knox": @knox "settings-sharelatex": @settings "./LocalFileWriter":@LocalFileWriter @@ -48,7 +50,7 @@ describe "S3PersistorManagerTests", -> end:-> ) @S3PersistorManager.getFileStream @bucketName, @key, @fsPath, (err)=> - @stubbedKnoxClient.get.calledWith(@key).should.equal true + @stubbedKnoxClient.get.calledWith(@key).should.equal true done() describe "sendFile", -> @@ -121,7 +123,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires it "should list the contents passing them onto multi delete", (done)-> - data = + data = Contents: [{Key:"1234"}, {Key: "456"}] @stubbedKnoxClient.list.callsArgWith(1, null, data) @stubbedKnoxClient.deleteMultiple.callsArgWith(1) @@ -138,7 +140,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.deleteFile @bucketName, @key, (err)=> opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) opts.method.should.equal "delete" opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}" @@ -162,7 +164,7 @@ describe "S3PersistorManagerTests", -> @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.s3.key, secret:@settings.s3.secret, bucket:@bucketName}) + assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) opts.method.should.equal "head" opts.timeout.should.equal (30*1000) opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"