Merge pull request #60 from overleaf/spd-integration-node10-decaf

Upgrade to Node 10 and decaffeinate

services/filestore/.dockerignore

@@ -5,5 +5,3 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
-app.js
-**/js/*

services/filestore/.eslintrc

@@ -0,0 +1,74 @@
+// this file was auto-generated, do not edit it directly.
+// instead run bin/update_build_scripts from
+// https://github.com/sharelatex/sharelatex-dev-environment
+// Version: 1.3
+{
+  "extends": [
+    "standard",
+    "prettier",
+    "prettier/standard"
+  ],
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
+  "plugins": [
+    "mocha",
+    "chai-expect",
+    "chai-friendly"
+  ],
+  "env": {
+    "node": true,
+    "mocha": true
+  },
+  "rules": {
+    // Swap the no-unused-expressions rule with a more chai-friendly one
+    "no-unused-expressions": 0,
+    "chai-friendly/no-unused-expressions": "error"
+  },
+  "overrides": [
+    {
+      // Test specific rules
+      "files": ["**/test/*/src/**/*.js"],
+      "globals": {
+        "expect": true
+      },
+      "rules": {
+        // mocha-specific rules
+        "mocha/handle-done-callback": "error",
+        "mocha/no-exclusive-tests": "error",
+        "mocha/no-global-tests": "error",
+        "mocha/no-identical-title": "error",
+        "mocha/no-nested-tests": "error",
+        "mocha/no-pending-tests": "error",
+        "mocha/no-skipped-tests": "error",
+        "mocha/no-mocha-arrows": "error",
+
+        // chai-specific rules
+        "chai-expect/missing-assertion": "error",
+        "chai-expect/terminating-properties": "error",
+
+        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
+        // we don't enforce this at the top-level - just in tests to manage `this` scope
+        // based on mocha's context mechanism
+        "mocha/prefer-arrow-callback": "error"
+      }
+    },
+    {
+      // Frontend test specific rules
+      "files": ["**/test/frontend/**/*.js"],
+      "globals": {
+        "expect": true,
+        "define": true,
+        "$": true
+      }
+    },
+    {
+      // Backend specific rules
+      "files": ["**/app/src/**/*.js"],
+      "rules": {
+        // don't allow console.log in backend code
+        "no-console": "error"
+      }
+    }
+  ]
+}

services/filestore/.gitignore

@@ -36,28 +36,11 @@ Icon?
 Thumbs.db
 
 /node_modules/*
-test/IntergrationTests/js/*
 data/*/*
 
-app.js
 **/*.map
-cluster.js
-app/js/*
-test/IntergrationTests/js/*
-test/UnitTests/js/*
 cookies.txt
 uploads/*
-public/js/editor.js
-public/js/home.js
-public/js/forms.js
-public/js/gui.js
-public/js/admin.js
-public/stylesheets/mainStyle.css
-public/minjs/
-test/unit/js/
-test/acceptence/js
-cluster.js
-test/acceptance/js/
 
 user_files/*
 template_files/*

services/filestore/.nvmrc

@@ -1 +1 @@
-6.9.5
+10.17.0

services/filestore/.prettierignore

@@ -0,0 +1 @@
+node_modules

services/filestore/.prettierrc

@@ -0,0 +1,8 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3
+{
+  "semi": false,
+  "singleQuote": true
+}

services/filestore/Dockerfile

@@ -1,4 +1,9 @@
-FROM node:6.9.5 as app
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3
+
+FROM node:10.17.0 as app
 
 WORKDIR /app
 
@@ -10,9 +15,8 @@ RUN npm install --quiet
 COPY . /app
 
 
-RUN npm run compile:all
 
-FROM node:6.9.5
+FROM node:10.17.0
 
 COPY --from=app /app /app
 

services/filestore/Jenkinsfile

@@ -16,6 +16,7 @@ pipeline {
   }
 
   stages {
+
     stage('Install') {
       steps {
         withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
@@ -36,6 +37,13 @@ pipeline {
       }
     }
 
+    stage('Linting') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
+      }
+    }
+
     stage('Unit Tests') {
       steps {
         sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

services/filestore/Makefile

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 clean:
 	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	rm -f app.js
-	rm -rf app/js
-	rm -rf test/unit/js
-	rm -rf test/acceptance/js
 
-test: test_unit test_acceptance
+format:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+
+format_fix:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+
+lint:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+
+test: format lint test_unit test_acceptance
 
 test_unit:
 	@[ ! -d test/unit ] && echo "filestore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
@@ -36,6 +41,7 @@ test_clean:
 
 test_acceptance_pre_run:
 	@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "filestore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+
 build:
 	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
 		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
@@ -48,4 +54,5 @@ publish:
 
 	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
+
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/filestore/app.coffee

@@ -1,147 +0,0 @@
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("filestore")
-express = require('express')
-bodyParser = require "body-parser"
-logger = require('logger-sharelatex')
-logger.initialize("filestore")
-settings = require("settings-sharelatex")
-request = require("request")
-fileController = require("./app/js/FileController")
-bucketController = require("./app/js/BucketController")
-keyBuilder = require("./app/js/KeyBuilder")
-healthCheckController = require("./app/js/HealthCheckController")
-domain = require("domain")
-appIsOk = true
-app = express()
-
-if settings.sentry?.dsn?
-	logger.initializeErrorReporting(settings.sentry.dsn)
-
-Metrics.open_sockets.monitor(logger)
-Metrics.event_loop?.monitor(logger)
-Metrics.memory.monitor(logger)
-
-app.use Metrics.http.monitor(logger)
-
-app.use (req, res, next)->
-	Metrics.inc "http-request"
-	next()
-
-app.use (req, res, next) ->
-	requestDomain = domain.create()
-	requestDomain.add req
-	requestDomain.add res
-	requestDomain.on "error", (err)->
-		try
-			# request a shutdown to prevent memory leaks
-			beginShutdown()
-			if !res.headerSent
-				res.send(500, "uncaught exception")
-			logger = require('logger-sharelatex')
-			req =
-				body:req.body
-				headers:req.headers
-				url:req.url
-				key: req.key
-				statusCode: req.statusCode
-			err =
-				message: err.message
-				stack: err.stack
-				name: err.name
-				type: err.type
-				arguments: err.arguments
-			logger.err err:err, req:req, res:res, "uncaught exception thrown on request"
-		catch exception
-			logger.err err: exception, "exception in request domain handler"
-	requestDomain.run next
-
-app.use (req, res, next) ->
-	if not appIsOk
-		# when shutting down, close any HTTP keep-alive connections
-		res.set 'Connection', 'close'
-	next()
-
-Metrics.injectMetricsRoute(app)
-
-app.head "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFileHead
-app.get  "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile
-app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile
-app.put  "/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile
-app.del  "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile
-
-app.head "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFileHead
-app.get  "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile
-app.get  "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile
-app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile
-
-
-app.head "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFileHead
-app.get  "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile
-app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile
-app.put  "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile
-app.del  "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile
-
-app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize
-
-app.get "/bucket/:bucket/key/*", bucketController.getFile
-
-app.get "/heapdump", (req, res)->
-	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)->
-		res.send filename
-
-app.post "/shutdown", (req, res)->
-	appIsOk = false
-	res.send()
-
-app.get '/status', (req, res)->
-	if appIsOk
-		res.send('filestore sharelatex up')
-	else
-		logger.log "app is not ok - shutting down"
-		res.send("server is being shut down", 500)
-
-
-app.get "/health_check", healthCheckController.check
-
-
-app.get '*', (req, res)->
-	res.send 404
-
-
-
-
-beginShutdown = () ->
-	if appIsOk
-		appIsOk = false
-		# hard-terminate this process if graceful shutdown fails
-		killTimer = setTimeout () ->
-			process.exit 1
-		, 120*1000
-		killTimer.unref?() # prevent timer from keeping process alive
-		server.close () ->
-			logger.log "closed all connections"
-			Metrics.close()
-			process.disconnect?()
-		logger.log "server will stop accepting connections"
-
-
-port = settings.internal.filestore.port or 3009
-host = "0.0.0.0"
-
-if !module.parent # Called directly
-	server = app.listen port, host, (error) ->
-		logger.info "Filestore starting up, listening on #{host}:#{port}"
-
-
-module.exports = app
-
-process.on 'SIGTERM', () ->
-	logger.log("filestore got SIGTERM, shutting down gracefully")
-	beginShutdown()
-
-if global.gc?
-	gcTimer = setInterval () ->
-		global.gc()
-		logger.log process.memoryUsage(), "global.gc"
-	, 3 * oneMinute = 60 * 1000
-	gcTimer.unref()

services/filestore/app.js

@@ -0,0 +1,154 @@
+const Metrics = require('metrics-sharelatex')
+const logger = require('logger-sharelatex')
+
+Metrics.initialize('filestore')
+logger.initialize('filestore')
+
+const settings = require('settings-sharelatex')
+const express = require('express')
+const bodyParser = require('body-parser')
+
+const fileController = require('./app/js/FileController')
+const keyBuilder = require('./app/js/KeyBuilder')
+const healthCheckController = require('./app/js/HealthCheckController')
+
+const app = express()
+
+if (settings.sentry && settings.sentry.dsn) {
+  logger.initializeErrorReporting(settings.sentry.dsn)
+}
+
+Metrics.open_sockets.monitor(logger)
+Metrics.memory.monitor(logger)
+if (Metrics.event_loop) {
+  Metrics.event_loop.monitor(logger)
+}
+
+app.use(Metrics.http.monitor(logger))
+app.use(function(req, res, next) {
+  Metrics.inc('http-request')
+  next()
+})
+
+Metrics.injectMetricsRoute(app)
+
+app.head(
+  '/project/:project_id/file/:file_id',
+  keyBuilder.userFileKeyMiddleware,
+  fileController.getFileHead
+)
+app.get(
+  '/project/:project_id/file/:file_id',
+  keyBuilder.userFileKeyMiddleware,
+  fileController.getFile
+)
+app.post(
+  '/project/:project_id/file/:file_id',
+  keyBuilder.userFileKeyMiddleware,
+  fileController.insertFile
+)
+app.put(
+  '/project/:project_id/file/:file_id',
+  keyBuilder.userFileKeyMiddleware,
+  bodyParser.json(),
+  fileController.copyFile
+)
+app.delete(
+  '/project/:project_id/file/:file_id',
+  keyBuilder.userFileKeyMiddleware,
+  fileController.deleteFile
+)
+
+app.head(
+  '/template/:template_id/v/:version/:format',
+  keyBuilder.templateFileKeyMiddleware,
+  fileController.getFileHead
+)
+app.get(
+  '/template/:template_id/v/:version/:format',
+  keyBuilder.templateFileKeyMiddleware,
+  fileController.getFile
+)
+app.get(
+  '/template/:template_id/v/:version/:format/:sub_type',
+  keyBuilder.templateFileKeyMiddleware,
+  fileController.getFile
+)
+app.post(
+  '/template/:template_id/v/:version/:format',
+  keyBuilder.templateFileKeyMiddleware,
+  fileController.insertFile
+)
+
+app.head(
+  '/project/:project_id/public/:public_file_id',
+  keyBuilder.publicFileKeyMiddleware,
+  fileController.getFileHead
+)
+app.get(
+  '/project/:project_id/public/:public_file_id',
+  keyBuilder.publicFileKeyMiddleware,
+  fileController.getFile
+)
+app.post(
+  '/project/:project_id/public/:public_file_id',
+  keyBuilder.publicFileKeyMiddleware,
+  fileController.insertFile
+)
+app.put(
+  '/project/:project_id/public/:public_file_id',
+  keyBuilder.publicFileKeyMiddleware,
+  bodyParser.json(),
+  fileController.copyFile
+)
+app.delete(
+  '/project/:project_id/public/:public_file_id',
+  keyBuilder.publicFileKeyMiddleware,
+  fileController.deleteFile
+)
+
+app.get(
+  '/project/:project_id/size',
+  keyBuilder.publicProjectKeyMiddleware,
+  fileController.directorySize
+)
+
+app.get(
+  '/bucket/:bucket/key/*',
+  keyBuilder.bucketFileKeyMiddleware,
+  fileController.getFile
+)
+
+app.get('/heapdump', (req, res, next) =>
+  require('heapdump').writeSnapshot(
+    '/tmp/' + Date.now() + '.filestore.heapsnapshot',
+    (err, filename) => {
+      if (err) {
+        return next(err)
+      }
+      res.send(filename)
+    }
+  )
+)
+
+app.get('/status', function(req, res) {
+  res.send('filestore sharelatex up')
+})
+
+app.get('/health_check', healthCheckController.check)
+
+const port = settings.internal.filestore.port || 3009
+const host = '0.0.0.0'
+
+if (!module.parent) {
+  // Called directly
+  app.listen(port, host, error => {
+    if (error) {
+      logger.error('Error starting Filestore', error)
+      throw error
+    }
+    logger.info(`Filestore starting up, listening on ${host}:${port}`)
+  })
+}
+
+module.exports = app

services/filestore/app/coffee/AWSSDKPersistorManager.coffee

@@ -1,106 +0,0 @@
-# This  module is not used in production, which currently uses
-# S3PersistorManager.  The intention is to migrate S3PersistorManager to use the
-# latest aws-sdk and delete this module so that PersistorManager would load the
-# same backend for both the 's3' and 'aws-sdk' options.
-
-logger = require "logger-sharelatex"
-aws = require "aws-sdk"
-_ = require "underscore"
-fs = require "fs"
-Errors = require "./Errors"
-
-s3 = new aws.S3()
-
-module.exports =
-	sendFile: (bucketName, key, fsPath, callback)->
-		logger.log bucketName:bucketName, key:key, "send file data to s3"
-		stream = fs.createReadStream fsPath
-		s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
-			if err?
-				logger.err err: err, Bucket: bucketName, Key: key, "error sending file data to s3"
-			callback err
-
-	sendStream: (bucketName, key, stream, callback)->
-		logger.log bucketName:bucketName, key:key, "send file stream to s3"
-		s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
-			if err?
-				logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3"
-			callback err
-
-	getFileStream: (bucketName, key, opts, callback = (err, res)->)->
-		logger.log bucketName:bucketName, key:key, "get file stream from s3"
-		callback = _.once callback
-		params =
-			Bucket:bucketName
-			Key: key
-		if opts.start? and opts.end?
-			params['Range'] = "bytes=#{opts.start}-#{opts.end}"
-		request = s3.getObject params
-		stream = request.createReadStream()
-		stream.on 'readable', () ->
-			callback null, stream
-		stream.on 'error', (err) ->
-			logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3"
-			if err.code == 'NoSuchKey'
-				return callback new Errors.NotFoundError "File not found in S3: #{bucketName}:#{key}"
-			callback err
-
-	copyFile: (bucketName, sourceKey, destKey, callback)->
-		logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3"
-		source = bucketName + '/' + sourceKey
-		s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) ->
-			if err?
-				logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in s3"
-			callback err
-
-	deleteFile: (bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "delete file in s3"
-		s3.deleteObject {Bucket: bucketName, Key: key}, (err) ->
-			if err?
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting file in s3"
-			callback err
-
-	deleteDirectory: (bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "delete directory in s3"
-		s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) ->
-			if err?
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3"
-				return callback err
-			if data.Contents.length == 0
-				logger.log bucketName:bucketName, key:key, "the directory is empty"
-				return callback()
-			keys = _.map data.Contents, (entry)->
-				Key: entry.Key
-			s3.deleteObjects
-				Bucket: bucketName
-				Delete:
-					Objects: keys
-					Quiet: true
-			, (err) ->
-					if err?
-						logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3"
-					callback err
-
-	checkIfFileExists:(bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "check file existence in s3"
-		s3.headObject {Bucket: bucketName, Key: key}, (err, data) ->
-			if err?
-				return (callback null, false) if err.code == 'NotFound'
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong checking head in s3"
-				return callback err
-			callback null, data.ETag?
-
-	directorySize:(bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "get project size in s3"
-		s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) ->
-			if err?
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3"
-				return callback err
-			if data.Contents.length == 0
-				logger.log bucketName:bucketName, key:key, "the directory is empty"
-				return callback()
-			totalSize = 0
-			_.each data.Contents, (entry)->
-				totalSize += entry.Size
-			callback null, totalSize
-

services/filestore/app/coffee/BucketController.coffee

@@ -1,29 +0,0 @@
-settings = require("settings-sharelatex")
-logger = require("logger-sharelatex")
-FileHandler = require("./FileHandler")
-metrics = require("metrics-sharelatex")
-Errors = require('./Errors')
-
-module.exports = BucketController =
-
-	getFile: (req, res)->
-		{bucket} = req.params
-		key = req.params[0]
-		credentials = settings.filestore.s3BucketCreds?[bucket]
-		options = {
-			key: key,
-			bucket: bucket,
-			credentials: credentials
-		}
-		metrics.inc "#{bucket}.getFile"
-		logger.log key:key, bucket:bucket, "receiving request to get file from bucket"
-		FileHandler.getFile bucket, key, options, (err, fileStream)->
-			if err?
-				logger.err err:err, key:key, bucket:bucket, "problem getting file from bucket"
-				if err instanceof Errors.NotFoundError
-					return res.send 404
-				else
-					return res.send 500
-			else
-				logger.log key:key, bucket:bucket, "sending bucket file to response"
-				fileStream.pipe res

services/filestore/app/coffee/Errors.coffee

@@ -1,9 +0,0 @@
-NotFoundError = (message) ->
-	error = new Error(message)
-	error.name = "NotFoundError"
-	error.__proto__ = NotFoundError.prototype
-	return error
-NotFoundError.prototype.__proto__ = Error.prototype
-
-module.exports = Errors =
-	NotFoundError: NotFoundError

services/filestore/app/coffee/FSPersistorManager.coffee

@@ -1,128 +0,0 @@
-logger = require("logger-sharelatex")
-fs = require("fs")
-path = require("path")
-LocalFileWriter = require("./LocalFileWriter")
-Errors = require('./Errors')
-rimraf = require("rimraf")
-_ = require "underscore"
-
-filterName = (key) ->
-  return key.replace /\//g, "_"
-
-
-module.exports =
-  sendFile: ( location, target, source, callback = (err)->) ->
-    filteredTarget = filterName target
-    logger.log location:location, target:filteredTarget, source:source, "sending file"
-    done = _.once (err) ->
-      if err?
-        logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file"
-      callback(err)
-    # actually copy the file (instead of moving it) to maintain consistent behaviour
-    # between the different implementations
-    sourceStream = fs.createReadStream source
-    sourceStream.on 'error', done
-    targetStream = fs.createWriteStream "#{location}/#{filteredTarget}"
-    targetStream.on 'error', done
-    targetStream.on 'finish', () ->
-      done()
-    sourceStream.pipe targetStream
-
-  sendStream: ( location, target, sourceStream, callback = (err)->) ->
-    logger.log location:location, target:target, "sending file stream"
-    sourceStream.on "error", (err)->
-      logger.err location:location, target:target, err:err "error on stream to send"
-    LocalFileWriter.writeStream sourceStream, null, (err, fsPath)=>
-      if err?
-        logger.err  location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
-        return callback err
-      @sendFile location, target, fsPath, (err) ->
-        # delete the temporary file created above and return the original error
-        LocalFileWriter.deleteFile fsPath, () ->
-          callback(err)
-
-  # opts may be {start: Number, end: Number}
-  getFileStream: (location, name, opts, callback = (err, res)->) ->
-    filteredName = filterName name
-    logger.log location:location, filteredName:filteredName, "getting file"
-    fs.open "#{location}/#{filteredName}", 'r', (err, fd) ->
-      if err?
-        logger.err err:err, location:location, filteredName:name, "Error reading from file"
-        if err.code == 'ENOENT'
-          return callback new Errors.NotFoundError(err.message), null
-        else
-          return callback err, null
-      opts.fd = fd
-      sourceStream = fs.createReadStream null, opts
-      return callback null, sourceStream
-
-  getFileSize: (location, filename, callback) ->
-    fullPath = path.join(location, filterName(filename))
-    fs.stat fullPath, (err, stats) ->
-      if err?
-        if err.code == 'ENOENT'
-          logger.log({location:location, filename:filename}, "file not found")
-          callback(new Errors.NotFoundError(err.message))
-        else
-          logger.err({err:err, location:location, filename:filename}, "failed to stat file")
-          callback(err)
-        return
-      callback(null, stats.size)
-
-  copyFile: (location, fromName, toName, callback = (err)->)->
-    filteredFromName=filterName fromName
-    filteredToName=filterName toName
-    logger.log location:location, fromName:filteredFromName, toName:filteredToName, "copying file"
-    sourceStream = fs.createReadStream "#{location}/#{filteredFromName}"
-    sourceStream.on 'error', (err) ->
-      logger.err err:err, location:location, key:filteredFromName, "Error reading from file"
-      callback err
-    targetStream = fs.createWriteStream "#{location}/#{filteredToName}"
-    targetStream.on 'error', (err) ->
-      logger.err err:err, location:location, key:filteredToName, "Error writing to file"
-      callback err
-    targetStream.on 'finish', () ->
-      callback null
-    sourceStream.pipe targetStream
-
-  deleteFile: (location, name, callback)->
-    filteredName = filterName name
-    logger.log location:location, filteredName:filteredName, "delete file"
-    fs.unlink "#{location}/#{filteredName}", (err) ->
-      if err?
-        logger.err err:err, location:location, filteredName:filteredName, "Error on delete."
-        callback err
-      else
-        callback()
-
-  deleteDirectory: (location, name, callback = (err)->)->
-    filteredName = filterName name.replace(/\/$/,'')
-    rimraf "#{location}/#{filteredName}", (err) ->
-      if err?
-        logger.err err:err, location:location, filteredName:filteredName, "Error on rimraf rmdir."
-        callback err
-      else
-        callback()
-
-  checkIfFileExists:(location, name, callback = (err,exists)->)->
-    filteredName = filterName name
-    logger.log location:location, filteredName:filteredName, "checking if file exists"
-    fs.exists "#{location}/#{filteredName}", (exists) ->
-      logger.log location:location, filteredName:filteredName, exists:exists, "checked if file exists"
-      callback null, exists
-
-  directorySize:(location, name, callback)->
-    filteredName = filterName name.replace(/\/$/,'')
-    logger.log location:location, filteredName:filteredName, "get project size in file system"
-    fs.readdir "#{location}/#{filteredName}", (err, files) ->
-      if err?
-        logger.err err:err, location:location, filteredName:filteredName, "something went wrong listing prefix in aws"
-        return callback(err)
-      totalSize = 0
-      _.each files, (entry)->
-        fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r'
-        fileStats = fs.fstatSync(fd)
-        totalSize += fileStats.size
-        fs.closeSync fd
-      logger.log totalSize:totalSize, "total size", files:files
-      callback null, totalSize

services/filestore/app/coffee/FileController.coffee

@@ -1,113 +0,0 @@
-PersistorManager = require("./PersistorManager")
-settings = require("settings-sharelatex")
-logger = require("logger-sharelatex")
-FileHandler = require("./FileHandler")
-metrics = require("metrics-sharelatex")
-parseRange = require('range-parser')
-Errors = require('./Errors')
-
-oneDayInSeconds = 60 * 60 * 24
-maxSizeInBytes = 1024 * 1024 * 1024 # 1GB
-
-module.exports = FileController =
-
-	getFile: (req, res)->
-		{key, bucket} = req
-		{format, style} = req.query
-		options = {
-			key: key,
-			bucket: bucket,
-			format: format,
-			style: style,
-		}
-		metrics.inc "getFile"
-		logger.log key:key, bucket:bucket, format:format, style: style, "receiving request to get file"
-		if req.headers.range?
-			range = FileController._get_range(req.headers.range)
-			options.start = range.start
-			options.end = range.end
-			logger.log start: range.start, end: range.end, "getting range of bytes from file"
-		FileHandler.getFile bucket, key, options, (err, fileStream)->
-			if err?
-				if err instanceof Errors.NotFoundError
-					return res.send 404
-				else
-					logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
-					return res.send 500
-			else if req.query.cacheWarm
-				logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream"
-				res.send 200
-			else
-				logger.log key:key, bucket:bucket, format:format, style:style, "sending file to response"
-				fileStream.pipe res
-
-	getFileHead: (req, res) ->
-		{key, bucket} = req
-		metrics.inc("getFileSize")
-		logger.log({ key: key, bucket: bucket }, "receiving request to get file metadata")
-		FileHandler.getFileSize bucket, key, (err, fileSize) ->
-			if err?
-				if err instanceof Errors.NotFoundError
-					res.status(404).end()
-				else
-					res.status(500).end()
-				return
-			res.set("Content-Length", fileSize)
-			res.status(200).end()
-
-	insertFile: (req, res)->
-		metrics.inc "insertFile"
-		{key, bucket} = req
-		logger.log key:key, bucket:bucket, "receiving request to insert file"
-		FileHandler.insertFile bucket, key, req, (err)->
-			if err?
-				logger.log err: err, key: key, bucket: bucket, "error inserting file"
-				res.send 500
-			else
-				res.send 200
-
-	copyFile: (req, res)->
-		metrics.inc "copyFile"
-		{key, bucket} = req
-		oldProject_id = req.body.source.project_id
-		oldFile_id = req.body.source.file_id
-		logger.log key:key, bucket:bucket, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "receiving request to copy file"
-		PersistorManager.copyFile bucket, "#{oldProject_id}/#{oldFile_id}", key, (err)->
-			if err?
-				if err instanceof Errors.NotFoundError
-					res.send 404
-				else
-					logger.log err:err, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "something went wrong copying file"
-					res.send 500
-			else
-				res.send 200
-
-	deleteFile: (req, res)->
-		metrics.inc "deleteFile"
-		{key, bucket} = req
-		logger.log key:key, bucket:bucket,  "receiving request to delete file"
-		FileHandler.deleteFile bucket, key, (err)->
-			if err?
-				logger.log err:err, key:key, bucket:bucket, "something went wrong deleting file"
-				res.send 500
-			else
-				res.send 204
-
-	_get_range: (header) ->
-		parsed = parseRange(maxSizeInBytes, header)
-		if parsed == -1 or parsed == -2 or parsed.type != 'bytes'
-			null
-		else
-			range = parsed[0]
-			{start: range.start, end: range.end}
-
-	directorySize: (req, res)->
-		metrics.inc "projectSize"
-		{project_id, bucket} = req
-		logger.log project_id:project_id, bucket:bucket, "receiving request to project size"
-		FileHandler.getDirectorySize bucket, project_id, (err, size)->
-			if err?
-				logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file"
-				res.send 500
-			else
-				res.json {'total bytes' : size}

services/filestore/app/coffee/FileConverter.coffee

@@ -1,62 +0,0 @@
-_ = require("underscore")
-metrics = require("metrics-sharelatex")
-logger = require("logger-sharelatex")
-safe_exec = require("./SafeExec")
-approvedFormats = ["png"]
-Settings = require "settings-sharelatex"
-
-fourtySeconds = 40 * 1000
-
-childProcessOpts =
-	killSignal: "SIGTERM"
-	timeout: fourtySeconds
-
-
-module.exports =
-
-	convert: (sourcePath, requestedFormat, callback)->
-		logger.log sourcePath:sourcePath, requestedFormat:requestedFormat, "converting file format"
-		timer = new metrics.Timer("imageConvert")
-		destPath = "#{sourcePath}.#{requestedFormat}"
-		sourcePath = "#{sourcePath}[0]"
-		if !_.include approvedFormats, requestedFormat
-			err = new Error("invalid format requested")
-			return callback err
-		width = "600x"
-		command = ["convert", "-define", "pdf:fit-page=#{width}", "-flatten", "-density", "300", sourcePath, destPath]
-		command = Settings.commands.convertCommandPrefix.concat(command)
-		safe_exec command, childProcessOpts, (err, stdout, stderr)->
-			timer.done()
-			if err?
-				logger.err err:err, stderr:stderr, sourcePath:sourcePath, requestedFormat:requestedFormat, destPath:destPath,  "something went wrong converting file"
-			else
-				logger.log  sourcePath:sourcePath, requestedFormat:requestedFormat, destPath:destPath,  "finished converting file"
-			callback(err, destPath)
-
-	thumbnail: (sourcePath, callback)->
-		destPath = "#{sourcePath}.png"
-		sourcePath = "#{sourcePath}[0]"
-		width = "260x"
-		command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", "pdf:fit-page=#{width}", sourcePath, "-resize", width, destPath]
-		logger.log sourcePath:sourcePath, destPath:destPath,  command:command, "thumbnail convert file"
-		command = Settings.commands.convertCommandPrefix.concat(command)
-		safe_exec command, childProcessOpts, (err, stdout, stderr)->
-			if err?
-				logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to thumbnail"
-			else
-				logger.log  sourcePath:sourcePath, destPath:destPath, "finished thumbnailing file"
-			callback(err, destPath)	
-
-	preview: (sourcePath, callback)->
-		logger.log sourcePath:sourcePath, "preview convert file"
-		destPath = "#{sourcePath}.png"
-		sourcePath = "#{sourcePath}[0]"
-		width = "548x"
-		command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", "pdf:fit-page=#{width}", sourcePath, "-resize", width, destPath]
-		command = Settings.commands.convertCommandPrefix.concat(command)
-		safe_exec command, childProcessOpts, (err, stdout, stderr)->
-			if err?
-				logger.err err:err, stderr:stderr, sourcePath:sourcePath, destPath:destPath, "something went wrong converting file to preview"
-			else
-				logger.log  sourcePath:sourcePath, destPath:destPath, "finished converting file to preview"
-			callback(err, destPath)

services/filestore/app/coffee/FileHandler.coffee

@@ -1,129 +0,0 @@
-settings = require("settings-sharelatex")
-PersistorManager = require("./PersistorManager")
-LocalFileWriter = require("./LocalFileWriter")
-logger = require("logger-sharelatex")
-FileConverter = require("./FileConverter")
-KeyBuilder = require("./KeyBuilder")
-async = require("async")
-ImageOptimiser = require("./ImageOptimiser")
-Errors = require('./Errors')
-
-module.exports = FileHandler =
-
-	insertFile: (bucket, key, stream, callback)->
-		convertedKey = KeyBuilder.getConvertedFolderKey key
-		PersistorManager.deleteDirectory bucket, convertedKey, (error) ->
-			return callback(error) if error?
-			PersistorManager.sendStream bucket, key, stream, callback
-
-	deleteFile: (bucket, key, callback)->
-		convertedKey = KeyBuilder.getConvertedFolderKey key
-		async.parallel [
-			(done)-> PersistorManager.deleteFile bucket, key, done
-			(done)-> PersistorManager.deleteDirectory bucket, convertedKey, done
-		], callback
-
-	getFile: (bucket, key, opts = {}, callback)->
-		# In this call, opts can contain credentials
-		logger.log bucket:bucket, key:key, opts:@_scrubSecrets(opts), "getting file"
-		if !opts.format? and !opts.style?
-			@_getStandardFile bucket, key, opts, callback
-		else
-			@_getConvertedFile bucket, key, opts, callback
-
-	getFileSize: (bucket, key, callback) ->
-		PersistorManager.getFileSize(bucket, key, callback)
-
-	_getStandardFile: (bucket, key, opts, callback)->
-		PersistorManager.getFileStream bucket, key, opts, (err, fileStream)->
-			if err? and !(err instanceof Errors.NotFoundError)
-				logger.err  bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream"
-			callback err, fileStream
-
-	_getConvertedFile: (bucket, key, opts, callback)->
-		convertedKey = KeyBuilder.addCachingToKey key, opts
-		PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=>
-			if err?
-				return callback err
-			if exists
-				PersistorManager.getFileStream bucket, convertedKey, opts, callback
-			else
-				@_getConvertedFileAndCache bucket, key, convertedKey, opts, callback
-
-	_getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)->
-		convertedFsPath = ""
-		originalFsPath = ""
-		async.series [
-			(cb) =>
-				@_convertFile bucket, key, opts, (err, fileSystemPath, originalFsPath) ->
-					convertedFsPath = fileSystemPath
-					originalFsPath = originalFsPath
-					cb err
-			(cb)->
-				ImageOptimiser.compressPng convertedFsPath, cb
-			(cb)->
-				PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb
-		], (err)->
-			if err?
-				LocalFileWriter.deleteFile convertedFsPath, ->
-				LocalFileWriter.deleteFile originalFsPath, ->
-				return callback(err)
-			# Send back the converted file from the local copy to avoid problems
-			# with the file not being present in S3 yet.  As described in the
-			# documentation below, we have already made a 'HEAD' request in
-			# checkIfFileExists so we only have "eventual consistency" if we try
-			# to stream it from S3 here.  This was a cause of many 403 errors.
-			#
-			# "Amazon S3 provides read-after-write consistency for PUTS of new
-			# objects in your S3 bucket in all regions with one caveat. The
-			# caveat is that if you make a HEAD or GET request to the key name
-			# (to find if the object exists) before creating the object, Amazon
-			# S3 provides eventual consistency for read-after-write.""
-			# https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
-			LocalFileWriter.getStream convertedFsPath, (err, readStream) ->
-				return callback(err) if err?
-				readStream.on 'end', () ->
-					logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file"
-					LocalFileWriter.deleteFile convertedFsPath, ->
-				callback(null, readStream)
-
-	_convertFile: (bucket, originalKey, opts, callback)->
-		@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->
-			if err?
-				return callback(err)
-			done = (err, destPath)->
-				if err?
-					logger.err err:err, bucket:bucket, originalKey:originalKey, opts:FileHandler._scrubSecrets(opts), "error converting file"
-					return callback(err)
-				LocalFileWriter.deleteFile originalFsPath, ->
-				callback(err, destPath, originalFsPath)
-
-			logger.log opts:opts, "converting file depending on opts"
-			
-			if opts.format?
-				FileConverter.convert originalFsPath, opts.format, done
-			else if opts.style == "thumbnail"
-				FileConverter.thumbnail originalFsPath, done
-			else if opts.style == "preview"
-				FileConverter.preview originalFsPath, done
-			else
-				return callback(new Error("should have specified opts to convert file with #{JSON.stringify(opts)}"))
-
-
-	_writeS3FileToDisk: (bucket, key, opts, callback)->
-		PersistorManager.getFileStream bucket, key, opts, (err, fileStream)->
-			if err?
-				return callback(err)
-			LocalFileWriter.writeStream fileStream, key, callback
-
-	getDirectorySize: (bucket, project_id, callback)->
-		logger.log bucket:bucket, project_id:project_id, "getting project size"
-		PersistorManager.directorySize bucket, project_id, (err, size)->
-			if err?
-				logger.err  bucket:bucket, project_id:project_id, "error getting size"
-			callback err, size
-
-	_scrubSecrets: (opts)->
-		safe = Object.assign {}, opts
-		delete safe.credentials
-		safe

services/filestore/app/coffee/HealthCheckController.coffee

@@ -1,57 +0,0 @@
-fs = require("fs-extra")
-path = require("path")
-async = require("async")
-fileConverter = require("./FileConverter")
-keyBuilder = require("./KeyBuilder")
-fileController = require("./FileController")
-logger = require('logger-sharelatex')
-settings = require("settings-sharelatex")
-streamBuffers = require("stream-buffers")
-_ = require('underscore')
-
-
-checkCanStoreFiles = (callback)->
-	callback = _.once(callback)
-	req = {params:{}, query:{}, headers:{}}
-	req.params.project_id = settings.health_check.project_id
-	req.params.file_id = settings.health_check.file_id
-	myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer(initialSize: 100)
-	res = {
-		send: (code) ->
-			if code != 200
-				callback(new Error("non-200 code from getFile: #{code}"))
-	}
-	myWritableStreamBuffer.send = res.send
-	keyBuilder.userFileKey req, res, ->
-		fileController.getFile req, myWritableStreamBuffer
-		myWritableStreamBuffer.on "close", ->
-			if myWritableStreamBuffer.size() > 0
-				callback()
-			else
-				err = "no data in write stream buffer for health check"
-				logger.err {err,}, "error performing health check"
-				callback(err)
-
-checkFileConvert = (callback)->
-	if !settings.enableConversions
-		return callback()
-	imgPath = path.join(settings.path.uploadFolder, "/tiny.pdf")
-	async.waterfall [
-		(cb)->
-			fs.copy("./tiny.pdf", imgPath, cb)
-		(cb)-> fileConverter.thumbnail imgPath, cb
-		(resultPath, cb)-> fs.unlink resultPath, cb
-		(cb)-> fs.unlink imgPath, cb
-	], callback
-
-
-module.exports =
-
-	check: (req, res) ->
-		logger.log {}, "performing health check"
-		async.parallel [checkFileConvert, checkCanStoreFiles], (err)->
-			if err?
-				logger.err err:err, "Health check: error running"
-				res.send 500
-			else
-				res.send 200

services/filestore/app/coffee/ImageOptimiser.coffee

@@ -1,25 +0,0 @@
-exec = require('child_process').exec
-logger = require("logger-sharelatex")
-Settings = require "settings-sharelatex"
-
-module.exports = 
-
-	compressPng: (localPath, callback)->
-		startTime = new Date()
-		logger.log localPath:localPath, "optimising png path"
-		args = "optipng #{localPath}"
-		opts =
-			timeout: 30 * 1000
-			killSignal: "SIGKILL"
-		if !Settings.enableConversions
-			error = new Error("Image conversions are disabled")
-			return callback(error)
-		exec args, opts,(err, stdout, stderr)->
-			if err? and err.signal == 'SIGKILL'
-				logger.warn {err: err, stderr: stderr, localPath: localPath}, "optimiser timeout reached"
-				err = null
-			else if err?
-				logger.err err:err, stderr:stderr, localPath:localPath, "something went wrong converting compressPng"
-			else
-				logger.log  localPath:localPath, "finished compressPng file"
-			callback(err)

services/filestore/app/coffee/KeyBuilder.coffee

@@ -1,50 +0,0 @@
-settings = require("settings-sharelatex")
-
-module.exports =
-
-
-	getConvertedFolderKey: (key)->
-		key = "#{key}-converted-cache/"
-
-	addCachingToKey: (key, opts)->
-		key = @getConvertedFolderKey(key)
-		if opts.format? and !opts.style?
-			key = "#{key}format-#{opts.format}"
-		if opts.style? and !opts.format?
-			key = "#{key}style-#{opts.style}"
-		if opts.style? and opts.format?
-			key = "#{key}format-#{opts.format}-style-#{opts.style}"
-		return key
-
-
-	userFileKey: (req, res, next)->
-		{project_id, file_id} = req.params
-		req.key = "#{project_id}/#{file_id}"
-		req.bucket = settings.filestore.stores.user_files
-		next()
-		
-	publicFileKey: (req, res, next)->
-		{project_id, public_file_id} = req.params
-		if not settings.filestore.stores.public_files?
-			res.status(501).send("public files not available")
-		else
-			req.key = "#{project_id}/#{public_file_id}"
-			req.bucket = settings.filestore.stores.public_files
-			next()
-
-	templateFileKey: (req, res, next)->
-		{template_id, format, version, sub_type} = req.params
-		req.key = "#{template_id}/v/#{version}/#{format}"
-		if sub_type?
-			req.key = "#{req.key}/#{sub_type}"
-		req.bucket = settings.filestore.stores.template_files
-		req.version = version
-		opts = req.query
-		next()
-
-	publicProjectKey: (req, res, next)->
-		{project_id} = req.params
-		req.project_id = project_id
-		req.bucket = settings.filestore.stores.user_files
-		next()
-	

services/filestore/app/coffee/LocalFileWriter.coffee

@@ -1,56 +0,0 @@
-fs = require("fs")
-uuid = require('node-uuid')
-path = require("path")
-_ = require("underscore")
-logger = require("logger-sharelatex")
-metrics = require("metrics-sharelatex")
-Settings = require("settings-sharelatex")
-Errors = require "./Errors"
-
-module.exports = 
-
-	writeStream: (stream, key, callback)->
-		timer = new metrics.Timer("writingFile")
-		callback = _.once callback
-		fsPath = @_getPath(key)
-		logger.log fsPath:fsPath, "writing file locally"
-		writeStream = fs.createWriteStream(fsPath)
-		writeStream.on "finish", ->
-			timer.done()
-			logger.log fsPath:fsPath, "finished writing file locally"
-			callback(null, fsPath)
-		writeStream.on "error", (err)->
-			logger.err err:err, fsPath:fsPath, "problem writing file locally, with write stream"
-			callback err
-		stream.on "error", (err)->
-			logger.log err:err, fsPath:fsPath, "problem writing file locally, with read stream"
-			callback err
-		stream.pipe writeStream
-
-	getStream: (fsPath, _callback = (err, res)->) ->
-		callback = _.once _callback
-		timer = new metrics.Timer("readingFile")
-		logger.log fsPath:fsPath, "reading file locally"
-		readStream = fs.createReadStream(fsPath)
-		readStream.on "end", ->
-			timer.done()
-			logger.log fsPath:fsPath, "finished reading file locally"
-		readStream.on "error", (err)->
-			logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream"
-			if err.code == 'ENOENT'
-				callback new Errors.NotFoundError(err.message), null
-			else
-				callback err
-		callback null, readStream
-
-	deleteFile: (fsPath, callback)->
-		if !fsPath? or fsPath == ""
-			return callback()
-		logger.log fsPath:fsPath, "removing local temp file"
-		fs.unlink fsPath, callback
-
-	_getPath : (key)->
-		if !key?
-			key = uuid.v1()
-		key = key.replace(/\//g,"-")
-		path.join(Settings.path.uploadFolder, key)

services/filestore/app/coffee/PersistorManager.coffee

@@ -1,16 +0,0 @@
-settings = require("settings-sharelatex")
-logger = require("logger-sharelatex")
-
-# assume s3 if none specified
-settings?.filestore?.backend ||= "s3"
-
-logger.log backend:settings?.filestore?.backend, "Loading backend"
-module.exports = switch settings?.filestore?.backend
-	when "aws-sdk"
-		require "./AWSSDKPersistorManager"
-	when "s3"
-		require("./S3PersistorManager")
-	when "fs"
-		require("./FSPersistorManager")
-	else
-		throw new Error( "Unknown filestore backend: #{settings.filestore.backend}" )

services/filestore/app/coffee/S3PersistorManager.coffee

@@ -1,238 +0,0 @@
-# This module is the one which is used in production.  It needs to be migrated
-# to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
-# details. The knox library is unmaintained and has bugs.
-
-http = require('http')
-http.globalAgent.maxSockets = 300
-https = require('https')
-https.globalAgent.maxSockets = 300
-settings = require("settings-sharelatex")
-request = require("request")
-logger = require("logger-sharelatex")
-metrics = require("metrics-sharelatex")
-fs = require("fs")
-knox = require("knox")
-path = require("path")
-LocalFileWriter = require("./LocalFileWriter")
-Errors = require("./Errors")
-_ = require("underscore")
-awsS3 = require "aws-sdk/clients/s3"
-URL = require('url')
-
-thirtySeconds = 30 * 1000
-
-buildDefaultOptions = (bucketName, method, key)->
-	if settings.filestore.s3.endpoint
-		endpoint = "#{settings.filestore.s3.endpoint}/#{bucketName}"
-	else
-		endpoint = "https://#{bucketName}.s3.amazonaws.com"
-	return {
-			aws:
-				key: settings.filestore.s3.key
-				secret: settings.filestore.s3.secret
-				bucket: bucketName
-			method: method
-			timeout: thirtySeconds
-			uri:"#{endpoint}/#{key}"
-	}
-
-getS3Options = (credentials) ->
-	options =
-		credentials:
-			accessKeyId: credentials.auth_key
-			secretAccessKey: credentials.auth_secret
-
-	if settings.filestore.s3.endpoint
-		endpoint = URL.parse(settings.filestore.s3.endpoint)
-		options.endpoint = settings.filestore.s3.endpoint
-		options.sslEnabled = endpoint.protocol == 'https'
-
-	return options
-
-defaultS3Client = new awsS3(getS3Options({
-	auth_key: settings.filestore.s3.key,
-	auth_secret: settings.filestore.s3.secret
-}))
-
-getS3Client = (credentials) ->
-	if credentials?
-		return new awsS3(getS3Options(credentials))
-	else
-		return defaultS3Client
-
-getKnoxClient = (bucketName) =>
-	options =
-		key: settings.filestore.s3.key
-		secret: settings.filestore.s3.secret
-		bucket: bucketName
-	if settings.filestore.s3.endpoint
-		endpoint = URL.parse(settings.filestore.s3.endpoint)
-		options.endpoint = endpoint.hostname
-		options.port = endpoint.port
-	return knox.createClient(options)
-
-module.exports =
-
-	sendFile: (bucketName, key, fsPath, callback)->
-		s3Client = getKnoxClient(bucketName)
-		uploaded = 0
-		putEventEmiter = s3Client.putFile fsPath, key, (err, res)->
-			metrics.count 's3.egress', uploaded
-			if err?
-				logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath,"something went wrong uploading file to s3"
-				return callback(err)
-			if !res?
-				logger.err err:err, res:res, bucketName:bucketName, key:key, fsPath:fsPath, "no response from s3 put file"
-				return callback("no response from put file")
-			if res.statusCode != 200
-				logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
-				return callback("non 200 response from s3 on put file")
-			logger.log res:res,  bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
-			callback(err)
-		putEventEmiter.on "error", (err)->
-			logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
-			callback err
-		putEventEmiter.on "progress", (progress)->
-			uploaded = progress.written
-
-	sendStream: (bucketName, key, readStream, callback)->
-		logger.log bucketName:bucketName, key:key, "sending file to s3"
-		readStream.on "error", (err)->
-			logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
-		LocalFileWriter.writeStream readStream, null, (err, fsPath)=>
-			if err?
-				logger.err  bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
-				return callback(err)
-			@sendFile bucketName, key, fsPath, (err) ->
-				# delete the temporary file created above and return the original error
-				LocalFileWriter.deleteFile fsPath, () ->
-					callback(err)
-
-	# opts may be {start: Number, end: Number}
-	getFileStream: (bucketName, key, opts, callback = (err, res)->)->
-		opts = opts || {}
-		callback = _.once(callback)
-		logger.log bucketName:bucketName, key:key, "getting file from s3"
-
-		s3 = getS3Client(opts.credentials)
-		s3Params = {
-			Bucket: bucketName
-			Key: key
-		}
-		if opts.start? and opts.end?
-			s3Params['Range'] = "bytes=#{opts.start}-#{opts.end}"
-		s3Request = s3.getObject(s3Params)
-
-		s3Request.on 'httpHeaders', (statusCode, headers, response, statusMessage) =>
-			if statusCode in [403, 404]
-				# S3 returns a 403 instead of a 404 when the user doesn't have
-				# permission to list the bucket contents.
-				logger.log({ bucketName: bucketName, key: key }, "file not found in s3")
-				return callback(new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null)
-			if statusCode not in [200, 206]
-				logger.log({bucketName: bucketName, key: key }, "error getting file from s3: #{statusCode}")
-				return callback(new Error("Got non-200 response from S3: #{statusCode} #{statusMessage}"), null)
-			stream = response.httpResponse.createUnbufferedStream()
-			stream.on 'data', (data) ->
-				metrics.count 's3.ingress', data.byteLength
-
-			callback(null, stream)
-
-		s3Request.on 'error', (err) =>
-			logger.err({ err: err, bucketName: bucketName, key: key }, "error getting file stream from s3")
-			callback(err)
-
-		s3Request.send()
-
-	getFileSize: (bucketName, key, callback) ->
-		logger.log({ bucketName: bucketName, key: key }, "getting file size from S3")
-		s3 = getS3Client()
-		s3.headObject { Bucket: bucketName, Key: key }, (err, data) ->
-			if err?
-				if err.statusCode in [403, 404]
-					# S3 returns a 403 instead of a 404 when the user doesn't have
-					# permission to list the bucket contents.
-					logger.log({
-						bucketName: bucketName,
-						key: key
-					}, "file not found in s3")
-					callback(
-						new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}")
-					)
-				else
-					logger.err({
-						bucketName: bucketName,
-						key: key,
-						err: err
-					}, "error performing S3 HeadObject")
-					callback(err)
-				return
-			callback(null, data.ContentLength)
-
-	copyFile: (bucketName, sourceKey, destKey, callback)->
-		logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3"
-		source = bucketName + '/' + sourceKey
-		# use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
-		s3 = getS3Client()
-		s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) ->
-			if err?
-				if err.code is 'NoSuchKey'
-					logger.err bucketName:bucketName, sourceKey:sourceKey, "original file not found in s3 when copying"
-					callback(new Errors.NotFoundError("original file not found in S3 when copying"))
-				else
-					logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in aws"
-					callback(err)
-			else
-				callback()
-
-	deleteFile: (bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "delete file in s3"
-		options = buildDefaultOptions(bucketName, "delete", key)
-		request options, (err, res)->
-			if err?
-				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong deleting file in aws"
-			callback(err)
-
-	deleteDirectory: (bucketName, key, _callback)->
-		# deleteMultiple can call the callback multiple times so protect against this.
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
-
-		logger.log key: key, bucketName: bucketName, "deleting directory"
-		s3Client = getKnoxClient(bucketName)
-		s3Client.list prefix:key, (err, data)->
-			if err?
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws"
-				return callback(err)
-			keys = _.map data.Contents, (entry)->
-				return entry.Key
-			s3Client.deleteMultiple keys, callback
-
-	checkIfFileExists:(bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "checking if file exists in s3"
-		options = buildDefaultOptions(bucketName, "head", key)
-		request options, (err, res)->
-			if err?
-				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong checking file in aws"
-				return callback(err)
-			if !res?
-				logger.err err:err, res:res, bucketName:bucketName, key:key, "no response object returned when checking if file exists"
-				err = new Error("no response from s3 #{bucketName} #{key}")
-				return callback(err)
-			exists = res.statusCode == 200
-			logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3"
-			callback(err, exists)
-
-	directorySize:(bucketName, key, callback)->
-		logger.log bucketName:bucketName, key:key, "get project size in s3"
-		s3Client = getKnoxClient(bucketName)
-		s3Client.list prefix:key, (err, data)->
-			if err?
-				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws"
-				return callback(err)
-			totalSize = 0
-			_.each data.Contents, (entry)->
-				totalSize += entry.Size
-			logger.log totalSize:totalSize, "total size"
-			callback null, totalSize

services/filestore/app/coffee/SafeExec.coffee

@@ -1,48 +0,0 @@
-_ = require("underscore")
-logger = require("logger-sharelatex")
-child_process = require('child_process')
-Settings = require "settings-sharelatex"
-
-# execute a command in the same way as 'exec' but with a timeout that
-# kills all child processes
-#
-# we spawn the command with 'detached:true' to make a new process
-# group, then we can kill everything in that process group.
-
-module.exports = (command, options, callback = (err, stdout, stderr) ->) ->
-	if !Settings.enableConversions
-		error = new Error("Image conversions are disabled")
-		return callback(error)
-
-	# options are {timeout:  number-of-milliseconds, killSignal: signal-name}
-	[cmd, args...] = command
-
-	child = child_process.spawn cmd, args, {detached:true}
-	stdout = ""
-	stderr = ""
-
-	cleanup = _.once (err) ->
-		clearTimeout killTimer if killTimer?
-		callback err, stdout, stderr
-
-	if options.timeout?
-		killTimer = setTimeout () ->
-			try
-				# use negative process id to kill process group
-				process.kill -child.pid, options.killSignal || "SIGTERM"
-			catch error
-				logger.log process: child.pid, kill_error: error, "error killing process"
-		, options.timeout
-
-	child.on 'close', (code, signal) ->
-		err = if code then new Error("exit status #{code}") else signal
-		cleanup err
-
-	child.on 'error', (err) ->
-		cleanup err
-
-	child.stdout.on 'data', (chunk) ->
-		stdout += chunk
-
-	child.stderr.on 'data', (chunk) ->
-		stderr += chunk

services/filestore/app/js/Errors.js

@@ -0,0 +1,52 @@
+const OError = require('@overleaf/o-error')
+
+// Error class for legacy errors so they inherit OError while staying
+// backward-compatible (can be instantiated with string as argument instead
+// of object)
+class BackwardCompatibleError extends OError {
+  constructor(messageOrOptions) {
+    let options
+    if (typeof messageOrOptions === 'string') {
+      options = { message: messageOrOptions }
+    } else if (!messageOrOptions) {
+      options = {}
+    } else {
+      options = messageOrOptions
+    }
+    super(options)
+  }
+}
+
+class NotFoundError extends BackwardCompatibleError {}
+class WriteError extends BackwardCompatibleError {}
+class ReadError extends BackwardCompatibleError {}
+class HealthCheckError extends BackwardCompatibleError {}
+class ConversionsDisabledError extends BackwardCompatibleError {}
+class ConversionError extends BackwardCompatibleError {}
+class SettingsError extends BackwardCompatibleError {}
+
+class FailedCommandError extends OError {
+  constructor(command, code, stdout, stderr) {
+    super({
+      message: 'command failed with error exit code',
+      info: {
+        command,
+        code
+      }
+    })
+    this.stdout = stdout
+    this.stderr = stderr
+    this.code = code
+  }
+}
+
+module.exports = {
+  NotFoundError,
+  FailedCommandError,
+  ConversionsDisabledError,
+  WriteError,
+  ReadError,
+  ConversionError,
+  HealthCheckError,
+  SettingsError
+}

services/filestore/app/js/FSPersistorManager.js

@@ -0,0 +1,229 @@
+const fs = require('fs')
+const glob = require('glob')
+const logger = require('logger-sharelatex')
+const path = require('path')
+const rimraf = require('rimraf')
+const Stream = require('stream')
+const { promisify, callbackify } = require('util')
+
+const LocalFileWriter = require('./LocalFileWriter').promises
+const { NotFoundError, ReadError, WriteError } = require('./Errors')
+
+const pipeline = promisify(Stream.pipeline)
+const fsUnlink = promisify(fs.unlink)
+const fsOpen = promisify(fs.open)
+const fsStat = promisify(fs.stat)
+const fsGlob = promisify(glob)
+const rmrf = promisify(rimraf)
+
+const filterName = key => key.replace(/\//g, '_')
+
+async function sendFile(location, target, source) {
+  const filteredTarget = filterName(target)
+  logger.log({ location, target: filteredTarget, source }, 'sending file')
+
+  // actually copy the file (instead of moving it) to maintain consistent behaviour
+  // between the different implementations
+  try {
+    const sourceStream = fs.createReadStream(source)
+    const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`)
+    await pipeline(sourceStream, targetStream)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to copy the specified file',
+      { location, target, source },
+      WriteError
+    )
+  }
+}
+
+async function sendStream(location, target, sourceStream) {
+  logger.log({ location, target }, 'sending file stream')
+
+  const fsPath = await LocalFileWriter.writeStream(sourceStream)
+
+  try {
+    await sendFile(location, target, fsPath)
+  } finally {
+    await LocalFileWriter.deleteFile(fsPath)
+  }
+}
+
+// opts may be {start: Number, end: Number}
+async function getFileStream(location, name, opts) {
+  const filteredName = filterName(name)
+  logger.log({ location, filteredName }, 'getting file')
+
+  try {
+    opts.fd = await fsOpen(`${location}/${filteredName}`, 'r')
+  } catch (err) {
+    logger.err({ err, location, filteredName: name }, 'Error reading from file')
+
+    throw _wrapError(
+      err,
+      'failed to open file for streaming',
+      { location, filteredName, opts },
+      ReadError
+    )
+  }
+
+  return fs.createReadStream(null, opts)
+}
+
+async function getFileSize(location, filename) {
+  const fullPath = path.join(location, filterName(filename))
+
+  try {
+    const stat = await fsStat(fullPath)
+    return stat.size
+  } catch (err) {
+    logger.err({ err, location, filename }, 'failed to stat file')
+
+    throw _wrapError(
+      err,
+      'failed to stat file',
+      { location, filename },
+      ReadError
+    )
+  }
+}
+
+async function copyFile(location, fromName, toName) {
+  const filteredFromName = filterName(fromName)
+  const filteredToName = filterName(toName)
+  logger.log({ location, filteredFromName, filteredToName }, 'copying file')
+
+  try {
+    const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`)
+    const targetStream = fs.createWriteStream(`${location}/${filteredToName}`)
+    await pipeline(sourceStream, targetStream)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to copy file',
+      { location, filteredFromName, filteredToName },
+      WriteError
+    )
+  }
+}
+
+async function deleteFile(location, name) {
+  const filteredName = filterName(name)
+  logger.log({ location, filteredName }, 'delete file')
+  try {
+    await fsUnlink(`${location}/${filteredName}`)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to delete file',
+      { location, filteredName },
+      WriteError
+    )
+  }
+}
+
+// this is only called internally for clean-up by `FileHandler` and isn't part of the external API
+async function deleteDirectory(location, name) {
+  const filteredName = filterName(name.replace(/\/$/, ''))
+
+  logger.log({ location, filteredName }, 'deleting directory')
+
+  try {
+    await rmrf(`${location}/${filteredName}`)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to delete directory',
+      { location, filteredName },
+      WriteError
+    )
+  }
+}
+
+async function checkIfFileExists(location, name) {
+  const filteredName = filterName(name)
+  try {
+    const stat = await fsStat(`${location}/${filteredName}`)
+    return !!stat
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return false
+    }
+    throw _wrapError(
+      err,
+      'failed to stat file',
+      { location, filteredName },
+      ReadError
+    )
+  }
+}
+
+// note, does not recurse into subdirectories, as we use a flattened directory structure
+async function directorySize(location, name) {
+  const filteredName = filterName(name.replace(/\/$/, ''))
+  let size = 0
+
+  try {
+    const files = await fsGlob(`${location}/${filteredName}_*`)
+    for (const file of files) {
+      try {
+        const stat = await fsStat(file)
+        if (stat.isFile()) {
+          size += stat.size
+        }
+      } catch (err) {
+        // ignore files that may have just been deleted
+        if (err.code !== 'ENOENT') {
+          throw err
+        }
+      }
+    }
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to get directory size',
+      { location, name },
+      ReadError
+    )
+  }
+
+  return size
+}
+
+function _wrapError(error, message, params, ErrorType) {
+  if (error.code === 'ENOENT') {
+    return new NotFoundError({
+      message: 'no such file or directory',
+      info: params
+    }).withCause(error)
+  } else {
+    return new ErrorType({
+      message: message,
+      info: params
+    }).withCause(error)
+  }
+}
+
+module.exports = {
+  sendFile: callbackify(sendFile),
+  sendStream: callbackify(sendStream),
+  getFileStream: callbackify(getFileStream),
+  getFileSize: callbackify(getFileSize),
+  copyFile: callbackify(copyFile),
+  deleteFile: callbackify(deleteFile),
+  deleteDirectory: callbackify(deleteDirectory),
+  checkIfFileExists: callbackify(checkIfFileExists),
+  directorySize: callbackify(directorySize),
+  promises: {
+    sendFile,
+    sendStream,
+    getFileStream,
+    getFileSize,
+    copyFile,
+    deleteFile,
+    deleteDirectory,
+    checkIfFileExists,
+    directorySize
+  }
+}

services/filestore/app/js/FileController.js

@@ -0,0 +1,171 @@
+const PersistorManager = require('./PersistorManager')
+const logger = require('logger-sharelatex')
+const FileHandler = require('./FileHandler')
+const metrics = require('metrics-sharelatex')
+const parseRange = require('range-parser')
+const Errors = require('./Errors')
+const { pipeline } = require('stream')
+
+const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
+
+module.exports = {
+  getFile,
+  getFileHead,
+  insertFile,
+  copyFile,
+  deleteFile,
+  directorySize
+}
+
+function getFile(req, res, next) {
+  const { key, bucket } = req
+  const { format, style } = req.query
+  const options = {
+    key,
+    bucket,
+    format,
+    style
+  }
+  metrics.inc('getFile')
+  logger.log({ key, bucket, format, style }, 'receiving request to get file')
+
+  if (req.headers.range) {
+    const range = _getRange(req.headers.range)
+    if (range) {
+      options.start = range.start
+      options.end = range.end
+      logger.log(
+        { start: range.start, end: range.end },
+        'getting range of bytes from file'
+      )
+    }
+  }
+
+  FileHandler.getFile(bucket, key, options, function(err, fileStream) {
+    if (err) {
+      if (err instanceof Errors.NotFoundError) {
+        res.sendStatus(404)
+      } else {
+        logger.err({ err, key, bucket, format, style }, 'problem getting file')
+        res.sendStatus(500)
+      }
+      return
+    }
+
+    if (req.query.cacheWarm) {
+      logger.log(
+        { key, bucket, format, style },
+        'request is only for cache warm so not sending stream'
+      )
+      return res.sendStatus(200)
+    }
+
+    logger.log({ key, bucket, format, style }, 'sending file to response')
+
+    // pass 'next' as a callback to 'pipeline' to receive any errors
+    pipeline(fileStream, res, next)
+  })
+}
+
+function getFileHead(req, res) {
+  const { key, bucket } = req
+  metrics.inc('getFileSize')
+  logger.log({ key, bucket }, 'receiving request to get file metadata')
+  FileHandler.getFileSize(bucket, key, function(err, fileSize) {
+    if (err) {
+      if (err instanceof Errors.NotFoundError) {
+        res.sendStatus(404)
+      } else {
+        res.sendStatus(500)
+      }
+      return
+    }
+    res.set('Content-Length', fileSize)
+    res.status(200).end()
+  })
+}
+
+function insertFile(req, res) {
+  metrics.inc('insertFile')
+  const { key, bucket } = req
+  logger.log({ key, bucket }, 'receiving request to insert file')
+  FileHandler.insertFile(bucket, key, req, function(err) {
+    if (err) {
+      logger.log({ err, key, bucket }, 'error inserting file')
+      res.sendStatus(500)
+    } else {
+      res.sendStatus(200)
+    }
+  })
+}
+
+function copyFile(req, res) {
+  metrics.inc('copyFile')
+  const { key, bucket } = req
+  const oldProjectId = req.body.source.project_id
+  const oldFileId = req.body.source.file_id
+  logger.log(
+    { key, bucket, oldProject_id: oldProjectId, oldFile_id: oldFileId },
+    'receiving request to copy file'
+  )
+
+  PersistorManager.copyFile(
+    bucket,
+    `${oldProjectId}/${oldFileId}`,
+    key,
+    function(err) {
+      if (err) {
+        if (err instanceof Errors.NotFoundError) {
+          res.sendStatus(404)
+        } else {
+          logger.log(
+            { err, oldProject_id: oldProjectId, oldFile_id: oldFileId },
+            'something went wrong copying file'
+          )
+          res.sendStatus(500)
+        }
+        return
+      }
+
+      res.sendStatus(200)
+    }
+  )
+}
+
+function deleteFile(req, res) {
+  metrics.inc('deleteFile')
+  const { key, bucket } = req
+  logger.log({ key, bucket }, 'receiving request to delete file')
+  return FileHandler.deleteFile(bucket, key, function(err) {
+    if (err != null) {
+      logger.log({ err, key, bucket }, 'something went wrong deleting file')
+      return res.sendStatus(500)
+    } else {
+      return res.sendStatus(204)
+    }
+  })
+}
+
+function directorySize(req, res) {
+  metrics.inc('projectSize')
+  const { project_id: projectId, bucket } = req
+  logger.log({ projectId, bucket }, 'receiving request to project size')
+  FileHandler.getDirectorySize(bucket, projectId, function(err, size) {
+    if (err) {
+      logger.log({ err, projectId, bucket }, 'error inserting file')
+      return res.sendStatus(500)
+    }
+
+    res.json({ 'total bytes': size })
+  })
+}
+
+function _getRange(header) {
+  const parsed = parseRange(maxSizeInBytes, header)
+  if (parsed === -1 || parsed === -2 || parsed.type !== 'bytes') {
+    return null
+  } else {
+    const range = parsed[0]
+    return { start: range.start, end: range.end }
+  }
+}

services/filestore/app/js/FileConverter.js

@@ -0,0 +1,105 @@
+const metrics = require('metrics-sharelatex')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+const { callbackify } = require('util')
+
+const safeExec = require('./SafeExec').promises
+const { ConversionError } = require('./Errors')
+
+const APPROVED_FORMATS = ['png']
+const FOURTY_SECONDS = 40 * 1000
+const KILL_SIGNAL = 'SIGTERM'
+
+module.exports = {
+  convert: callbackify(convert),
+  thumbnail: callbackify(thumbnail),
+  preview: callbackify(preview),
+  promises: {
+    convert,
+    thumbnail,
+    preview
+  }
+}
+
+async function convert(sourcePath, requestedFormat) {
+  const width = '600x'
+  return _convert(sourcePath, requestedFormat, [
+    'convert',
+    '-define',
+    `pdf:fit-page=${width}`,
+    '-flatten',
+    '-density',
+    '300',
+    `${sourcePath}[0]`
+  ])
+}
+
+async function thumbnail(sourcePath) {
+  const width = '260x'
+  return convert(sourcePath, 'png', [
+    'convert',
+    '-flatten',
+    '-background',
+    'white',
+    '-density',
+    '300',
+    '-define',
+    `pdf:fit-page=${width}`,
+    `${sourcePath}[0]`,
+    '-resize',
+    width
+  ])
+}
+
+async function preview(sourcePath) {
+  const width = '548x'
+  return convert(sourcePath, 'png', [
+    'convert',
+    '-flatten',
+    '-background',
+    'white',
+    '-density',
+    '300',
+    '-define',
+    `pdf:fit-page=${width}`,
+    `${sourcePath}[0]`,
+    '-resize',
+    width
+  ])
+}
+
+async function _convert(sourcePath, requestedFormat, command) {
+  logger.log({ sourcePath, requestedFormat }, 'converting file format')
+
+  if (!APPROVED_FORMATS.includes(requestedFormat)) {
+    throw new ConversionError({
+      message: 'invalid format requested',
+      info: { format: requestedFormat }
+    })
+  }
+
+  const timer = new metrics.Timer('imageConvert')
+  const destPath = `${sourcePath}.${requestedFormat}`
+
+  command.push(destPath)
+  command = Settings.commands.convertCommandPrefix.concat(command)
+
+  try {
+    await safeExec(command, {
+      killSignal: KILL_SIGNAL,
+      timeout: FOURTY_SECONDS
+    })
+  } catch (err) {
+    throw new ConversionError({
+      message: 'something went wrong converting file',
+      info: { stderr: err.stderr, sourcePath, requestedFormat, destPath }
+    }).withCause(err)
+  }
+
+  timer.done()
+  logger.log(
+    { sourcePath, requestedFormat, destPath },
+    'finished converting file'
+  )
+  return destPath
+}

services/filestore/app/js/FileHandler.js

@@ -0,0 +1,208 @@
+const { promisify } = require('util')
+const fs = require('fs')
+const PersistorManager = require('./PersistorManager')
+const LocalFileWriter = require('./LocalFileWriter')
+const logger = require('logger-sharelatex')
+const FileConverter = require('./FileConverter')
+const KeyBuilder = require('./KeyBuilder')
+const async = require('async')
+const ImageOptimiser = require('./ImageOptimiser')
+const {
+  WriteError,
+  ReadError,
+  ConversionError,
+  NotFoundError
+} = require('./Errors')
+
+module.exports = {
+  insertFile,
+  deleteFile,
+  getFile,
+  getFileSize,
+  getDirectorySize,
+  promises: {
+    getFile: promisify(getFile),
+    insertFile: promisify(insertFile),
+    deleteFile: promisify(deleteFile),
+    getFileSize: promisify(getFileSize),
+    getDirectorySize: promisify(getDirectorySize)
+  }
+}
+
+function insertFile(bucket, key, stream, callback) {
+  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
+  PersistorManager.deleteDirectory(bucket, convertedKey, function(error) {
+    if (error) {
+      return callback(new WriteError('error inserting file').withCause(error))
+    }
+    PersistorManager.sendStream(bucket, key, stream, callback)
+  })
+}
+
+function deleteFile(bucket, key, callback) {
+  const convertedKey = KeyBuilder.getConvertedFolderKey(key)
+  async.parallel(
+    [
+      done => PersistorManager.deleteFile(bucket, key, done),
+      done => PersistorManager.deleteDirectory(bucket, convertedKey, done)
+    ],
+    callback
+  )
+}
+
+function getFile(bucket, key, opts, callback) {
+  // In this call, opts can contain credentials
+  if (!opts) {
+    opts = {}
+  }
+  logger.log({ bucket, key, opts: _scrubSecrets(opts) }, 'getting file')
+  if (!opts.format && !opts.style) {
+    _getStandardFile(bucket, key, opts, callback)
+  } else {
+    _getConvertedFile(bucket, key, opts, callback)
+  }
+}
+
+function getFileSize(bucket, key, callback) {
+  PersistorManager.getFileSize(bucket, key, callback)
+}
+
+function getDirectorySize(bucket, projectId, callback) {
+  logger.log({ bucket, project_id: projectId }, 'getting project size')
+  PersistorManager.directorySize(bucket, projectId, function(err, size) {
+    if (err) {
+      return callback(
+        new ReadError('error getting project size').withCause(err)
+      )
+    }
+    callback(null, size)
+  })
+}
+
+function _getStandardFile(bucket, key, opts, callback) {
+  PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) {
+    if (err && !(err instanceof NotFoundError)) {
+      logger.err(
+        { bucket, key, opts: _scrubSecrets(opts) },
+        'error getting fileStream'
+      )
+    }
+    callback(err, fileStream)
+  })
+}
+
+function _getConvertedFile(bucket, key, opts, callback) {
+  const convertedKey = KeyBuilder.addCachingToKey(key, opts)
+  PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists) => {
+    if (err) {
+      return callback(err)
+    }
+
+    if (exists) {
+      PersistorManager.getFileStream(bucket, convertedKey, opts, callback)
+    } else {
+      _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback)
+    }
+  })
+}
+
+function _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) {
+  let convertedFsPath
+
+  async.series(
+    [
+      cb => {
+        _convertFile(bucket, key, opts, function(err, fileSystemPath) {
+          convertedFsPath = fileSystemPath
+          cb(err)
+        })
+      },
+      cb => ImageOptimiser.compressPng(convertedFsPath, cb),
+      cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb)
+    ],
+    function(err) {
+      if (err) {
+        LocalFileWriter.deleteFile(convertedFsPath, function() {})
+        return callback(
+          new ConversionError('failed to convert file').withCause(err)
+        )
+      }
+      // Send back the converted file from the local copy to avoid problems
+      // with the file not being present in S3 yet.  As described in the
+      // documentation below, we have already made a 'HEAD' request in
+      // checkIfFileExists so we only have "eventual consistency" if we try
+      // to stream it from S3 here.  This was a cause of many 403 errors.
+      //
+      // "Amazon S3 provides read-after-write consistency for PUTS of new
+      // objects in your S3 bucket in all regions with one caveat. The
+      // caveat is that if you make a HEAD or GET request to the key name
+      // (to find if the object exists) before creating the object, Amazon
+      // S3 provides eventual consistency for read-after-write.""
+      // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
+      const readStream = fs.createReadStream(convertedFsPath)
+      readStream.on('end', function() {
+        LocalFileWriter.deleteFile(convertedFsPath, function() {})
+      })
+      callback(null, readStream)
+    }
+  )
+}
+
+function _convertFile(bucket, originalKey, opts, callback) {
+  _writeFileToDisk(bucket, originalKey, opts, function(err, originalFsPath) {
+    if (err) {
+      return callback(
+        new ConversionError('unable to write file to disk').withCause(err)
+      )
+    }
+
+    const done = function(err, destPath) {
+      if (err) {
+        logger.err(
+          { err, bucket, originalKey, opts: _scrubSecrets(opts) },
+          'error converting file'
+        )
+        return callback(
+          new ConversionError('error converting file').withCause(err)
+        )
+      }
+      LocalFileWriter.deleteFile(originalFsPath, function() {})
+      callback(err, destPath)
+    }
+
+    logger.log({ opts }, 'converting file depending on opts')
+
+    if (opts.format) {
+      FileConverter.convert(originalFsPath, opts.format, done)
+    } else if (opts.style === 'thumbnail') {
+      FileConverter.thumbnail(originalFsPath, done)
+    } else if (opts.style === 'preview') {
+      FileConverter.preview(originalFsPath, done)
+    } else {
+      callback(
+        new ConversionError(
+          `should have specified opts to convert file with ${JSON.stringify(
+            opts
+          )}`
+        )
+      )
+    }
+  })
+}
+
+function _writeFileToDisk(bucket, key, opts, callback) {
+  PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) {
+    if (err) {
+      return callback(
+        new ReadError('unable to get read stream for file').withCause(err)
+      )
+    }
+    LocalFileWriter.writeStream(fileStream, key, callback)
+  })
+}
+
+function _scrubSecrets(opts) {
+  const safe = Object.assign({}, opts)
+  delete safe.credentials
+  return safe
+}

services/filestore/app/js/HealthCheckController.js

@@ -0,0 +1,72 @@
+const fs = require('fs-extra')
+const path = require('path')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+const streamBuffers = require('stream-buffers')
+const { promisify } = require('util')
+const Stream = require('stream')
+
+const pipeline = promisify(Stream.pipeline)
+const fsCopy = promisify(fs.copy)
+const fsUnlink = promisify(fs.unlink)
+
+const { HealthCheckError } = require('./Errors')
+const FileConverter = require('./FileConverter').promises
+const FileHandler = require('./FileHandler').promises
+
+async function checkCanGetFiles() {
+  if (!Settings.health_check) {
+    return
+  }
+
+  const projectId = Settings.health_check.project_id
+  const fileId = Settings.health_check.file_id
+  const key = `${projectId}/${fileId}`
+  const bucket = Settings.filestore.stores.user_files
+
+  const buffer = new streamBuffers.WritableStreamBuffer({
+    initialSize: 100
+  })
+
+  const sourceStream = await FileHandler.getFile(bucket, key, {})
+  try {
+    await pipeline(sourceStream, buffer)
+  } catch (err) {
+    throw new HealthCheckError('failed to get health-check file').withCause(err)
+  }
+
+  if (!buffer.size()) {
+    throw new HealthCheckError('no bytes written to download stream')
+  }
+}
+
+async function checkFileConvert() {
+  if (!Settings.enableConversions) {
+    return
+  }
+
+  const imgPath = path.join(Settings.path.uploadFolder, '/tiny.pdf')
+
+  let resultPath
+  try {
+    await fsCopy('./tiny.pdf', imgPath)
+    resultPath = await FileConverter.thumbnail(imgPath)
+  } finally {
+    if (resultPath) {
+      await fsUnlink(resultPath)
+    }
+    await fsUnlink(imgPath)
+  }
+}
+
+module.exports = {
+  check(req, res) {
+    logger.log({}, 'performing health check')
+    Promise.all([checkCanGetFiles(), checkFileConvert()])
+      .then(() => res.sendStatus(200))
+      .catch(err => {
+        logger.err({ err }, 'Health check: error running')
+        res.sendStatus(500)
+      })
+  }
+}

services/filestore/app/js/ImageOptimiser.js

@@ -0,0 +1,41 @@
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const { callbackify } = require('util')
+const safeExec = require('./SafeExec').promises
+
+module.exports = {
+  compressPng: callbackify(compressPng),
+  promises: {
+    compressPng
+  }
+}
+
+async function compressPng(localPath, callback) {
+  const timer = new metrics.Timer('compressPng')
+  logger.log({ localPath }, 'optimising png path')
+
+  const args = ['optipng', localPath]
+  const opts = {
+    timeout: 30 * 1000,
+    killSignal: 'SIGKILL'
+  }
+
+  try {
+    await safeExec(args, opts)
+    timer.done()
+    logger.log({ localPath }, 'finished compressing png')
+  } catch (err) {
+    if (err.code === 'SIGKILL') {
+      logger.warn(
+        { err, stderr: err.stderr, localPath },
+        'optimiser timeout reached'
+      )
+    } else {
+      logger.err(
+        { err, stderr: err.stderr, localPath },
+        'something went wrong compressing png'
+      )
+      throw err
+    }
+  }
+}

services/filestore/app/js/KeyBuilder.js

@@ -0,0 +1,85 @@
+const settings = require('settings-sharelatex')
+
+module.exports = {
+  getConvertedFolderKey,
+  addCachingToKey,
+  userFileKeyMiddleware,
+  publicFileKeyMiddleware,
+  publicProjectKeyMiddleware,
+  bucketFileKeyMiddleware,
+  templateFileKeyMiddleware
+}
+
+function getConvertedFolderKey(key) {
+  return `${key}-converted-cache/`
+}
+
+function addCachingToKey(key, opts) {
+  key = this.getConvertedFolderKey(key)
+
+  if (opts.format && !opts.style) {
+    key = `${key}format-${opts.format}`
+  }
+  if (opts.style && !opts.format) {
+    key = `${key}style-${opts.style}`
+  }
+  if (opts.style && opts.format) {
+    key = `${key}format-${opts.format}-style-${opts.style}`
+  }
+
+  return key
+}
+
+function userFileKeyMiddleware(req, res, next) {
+  const { project_id: projectId, file_id: fileId } = req.params
+  req.key = `${projectId}/${fileId}`
+  req.bucket = settings.filestore.stores.user_files
+  next()
+}
+
+function publicFileKeyMiddleware(req, res, next) {
+  if (settings.filestore.stores.public_files == null) {
+    return res.status(501).send('public files not available')
+  }
+
+  const { project_id: projectId, public_file_id: publicFileId } = req.params
+  req.key = `${projectId}/${publicFileId}`
+  req.bucket = settings.filestore.stores.public_files
+
+  next()
+}
+
+function bucketFileKeyMiddleware(req, res, next) {
+  req.bucket = req.params.bucket
+  req.key = req.params[0]
+  next()
+}
+
+function templateFileKeyMiddleware(req, res, next) {
+  const {
+    template_id: templateId,
+    format,
+    version,
+    sub_type: subType
+  } = req.params
+
+  req.key = `${templateId}/v/${version}/${format}`
+
+  if (subType) {
+    req.key = `${req.key}/${subType}`
+  }
+
+  req.bucket = settings.filestore.stores.template_files
+  req.version = version
+
+  next()
+}
+
+function publicProjectKeyMiddleware(req, res, next) {
+  const { project_id: projectId } = req.params
+
+  req.project_id = projectId
+  req.bucket = settings.filestore.stores.user_files
+
+  next()
+}

services/filestore/app/js/LocalFileWriter.js

@@ -0,0 +1,68 @@
+const fs = require('fs')
+const uuid = require('node-uuid')
+const path = require('path')
+const Stream = require('stream')
+const { callbackify, promisify } = require('util')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const Settings = require('settings-sharelatex')
+const { WriteError } = require('./Errors')
+
+module.exports = {
+  promises: {
+    writeStream,
+    deleteFile
+  },
+  writeStream: callbackify(writeStream),
+  deleteFile: callbackify(deleteFile)
+}
+
+const pipeline = promisify(Stream.pipeline)
+
+async function writeStream(stream, key) {
+  const timer = new metrics.Timer('writingFile')
+  const fsPath = _getPath(key)
+
+  logger.log({ fsPath }, 'writing file locally')
+
+  const writeStream = fs.createWriteStream(fsPath)
+  try {
+    await pipeline(stream, writeStream)
+    timer.done()
+    logger.log({ fsPath }, 'finished writing file locally')
+    return fsPath
+  } catch (err) {
+    await deleteFile(fsPath)
+
+    logger.err({ err, fsPath }, 'problem writing file locally')
+    throw new WriteError({
+      message: 'problem writing file locally',
+      info: { err, fsPath }
+    }).withCause(err)
+  }
+}
+
+async function deleteFile(fsPath) {
+  if (!fsPath) {
+    return
+  }
+  logger.log({ fsPath }, 'removing local temp file')
+  try {
+    await promisify(fs.unlink)(fsPath)
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      throw new WriteError({
+        message: 'failed to delete file',
+        info: { fsPath }
+      }).withCause(err)
+    }
+  }
+}
+
+function _getPath(key) {
+  if (key == null) {
+    key = uuid.v1()
+  }
+  key = key.replace(/\//g, '-')
+  return path.join(Settings.path.uploadFolder, key)
+}

services/filestore/app/js/PersistorManager.js

@@ -0,0 +1,24 @@
+const settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+
+logger.log(
+  {
+    backend: settings.filestore.backend
+  },
+  'Loading backend'
+)
+if (!settings.filestore.backend) {
+  throw new Error('no backend specified - config incomplete')
+}
+
+switch (settings.filestore.backend) {
+  case 'aws-sdk':
+  case 's3':
+    module.exports = require('./S3PersistorManager')
+    break
+  case 'fs':
+    module.exports = require('./FSPersistorManager')
+    break
+  default:
+    throw new Error(`unknown filestore backend: ${settings.filestore.backend}`)
+}

services/filestore/app/js/S3PersistorManager.js

@@ -0,0 +1,313 @@
+const http = require('http')
+const https = require('https')
+http.globalAgent.maxSockets = 300
+https.globalAgent.maxSockets = 300
+
+const settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+const meter = require('stream-meter')
+const fs = require('fs')
+const S3 = require('aws-sdk/clients/s3')
+const { URL } = require('url')
+const { callbackify } = require('util')
+const {
+  WriteError,
+  ReadError,
+  NotFoundError,
+  SettingsError
+} = require('./Errors')
+
+module.exports = {
+  sendFile: callbackify(sendFile),
+  sendStream: callbackify(sendStream),
+  getFileStream: callbackify(getFileStream),
+  deleteDirectory: callbackify(deleteDirectory),
+  getFileSize: callbackify(getFileSize),
+  deleteFile: callbackify(deleteFile),
+  copyFile: callbackify(copyFile),
+  checkIfFileExists: callbackify(checkIfFileExists),
+  directorySize: callbackify(directorySize),
+  promises: {
+    sendFile,
+    sendStream,
+    getFileStream,
+    deleteDirectory,
+    getFileSize,
+    deleteFile,
+    copyFile,
+    checkIfFileExists,
+    directorySize
+  }
+}
+
+async function sendFile(bucketName, key, fsPath) {
+  let readStream
+  try {
+    readStream = fs.createReadStream(fsPath)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'error reading file from disk',
+      { bucketName, key, fsPath },
+      ReadError
+    )
+  }
+  return sendStream(bucketName, key, readStream)
+}
+
+async function sendStream(bucketName, key, readStream) {
+  try {
+    const meteredStream = meter()
+    meteredStream.on('finish', () => {
+      metrics.count('s3.egress', meteredStream.bytes)
+    })
+
+    const response = await _getClientForBucket(bucketName)
+      .upload({
+        Bucket: bucketName,
+        Key: key,
+        Body: readStream.pipe(meteredStream)
+      })
+      .promise()
+
+    logger.log({ response, bucketName, key }, 'data uploaded to s3')
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'upload to S3 failed',
+      { bucketName, key },
+      WriteError
+    )
+  }
+}
+
+async function getFileStream(bucketName, key, opts) {
+  opts = opts || {}
+
+  const params = {
+    Bucket: bucketName,
+    Key: key
+  }
+  if (opts.start != null && opts.end != null) {
+    params.Range = `bytes=${opts.start}-${opts.end}`
+  }
+
+  return new Promise((resolve, reject) => {
+    const stream = _getClientForBucket(bucketName)
+      .getObject(params)
+      .createReadStream()
+
+    const meteredStream = meter()
+    meteredStream.on('finish', () => {
+      metrics.count('s3.ingress', meteredStream.bytes)
+    })
+
+    const onStreamReady = function() {
+      stream.removeListener('readable', onStreamReady)
+      resolve(stream.pipe(meteredStream))
+    }
+    stream.on('readable', onStreamReady)
+    stream.on('error', err => {
+      reject(_wrapError(err, 'error reading from S3', params, ReadError))
+    })
+  })
+}
+
+async function deleteDirectory(bucketName, key) {
+  logger.log({ key, bucketName }, 'deleting directory')
+  let response
+
+  try {
+    response = await _getClientForBucket(bucketName)
+      .listObjects({ Bucket: bucketName, Prefix: key })
+      .promise()
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to list objects in S3',
+      { bucketName, key },
+      ReadError
+    )
+  }
+
+  const objects = response.Contents.map(item => ({ Key: item.Key }))
+  if (objects.length) {
+    try {
+      await _getClientForBucket(bucketName)
+        .deleteObjects({
+          Bucket: bucketName,
+          Delete: {
+            Objects: objects,
+            Quiet: true
+          }
+        })
+        .promise()
+    } catch (err) {
+      throw _wrapError(
+        err,
+        'failed to delete objects in S3',
+        { bucketName, key },
+        WriteError
+      )
+    }
+  }
+}
+
+async function getFileSize(bucketName, key) {
+  try {
+    const response = await _getClientForBucket(bucketName)
+      .headObject({ Bucket: bucketName, Key: key })
+      .promise()
+    return response.ContentLength
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'error getting size of s3 object',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+async function deleteFile(bucketName, key) {
+  try {
+    await _getClientForBucket(bucketName)
+      .deleteObject({ Bucket: bucketName, Key: key })
+      .promise()
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'failed to delete file in S3',
+      { bucketName, key },
+      WriteError
+    )
+  }
+}
+
+async function copyFile(bucketName, sourceKey, destKey) {
+  const params = {
+    Bucket: bucketName,
+    Key: destKey,
+    CopySource: `${bucketName}/${sourceKey}`
+  }
+  try {
+    await _getClientForBucket(bucketName)
+      .copyObject(params)
+      .promise()
+  } catch (err) {
+    throw _wrapError(err, 'failed to copy file in S3', params, WriteError)
+  }
+}
+
+async function checkIfFileExists(bucketName, key) {
+  try {
+    await getFileSize(bucketName, key)
+    return true
+  } catch (err) {
+    if (err instanceof NotFoundError) {
+      return false
+    }
+    throw _wrapError(
+      err,
+      'error checking whether S3 object exists',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+async function directorySize(bucketName, key) {
+  try {
+    const response = await _getClientForBucket(bucketName)
+      .listObjects({ Bucket: bucketName, Prefix: key })
+      .promise()
+
+    return response.Contents.reduce((acc, item) => item.Size + acc, 0)
+  } catch (err) {
+    throw _wrapError(
+      err,
+      'error getting directory size in S3',
+      { bucketName, key },
+      ReadError
+    )
+  }
+}
+
+function _wrapError(error, message, params, ErrorType) {
+  if (['NoSuchKey', 'NotFound', 'ENOENT'].includes(error.code)) {
+    return new NotFoundError({
+      message: 'no such file',
+      info: params
+    }).withCause(error)
+  } else {
+    return new ErrorType({
+      message: message,
+      info: params
+    }).withCause(error)
+  }
+}
+
+const _clients = new Map()
+let _defaultClient
+
+function _getClientForBucket(bucket) {
+  if (_clients[bucket]) {
+    return _clients[bucket]
+  }
+
+  if (
+    settings.filestore.s3BucketCreds &&
+    settings.filestore.s3BucketCreds[bucket]
+  ) {
+    _clients[bucket] = new S3(
+      _buildClientOptions(settings.filestore.s3BucketCreds[bucket])
+    )
+    return _clients[bucket]
+  }
+
+  // no specific credentials for the bucket
+  if (_defaultClient) {
+    return _defaultClient
+  }
+
+  if (settings.filestore.s3.key) {
+    _defaultClient = new S3(_buildClientOptions())
+    return _defaultClient
+  }
+
+  throw new SettingsError({
+    message: 'no bucket-specific or default credentials provided',
+    info: { bucket }
+  })
+}
+
+function _buildClientOptions(bucketCredentials) {
+  const options = {}
+
+  if (bucketCredentials) {
+    options.credentials = {
+      accessKeyId: bucketCredentials.auth_key,
+      secretAccessKey: bucketCredentials.auth_secret
+    }
+  } else {
+    options.credentials = {
+      accessKeyId: settings.filestore.s3.key,
+      secretAccessKey: settings.filestore.s3.secret
+    }
+  }
+
+  if (settings.filestore.s3.endpoint) {
+    const endpoint = new URL(settings.filestore.s3.endpoint)
+    options.endpoint = settings.filestore.s3.endpoint
+    options.sslEnabled = endpoint.protocol === 'https'
+  }
+
+  // path-style access is only used for acceptance tests
+  if (settings.filestore.s3.pathStyle) {
+    options.s3ForcePathStyle = true
+  }
+
+  return options
+}

services/filestore/app/js/SafeExec.js

@@ -0,0 +1,83 @@
+const _ = require('underscore')
+const logger = require('logger-sharelatex')
+const childProcess = require('child_process')
+const Settings = require('settings-sharelatex')
+const { ConversionsDisabledError, FailedCommandError } = require('./Errors')
+
+// execute a command in the same way as 'exec' but with a timeout that
+// kills all child processes
+//
+// we spawn the command with 'detached:true' to make a new process
+// group, then we can kill everything in that process group.
+
+module.exports = safeExec
+module.exports.promises = safeExecPromise
+
+// options are {timeout:  number-of-milliseconds, killSignal: signal-name}
+function safeExec(command, options, callback) {
+  if (!Settings.enableConversions) {
+    return callback(
+      new ConversionsDisabledError('image conversions are disabled')
+    )
+  }
+
+  const [cmd, ...args] = command
+
+  const child = childProcess.spawn(cmd, args, { detached: true })
+  let stdout = ''
+  let stderr = ''
+
+  let killTimer
+
+  if (options.timeout) {
+    killTimer = setTimeout(function() {
+      try {
+        // use negative process id to kill process group
+        process.kill(-child.pid, options.killSignal || 'SIGTERM')
+      } catch (error) {
+        logger.log(
+          { process: child.pid, kill_error: error },
+          'error killing process'
+        )
+      }
+    }, options.timeout)
+  }
+
+  const cleanup = _.once(function(err) {
+    if (killTimer) {
+      clearTimeout(killTimer)
+    }
+    callback(err, stdout, stderr)
+  })
+
+  child.on('close', function(code, signal) {
+    if (code || signal) {
+      return cleanup(
+        new FailedCommandError(command, code || signal, stdout, stderr)
+      )
+    }
+
+    cleanup()
+  })
+
+  child.on('error', err => {
+    cleanup(err)
+  })
+  child.stdout.on('data', chunk => {
+    stdout += chunk
+  })
+  child.stderr.on('data', chunk => {
+    stderr += chunk
+  })
+}
+
+function safeExecPromise(command, options) {
+  return new Promise((resolve, reject) => {
+    safeExec(command, options, (err, stdout, stderr) => {
+      if (err) {
+        reject(err)
+      }
+      resolve({ stdout, stderr })
+    })
+  })
+}

services/filestore/buildscript.txt

@@ -1,10 +1,10 @@
 filestore
---language=coffeescript
---node-version=6.9.5
---acceptance-creds=None
---dependencies=mongo,redis
+--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,AWS_S3_PUBLIC_FILES_BUCKET_NAME=fake_public_files,AWS_S3_ENDPOINT=http://s3:9090,AWS_ACCESS_KEY_ID=fake,AWS_SECRET_ACCESS_KEY=fake
+--dependencies=s3
+--acceptance-creds=
+--script-version=1.3
+--node-version=10.17.0
+--language=es
 --docker-repos=gcr.io/overleaf-ops
---build-target=docker
---script-version=1.1.24
---env-pass-through=
 --public-repo=True
+--env-pass-through=

services/filestore/cluster.coffee

@@ -1,9 +0,0 @@
-recluster = require "recluster" # https://github.com/doxout/recluster
-path = require "path"
-
-cluster = recluster path.join(__dirname, 'app.js'), {
-		workers: 2,
-		backoff: 0,
-		readyWhen: "listening"
-}
-cluster.run()

services/filestore/config/settings.defaults.coffee

@@ -1,5 +1,12 @@
 Path = require "path"
 
+# environment variables renamed for consistency
+# use AWS_ACCESS_KEY_ID-style going forward
+if process.env['AWS_KEY'] && !process.env['AWS_ACCESS_KEY_ID']
+	process.env['AWS_ACCESS_KEY_ID'] = process.env['AWS_KEY']
+if process.env['AWS_SECRET'] && !process.env['AWS_SECRET_ACCESS_KEY']
+	process.env['AWS_SECRET_ACCESS_KEY'] = process.env['AWS_SECRET']
+
 settings =
 	internal:
 		filestore:
@@ -11,11 +18,11 @@ settings =
 		# Choices are
 		# s3 - Amazon S3
 		# fs - local filesystem
-		if process.env['AWS_KEY']? or process.env['S3_BUCKET_CREDENTIALS']?
+		if process.env['AWS_ACCESS_KEY_ID']? or process.env['S3_BUCKET_CREDENTIALS']?
 			backend: "s3"
 			s3:
-				key: process.env['AWS_KEY']
-				secret: process.env['AWS_SECRET']
+				key: process.env['AWS_ACCESS_KEY_ID']
+				secret: process.env['AWS_SECRET_ACCESS_KEY']
 				endpoint: process.env['AWS_S3_ENDPOINT']
 			stores:
 				user_files: process.env['AWS_S3_USER_FILES_BUCKET_NAME']

services/filestore/docker-compose.ci.yml

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3
 
 version: "2.1"
 
@@ -22,28 +22,23 @@ services:
       REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      ENABLE_CONVERSIONS: "true"
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
+      ENABLE_CONVERSIONS: "true"
       USE_PROM_METRICS: "true"
-      AWS_KEY: fake
-      AWS_SECRET: fake
       AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
       AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
       AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
-      AWS_S3_ENDPOINT: http://fakes3:9090
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
     depends_on:
-      mongo:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-      fakes3:
+      s3:
         condition: service_healthy
     user: node
     command: npm run test:acceptance:_run
 
 
-
   tar:
     build: .
     image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
@@ -51,17 +46,9 @@ services:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
-
-  redis:
-    image: redis
-
-  mongo:
-    image: mongo:3.4
-
-  fakes3:
+  s3:
     image: adobe/s3mock
     environment:
       - initialBuckets=fake_user_files,fake_template_files,fake_public_files
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9090"]
-

services/filestore/docker-compose.yml

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3
 
 version: "2.1"
 
@@ -28,47 +28,25 @@ services:
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
-      ENABLE_CONVERSIONS: "true"
       LOG_LEVEL: ERROR
       NODE_ENV: test
+      ENABLE_CONVERSIONS: "true"
       USE_PROM_METRICS: "true"
-      AWS_KEY: fake
-      AWS_SECRET: fake
       AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
       AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
       AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
-      AWS_S3_ENDPOINT: http://fakes3:9090
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
     user: node
     depends_on:
-      mongo:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-      fakes3:
+      s3:
         condition: service_healthy
     command: npm run test:acceptance
 
-
-
-  tar:
-    build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - ./:/tmp/build/
-    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
-    user: root
-
-  redis:
-    image: redis
-
-  mongo:
-    image: mongo:3.4
-
-  fakes3:
+  s3:
     image: adobe/s3mock
     environment:
       - initialBuckets=fake_user_files,fake_template_files,fake_public_files
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9090"]
-
-

services/filestore/nodemon.json

@@ -10,10 +10,9 @@
   },
 
   "watch": [
-    "app/coffee/",
-    "app.coffee",
+    "app/js/",
+    "app.js",
     "config/"
   ],
-  "ext": "coffee"
-
+  "ext": "js"
 }

services/filestore/package.json

@@ -8,25 +8,25 @@
   },
   "scripts": {
     "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000  $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
     "test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
-    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
-    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
-    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "start": "node $NODE_APP_OPTIONS app.js",
     "nodemon": "nodemon --config nodemon.json",
-    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
-    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
-    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+    "lint": "node_modules/.bin/eslint .",
+    "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
+    "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js"
   },
   "dependencies": {
+    "@overleaf/o-error": "^2.1.0",
     "async": "~0.2.10",
     "aws-sdk": "^2.1.39",
     "body-parser": "^1.2.0",
     "express": "^4.2.0",
     "fs-extra": "^1.0.0",
+    "glob": "^7.1.6",
     "heapdump": "^0.3.2",
     "knox": "~0.9.1",
     "logger-sharelatex": "^1.7.0",
@@ -37,18 +37,37 @@
     "pngcrush": "0.0.3",
     "range-parser": "^1.0.2",
     "recluster": "^0.3.7",
-    "request": "2.14.0",
+    "request": "^2.88.0",
+    "request-promise-native": "^1.0.8",
     "response": "0.14.0",
     "rimraf": "2.2.8",
     "settings-sharelatex": "^1.1.0",
     "stream-browserify": "^2.0.1",
     "stream-buffers": "~0.2.5",
+    "stream-meter": "^1.0.4",
     "underscore": "~1.5.2"
   },
   "devDependencies": {
+    "babel-eslint": "^10.0.3",
     "bunyan": "^1.3.5",
     "chai": "4.2.0",
+    "chai-as-promised": "^7.1.1",
+    "disrequire": "^1.1.0",
+    "eslint": "^6.4.0",
+    "eslint-config-prettier": "^6.7.0",
+    "eslint-config-standard": "^14.1.0",
+    "eslint-plugin-chai-expect": "^2.1.0",
+    "eslint-plugin-chai-friendly": "^0.5.0",
+    "eslint-plugin-import": "^2.19.1",
+    "eslint-plugin-mocha": "^6.2.2",
+    "eslint-plugin-node": "^10.0.0",
+    "eslint-plugin-prettier": "^3.1.2",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.1",
+    "prettier-eslint": "^9.0.1",
+    "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "2.0.3",
-    "sinon": "7.1.1"
+    "sinon": "7.1.1",
+    "sinon-chai": "^3.3.0"
   }
 }

services/filestore/test/acceptance/coffee/FilestoreApp.coffee

@@ -1,45 +0,0 @@
-app = require('../../../app')
-require("logger-sharelatex").logger.level("info")
-logger = require("logger-sharelatex")
-Settings = require("settings-sharelatex")
-request = require('request')
-
-S3_TRIES = 30
-
-module.exports =
-	running: false
-	initing: false
-	callbacks: []
-	ensureRunning: (callback = (error) ->) ->
-		if @running
-			return callback()
-		else if @initing
-			@callbacks.push callback
-		else
-			@initing = true
-			@callbacks.push callback
-			app.listen Settings.internal?.filestore?.port, "localhost", (error) => 
-				throw error if error?
-				@running = true
-				logger.log("filestore running in dev mode")
-
-				for callback in @callbacks
-					callback()
-
-	waitForS3: (callback, tries) ->
-		return callback() unless Settings.filestore.s3?.endpoint
-		tries = 1 unless tries
-
-		request.get "#{Settings.filestore.s3.endpoint}/", (err, response) =>
-			console.log(err, response?.statusCode, tries)
-			if !err && [200, 404].includes(response?.statusCode)
-				return callback()
-
-			if tries == S3_TRIES
-				return callback('timed out waiting for S3')
-
-			setTimeout(
-				() =>
-					@waitForS3 callback, tries + 1
-				1000
-			)

services/filestore/test/acceptance/coffee/SendingFileTest.coffee

@@ -1,226 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/LocalFileWriter.js"
-SandboxedModule = require('sandboxed-module')
-fs = require("fs")
-request = require("request")
-settings = require("settings-sharelatex")
-FilestoreApp = require "./FilestoreApp"
-async = require('async')
-
-
-getMetric = (filestoreUrl, metric, cb) ->
-	request.get "#{filestoreUrl}/metrics", (err, res) ->
-		expect(res.statusCode).to.equal 200
-		metricRegex = new RegExp("^#{metric}{[^}]+} ([0-9]+)$", "m")
-		cb(parseInt(metricRegex.exec(res.body)?[1] || '0'))
-
-describe "Filestore", ->
-	before (done)->
-		@localFileReadPath = "/tmp/filestore_acceptence_tests_file_read.txt"
-		@localFileWritePath = "/tmp/filestore_acceptence_tests_file_write.txt"
-
-		@constantFileContent = [
-			"hello world"
-			"line 2 goes here #{Math.random()}"
-			"there are 3 lines in all"
-		].join("\n")
-
-		@filestoreUrl = "http://localhost:#{settings.internal.filestore.port}"
-		fs.writeFile @localFileReadPath, @constantFileContent, (err) ->
-			return done(err) if err
-			FilestoreApp.waitForS3(done)
-
-	beforeEach (done)->
-		FilestoreApp.ensureRunning =>
-			async.parallel [
-				(cb) =>
-					fs.unlink @localFileWritePath, () ->
-						cb()
-				(cb) =>
-					getMetric @filestoreUrl, 's3_egress', (metric) =>
-						@previousEgress = metric
-						cb()
-				(cb) =>
-					getMetric @filestoreUrl, 's3_ingress', (metric) =>
-						@previousIngress = metric
-						cb()
-			], done
-
-	it "should send a 200 for status endpoint", (done)->
-		request "#{@filestoreUrl}/status", (err, response, body)->
-			response.statusCode.should.equal 200
-			body.indexOf("filestore").should.not.equal -1
-			body.indexOf("up").should.not.equal -1
-			done()
-
-	describe "with a file on the server", ->
-
-		beforeEach (done)->
-			@timeout(1000 * 10)
-			@file_id = Math.random()
-			@fileUrl = "#{@filestoreUrl}/project/acceptence_tests/file/#{@file_id}"
-
-			writeStream = request.post(@fileUrl)
-
-			writeStream.on "end", done
-			fs.createReadStream(@localFileReadPath).pipe writeStream
-
-		it "should return 404 for a non-existant id", (done) ->
-			@timeout(1000 * 20)
-			options =
-				uri: @fileUrl + '___this_is_clearly_wrong___'
-			request.get options, (err, response, body) =>
-				response.statusCode.should.equal 404
-				done()
-
-		it 'should record an egress metric for the upload', (done) ->
-			getMetric @filestoreUrl, 's3_egress', (metric) =>
-				expect(metric - @previousEgress).to.equal @constantFileContent.length
-				done()
-
-		it "should return the file size on a HEAD request", (done) ->
-			expectedLength = Buffer.byteLength(@constantFileContent)
-			request.head @fileUrl, (err, res) =>
-				expect(res.statusCode).to.equal(200)
-				expect(res.headers['content-length']).to.equal(expectedLength.toString())
-				done()
-
-		it "should be able get the file back", (done)->
-			@timeout(1000 * 10)
-			request.get @fileUrl, (err, response, body)=>
-				body.should.equal @constantFileContent
-				done()
-
-		it "should record an ingress metric when downloading the file", (done)->
-			@timeout(1000 * 10)
-			request.get @fileUrl, () =>
-				getMetric @filestoreUrl, 's3_ingress', (metric) =>
-					expect(metric - @previousIngress).to.equal @constantFileContent.length
-					done()
-
-		it "should be able to get back the first 9 bytes of the file", (done) ->
-			@timeout(1000 * 10)
-			options =
-				uri: @fileUrl
-				headers:
-					'Range': 'bytes=0-8'
-			request.get options, (err, response, body)=>
-				body.should.equal 'hello wor'
-				done()
-
-		it "should record an ingress metric for a partial download", (done)->
-			@timeout(1000 * 10)
-			options =
-				uri: @fileUrl
-				headers:
-					'Range': 'bytes=0-8'
-			request.get options, ()=>
-				getMetric @filestoreUrl, 's3_ingress', (metric) =>
-					expect(metric - @previousIngress).to.equal 9
-					done()
-
-		it "should be able to get back bytes 4 through 10 of the file", (done) ->
-			@timeout(1000 * 10)
-			options =
-				uri: @fileUrl
-				headers:
-					'Range': 'bytes=4-10'
-			request.get options, (err, response, body)=>
-				body.should.equal 'o world'
-				done()
-
-		it "should be able to delete the file", (done)->
-			@timeout(1000 * 20)
-			request.del @fileUrl, (err, response, body)=>
-				response.statusCode.should.equal 204
-				request.get @fileUrl, (err, response, body)=>
-					response.statusCode.should.equal 404
-					done()
-
-		it "should be able to copy files", (done)->
-			@timeout(1000 * 20)
-
-			newProjectID = "acceptence_tests_copyied_project"
-			newFileId = Math.random()
-			newFileUrl = "#{@filestoreUrl}/project/#{newProjectID}/file/#{newFileId}"
-			opts =
-				method: 'put'
-				uri: newFileUrl
-				json:
-					source:
-						project_id:"acceptence_tests"
-						file_id: @file_id
-			request opts, (err, response, body)=>
-				response.statusCode.should.equal 200
-				request.del @fileUrl, (err, response, body)=>
-					response.statusCode.should.equal 204
-					request.get newFileUrl, (err, response, body)=>
-						body.should.equal @constantFileContent
-						done()
-
-	describe "with a pdf file", ->
-
-		beforeEach (done)->
-			@timeout(1000 * 10)
-			@file_id = Math.random()
-			@fileUrl = "#{@filestoreUrl}/project/acceptence_tests/file/#{@file_id}"
-			@localFileReadPath = __dirname + '/../../fixtures/test.pdf'
-			fs.stat @localFileReadPath, (err, stat) =>
-				@localFileSize = stat.size
-				writeStream = request.post(@fileUrl)
-
-				writeStream.on "end", done
-				fs.createReadStream(@localFileReadPath).pipe writeStream
-
-		it 'should record an egress metric for the upload', (done) ->
-			getMetric @filestoreUrl, 's3_egress', (metric) =>
-				expect(metric - @previousEgress).to.equal @localFileSize
-				done()
-
-		it "should be able get the file back", (done)->
-			@timeout(1000 * 10)
-			request.get @fileUrl, (err, response, body)=>
-				expect(body.substring(0, 8)).to.equal '%PDF-1.5'
-				done()
-
-		describe "getting the preview image", ->
-
-			beforeEach ->
-				@previewFileUrl = "#{@fileUrl}?style=preview"
-
-			it "should not time out", (done) ->
-				@timeout(1000 * 20)
-				request.get @previewFileUrl, (err, response, body) =>
-					expect(response).to.not.equal null
-					done()
-
-			it "should respond with image data", (done) ->
-				# note: this test relies of the imagemagick conversion working
-				@timeout(1000 * 20)
-				request.get @previewFileUrl, (err, response, body) =>
-					expect(response.statusCode).to.equal 200
-					expect(body.length).to.be.greaterThan 400
-					done()
-
-		describe "warming the cache", ->
-
-			beforeEach ->
-				@fileUrl = @fileUrl + '?style=preview&cacheWarm=true'
-
-			it "should not time out", (done) ->
-				@timeout(1000 * 20)
-				request.get @fileUrl, (err, response, body) =>
-					expect(response).to.not.equal null
-					done()
-
-			it "should respond with only an 'OK'", (done) ->
-				# note: this test relies of the imagemagick conversion working
-				@timeout(1000 * 20)
-				request.get @fileUrl, (err, response, body) =>
-					expect(response.statusCode).to.equal 200
-					body.should.equal 'OK'
-					done()

services/filestore/test/acceptance/js/FilestoreApp.js

@@ -0,0 +1,114 @@
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+const fs = require('fs')
+const Path = require('path')
+const { promisify } = require('util')
+const disrequire = require('disrequire')
+const rp = require('request-promise-native').defaults({
+  resolveWithFullResponse: true
+})
+
+const S3_TRIES = 30
+
+logger.logger.level('info')
+
+const fsReaddir = promisify(fs.readdir)
+const sleep = promisify(setTimeout)
+
+class FilestoreApp {
+  constructor() {
+    this.running = false
+    this.initing = false
+  }
+
+  async runServer() {
+    if (this.running) {
+      return
+    }
+
+    if (this.initing) {
+      return this.waitForInit()
+    }
+    this.initing = true
+
+    this.app = await FilestoreApp.requireApp()
+
+    await new Promise((resolve, reject) => {
+      this.server = this.app.listen(
+        Settings.internal.filestore.port,
+        'localhost',
+        err => {
+          if (err) {
+            return reject(err)
+          }
+          resolve()
+        }
+      )
+    })
+
+    if (Settings.filestore.backend === 's3') {
+      try {
+        await FilestoreApp.waitForS3()
+      } catch (err) {
+        await this.stop()
+        throw err
+      }
+    }
+
+    this.initing = false
+  }
+
+  async waitForInit() {
+    while (this.initing) {
+      await sleep(1000)
+    }
+  }
+
+  async stop() {
+    const closeServer = promisify(this.server.close).bind(this.server)
+    try {
+      await closeServer()
+    } finally {
+      delete this.server
+    }
+  }
+
+  static async waitForS3() {
+    let tries = 0
+    if (!Settings.filestore.s3.endpoint) {
+      return
+    }
+
+    let s3Available = false
+
+    while (tries < S3_TRIES && !s3Available) {
+      try {
+        const response = await rp.get(`${Settings.filestore.s3.endpoint}/`)
+        if ([200, 404].includes(response.statusCode)) {
+          s3Available = true
+        }
+      } catch (err) {
+        // swallow errors, as we may experience them until fake-s3 is running
+      } finally {
+        tries++
+        if (!s3Available) {
+          await sleep(1000)
+        }
+      }
+    }
+  }
+
+  static async requireApp() {
+    // unload the app, as we may be doing this on multiple runs with
+    // different settings, which affect startup in some cases
+    const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js'))
+    files.forEach(file => {
+      disrequire(Path.resolve(__dirname, '../../../app/js', file))
+    })
+    disrequire(Path.resolve(__dirname, '../../../app'))
+
+    return require('../../../app')
+  }
+}
+
+module.exports = FilestoreApp

services/filestore/test/acceptance/js/FilestoreTests.js

@@ -0,0 +1,408 @@
+const chai = require('chai')
+const { expect } = chai
+const fs = require('fs')
+const Settings = require('settings-sharelatex')
+const Path = require('path')
+const FilestoreApp = require('./FilestoreApp')
+const rp = require('request-promise-native').defaults({
+  resolveWithFullResponse: true
+})
+const S3 = require('aws-sdk/clients/s3')
+const Stream = require('stream')
+const request = require('request')
+const { promisify } = require('util')
+chai.use(require('chai-as-promised'))
+
+const fsWriteFile = promisify(fs.writeFile)
+const fsStat = promisify(fs.stat)
+const pipeline = promisify(Stream.pipeline)
+
+async function getMetric(filestoreUrl, metric) {
+  const res = await rp.get(`${filestoreUrl}/metrics`)
+  expect(res.statusCode).to.equal(200)
+  const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
+  const found = metricRegex.exec(res.body)
+  return parseInt(found ? found[1] : 0) || 0
+}
+
+// store settings for multiple backends, so that we can test each one.
+// fs will always be available - add others if they are configured
+const BackendSettings = {
+  FSPersistor: {
+    backend: 'fs',
+    stores: {
+      user_files: Path.resolve(__dirname, '../../../user_files'),
+      public_files: Path.resolve(__dirname, '../../../public_files'),
+      template_files: Path.resolve(__dirname, '../../../template_files')
+    }
+  }
+}
+
+if (process.env.AWS_ACCESS_KEY_ID) {
+  BackendSettings.S3Persistor = {
+    backend: 's3',
+    s3: {
+      key: process.env.AWS_ACCESS_KEY_ID,
+      secret: process.env.AWS_SECRET_ACCESS_KEY,
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      pathStyle: true
+    },
+    stores: {
+      user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
+      template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
+      public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
+    }
+  }
+}
+
+describe('Filestore', function() {
+  this.timeout(1000 * 10)
+  const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}`
+  const directoryName = 'directory'
+
+  // redefine the test suite for every available backend
+  Object.keys(BackendSettings).forEach(backend => {
+    describe(backend, function() {
+      let app, previousEgress, previousIngress
+
+      before(async function() {
+        // create the app with the relevant filestore settings
+        Settings.filestore = BackendSettings[backend]
+        app = new FilestoreApp()
+        await app.runServer()
+      })
+
+      after(async function() {
+        return app.stop()
+      })
+
+      beforeEach(async function() {
+        // retrieve previous metrics from the app
+        if (Settings.filestore.backend === 's3') {
+          ;[previousEgress, previousIngress] = await Promise.all([
+            getMetric(filestoreUrl, 's3_egress'),
+            getMetric(filestoreUrl, 's3_ingress')
+          ])
+        }
+      })
+
+      it('should send a 200 for the status endpoint', async function() {
+        const response = await rp(`${filestoreUrl}/status`)
+        expect(response.statusCode).to.equal(200)
+        expect(response.body).to.contain('filestore')
+        expect(response.body).to.contain('up')
+      })
+
+      it('should send a 200 for the health-check endpoint', async function() {
+        const response = await rp(`${filestoreUrl}/health_check`)
+        expect(response.statusCode).to.equal(200)
+        expect(response.body).to.equal('OK')
+      })
+
+      describe('with a file on the server', function() {
+        let fileId, fileUrl
+
+        const localFileReadPath =
+          '/tmp/filestore_acceptance_tests_file_read.txt'
+        const constantFileContent = [
+          'hello world',
+          `line 2 goes here ${Math.random()}`,
+          'there are 3 lines in all'
+        ].join('\n')
+
+        before(async function() {
+          await fsWriteFile(localFileReadPath, constantFileContent)
+        })
+
+        beforeEach(async function() {
+          fileId = Math.random()
+          fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${directoryName}%2F${fileId}`
+
+          const writeStream = request.post(fileUrl)
+          const readStream = fs.createReadStream(localFileReadPath)
+          // hack to consume the result to ensure the http request has been fully processed
+          const resultStream = fs.createWriteStream('/dev/null')
+          await pipeline(readStream, writeStream, resultStream)
+        })
+
+        it('should return 404 for a non-existant id', async function() {
+          const options = { uri: fileUrl + '___this_is_clearly_wrong___' }
+          await expect(
+            rp.get(options)
+          ).to.eventually.be.rejected.and.have.property('statusCode', 404)
+        })
+
+        it('should return the file size on a HEAD request', async function() {
+          const expectedLength = Buffer.byteLength(constantFileContent)
+          const res = await rp.head(fileUrl)
+          expect(res.statusCode).to.equal(200)
+          expect(res.headers['content-length']).to.equal(
+            expectedLength.toString()
+          )
+        })
+
+        it('should be able get the file back', async function() {
+          const res = await rp.get(fileUrl)
+          expect(res.body).to.equal(constantFileContent)
+        })
+
+        it('should be able to get back the first 9 bytes of the file', async function() {
+          const options = {
+            uri: fileUrl,
+            headers: {
+              Range: 'bytes=0-8'
+            }
+          }
+          const res = await rp.get(options)
+          expect(res.body).to.equal('hello wor')
+        })
+
+        it('should be able to get back bytes 4 through 10 of the file', async function() {
+          const options = {
+            uri: fileUrl,
+            headers: {
+              Range: 'bytes=4-10'
+            }
+          }
+          const res = await rp.get(options)
+          expect(res.body).to.equal('o world')
+        })
+
+        it('should be able to delete the file', async function() {
+          const response = await rp.del(fileUrl)
+          expect(response.statusCode).to.equal(204)
+          await expect(
+            rp.get(fileUrl)
+          ).to.eventually.be.rejected.and.have.property('statusCode', 404)
+        })
+
+        it('should be able to copy files', async function() {
+          const newProjectID = 'acceptance_tests_copyied_project'
+          const newFileId = Math.random()
+          const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}`
+          const opts = {
+            method: 'put',
+            uri: newFileUrl,
+            json: {
+              source: {
+                project_id: 'acceptance_tests',
+                file_id: `${directoryName}/${fileId}`
+              }
+            }
+          }
+          let response = await rp(opts)
+          expect(response.statusCode).to.equal(200)
+          response = await rp.del(fileUrl)
+          expect(response.statusCode).to.equal(204)
+          response = await rp.get(newFileUrl)
+          expect(response.body).to.equal(constantFileContent)
+        })
+
+        if (backend === 'S3Persistor') {
+          it('should record an egress metric for the upload', async function() {
+            const metric = await getMetric(filestoreUrl, 's3_egress')
+            expect(metric - previousEgress).to.equal(constantFileContent.length)
+          })
+
+          it('should record an ingress metric when downloading the file', async function() {
+            await rp.get(fileUrl)
+            const metric = await getMetric(filestoreUrl, 's3_ingress')
+            expect(metric - previousIngress).to.equal(
+              constantFileContent.length
+            )
+          })
+
+          it('should record an ingress metric for a partial download', async function() {
+            const options = {
+              uri: fileUrl,
+              headers: {
+                Range: 'bytes=0-8'
+              }
+            }
+            await rp.get(options)
+            const metric = await getMetric(filestoreUrl, 's3_ingress')
+            expect(metric - previousIngress).to.equal(9)
+          })
+        }
+      })
+
+      describe('with multiple files', function() {
+        let fileIds, fileUrls, project
+        const directoryName = 'directory'
+        const localFileReadPaths = [
+          '/tmp/filestore_acceptance_tests_file_read_1.txt',
+          '/tmp/filestore_acceptance_tests_file_read_2.txt'
+        ]
+        const constantFileContents = [
+          [
+            'hello world',
+            `line 2 goes here ${Math.random()}`,
+            'there are 3 lines in all'
+          ].join('\n'),
+          [
+            `for reference: ${Math.random()}`,
+            'cats are the best animals',
+            'wombats are a close second'
+          ].join('\n')
+        ]
+
+        before(async function() {
+          return Promise.all([
+            fsWriteFile(localFileReadPaths[0], constantFileContents[0]),
+            fsWriteFile(localFileReadPaths[1], constantFileContents[1])
+          ])
+        })
+
+        beforeEach(async function() {
+          project = `acceptance_tests_${Math.random()}`
+          fileIds = [Math.random(), Math.random()]
+          fileUrls = [
+            `${filestoreUrl}/project/${project}/file/${directoryName}%2F${fileIds[0]}`,
+            `${filestoreUrl}/project/${project}/file/${directoryName}%2F${fileIds[1]}`
+          ]
+
+          const writeStreams = [
+            request.post(fileUrls[0]),
+            request.post(fileUrls[1])
+          ]
+          const readStreams = [
+            fs.createReadStream(localFileReadPaths[0]),
+            fs.createReadStream(localFileReadPaths[1])
+          ]
+          // hack to consume the result to ensure the http request has been fully processed
+          const resultStreams = [
+            fs.createWriteStream('/dev/null'),
+            fs.createWriteStream('/dev/null')
+          ]
+          return Promise.all([
+            pipeline(readStreams[0], writeStreams[0], resultStreams[0]),
+            pipeline(readStreams[1], writeStreams[1], resultStreams[1])
+          ])
+        })
+
+        it('should get the directory size', async function() {
+          const response = await rp.get(
+            `${filestoreUrl}/project/${project}/size`
+          )
+          expect(parseInt(JSON.parse(response.body)['total bytes'])).to.equal(
+            constantFileContents[0].length + constantFileContents[1].length
+          )
+        })
+      })
+
+      if (backend === 'S3Persistor') {
+        describe('with a file in a specific bucket', function() {
+          let constantFileContents, fileId, fileUrl, bucketName
+
+          beforeEach(async function() {
+            constantFileContents = `This is a file in a different S3 bucket ${Math.random()}`
+            fileId = Math.random().toString()
+            bucketName = Math.random().toString()
+            fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
+
+            const s3ClientSettings = {
+              credentials: {
+                accessKeyId: 'fake',
+                secretAccessKey: 'fake'
+              },
+              endpoint: process.env.AWS_S3_ENDPOINT,
+              sslEnabled: false,
+              s3ForcePathStyle: true
+            }
+
+            const s3 = new S3(s3ClientSettings)
+            await s3
+              .createBucket({
+                Bucket: bucketName
+              })
+              .promise()
+            await s3
+              .upload({
+                Bucket: bucketName,
+                Key: fileId,
+                Body: constantFileContents
+              })
+              .promise()
+          })
+
+          it('should get the file from the specified bucket', async function() {
+            const response = await rp.get(fileUrl)
+            expect(response.body).to.equal(constantFileContents)
+          })
+        })
+      }
+
+      describe('with a pdf file', function() {
+        let fileId, fileUrl, localFileSize
+        const localFileReadPath = Path.resolve(
+          __dirname,
+          '../../fixtures/test.pdf'
+        )
+
+        beforeEach(async function() {
+          fileId = Math.random()
+          fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${directoryName}%2F${fileId}`
+          const stat = await fsStat(localFileReadPath)
+          localFileSize = stat.size
+          const writeStream = request.post(fileUrl)
+          const endStream = fs.createWriteStream('/dev/null')
+          const readStream = fs.createReadStream(localFileReadPath)
+          await pipeline(readStream, writeStream, endStream)
+        })
+
+        it('should be able get the file back', async function() {
+          const response = await rp.get(fileUrl)
+          expect(response.body.substring(0, 8)).to.equal('%PDF-1.5')
+        })
+
+        if (backend === 'S3Persistor') {
+          it('should record an egress metric for the upload', async function() {
+            const metric = await getMetric(filestoreUrl, 's3_egress')
+            expect(metric - previousEgress).to.equal(localFileSize)
+          })
+        }
+
+        describe('getting the preview image', function() {
+          this.timeout(1000 * 20)
+          let previewFileUrl
+
+          beforeEach(function() {
+            previewFileUrl = `${fileUrl}?style=preview`
+          })
+
+          it('should not time out', async function() {
+            const response = await rp.get(previewFileUrl)
+            expect(response.statusCode).to.equal(200)
+          })
+
+          it('should respond with image data', async function() {
+            // note: this test relies of the imagemagick conversion working
+            const response = await rp.get(previewFileUrl)
+            expect(response.body.length).to.be.greaterThan(400)
+            expect(response.body.substr(1, 3)).to.equal('PNG')
+          })
+        })
+
+        describe('warming the cache', function() {
+          this.timeout(1000 * 20)
+          let previewFileUrl
+
+          beforeEach(function() {
+            previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
+          })
+
+          it('should not time out', async function() {
+            const response = await rp.get(previewFileUrl)
+            expect(response.statusCode).to.equal(200)
+          })
+
+          it("should respond with only an 'OK'", async function() {
+            // note: this test relies of the imagemagick conversion working
+            const response = await rp.get(previewFileUrl)
+            expect(response.body).to.equal('OK')
+          })
+        })
+      })
+    })
+  })
+})

services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee

@@ -1,272 +0,0 @@
-sinon = require 'sinon'
-chai = require 'chai'
-
-should = chai.should()
-expect = chai.expect
-
-modulePath = "../../../app/js/AWSSDKPersistorManager.js"
-SandboxedModule = require 'sandboxed-module'
-
-describe "AWSSDKPersistorManager", ->
-	beforeEach ->
-		@settings =
-			filestore:
-				backend: "aws-sdk"
-		@s3 =
-			upload: sinon.stub()
-			getObject: sinon.stub()
-			copyObject: sinon.stub()
-			deleteObject: sinon.stub()
-			listObjects: sinon.stub()
-			deleteObjects: sinon.stub()
-			headObject: sinon.stub()
-		@awssdk =
-			S3: sinon.stub().returns @s3
-
-		@requires =
-			"aws-sdk": @awssdk
-			"settings-sharelatex": @settings
-			"logger-sharelatex":
-				log:->
-				err:->
-			"fs": @fs =
-				createReadStream: sinon.stub()
-			"./Errors": @Errors =
-				NotFoundError: sinon.stub()
-		@key = "my/key"
-		@bucketName = "my-bucket"
-		@error = "my error"
-		@AWSSDKPersistorManager = SandboxedModule.require modulePath, requires: @requires
-
-	describe "sendFile", ->
-		beforeEach ->
-			@stream = {}
-			@fsPath = "/usr/local/some/file"
-			@fs.createReadStream.returns @stream
-
-		it "should put the file with s3.upload", (done) ->
-			@s3.upload.callsArgWith 1
-			@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.upload.calledOnce, "called only once").to.be.true
-				expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream)
-							 , "called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.upload", (done) ->
-			@s3.upload.callsArgWith 1, @error
-			@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
-				expect(err).to.equal @error
-				done()
-
-
-	describe "sendStream", ->
-		beforeEach ->
-			@stream = {}
-
-		it "should put the file with s3.upload", (done) ->
-			@s3.upload.callsArgWith 1
-			@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.upload.calledOnce, "called only once").to.be.true
-				expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream),
-							 "called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.upload", (done) ->
-			@s3.upload.callsArgWith 1, @error
-			@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
-				expect(err).to.equal @error
-				done()
-
-	describe "getFileStream", ->
-		beforeEach ->
-			@opts = {}
-			@stream = {}
-			@read_stream =
-				on: @read_stream_on = sinon.stub()
-			@object =
-				createReadStream: sinon.stub().returns @read_stream
-			@s3.getObject.returns @object
-
-		it "should return a stream from s3.getObject", (done) ->
-			@read_stream_on.withArgs('readable').callsArgWith 1
-
-			@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
-				expect(@read_stream_on.calledTwice)
-				expect(err).to.not.be.ok
-				expect(stream, "returned the stream").to.equal @read_stream
-				expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key),
-							 "called with correct arguments").to.be.true
-				done()
-
-		describe "with start and end options", ->
-			beforeEach ->
-				@opts =
-					start: 0
-					end: 8
-			it "should pass headers to the s3.GetObject", (done) ->
-				@read_stream_on.withArgs('readable').callsArgWith 1
-				@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
-					expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key, Range: 'bytes=0-8'),
-						"called with correct arguments").to.be.true
-				done()
-
-		describe "error conditions", ->
-			describe "when the file doesn't exist", ->
-				beforeEach ->
-					@error = new Error()
-					@error.code = 'NoSuchKey'
-				it "should produce a NotFoundError", (done) ->
-					@read_stream_on.withArgs('error').callsArgWith 1, @error
-					@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
-						expect(stream).to.not.be.ok
-						expect(err).to.be.ok
-						expect(err instanceof @Errors.NotFoundError, "error is a correct instance").to.equal true
-						done()
-
-			describe "when there is some other error", ->
-				beforeEach ->
-					@error = new Error()
-				it "should dispatch the error from s3 object stream", (done) ->
-					@read_stream_on.withArgs('error').callsArgWith 1, @error
-					@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
-						expect(stream).to.not.be.ok
-						expect(err).to.be.ok
-						expect(err).to.equal @error
-						done()
-
-	describe "copyFile", ->
-		beforeEach ->
-			@destKey = "some/key"
-			@stream = {}
-
-		it "should copy the file with s3.copyObject", (done) ->
-			@s3.copyObject.callsArgWith 1
-			@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.copyObject.calledOnce, "called only once").to.be.true
-				expect((@s3.copyObject.calledWith Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key),
-					"called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.copyObject", (done) ->
-			@s3.copyObject.callsArgWith 1, @error
-			@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
-				expect(err).to.equal @error
-				done()
-
-	describe "deleteFile", ->
-		it "should delete the file with s3.deleteObject", (done) ->
-			@s3.deleteObject.callsArgWith 1
-			@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.deleteObject.calledOnce, "called only once").to.be.true
-				expect((@s3.deleteObject.calledWith Bucket: @bucketName, Key: @key),
-					"called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.deleteObject", (done) ->
-			@s3.deleteObject.callsArgWith 1, @error
-			@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
-				expect(err).to.equal @error
-				done()
-
-	describe "deleteDirectory", ->
-
-		it "should list the directory content using s3.listObjects", (done) ->
-			@s3.listObjects.callsArgWith 1, null, Contents: []
-			@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.listObjects.calledOnce, "called only once").to.be.true
-				expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key),
-					"called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.listObjects", (done) ->
-			@s3.listObjects.callsArgWith 1, @error
-			@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
-				expect(err).to.equal @error
-				done()
-
-		describe "with directory content", ->
-			beforeEach ->
-				@fileList = [
-					Key: 'foo'
-				, Key: 'bar'
-				, Key: 'baz'
-				]
-
-			it "should forward the file keys to s3.deleteObjects", (done) ->
-				@s3.listObjects.callsArgWith 1, null, Contents: @fileList
-				@s3.deleteObjects.callsArgWith 1
-				@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
-					expect(err).to.not.be.ok
-					expect(@s3.deleteObjects.calledOnce, "called only once").to.be.true
-					expect((@s3.deleteObjects.calledWith
-							Bucket: @bucketName
-							Delete:
-								Quiet: true
-								Objects: @fileList),
-						"called with correct arguments").to.be.true
-					done()
-
-			it "should dispatch the error from s3.deleteObjects", (done) ->
-				@s3.listObjects.callsArgWith 1, null, Contents: @fileList
-				@s3.deleteObjects.callsArgWith 1, @error
-				@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
-					expect(err).to.equal @error
-					done()
-
-
-	describe "checkIfFileExists", ->
-
-		it "should check for the file with s3.headObject", (done) ->
-			@s3.headObject.callsArgWith 1, null, {}
-			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
-				expect(err).to.not.be.ok
-				expect(@s3.headObject.calledOnce, "called only once").to.be.true
-				expect((@s3.headObject.calledWith Bucket: @bucketName, Key: @key),
-					"called with correct arguments").to.be.true
-				done()
-
-		it "should return false on an inexistant file", (done) ->
-			@s3.headObject.callsArgWith 1, null, {}
-			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
-				expect(exists).to.be.false
-				done()
-
-		it "should return true on an existing file", (done) ->
-			@s3.headObject.callsArgWith 1, null, ETag: "etag"
-			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
-				expect(exists).to.be.true
-				done()
-
-		it "should dispatch the error from s3.headObject", (done) ->
-			@s3.headObject.callsArgWith 1, @error
-			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
-				expect(err).to.equal @error
-				done()
-
-	describe "directorySize", ->
-
-		it "should list the directory content using s3.listObjects", (done) ->
-			@s3.listObjects.callsArgWith 1, null, Contents: []
-			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err) =>
-				expect(err).to.not.be.ok
-				expect(@s3.listObjects.calledOnce, "called only once").to.be.true
-				expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key),
-					"called with correct arguments").to.be.true
-				done()
-
-		it "should dispatch the error from s3.listObjects", (done) ->
-			@s3.listObjects.callsArgWith 1, @error
-			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err) =>
-				expect(err).to.equal @error
-				done()
-
-		it "should sum directory files sizes", (done) ->
-			@s3.listObjects.callsArgWith 1, null, Contents: [ { Size: 1024 }, { Size: 2048 }]
-			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err, size) =>
-				expect(size).to.equal 3072
-				done()

services/filestore/test/unit/coffee/BucketControllerTests.coffee

@@ -1,71 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/BucketController.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "BucketController", ->
-
-	beforeEach ->
-		@PersistorManager =
-			sendStream: sinon.stub()
-			copyFile: sinon.stub()
-			deleteFile:sinon.stub()
-
-		@settings =
-			s3:
-				buckets:
-					user_files:"user_files"
-			filestore:
-				backend: "s3"
-				s3:
-					secret: "secret"
-					key: "this_key"
-
-		@FileHandler =
-			getFile: sinon.stub()
-			deleteFile: sinon.stub()
-			insertFile: sinon.stub()
-			getDirectorySize: sinon.stub()
-		@LocalFileWriter = {}
-		@controller = SandboxedModule.require modulePath, requires:
-			"./LocalFileWriter":@LocalFileWriter
-			"./FileHandler": @FileHandler
-			"./PersistorManager":@PersistorManager
-			"settings-sharelatex": @settings
-			"metrics-sharelatex": 
-				inc:->
-			"logger-sharelatex":
-				log:->
-				err:->
-		@project_id = "project_id"
-		@file_id = "file_id"
-		@bucket = "user_files"
-		@key = "#{@project_id}/#{@file_id}"
-		@req =
-			query:{}
-			params:
-				bucket: @bucket
-				0: @key
-			headers: {}
-		@res =
-			setHeader: ->
-		@fileStream = {}
-
-	describe "getFile", ->
-
-		it "should pipe the stream", (done)->
-			@FileHandler.getFile.callsArgWith(3, null, @fileStream)
-			@fileStream.pipe = (res)=>
-				res.should.equal @res
-				done()
-			@controller.getFile @req, @res
-
-		it "should send a 500 if there is a problem", (done)->
-			@FileHandler.getFile.callsArgWith(3, "error")
-			@res.send = (code)=>
-				code.should.equal 500
-				done()
-			@controller.getFile @req, @res

services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee

@@ -1,281 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should
-expect = chai.expect
-modulePath = "../../../app/js/FSPersistorManager.js"
-SandboxedModule = require('sandboxed-module')
-fs = require("fs")
-response = require("response")
-
-describe "FSPersistorManagerTests", ->
-
-  beforeEach ->
-    @Fs =
-      rename:sinon.stub()
-      createReadStream:sinon.stub()
-      createWriteStream:sinon.stub()
-      unlink:sinon.stub()
-      rmdir:sinon.stub()
-      exists:sinon.stub()
-      readdir:sinon.stub()
-      open:sinon.stub()
-      openSync:sinon.stub()
-      fstatSync:sinon.stub()
-      closeSync:sinon.stub()
-      stat:sinon.stub()
-    @Rimraf = sinon.stub()
-    @LocalFileWriter =
-      writeStream: sinon.stub()
-      deleteFile: sinon.stub()
-    @requires =
-      "./LocalFileWriter":@LocalFileWriter
-      "fs":@Fs
-      "logger-sharelatex":
-        log:->
-        err:->
-      "response":response
-      "rimraf":@Rimraf
-      "./Errors": @Errors =
-        NotFoundError: sinon.stub()
-    @location = "/tmp"
-    @name1 = "530f2407e7ef165704000007/530f838b46d9a9e859000008"
-    @name1Filtered ="530f2407e7ef165704000007_530f838b46d9a9e859000008"
-    @name2 = "second_file"
-    @error = "error_message"
-    @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires
-
-  describe "sendFile", ->
-    beforeEach ->
-      @Fs.createReadStream = sinon.stub().returns({
-        on: ->
-        pipe: ->
-      })
-
-    it "should copy the file", (done) ->
-      @Fs.createWriteStream =sinon.stub().returns({
-        on: (event, handler) ->
-          process.nextTick(handler) if event is 'finish'
-      })
-      @FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
-        @Fs.createReadStream.calledWith(@name2).should.equal true
-        @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
-        done()
-
-    it "should return an error if the file cannot be stored", (done) ->
-      @Fs.createWriteStream =sinon.stub().returns({
-        on: (event, handler) =>
-         if event is 'error'
-          process.nextTick () =>
-            handler(@error)
-      })
-      @FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
-        @Fs.createReadStream.calledWith(@name2).should.equal true
-        @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
-        err.should.equal @error
-        done()
-
-  describe "sendStream", ->
-    beforeEach ->
-      @FSPersistorManager.sendFile = sinon.stub().callsArgWith(3)
-      @LocalFileWriter.writeStream.callsArgWith(2, null, @name1)
-      @LocalFileWriter.deleteFile.callsArg(1)
-      @SourceStream =
-        on:->
-
-    it "should sent stream to LocalFileWriter", (done)->
-      @FSPersistorManager.sendStream @location, @name1, @SourceStream, =>
-        @LocalFileWriter.writeStream.calledWith(@SourceStream).should.equal true
-        done()
-
-    it "should return the error from LocalFileWriter", (done)->
-      @LocalFileWriter.writeStream.callsArgWith(2, @error)
-      @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=>
-        err.should.equal @error
-        done()
-
-    it "should send the file to the filestore", (done)->
-      @LocalFileWriter.writeStream.callsArgWith(2)
-      @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=>
-        @FSPersistorManager.sendFile.called.should.equal true
-        done()
-
-  describe "getFileStream", ->
-    beforeEach ->
-      @opts = {}
-
-    it "should use correct file location", (done) ->
-      @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res) =>
-      @Fs.open.calledWith("#{@location}/#{@name1Filtered}").should.equal true
-      done()
-
-    describe "with start and end options", ->
-
-      beforeEach ->
-        @fd = 2019
-        @opts_in = {start: 0, end: 8}
-        @opts = {start: 0, end: 8, fd: @fd}
-        @Fs.open.callsArgWith(2, null, @fd)
-
-      it 'should pass the options to createReadStream', (done) ->
-        @FSPersistorManager.getFileStream @location, @name1, @opts_in, (err,res)=>
-        @Fs.createReadStream.calledWith(null, @opts).should.equal true
-        done()
-
-    describe "error conditions", ->
-
-      describe "when the file does not exist", ->
-
-        beforeEach ->
-          @fakeCode = 'ENOENT'
-          err = new Error()
-          err.code = @fakeCode
-          @Fs.open.callsArgWith(2, err, null)
-
-        it "should give a NotFoundError", (done) ->
-          @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=>
-            expect(res).to.equal null
-            expect(err).to.not.equal null
-            expect(err instanceof @Errors.NotFoundError).to.equal true
-            done()
-
-      describe "when some other error happens", ->
-
-        beforeEach ->
-          @fakeCode = 'SOMETHINGHORRIBLE'
-          err = new Error()
-          err.code = @fakeCode
-          @Fs.open.callsArgWith(2, err, null)
-
-        it "should give an Error", (done) ->
-          @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=>
-            expect(res).to.equal null
-            expect(err).to.not.equal null
-            expect(err instanceof Error).to.equal true
-            done()
-
-  describe "getFileSize", ->
-    it "should return the file size", (done) ->
-      expectedFileSize = 75382
-      @Fs.stat.yields(new Error("fs.stat got unexpected arguments"))
-      @Fs.stat.withArgs("#{@location}/#{@name1Filtered}")
-        .yields(null, { size: expectedFileSize })
-
-      @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) =>
-        if err?
-          return done(err)
-        expect(fileSize).to.equal(expectedFileSize)
-        done()
-
-    it "should throw a NotFoundError if the file does not exist", (done) ->
-      error = new Error()
-      error.code = "ENOENT"
-      @Fs.stat.yields(error)
-
-      @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) =>
-        expect(err).to.be.instanceof(@Errors.NotFoundError)
-        done()
-
-    it "should rethrow any other error", (done) ->
-      error = new Error()
-      @Fs.stat.yields(error)
-
-      @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) =>
-        expect(err).to.equal(error)
-        done()
-
-  describe "copyFile", ->
-    beforeEach ->
-      @ReadStream=
-        on:->
-        pipe:sinon.stub()
-      @WriteStream=
-        on:->
-      @Fs.createReadStream.returns(@ReadStream)
-      @Fs.createWriteStream.returns(@WriteStream)
-
-    it "Should open the source for reading", (done) ->
-      @FSPersistorManager.copyFile @location, @name1, @name2, ->
-      @Fs.createReadStream.calledWith("#{@location}/#{@name1Filtered}").should.equal true
-      done()
-
-    it "Should open the target for writing", (done) ->
-      @FSPersistorManager.copyFile @location, @name1, @name2, ->
-      @Fs.createWriteStream.calledWith("#{@location}/#{@name2}").should.equal true
-      done()
-
-    it "Should pipe the source to the target", (done) ->
-      @FSPersistorManager.copyFile @location, @name1, @name2, ->
-      @ReadStream.pipe.calledWith(@WriteStream).should.equal true
-      done()
-
-  describe "deleteFile", ->
-    beforeEach ->
-      @Fs.unlink.callsArgWith(1,@error)
-
-    it "Should call unlink with correct options", (done) ->
-      @FSPersistorManager.deleteFile @location, @name1, (err) =>
-        @Fs.unlink.calledWith("#{@location}/#{@name1Filtered}").should.equal true
-        done()
-
-    it "Should propogate the error", (done) ->
-      @FSPersistorManager.deleteFile @location, @name1, (err) =>
-        err.should.equal @error
-        done()
-
-
-  describe "deleteDirectory", ->
-    beforeEach ->
-      @Rimraf.callsArgWith(1,@error)
-
-    it "Should call rmdir(rimraf) with correct options", (done) ->
-      @FSPersistorManager.deleteDirectory @location, @name1, (err) =>
-        @Rimraf.calledWith("#{@location}/#{@name1Filtered}").should.equal true
-        done()
-
-    it "Should propogate the error", (done) ->
-      @FSPersistorManager.deleteDirectory @location, @name1, (err) =>
-        err.should.equal @error
-        done()
-
-  describe "checkIfFileExists", ->
-    beforeEach ->
-      @Fs.exists.callsArgWith(1,true)
-
-    it "Should call exists with correct options", (done) ->
-      @FSPersistorManager.checkIfFileExists @location, @name1, (exists) =>
-        @Fs.exists.calledWith("#{@location}/#{@name1Filtered}").should.equal true
-        done()
-
-    # fs.exists simply returns false on any error, so...
-    it "should not return an error", (done) ->
-      @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) =>
-        expect(err).to.be.null
-        done()
-
-    it "Should return true for existing files", (done) ->
-      @Fs.exists.callsArgWith(1,true)
-      @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) =>
-        exists.should.be.true
-        done()
-
-    it "Should return false for non-existing files", (done) ->
-      @Fs.exists.callsArgWith(1,false)
-      @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) =>
-        exists.should.be.false
-        done()
-
-  describe "directorySize", ->
-
-    it "should propogate the error", (done) ->
-      @Fs.readdir.callsArgWith(1, @error)
-      @FSPersistorManager.directorySize @location, @name1, (err, totalsize) =>
-        err.should.equal @error
-        done()
-
-    it "should sum directory files size", (done) ->
-      @Fs.readdir.callsArgWith(1, null, [ {'file1'}, {'file2'} ])
-      @Fs.fstatSync.returns({size : 1024})
-      @FSPersistorManager.directorySize @location, @name1, (err, totalsize) =>
-        expect(totalsize).to.equal 2048
-        done()

services/filestore/test/unit/coffee/FileControllerTests.coffee

@@ -1,216 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/FileController.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "FileController", ->
-
-	beforeEach ->
-		@PersistorManager =
-			sendStream: sinon.stub()
-			copyFile: sinon.stub()
-			deleteFile:sinon.stub()
-
-		@settings =
-			s3:
-				buckets:
-					user_files:"user_files"
-		@FileHandler =
-			getFile: sinon.stub()
-			getFileSize: sinon.stub()
-			deleteFile: sinon.stub()
-			insertFile: sinon.stub()
-			getDirectorySize: sinon.stub()
-		@LocalFileWriter = {}
-		@controller = SandboxedModule.require modulePath, requires:
-			"./LocalFileWriter":@LocalFileWriter
-			"./FileHandler": @FileHandler
-			"./PersistorManager":@PersistorManager
-			"./Errors": @Errors =
-				NotFoundError: sinon.stub()
-			"settings-sharelatex": @settings
-			"metrics-sharelatex": 
-				inc:->
-			"logger-sharelatex":
-				log:->
-				err:->
-		@project_id = "project_id"
-		@file_id = "file_id"
-		@bucket = "user_files"
-		@key = "#{@project_id}/#{@file_id}"
-		@req =
-			key:@key
-			bucket:@bucket
-			query:{}
-			params:
-				project_id:@project_id
-				file_id:@file_id
-			headers: {}
-		@res =
-			set: sinon.stub().returnsThis()
-			status: sinon.stub().returnsThis()
-		@fileStream = {}
-
-	describe "getFile", ->
-
-		it "should pipe the stream", (done)->
-			@FileHandler.getFile.callsArgWith(3, null, @fileStream)
-			@fileStream.pipe = (res)=>
-				res.should.equal @res
-				done()
-			@controller.getFile @req, @res
-
-		it "should send a 200 if the cacheWarm param is true", (done)->
-			@req.query.cacheWarm = true
-			@FileHandler.getFile.callsArgWith(3, null, @fileStream)
-			@res.send = (statusCode)=>
-				statusCode.should.equal 200
-				done()
-			@controller.getFile @req, @res
-
-		it "should send a 500 if there is a problem", (done)->
-			@FileHandler.getFile.callsArgWith(3, "error")
-			@res.send = (code)=>
-				code.should.equal 500
-				done()
-			@controller.getFile @req, @res
-
-		describe "with a 'Range' header set", ->
-
-			beforeEach ->
-				@req.headers.range = 'bytes=0-8'
-
-			it "should pass 'start' and 'end' options to FileHandler", (done) ->
-				@FileHandler.getFile.callsArgWith(3, null, @fileStream)
-				@fileStream.pipe = (res)=>
-					expect(@FileHandler.getFile.lastCall.args[2].start).to.equal 0
-					expect(@FileHandler.getFile.lastCall.args[2].end).to.equal 8
-					done()
-				@controller.getFile @req, @res
-
-	describe "getFileHead", ->
-		it "should return the file size in a Content-Length header", (done) ->
-			expectedFileSize = 84921
-			@FileHandler.getFileSize.yields(
-				new Error("FileHandler.getFileSize: unexpected arguments")
-			)
-			@FileHandler.getFileSize.withArgs(@bucket, @key).yields(null, expectedFileSize)
-
-			@res.end = () =>
-				expect(@res.status.lastCall.args[0]).to.equal(200)
-				expect(@res.set.calledWith("Content-Length", expectedFileSize)).to.equal(true)
-				done()
-
-			@controller.getFileHead(@req, @res)
-
-		it "should return a 404 is the file is not found", (done) ->
-			@FileHandler.getFileSize.yields(new @Errors.NotFoundError())
-
-			@res.end = () =>
-				expect(@res.status.lastCall.args[0]).to.equal(404)
-				done()
-
-			@controller.getFileHead(@req, @res)
-
-		it "should return a 500 on internal errors", (done) ->
-			@FileHandler.getFileSize.yields(new Error())
-
-			@res.end = () =>
-				expect(@res.status.lastCall.args[0]).to.equal(500)
-				done()
-
-			@controller.getFileHead(@req, @res)
-
-	describe "insertFile", ->
-
-		it "should send bucket name key and res to PersistorManager", (done)->
-			@FileHandler.insertFile.callsArgWith(3)
-			@res.send = =>
-				@FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true
-				done()
-			@controller.insertFile @req, @res
-
-	describe "copyFile", ->
-		beforeEach ->
-			@oldFile_id = "old_file_id"
-			@oldProject_id = "old_project_id"
-			@req.body =
-				source:
-					project_id: @oldProject_id
-					file_id: @oldFile_id
-
-		it "should send bucket name and both keys to PersistorManager", (done)->
-			@PersistorManager.copyFile.callsArgWith(3)
-			@res.send = (code)=>
-				code.should.equal 200
-				@PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true
-				done()
-			@controller.copyFile @req, @res
-
-		it "should send a 404 if the original file was not found", (done) ->
-			@PersistorManager.copyFile.callsArgWith(3, new @Errors.NotFoundError())
-			@res.send = (code)=>
-				code.should.equal 404
-				done()
-			@controller.copyFile @req, @res
-
-		it "should send a 500 if there was an error", (done)->
-			@PersistorManager.copyFile.callsArgWith(3, "error")
-			@res.send = (code)=>
-				code.should.equal 500
-				done()
-			@controller.copyFile @req, @res
-
-	describe "delete file", ->
-
-		it "should tell the file handler", (done)->
-			@FileHandler.deleteFile.callsArgWith(2)
-			@res.send = (code)=>
-				code.should.equal 204
-				@FileHandler.deleteFile.calledWith(@bucket, @key).should.equal true
-				done()
-			@controller.deleteFile @req, @res
-
-		it "should send a 500 if there was an error", (done)->
-			@FileHandler.deleteFile.callsArgWith(2, "error")
-			@res.send = (code)->
-				code.should.equal 500
-				done()
-			@controller.deleteFile @req, @res
-
-	describe "_get_range", ->
-
-		it "should parse a valid Range header", (done) ->
-			result = @controller._get_range('bytes=0-200')
-			expect(result).to.not.equal null
-			expect(result.start).to.equal 0
-			expect(result.end).to.equal 200
-			done()
-
-		it "should return null for an invalid Range header", (done) ->
-			result = @controller._get_range('wat')
-			expect(result).to.equal null
-			done()
-
-		it "should return null for any type other than 'bytes'", (done) ->
-			result = @controller._get_range('carrots=0-200')
-			expect(result).to.equal null
-			done()
-
-	describe "directorySize", ->
-
-		it "should return total directory size bytes", (done) ->
-			@FileHandler.getDirectorySize.callsArgWith(2, null, 1024)
-			@controller.directorySize @req, json:(result)=>
-				expect(result['total bytes']).to.equal 1024
-				done()
-
-		it "should send a 500 if there was an error", (done)->
-			@FileHandler.getDirectorySize.callsArgWith(2, "error")
-			@res.send = (code)->
-				code.should.equal 500
-				done()
-			@controller.directorySize @req, @res

services/filestore/test/unit/coffee/FileConverterTests.coffee

@@ -1,81 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/FileConverter.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "FileConverter", ->
-
-	beforeEach ->
-
-		@safe_exec = sinon.stub()
-		@converter = SandboxedModule.require modulePath, requires:
-			"./SafeExec": @safe_exec
-			"logger-sharelatex":
-				log:->
-				err:->
-			"metrics-sharelatex": 
-				inc:->
-				Timer:->
-					done:->
-			"settings-sharelatex": @Settings =
-				commands:
-					convertCommandPrefix: []
-
-		@sourcePath = "/this/path/here.eps"
-		@format = "png"
-		@error = "Error"
-
-	describe "convert", ->
-
-		it "should convert the source to the requested format", (done)->
-			@safe_exec.callsArgWith(2)
-			@converter.convert @sourcePath, @format, (err)=>
-				args = @safe_exec.args[0][0]
-				args.indexOf("#{@sourcePath}[0]").should.not.equal -1 
-				args.indexOf("#{@sourcePath}.#{@format}").should.not.equal -1 
-				done()
-
-		it "should return the dest path", (done)->
-			@safe_exec.callsArgWith(2)
-			@converter.convert @sourcePath, @format, (err, destPath)=>
-				destPath.should.equal "#{@sourcePath}.#{@format}"
-				done()
-
-		it "should return the error from convert", (done)->
-			@safe_exec.callsArgWith(2, @error)
-			@converter.convert @sourcePath, @format, (err)=>
-				err.should.equal @error
-				done()
-
-		it "should not accapt an non aproved format", (done)->
-			@safe_exec.callsArgWith(2)
-			@converter.convert @sourcePath, "ahhhhh", (err)=>
-				expect(err).to.exist
-				done()
-		
-		it "should prefix the command with Settings.commands.convertCommandPrefix", (done) ->
-			@safe_exec.callsArgWith(2)
-			@Settings.commands.convertCommandPrefix = ["nice"]
-			@converter.convert @sourcePath, @format, (err)=>
-				command = @safe_exec.args[0][0]
-				command[0].should.equal "nice"
-				done()
-
-	describe "thumbnail", ->
-		it "should call converter resize with args", (done)->
-			@safe_exec.callsArgWith(2)
-			@converter.thumbnail @sourcePath, (err)=>
-				args = @safe_exec.args[0][0]
-				args.indexOf("#{@sourcePath}[0]").should.not.equal -1 
-				done()
-
-	describe "preview", ->
-		it "should call converter resize with args", (done)->
-			@safe_exec.callsArgWith(2)
-			@converter.preview @sourcePath, (err)=>
-				args = @safe_exec.args[0][0]
-				args.indexOf("#{@sourcePath}[0]").should.not.equal -1 
-				done()

services/filestore/test/unit/coffee/FileHandlerTests.coffee

@@ -1,208 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/FileHandler.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "FileHandler", ->
-
-	beforeEach ->
-		@settings =
-			s3:
-				buckets:
-					user_files:"user_files"
-		@PersistorManager =
-			getFileStream: sinon.stub()
-			checkIfFileExists: sinon.stub()
-			deleteFile: sinon.stub()
-			deleteDirectory: sinon.stub()
-			sendStream: sinon.stub()
-			insertFile: sinon.stub()
-			directorySize: sinon.stub()
-		@LocalFileWriter =
-			writeStream: sinon.stub()
-			getStream: sinon.stub()
-			deleteFile: sinon.stub()
-		@FileConverter =
-			convert: sinon.stub()
-			thumbnail: sinon.stub()
-			preview: sinon.stub()
-		@keyBuilder =
-			addCachingToKey: sinon.stub()
-			getConvertedFolderKey: sinon.stub()
-		@ImageOptimiser =
-			compressPng: sinon.stub()
-		@handler = SandboxedModule.require modulePath, requires:
-			"settings-sharelatex": @settings
-			"./PersistorManager":@PersistorManager
-			"./LocalFileWriter":@LocalFileWriter
-			"./FileConverter":@FileConverter
-			"./KeyBuilder": @keyBuilder
-			"./ImageOptimiser":@ImageOptimiser
-			"logger-sharelatex":
-				log:->
-				err:->
-		@bucket = "my_bucket"
-		@key = "key/here"
-		@stubbedPath = "/var/somewhere/path"
-		@format = "png"
-		@formattedStubbedPath = "#{@stubbedPath}.#{@format}"
-
-	describe "insertFile", ->
-		beforeEach ->
-			@stream = {}
-			@PersistorManager.deleteDirectory.callsArgWith(2)
-			@PersistorManager.sendStream.callsArgWith(3)
-
-		it "should send file to the filestore", (done)->
-			@handler.insertFile @bucket, @key, @stream, =>
-				@PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true
-				done()
-
-		it "should delete the convetedKey folder", (done)->
-			@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
-			@handler.insertFile @bucket, @key, @stream, =>
-				@PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
-				done()
-
-	describe "deleteFile", ->
-		beforeEach ->
-			@keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey)
-			@PersistorManager.deleteFile.callsArgWith(2)
-			@PersistorManager.deleteDirectory.callsArgWith(2)
-
-		it "should tell the filestore manager to delete the file", (done)->
-			@handler.deleteFile @bucket, @key, =>
-				@PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true
-				done()
-
-		it "should tell the filestore manager to delete the cached foler", (done)->
-			@handler.deleteFile @bucket, @key, =>
-				@PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true
-				done()
-
-	describe "getFile", ->
-		beforeEach ->
-			@handler._getStandardFile = sinon.stub().callsArgWith(3)
-			@handler._getConvertedFile = sinon.stub().callsArgWith(3)
-
-		it "should call _getStandardFile if no format or style are defined", (done)->
-
-			@handler.getFile @bucket, @key, null, =>
-				@handler._getStandardFile.called.should.equal true
-				@handler._getConvertedFile.called.should.equal false
-				done()
-
-		it "should pass options to _getStandardFile", (done) ->
-			options = {start: 0, end: 8}
-			@handler.getFile @bucket, @key, options, =>
-				expect(@handler._getStandardFile.lastCall.args[2].start).to.equal 0
-				expect(@handler._getStandardFile.lastCall.args[2].end).to.equal 8
-				done()
-
-		it "should call _getConvertedFile if a format is defined", (done)->
-			@handler.getFile @bucket, @key, format:"png", =>
-				@handler._getStandardFile.called.should.equal false
-				@handler._getConvertedFile.called.should.equal true
-				done()
-
-	describe "_getStandardFile", ->
-
-		beforeEach ->
-			@fileStream = {on:->}
-			@PersistorManager.getFileStream.callsArgWith(3, "err", @fileStream)
-
-		it "should get the stream", (done)->
-			@handler.getFile @bucket, @key, null, =>
-				@PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true
-				done()
-
-		it "should return the stream and error", (done)->
-			@handler.getFile @bucket, @key, null, (err, stream)=>
-				err.should.equal "err"
-				stream.should.equal @fileStream
-				done()
-
-		it "should pass options to PersistorManager", (done) ->
-			@handler.getFile @bucket, @key, {start: 0, end: 8}, =>
-				expect(@PersistorManager.getFileStream.lastCall.args[2].start).to.equal 0
-				expect(@PersistorManager.getFileStream.lastCall.args[2].end).to.equal 8
-				done()
-
-
-	describe "_getConvertedFile", ->
-
-		it "should getFileStream if it does exists", (done)->
-			@PersistorManager.checkIfFileExists.callsArgWith(2, null, true)
-			@PersistorManager.getFileStream.callsArgWith(3)
-			@handler._getConvertedFile @bucket, @key, {}, =>
-				@PersistorManager.getFileStream.calledWith(@bucket).should.equal true
-				done()
-
-		it "should call _getConvertedFileAndCache if it does exists", (done)->
-			@PersistorManager.checkIfFileExists.callsArgWith(2, null, false)
-			@handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4)
-			@handler._getConvertedFile @bucket, @key, {}, =>
-				@handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true
-				done()
-
-	describe "_getConvertedFileAndCache", ->
-
-		it "should _convertFile ", (done)->
-			@stubbedStream = {"something":"here"}
-			@localStream = {
-				on: ->
-			}
-			@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
-			@LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream)
-			@convetedKey = @key+"converted"
-			@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
-			@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
-			@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=>
-				@handler._convertFile.called.should.equal true
-				@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
-				@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
-				@LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true
-				fsStream.should.equal @localStream
-				done()
-
-	describe "_convertFile", ->
-		beforeEach ->
-			@FileConverter.convert.callsArgWith(2, null, @formattedStubbedPath)
-			@FileConverter.thumbnail.callsArgWith(1, null, @formattedStubbedPath)
-			@FileConverter.preview.callsArgWith(1, null, @formattedStubbedPath)
-			@handler._writeS3FileToDisk = sinon.stub().callsArgWith(3, null, @stubbedPath)
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-
-		it "should call thumbnail on the writer path if style was thumbnail was specified", (done)->
-			@handler._convertFile @bucket, @key, style:"thumbnail", (err, path)=>
-				path.should.equal @formattedStubbedPath
-				@FileConverter.thumbnail.calledWith(@stubbedPath).should.equal true
-				@LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true
-				done()
-
-		it "should call preview on the writer path if style was preview was specified", (done)->
-			@handler._convertFile @bucket, @key, style:"preview", (err, path)=>
-				path.should.equal @formattedStubbedPath
-				@FileConverter.preview.calledWith(@stubbedPath).should.equal true
-				@LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true
-				done()
-
-		it "should call convert on the writer path if a format was specified", (done)->
-			@handler._convertFile @bucket, @key, format:@format, (err, path)=>
-				path.should.equal @formattedStubbedPath
-				@FileConverter.convert.calledWith(@stubbedPath, @format).should.equal true
-				@LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true
-				done()
-
-	describe "getDirectorySize", ->
-
-		beforeEach ->
-			@PersistorManager.directorySize.callsArgWith(2)
-
-		it "should call the filestore manager to get directory size", (done)->
-			@handler.getDirectorySize @bucket, @key, =>
-				@PersistorManager.directorySize.calledWith(@bucket, @key).should.equal true
-				done()

services/filestore/test/unit/coffee/ImageOptimiserTests.coffee

@@ -1,64 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/ImageOptimiser.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "ImageOptimiser", ->
-
-	beforeEach ->
-		@child_process =
-			exec : sinon.stub()
-		@settings = 
-			enableConversions:true
-		@optimiser = SandboxedModule.require modulePath, requires:
-			'child_process': @child_process
-			"logger-sharelatex":
-				log:->
-				err:->
-				warn:->
-			"settings-sharelatex": @settings
-				
-
-		@sourcePath = "/this/path/here.eps"
-		@error = "Error"
-
-	describe "compressPng", ->
-		
-
-		it "convert the file", (done)->
-			@child_process.exec.callsArgWith(2)
-			@optimiser.compressPng @sourcePath, (err)=>
-				args = @child_process.exec.args[0][0]
-				args.should.equal "optipng #{@sourcePath}"
-				done()
-
-
-		it "should return the error", (done)->
-			@child_process.exec.callsArgWith(2, @error)
-			@optimiser.compressPng @sourcePath, (err)=>
-				err.should.equal @error
-				done()
-
-	describe 'when enableConversions is disabled', ->
-
-		it 'should produce an error', (done) ->
-			@settings.enableConversions = false
-			@child_process.exec.callsArgWith(2)
-			@optimiser.compressPng @sourcePath, (err)=>
-				@child_process.exec.called.should.equal false
-				expect(err).to.exist
-				done()
-
-
-	describe 'when optimiser is sigkilled', ->
-
-		it 'should not produce an error', (done) ->
-			@error = new Error('woops')
-			@error.signal = 'SIGKILL'
-			@child_process.exec.callsArgWith(2, @error)
-			@optimiser.compressPng @sourcePath, (err)=>
-				expect(err).to.equal(null)
-				done()

services/filestore/test/unit/coffee/KeybuilderTests.coffee

@@ -1,39 +0,0 @@
-
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/KeyBuilder.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "LocalFileWriter", ->
-
-	beforeEach ->
-
-		@keyBuilder = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex":
-				log:->
-				err:->
-		@key = "123/456"
-		
-	describe "cachedKey", ->
-
-		it "should add the fomat on", ->
-			opts =
-				format: "png"
-			newKey = @keyBuilder.addCachingToKey @key, opts
-			newKey.should.equal "#{@key}-converted-cache/format-png"
-
-		it "should add the style on", ->
-			opts =
-				style: "thumbnail"
-			newKey = @keyBuilder.addCachingToKey @key, opts
-			newKey.should.equal "#{@key}-converted-cache/style-thumbnail"
-
-		it "should add format on first", ->
-			opts =
-				style: "thumbnail"
-				format: "png"
-			newKey = @keyBuilder.addCachingToKey @key, opts
-			newKey.should.equal "#{@key}-converted-cache/format-png-style-thumbnail"

services/filestore/test/unit/coffee/LocalFileWriterTests.coffee

@@ -1,83 +0,0 @@
-
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/LocalFileWriter.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "LocalFileWriter", ->
-
-	beforeEach ->
-
-		@writeStream = 
-			on: (type, cb)->
-				if type == "finish"
-					cb()
-		@readStream = 
-			on: ->
-		@fs = 
-			createWriteStream : sinon.stub().returns(@writeStream)
-			createReadStream: sinon.stub().returns(@readStream)
-			unlink: sinon.stub()
-		@settings =
-			path:
-				uploadFolder:"somewhere"
-		@writer = SandboxedModule.require modulePath, requires:
-			"fs": @fs
-			"logger-sharelatex":
-				log:->
-				err:->
-			"settings-sharelatex":@settings
-			"metrics-sharelatex": 
-				inc:->
-				Timer:->
-					done:->
-
-		@stubbedFsPath = "something/uploads/eio2k1j3"
-
-	describe "writeStrem", ->
-		beforeEach ->
-			@writer._getPath = sinon.stub().returns(@stubbedFsPath)
-
-		it "write the stream to ./uploads", (done)->
-			stream = 
-				pipe: (dest)=>
-					dest.should.equal @writeStream
-					done()
-				on: ->
-			@writer.writeStream stream, null, ()=>
-
-		it "should send the path in the callback", (done)->
-			stream = 
-				pipe: (dest)=>
-				on: (type, cb)->
-					if type == "end"
-						cb()
-			@writer.writeStream stream, null, (err, fsPath)=>
-				fsPath.should.equal @stubbedFsPath
-				done()
-
-	describe "getStream", ->
-
-		it "should read the stream from the file ", (done)->
-			@writer.getStream @stubbedFsPath, (err, stream)=>
-				@fs.createReadStream.calledWith(@stubbedFsPath).should.equal true
-				done()
-
-		it "should send the stream in the callback", (done)->
-			@writer.getStream @stubbedFsPath, (err, readStream)=>
-				readStream.should.equal @readStream
-				done()
-
-	describe "delete file", ->
-
-		it "should unlink the file", (done)->
-			error = "my error"
-			@fs.unlink.callsArgWith(1, error)
-			@writer.deleteFile @stubbedFsPath, (err)=>
-				@fs.unlink.calledWith(@stubbedFsPath).should.equal true
-				err.should.equal error
-				done()
-

services/filestore/test/unit/coffee/PersistorManagerTests.coffee

@@ -1,101 +0,0 @@
-logger = require("logger-sharelatex")
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/PersistorManager.js"
-SandboxedModule = require('sandboxed-module')
-
-
-describe "PersistorManagerTests", ->
-
-	beforeEach ->
-		@S3PersistorManager =
-			getFileStream: sinon.stub()
-			checkIfFileExists: sinon.stub()
-			deleteFile: sinon.stub()
-			deleteDirectory: sinon.stub()
-			sendStream: sinon.stub()
-			insertFile: sinon.stub()
-
-	describe "test s3 mixin", ->
-		beforeEach ->
-			@settings =
-				filestore:
-					backend: "s3"
-			@requires =
-				"./S3PersistorManager": @S3PersistorManager
-				"settings-sharelatex": @settings
-				"logger-sharelatex":
-					log:->
-					err:->
-			@PersistorManager = SandboxedModule.require modulePath, requires: @requires
-
-		it "should load getFileStream", (done) ->
-			@PersistorManager.should.respondTo("getFileStream")
-			@PersistorManager.getFileStream()
-			@S3PersistorManager.getFileStream.calledOnce.should.equal true
-			done()
-
-		it "should load checkIfFileExists", (done) ->
-			@PersistorManager.checkIfFileExists()
-			@S3PersistorManager.checkIfFileExists.calledOnce.should.equal true
-			done()
-
-		it "should load deleteFile", (done) ->
-			@PersistorManager.deleteFile()
-			@S3PersistorManager.deleteFile.calledOnce.should.equal true
-			done()
-
-		it "should load deleteDirectory", (done) ->
-			@PersistorManager.deleteDirectory()
-			@S3PersistorManager.deleteDirectory.calledOnce.should.equal true
-			done()
-
-		it "should load sendStream", (done) ->
-			@PersistorManager.sendStream()
-			@S3PersistorManager.sendStream.calledOnce.should.equal true
-			done()
-
-		it "should load insertFile", (done) ->
-			@PersistorManager.insertFile()
-			@S3PersistorManager.insertFile.calledOnce.should.equal true
-			done()
-
-	describe "test unspecified mixins", ->
-
-		it "should load s3 when no wrapper specified", (done) ->
-			@settings = {filestore:{}}
-			@requires =
-				"./S3PersistorManager": @S3PersistorManager
-				"settings-sharelatex": @settings
-				"logger-sharelatex":
-					log:->
-					err:->
-			@PersistorManager = SandboxedModule.require modulePath, requires: @requires
-			@PersistorManager.should.respondTo("getFileStream")
-			@PersistorManager.getFileStream()
-			@S3PersistorManager.getFileStream.calledOnce.should.equal true
-			done()
-
-	describe "test invalid mixins", ->
-		it "should not load an invalid wrapper", (done) ->
-			@settings =
-				filestore:
-					backend:"magic"
-			@requires =
-				"./S3PersistorManager": @S3PersistorManager
-				"settings-sharelatex": @settings
-				"logger-sharelatex":
-					log:->
-					err:->
-			@fsWrapper=null
-			try
-				@PersistorManager=SandboxedModule.require modulePath, requires: @requires
-			catch error
-				assert.equal("Unknown filestore backend: magic",error.message)
-			assert.isNull(@fsWrapper)
-			done()
-
-

services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee

@@ -1,343 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/S3PersistorManager.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "S3PersistorManagerTests", ->
-
-	beforeEach ->
-		@settings =
-			filestore:
-				backend: "s3"
-				s3:
-					secret: "secret"
-					key: "this_key"
-				stores:
-					user_files:"sl_user_files"
-		@knoxClient =
-			putFile:sinon.stub()
-			copyFile:sinon.stub()
-			list: sinon.stub()
-			deleteMultiple: sinon.stub()
-			get: sinon.stub()
-		@knox =
-			createClient: sinon.stub().returns(@knoxClient)
-		@s3EventHandlers = {}
-		@s3Request =
-			on: sinon.stub().callsFake (event, callback) =>
-				@s3EventHandlers[event] = callback
-			send: sinon.stub()
-		@s3Response =
-			httpResponse:
-				createUnbufferedStream: sinon.stub()
-		@s3Client =
-			copyObject: sinon.stub()
-			headObject: sinon.stub()
-			getObject: sinon.stub().returns(@s3Request)
-		@awsS3 = sinon.stub().returns(@s3Client)
-		@LocalFileWriter =
-			writeStream: sinon.stub()
-			deleteFile: sinon.stub()
-		@request = sinon.stub()
-		@requires =
-			"knox": @knox
-			"aws-sdk/clients/s3": @awsS3
-			"settings-sharelatex": @settings
-			"./LocalFileWriter":@LocalFileWriter
-			"logger-sharelatex":
-				log:->
-				err:->
-			"request": @request
-			"./Errors": @Errors =
-				NotFoundError: sinon.stub()
-		@key = "my/key"
-		@bucketName = "my-bucket"
-		@error = "my errror"
-		@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
-
-	describe "getFileStream", ->
-		describe "success", ->
-			beforeEach () ->
-				@expectedStream = { expectedStream: true }
-				@expectedStream.on = sinon.stub()
-				@s3Request.send.callsFake () =>
-					@s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK")
-				@s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream)
-
-			it "returns a stream", (done) ->
-				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-					if err?
-						return done(err)
-					expect(stream).to.equal(@expectedStream)
-					done()
-
-			it "sets the AWS client up with credentials from settings", (done) ->
-				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-					if err?
-						return done(err)
-					expect(@awsS3.lastCall.args).to.deep.equal([{
-						credentials:
-							accessKeyId: @settings.filestore.s3.key
-							secretAccessKey: @settings.filestore.s3.secret
-					}])
-					done()
-
-			it "fetches the right key from the right bucket", (done) ->
-				@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-					if err?
-						return done(err)
-					expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
-						Bucket: @bucketName,
-						Key: @key
-					}])
-					done()
-
-			it "accepts alternative credentials", (done) ->
-				accessKeyId = "that_key"
-				secret = "that_secret"
-				opts = {
-					credentials:
-						auth_key: accessKeyId
-						auth_secret: secret
-				}
-				@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
-					if err?
-						return done(err)
-					expect(@awsS3.lastCall.args).to.deep.equal([{
-						credentials:
-							accessKeyId: accessKeyId
-							secretAccessKey: secret
-					}])
-					expect(stream).to.equal(@expectedStream)
-					done()
-
-			it "accepts byte range", (done) ->
-				start = 0
-				end = 8
-				opts = { start: start, end: end }
-				@S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) =>
-					if err?
-						return done(err)
-					expect(@s3Client.getObject.lastCall.args).to.deep.equal([{
-						Bucket: @bucketName
-						Key: @key
-						Range: "bytes=#{start}-#{end}"
-					}])
-					expect(stream).to.equal(@expectedStream)
-					done()
-
-		describe "errors", ->
-			describe "when the file doesn't exist", ->
-				beforeEach ->
-					@s3Request.send.callsFake () =>
-						@s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found")
-
-				it "returns a NotFoundError that indicates the bucket and key", (done) ->
-					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-						expect(err).to.be.instanceof(@Errors.NotFoundError)
-						errMsg = @Errors.NotFoundError.lastCall.args[0]
-						expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*"))
-						expect(errMsg).to.match(new RegExp(".*#{@key}.*"))
-						done()
-
-			describe "when S3 encounters an unkown error", ->
-				beforeEach ->
-					@s3Request.send.callsFake () =>
-						@s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error")
-
-				it "returns an error", (done) ->
-					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-						expect(err).to.be.instanceof(Error)
-						done()
-
-			describe "when the S3 request errors out before receiving HTTP headers", ->
-				beforeEach ->
-					@s3Request.send.callsFake () =>
-						@s3EventHandlers.error(new Error("connection failed"))
-
-				it "returns an error", (done) ->
-					@S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) =>
-						expect(err).to.be.instanceof(Error)
-						done()
-
-	describe "getFileSize", ->
-		it "should obtain the file size from S3", (done) ->
-			expectedFileSize = 123
-			@s3Client.headObject.yields(new Error(
-				"s3Client.headObject got unexpected arguments"
-			))
-			@s3Client.headObject.withArgs({
-				Bucket: @bucketName
-				Key: @key
-			}).yields(null, { ContentLength: expectedFileSize })
-
-			@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
-				if err?
-					return done(err)
-				expect(fileSize).to.equal(expectedFileSize)
-				done()
-
-		[403, 404].forEach (statusCode) ->
-			it "should throw NotFoundError when S3 responds with #{statusCode}", (done) ->
-				error = new Error()
-				error.statusCode = statusCode
-				@s3Client.headObject.yields(error)
-
-				@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
-					expect(err).to.be.an.instanceof(@Errors.NotFoundError)
-					done()
-
-		it "should rethrow any other error", (done) ->
-			error = new Error()
-			@s3Client.headObject.yields(error)
-			@s3Client.headObject.yields(error)
-
-			@S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) =>
-				expect(err).to.equal(error)
-				done()
-
-	describe "sendFile", ->
-
-		beforeEach ->
-			@knoxClient.putFile.returns on:->
-
-		it "should put file with knox", (done)->
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-			@knoxClient.putFile.callsArgWith(2, @error)
-			@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
-				@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
-				err.should.equal @error
-				done()
-
-		it "should delete the file and pass the error with it", (done)->
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-			@knoxClient.putFile.callsArgWith(2, @error)
-			@S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=>
-				@knoxClient.putFile.calledWith(@fsPath, @key).should.equal true
-				err.should.equal @error
-				done()
-
-	describe "sendStream", ->
-		beforeEach ->
-			@fsPath = "to/some/where"
-			@origin =
-				on:->
-			@S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)
-
-		it "should send stream to LocalFileWriter", (done)->
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-			@LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath)
-			@S3PersistorManager.sendStream @bucketName, @key, @origin, =>
-				@LocalFileWriter.writeStream.calledWith(@origin).should.equal true
-				done()
-
-		it "should return the error from LocalFileWriter", (done)->
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-			@LocalFileWriter.writeStream.callsArgWith(2, @error)
-			@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
-				err.should.equal @error
-				done()
-
-		it "should send the file to the filestore", (done)->
-			@LocalFileWriter.deleteFile.callsArgWith(1)
-			@LocalFileWriter.writeStream.callsArgWith(2)
-			@S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=>
-				@S3PersistorManager.sendFile.called.should.equal true
-				done()
-
-	describe "copyFile", ->
-		beforeEach ->
-			@sourceKey = "my/key"
-			@destKey = "my/dest/key"
-
-		it "should use AWS SDK to copy file", (done)->
-			@s3Client.copyObject.callsArgWith(1, @error)
-			@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
-				err.should.equal @error
-				@s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true
-				done()
-
-		it "should return a NotFoundError object if the original file does not exist", (done)->
-			NoSuchKeyError = {code: "NoSuchKey"}
-			@s3Client.copyObject.callsArgWith(1, NoSuchKeyError)
-			@S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=>
-				expect(err instanceof @Errors.NotFoundError).to.equal true
-				done()
-
-	describe "deleteDirectory", ->
-
-		it "should list the contents passing them onto multi delete", (done)->
-			data =
-				Contents: [{Key:"1234"}, {Key: "456"}]
-			@knoxClient.list.callsArgWith(1, null, data)
-			@knoxClient.deleteMultiple.callsArgWith(1)
-			@S3PersistorManager.deleteDirectory @bucketName, @key, (err)=>
-				@knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true
-				done()
-
-	describe "deleteFile", ->
-
-		it "should use correct options", (done)->
-			@request.callsArgWith(1)
-
-			@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
-				opts = @request.args[0][0]
-				assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
-				opts.method.should.equal "delete"
-				opts.timeout.should.equal (30*1000)
-				opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
-				done()
-
-		it "should return the error", (done)->
-			@request.callsArgWith(1, @error)
-
-			@S3PersistorManager.deleteFile @bucketName, @key, (err)=>
-				err.should.equal @error
-				done()
-
-	describe "checkIfFileExists", ->
-
-		it "should use correct options", (done)->
-			@request.callsArgWith(1,  null, statusCode:200)
-
-			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
-				opts = @request.args[0][0]
-				assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName})
-				opts.method.should.equal "head"
-				opts.timeout.should.equal (30*1000)
-				opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}"
-				done()
-
-		it "should return true for a 200", (done)->
-			@request.callsArgWith(1, null, statusCode:200)
-
-			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
-				exists.should.equal true
-				done()
-
-		it "should return false for a non 200", (done)->
-			@request.callsArgWith(1, null, statusCode:404)
-
-			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=>
-				exists.should.equal false
-				done()
-
-		it "should return the error", (done)->
-			@request.callsArgWith(1, @error, {})
-
-			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
-				err.should.equal @error
-				done()
-
-	describe "directorySize", ->
-
-		it "should sum directory files size", (done) ->
-			data =
-				Contents: [ {Size: 1024}, {Size: 2048} ]
-			@knoxClient.list.callsArgWith(1, null, data)
-			@S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=>
-				totalSize.should.equal 3072
-				done()

services/filestore/test/unit/coffee/SafeExecTests.coffee

@@ -1,50 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/SafeExec.js"
-SandboxedModule = require('sandboxed-module')
-
-describe "SafeExec", ->
-
-	beforeEach ->
-		@settings = 
-			enableConversions:true
-		@safe_exec = SandboxedModule.require modulePath, requires:
-			"logger-sharelatex":
-				log:->
-				err:->
-			"settings-sharelatex": @settings
-		@options = {timeout: 10*1000, killSignal: "SIGTERM" }
-
-	describe "safe_exec", ->
-
-		it "should execute a valid command", (done) ->
-			@safe_exec ["/bin/echo", "hello"], @options, (err, stdout, stderr) =>
-				stdout.should.equal "hello\n"
-				should.not.exist(err)
-				done()
-
-		it "should error when conversions are disabled", (done) ->
-			@settings.enableConversions = false
-			@safe_exec ["/bin/echo", "hello"], @options, (err, stdout, stderr) =>
-				expect(err).to.exist
-				done()
-
-		it "should execute a command with non-zero exit status", (done) ->
-			@safe_exec ["/usr/bin/env", "false"], @options, (err, stdout, stderr) =>
-				stdout.should.equal ""
-				stderr.should.equal ""
-				err.message.should.equal "exit status 1"
-				done()
-
-		it "should handle an invalid command", (done) ->
-			@safe_exec ["/bin/foobar"], @options, (err, stdout, stderr) =>
-				err.code.should.equal "ENOENT"
-				done()
-
-		it "should handle a command that runs too long", (done) ->
-			@safe_exec ["/bin/sleep", "10"], {timeout: 500, killSignal: "SIGTERM"}, (err, stdout, stderr) =>
-				err.should.equal "SIGTERM"
-				done()

services/filestore/test/unit/coffee/SettingsTests.coffee

@@ -1,19 +0,0 @@
-assert = require("chai").assert
-sinon = require('sinon')
-chai = require('chai')
-should = chai.should()
-expect = chai.expect
-modulePath = "../../../app/js/BucketController.js"
-
-describe "Settings", ->
-	describe "s3", ->
-		it "should use JSONified env var if present", (done)->
-			s3_settings =
-				bucket1:
-					auth_key: 'bucket1_key'
-					auth_secret: 'bucket1_secret'
-			process.env['S3_BUCKET_CREDENTIALS'] = JSON.stringify s3_settings
-
-			settings = require("settings-sharelatex")
-			expect(settings.filestore.s3BucketCreds).to.deep.equal s3_settings
-			done()

services/filestore/test/unit/js/FSPersistorManagerTests.js

@@ -0,0 +1,328 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const SandboxedModule = require('sandboxed-module')
+const Errors = require('../../../app/js/Errors')
+
+chai.use(require('sinon-chai'))
+chai.use(require('chai-as-promised'))
+
+const modulePath = '../../../app/js/FSPersistorManager.js'
+
+describe('FSPersistorManagerTests', function() {
+  const stat = { size: 4, isFile: sinon.stub().returns(true) }
+  const fd = 1234
+  const readStream = 'readStream'
+  const writeStream = 'writeStream'
+  const remoteStream = 'remoteStream'
+  const tempFile = '/tmp/potato.txt'
+  const location = '/foo'
+  const error = new Error('guru meditation error')
+
+  const files = ['animals/wombat.tex', 'vegetables/potato.tex']
+  const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`]
+  const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex']
+  let fs, rimraf, stream, LocalFileWriter, FSPersistorManager, glob
+
+  beforeEach(function() {
+    fs = {
+      createReadStream: sinon.stub().returns(readStream),
+      createWriteStream: sinon.stub().returns(writeStream),
+      unlink: sinon.stub().yields(),
+      open: sinon.stub().yields(null, fd),
+      stat: sinon.stub().yields(null, stat)
+    }
+    glob = sinon.stub().yields(null, globs)
+    rimraf = sinon.stub().yields()
+    stream = { pipeline: sinon.stub().yields() }
+    LocalFileWriter = {
+      promises: {
+        writeStream: sinon.stub().resolves(tempFile),
+        deleteFile: sinon.stub().resolves()
+      }
+    }
+    FSPersistorManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './LocalFileWriter': LocalFileWriter,
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        },
+        './Errors': Errors,
+        fs,
+        glob,
+        rimraf,
+        stream
+      },
+      globals: { console }
+    })
+  })
+
+  describe('sendFile', function() {
+    const localFilesystemPath = '/path/to/local/file'
+    it('should copy the file', async function() {
+      await FSPersistorManager.promises.sendFile(
+        location,
+        files[0],
+        localFilesystemPath
+      )
+      expect(fs.createReadStream).to.have.been.calledWith(localFilesystemPath)
+      expect(fs.createWriteStream).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+      expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
+    })
+
+    it('should return an error if the file cannot be stored', async function() {
+      stream.pipeline.yields(error)
+      await expect(
+        FSPersistorManager.promises.sendFile(
+          location,
+          files[0],
+          localFilesystemPath
+        )
+      ).to.eventually.be.rejected.and.have.property('cause', error)
+    })
+  })
+
+  describe('sendStream', function() {
+    it('should send the stream to LocalFileWriter', async function() {
+      await FSPersistorManager.promises.sendStream(
+        location,
+        files[0],
+        remoteStream
+      )
+      expect(LocalFileWriter.promises.writeStream).to.have.been.calledWith(
+        remoteStream
+      )
+    })
+
+    it('should delete the temporary file', async function() {
+      await FSPersistorManager.promises.sendStream(
+        location,
+        files[0],
+        remoteStream
+      )
+      expect(LocalFileWriter.promises.deleteFile).to.have.been.calledWith(
+        tempFile
+      )
+    })
+
+    it('should return the error from LocalFileWriter', async function() {
+      LocalFileWriter.promises.writeStream.rejects(error)
+      await expect(
+        FSPersistorManager.promises.sendStream(location, files[0], remoteStream)
+      ).to.eventually.be.rejectedWith(error)
+    })
+
+    it('should send the temporary file to the filestore', async function() {
+      await FSPersistorManager.promises.sendStream(
+        location,
+        files[0],
+        remoteStream
+      )
+      expect(fs.createReadStream).to.have.been.calledWith(tempFile)
+    })
+  })
+
+  describe('getFileStream', function() {
+    it('should use correct file location', async function() {
+      await FSPersistorManager.promises.getFileStream(location, files[0], {})
+      expect(fs.open).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+    })
+
+    it('should pass the options to createReadStream', async function() {
+      await FSPersistorManager.promises.getFileStream(location, files[0], {
+        start: 0,
+        end: 8
+      })
+      expect(fs.createReadStream).to.have.been.calledWith(null, {
+        start: 0,
+        end: 8,
+        fd
+      })
+    })
+
+    it('should give a NotFoundError if the file does not exist', async function() {
+      const err = new Error()
+      err.code = 'ENOENT'
+      fs.open.yields(err)
+
+      await expect(
+        FSPersistorManager.promises.getFileStream(location, files[0], {})
+      )
+        .to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
+        .and.have.property('cause', err)
+    })
+
+    it('should wrap any other error', async function() {
+      fs.open.yields(error)
+      await expect(
+        FSPersistorManager.promises.getFileStream(location, files[0], {})
+      )
+        .to.eventually.be.rejectedWith('failed to open file for streaming')
+        .and.be.an.instanceOf(Errors.ReadError)
+        .and.have.property('cause', error)
+    })
+  })
+
+  describe('getFileSize', function() {
+    const badFilename = 'neenaw.tex'
+    const size = 65536
+    const noentError = new Error('not found')
+    noentError.code = 'ENOENT'
+
+    beforeEach(function() {
+      fs.stat
+        .yields(error)
+        .withArgs(`${location}/${filteredFilenames[0]}`)
+        .yields(null, { size })
+        .withArgs(`${location}/${badFilename}`)
+        .yields(noentError)
+    })
+
+    it('should return the file size', async function() {
+      expect(
+        await FSPersistorManager.promises.getFileSize(location, files[0])
+      ).to.equal(size)
+    })
+
+    it('should throw a NotFoundError if the file does not exist', async function() {
+      await expect(
+        FSPersistorManager.promises.getFileSize(location, badFilename)
+      ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
+    })
+
+    it('should wrap any other error', async function() {
+      await expect(FSPersistorManager.promises.getFileSize(location, 'raccoon'))
+        .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
+        .and.have.property('cause', error)
+    })
+  })
+
+  describe('copyFile', function() {
+    it('Should open the source for reading', async function() {
+      await FSPersistorManager.promises.copyFile(location, files[0], files[1])
+      expect(fs.createReadStream).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+    })
+
+    it('Should open the target for writing', async function() {
+      await FSPersistorManager.promises.copyFile(location, files[0], files[1])
+      expect(fs.createWriteStream).to.have.been.calledWith(
+        `${location}/${filteredFilenames[1]}`
+      )
+    })
+
+    it('Should pipe the source to the target', async function() {
+      await FSPersistorManager.promises.copyFile(location, files[0], files[1])
+      expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
+    })
+  })
+
+  describe('deleteFile', function() {
+    it('Should call unlink with correct options', async function() {
+      await FSPersistorManager.promises.deleteFile(location, files[0])
+      expect(fs.unlink).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+    })
+
+    it('Should propagate the error', async function() {
+      fs.unlink.yields(error)
+      await expect(
+        FSPersistorManager.promises.deleteFile(location, files[0])
+      ).to.eventually.be.rejected.and.have.property('cause', error)
+    })
+  })
+
+  describe('deleteDirectory', function() {
+    it('Should call rmdir(rimraf) with correct options', async function() {
+      await FSPersistorManager.promises.deleteDirectory(location, files[0])
+      expect(rimraf).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+    })
+
+    it('Should propagate the error', async function() {
+      rimraf.yields(error)
+      await expect(
+        FSPersistorManager.promises.deleteDirectory(location, files[0])
+      ).to.eventually.be.rejected.and.have.property('cause', error)
+    })
+  })
+
+  describe('checkIfFileExists', function() {
+    const badFilename = 'pototo'
+    const noentError = new Error('not found')
+    noentError.code = 'ENOENT'
+
+    beforeEach(function() {
+      fs.stat
+        .yields(error)
+        .withArgs(`${location}/${filteredFilenames[0]}`)
+        .yields(null, {})
+        .withArgs(`${location}/${badFilename}`)
+        .yields(noentError)
+    })
+
+    it('Should call stat with correct options', async function() {
+      await FSPersistorManager.promises.checkIfFileExists(location, files[0])
+      expect(fs.stat).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}`
+      )
+    })
+
+    it('Should return true for existing files', async function() {
+      expect(
+        await FSPersistorManager.promises.checkIfFileExists(location, files[0])
+      ).to.equal(true)
+    })
+
+    it('Should return false for non-existing files', async function() {
+      expect(
+        await FSPersistorManager.promises.checkIfFileExists(
+          location,
+          badFilename
+        )
+      ).to.equal(false)
+    })
+
+    it('should wrap the error if there is a problem', async function() {
+      await expect(
+        FSPersistorManager.promises.checkIfFileExists(location, 'llama')
+      )
+        .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
+        .and.have.property('cause', error)
+    })
+  })
+
+  describe('directorySize', function() {
+    it('should wrap the error', async function() {
+      glob.yields(error)
+      await expect(
+        FSPersistorManager.promises.directorySize(location, files[0])
+      )
+        .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
+        .and.include({ cause: error })
+        .and.have.property('info')
+        .which.includes({ location, name: files[0] })
+    })
+
+    it('should filter the directory name', async function() {
+      await FSPersistorManager.promises.directorySize(location, files[0])
+      expect(glob).to.have.been.calledWith(
+        `${location}/${filteredFilenames[0]}_*`
+      )
+    })
+
+    it('should sum directory files size', async function() {
+      expect(
+        await FSPersistorManager.promises.directorySize(location, files[0])
+      ).to.equal(stat.size * files.length)
+    })
+  })
+})

services/filestore/test/unit/js/FileControllerTests.js

@@ -0,0 +1,288 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const SandboxedModule = require('sandboxed-module')
+const Errors = require('../../../app/js/Errors')
+const modulePath = '../../../app/js/FileController.js'
+
+describe('FileController', function() {
+  let PersistorManager,
+    FileHandler,
+    LocalFileWriter,
+    FileController,
+    req,
+    res,
+    stream
+  const settings = {
+    s3: {
+      buckets: {
+        user_files: 'user_files'
+      }
+    }
+  }
+  const fileSize = 1234
+  const fileStream = 'fileStream'
+  const projectId = 'projectId'
+  const fileId = 'file_id'
+  const bucket = 'user_files'
+  const key = `${projectId}/${fileId}`
+
+  beforeEach(function() {
+    PersistorManager = {
+      sendStream: sinon.stub().yields(),
+      copyFile: sinon.stub().yields(),
+      deleteFile: sinon.stub().yields()
+    }
+
+    FileHandler = {
+      getFile: sinon.stub().yields(null, fileStream),
+      getFileSize: sinon.stub().yields(null, fileSize),
+      deleteFile: sinon.stub().yields(),
+      insertFile: sinon.stub().yields(),
+      getDirectorySize: sinon.stub().yields(null, fileSize)
+    }
+
+    LocalFileWriter = {}
+    stream = {
+      pipeline: sinon.stub()
+    }
+
+    FileController = SandboxedModule.require(modulePath, {
+      requires: {
+        './LocalFileWriter': LocalFileWriter,
+        './FileHandler': FileHandler,
+        './PersistorManager': PersistorManager,
+        './Errors': Errors,
+        stream: stream,
+        'settings-sharelatex': settings,
+        'metrics-sharelatex': {
+          inc() {}
+        },
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        }
+      },
+      globals: { console }
+    })
+
+    req = {
+      key: key,
+      bucket: bucket,
+      query: {},
+      params: {
+        project_id: projectId,
+        file_id: fileId
+      },
+      headers: {}
+    }
+
+    res = {
+      set: sinon.stub().returnsThis(),
+      sendStatus: sinon.stub().returnsThis(),
+      status: sinon.stub().returnsThis()
+    }
+  })
+
+  describe('getFile', function() {
+    it('should pipe the stream', function() {
+      FileController.getFile(req, res)
+      expect(stream.pipeline).to.have.been.calledWith(fileStream, res)
+    })
+
+    it('should send a 200 if the cacheWarm param is true', function(done) {
+      req.query.cacheWarm = true
+      res.sendStatus = statusCode => {
+        statusCode.should.equal(200)
+        done()
+      }
+      FileController.getFile(req, res)
+    })
+
+    it('should send a 500 if there is a problem', function(done) {
+      FileHandler.getFile.yields('error')
+      res.sendStatus = code => {
+        code.should.equal(500)
+        done()
+      }
+      FileController.getFile(req, res)
+    })
+
+    describe('with a range header', function() {
+      let expectedOptions
+
+      beforeEach(function() {
+        expectedOptions = {
+          bucket,
+          key,
+          format: undefined,
+          style: undefined
+        }
+      })
+
+      it('should pass range options to FileHandler', function() {
+        req.headers.range = 'bytes=0-8'
+        expectedOptions.start = 0
+        expectedOptions.end = 8
+
+        FileController.getFile(req, res)
+        expect(FileHandler.getFile).to.have.been.calledWith(
+          bucket,
+          key,
+          expectedOptions
+        )
+      })
+
+      it('should ignore an invalid range header', function() {
+        req.headers.range = 'potato'
+        FileController.getFile(req, res)
+        expect(FileHandler.getFile).to.have.been.calledWith(
+          bucket,
+          key,
+          expectedOptions
+        )
+      })
+
+      it("should ignore any type other than 'bytes'", function() {
+        req.headers.range = 'wombats=0-8'
+        FileController.getFile(req, res)
+        expect(FileHandler.getFile).to.have.been.calledWith(
+          bucket,
+          key,
+          expectedOptions
+        )
+      })
+    })
+  })
+
+  describe('getFileHead', function() {
+    it('should return the file size in a Content-Length header', function(done) {
+      res.end = () => {
+        expect(res.status).to.have.been.calledWith(200)
+        expect(res.set).to.have.been.calledWith('Content-Length', fileSize)
+        done()
+      }
+
+      FileController.getFileHead(req, res)
+    })
+
+    it('should return a 404 is the file is not found', function(done) {
+      FileHandler.getFileSize.yields(new Errors.NotFoundError())
+
+      res.sendStatus = code => {
+        expect(code).to.equal(404)
+        done()
+      }
+
+      FileController.getFileHead(req, res)
+    })
+
+    it('should return a 500 on internal errors', function(done) {
+      FileHandler.getFileSize.yields(new Error())
+
+      res.sendStatus = code => {
+        expect(code).to.equal(500)
+        done()
+      }
+
+      FileController.getFileHead(req, res)
+    })
+  })
+
+  describe('insertFile', function() {
+    it('should send bucket name key and res to PersistorManager', function(done) {
+      res.sendStatus = code => {
+        expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req)
+        expect(code).to.equal(200)
+        done()
+      }
+      FileController.insertFile(req, res)
+    })
+  })
+
+  describe('copyFile', function() {
+    const oldFileId = 'oldFileId'
+    const oldProjectId = 'oldProjectid'
+    const oldKey = `${oldProjectId}/${oldFileId}`
+
+    beforeEach(function() {
+      req.body = {
+        source: {
+          project_id: oldProjectId,
+          file_id: oldFileId
+        }
+      }
+    })
+
+    it('should send bucket name and both keys to PersistorManager', function(done) {
+      res.sendStatus = code => {
+        code.should.equal(200)
+        expect(PersistorManager.copyFile).to.have.been.calledWith(
+          bucket,
+          oldKey,
+          key
+        )
+        done()
+      }
+      FileController.copyFile(req, res)
+    })
+
+    it('should send a 404 if the original file was not found', function(done) {
+      PersistorManager.copyFile.yields(new Errors.NotFoundError())
+      res.sendStatus = code => {
+        code.should.equal(404)
+        done()
+      }
+      FileController.copyFile(req, res)
+    })
+
+    it('should send a 500 if there was an error', function(done) {
+      PersistorManager.copyFile.yields('error')
+      res.sendStatus = code => {
+        code.should.equal(500)
+        done()
+      }
+      FileController.copyFile(req, res)
+    })
+  })
+
+  describe('delete file', function() {
+    it('should tell the file handler', function(done) {
+      res.sendStatus = code => {
+        code.should.equal(204)
+        expect(FileHandler.deleteFile).to.have.been.calledWith(bucket, key)
+        done()
+      }
+      FileController.deleteFile(req, res)
+    })
+
+    it('should send a 500 if there was an error', function(done) {
+      FileHandler.deleteFile.yields('error')
+      res.sendStatus = code => {
+        code.should.equal(500)
+        done()
+      }
+      FileController.deleteFile(req, res)
+    })
+  })
+
+  describe('directorySize', function() {
+    it('should return total directory size bytes', function(done) {
+      FileController.directorySize(req, {
+        json: result => {
+          expect(result['total bytes']).to.equal(fileSize)
+          done()
+        }
+      })
+    })
+
+    it('should send a 500 if there was an error', function(done) {
+      FileHandler.getDirectorySize.callsArgWith(2, 'error')
+      res.sendStatus = code => {
+        code.should.equal(500)
+        done()
+      }
+      FileController.directorySize(req, res)
+    })
+  })
+})

services/filestore/test/unit/js/FileConverterTests.js

@@ -0,0 +1,107 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const SandboxedModule = require('sandboxed-module')
+
+const modulePath = '../../../app/js/FileConverter.js'
+
+describe('FileConverter', function() {
+  let SafeExec, FileConverter
+  const sourcePath = '/data/wombat.eps'
+  const destPath = '/tmp/dest.png'
+  const format = 'png'
+  const errorMessage = 'guru meditation error'
+  const Settings = {
+    commands: {
+      convertCommandPrefix: []
+    }
+  }
+
+  beforeEach(function() {
+    SafeExec = {
+      promises: sinon.stub().resolves(destPath)
+    }
+
+    FileConverter = SandboxedModule.require(modulePath, {
+      requires: {
+        './SafeExec': SafeExec,
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        },
+        'metrics-sharelatex': {
+          inc: sinon.stub(),
+          Timer: sinon.stub().returns({ done: sinon.stub() })
+        },
+        'settings-sharelatex': Settings
+      }
+    })
+  })
+
+  describe('convert', function() {
+    it('should convert the source to the requested format', async function() {
+      await FileConverter.promises.convert(sourcePath, format)
+      const args = SafeExec.promises.args[0][0]
+      expect(args).to.include(`${sourcePath}[0]`)
+      expect(args).to.include(`${sourcePath}.${format}`)
+    })
+
+    it('should return the dest path', async function() {
+      const destPath = await FileConverter.promises.convert(sourcePath, format)
+      destPath.should.equal(`${sourcePath}.${format}`)
+    })
+
+    it('should wrap the error from convert', async function() {
+      SafeExec.promises.rejects(errorMessage)
+      try {
+        await FileConverter.promises.convert(sourcePath, format)
+        expect('error should have been thrown').not.to.exist
+      } catch (err) {
+        expect(err.name).to.equal('ConversionError')
+        expect(err.cause.toString()).to.equal(errorMessage)
+      }
+    })
+
+    it('should not accept an non approved format', async function() {
+      try {
+        await FileConverter.promises.convert(sourcePath, 'potato')
+        expect('error should have been thrown').not.to.exist
+      } catch (err) {
+        expect(err.name).to.equal('ConversionError')
+      }
+    })
+
+    it('should prefix the command with Settings.commands.convertCommandPrefix', async function() {
+      Settings.commands.convertCommandPrefix = ['nice']
+      await FileConverter.promises.convert(sourcePath, format)
+    })
+
+    it('should convert the file when called as a callback', function(done) {
+      FileConverter.convert(sourcePath, format, (err, destPath) => {
+        expect(err).not.to.exist
+        destPath.should.equal(`${sourcePath}.${format}`)
+
+        const args = SafeExec.promises.args[0][0]
+        expect(args).to.include(`${sourcePath}[0]`)
+        expect(args).to.include(`${sourcePath}.${format}`)
+        done()
+      })
+    })
+  })
+
+  describe('thumbnail', function() {
+    it('should call converter resize with args', async function() {
+      await FileConverter.promises.thumbnail(sourcePath)
+      const args = SafeExec.promises.args[0][0]
+      expect(args).to.include(`${sourcePath}[0]`)
+    })
+  })
+
+  describe('preview', function() {
+    it('should call converter resize with args', async function() {
+      await FileConverter.promises.preview(sourcePath)
+      const args = SafeExec.promises.args[0][0]
+      expect(args).to.include(`${sourcePath}[0]`)
+    })
+  })
+})

services/filestore/test/unit/js/FileHandlerTests.js

@@ -0,0 +1,234 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const modulePath = '../../../app/js/FileHandler.js'
+const SandboxedModule = require('sandboxed-module')
+
+describe('FileHandler', function() {
+  let PersistorManager,
+    LocalFileWriter,
+    FileConverter,
+    KeyBuilder,
+    ImageOptimiser,
+    FileHandler,
+    fs
+  const settings = {
+    s3: {
+      buckets: {
+        user_files: 'user_files'
+      }
+    }
+  }
+
+  const bucket = 'my_bucket'
+  const key = 'key/here'
+  const convertedFolderKey = 'convertedFolder'
+  const sourceStream = 'sourceStream'
+  const convertedKey = 'convertedKey'
+  const readStream = {
+    stream: 'readStream',
+    on: sinon.stub()
+  }
+
+  beforeEach(function() {
+    PersistorManager = {
+      getFileStream: sinon.stub().yields(null, sourceStream),
+      checkIfFileExists: sinon.stub().yields(),
+      deleteFile: sinon.stub().yields(),
+      deleteDirectory: sinon.stub().yields(),
+      sendStream: sinon.stub().yields(),
+      insertFile: sinon.stub().yields(),
+      sendFile: sinon.stub().yields(),
+      directorySize: sinon.stub().yields()
+    }
+    LocalFileWriter = {
+      writeStream: sinon.stub().yields(),
+      deleteFile: sinon.stub().yields()
+    }
+    FileConverter = {
+      convert: sinon.stub().yields(),
+      thumbnail: sinon.stub().yields(),
+      preview: sinon.stub().yields()
+    }
+    KeyBuilder = {
+      addCachingToKey: sinon.stub().returns(convertedKey),
+      getConvertedFolderKey: sinon.stub().returns(convertedFolderKey)
+    }
+    ImageOptimiser = { compressPng: sinon.stub().yields() }
+    fs = {
+      createReadStream: sinon.stub().returns(readStream)
+    }
+
+    FileHandler = SandboxedModule.require(modulePath, {
+      requires: {
+        'settings-sharelatex': settings,
+        './PersistorManager': PersistorManager,
+        './LocalFileWriter': LocalFileWriter,
+        './FileConverter': FileConverter,
+        './KeyBuilder': KeyBuilder,
+        './ImageOptimiser': ImageOptimiser,
+        fs: fs,
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        }
+      },
+      globals: { console }
+    })
+  })
+
+  describe('insertFile', function() {
+    const stream = 'stream'
+
+    it('should send file to the filestore', function(done) {
+      FileHandler.insertFile(bucket, key, stream, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.sendStream).to.have.been.calledWith(
+          bucket,
+          key,
+          stream
+        )
+        done()
+      })
+    })
+
+    it('should delete the convertedKey folder', function(done) {
+      FileHandler.insertFile(bucket, key, stream, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
+          bucket,
+          convertedFolderKey
+        )
+        done()
+      })
+    })
+  })
+
+  describe('deleteFile', function() {
+    it('should tell the filestore manager to delete the file', function(done) {
+      FileHandler.deleteFile(bucket, key, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.deleteFile).to.have.been.calledWith(bucket, key)
+        done()
+      })
+    })
+
+    it('should tell the filestore manager to delete the cached folder', function(done) {
+      FileHandler.deleteFile(bucket, key, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
+          bucket,
+          convertedFolderKey
+        )
+        done()
+      })
+    })
+  })
+
+  describe('getFile', function() {
+    it('should return the source stream no format or style are defined', function(done) {
+      FileHandler.getFile(bucket, key, null, (err, stream) => {
+        expect(err).not.to.exist
+        expect(stream).to.equal(sourceStream)
+        done()
+      })
+    })
+
+    it('should pass options through to PersistorManager', function(done) {
+      const options = { start: 0, end: 8 }
+      FileHandler.getFile(bucket, key, options, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.getFileStream).to.have.been.calledWith(
+          bucket,
+          key,
+          options
+        )
+        done()
+      })
+    })
+
+    describe('when a format is defined', function() {
+      let result
+
+      describe('when the file is not cached', function() {
+        beforeEach(function(done) {
+          FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
+            result = { err, stream }
+            done()
+          })
+        })
+
+        it('should convert the file', function() {
+          expect(FileConverter.convert).to.have.been.called
+          expect(ImageOptimiser.compressPng).to.have.been.called
+        })
+
+        it('should return the the converted stream', function() {
+          expect(result.err).not.to.exist
+          expect(result.stream).to.equal(readStream)
+          expect(PersistorManager.getFileStream).to.have.been.calledWith(
+            bucket,
+            key
+          )
+        })
+      })
+
+      describe('when the file is cached', function() {
+        beforeEach(function(done) {
+          PersistorManager.checkIfFileExists = sinon.stub().yields(null, true)
+          FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
+            result = { err, stream }
+            done()
+          })
+        })
+
+        it('should not convert the file', function() {
+          expect(FileConverter.convert).not.to.have.been.called
+          expect(ImageOptimiser.compressPng).not.to.have.been.called
+        })
+
+        it('should return the cached stream', function() {
+          expect(result.err).not.to.exist
+          expect(result.stream).to.equal(sourceStream)
+          expect(PersistorManager.getFileStream).to.have.been.calledWith(
+            bucket,
+            convertedKey
+          )
+        })
+      })
+    })
+
+    describe('when a style is defined', function() {
+      it('generates a thumbnail when requested', function(done) {
+        FileHandler.getFile(bucket, key, { style: 'thumbnail' }, err => {
+          expect(err).not.to.exist
+          expect(FileConverter.thumbnail).to.have.been.called
+          expect(FileConverter.preview).not.to.have.been.called
+          done()
+        })
+      })
+
+      it('generates a preview when requested', function(done) {
+        FileHandler.getFile(bucket, key, { style: 'preview' }, err => {
+          expect(err).not.to.exist
+          expect(FileConverter.thumbnail).not.to.have.been.called
+          expect(FileConverter.preview).to.have.been.called
+          done()
+        })
+      })
+    })
+  })
+
+  describe('getDirectorySize', function() {
+    it('should call the filestore manager to get directory size', function(done) {
+      FileHandler.getDirectorySize(bucket, key, err => {
+        expect(err).not.to.exist
+        expect(PersistorManager.directorySize).to.have.been.calledWith(
+          bucket,
+          key
+        )
+        done()
+      })
+    })
+  })
+})

services/filestore/test/unit/js/ImageOptimiserTests.js

@@ -0,0 +1,59 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const modulePath = '../../../app/js/ImageOptimiser.js'
+const { FailedCommandError } = require('../../../app/js/Errors')
+const SandboxedModule = require('sandboxed-module')
+
+describe('ImageOptimiser', function() {
+  let ImageOptimiser, SafeExec
+  const sourcePath = '/wombat/potato.eps'
+
+  beforeEach(function() {
+    SafeExec = {
+      promises: sinon.stub().resolves()
+    }
+    ImageOptimiser = SandboxedModule.require(modulePath, {
+      requires: {
+        './SafeExec': SafeExec,
+        'logger-sharelatex': {
+          log() {},
+          err() {},
+          warn() {}
+        }
+      }
+    })
+  })
+
+  describe('compressPng', function() {
+    it('should convert the file', function(done) {
+      ImageOptimiser.compressPng(sourcePath, err => {
+        expect(err).not.to.exist
+        expect(SafeExec.promises).to.have.been.calledWith([
+          'optipng',
+          sourcePath
+        ])
+        done()
+      })
+    })
+
+    it('should return the error', function(done) {
+      SafeExec.promises.rejects('wombat herding failure')
+      ImageOptimiser.compressPng(sourcePath, err => {
+        expect(err.toString()).to.equal('wombat herding failure')
+        done()
+      })
+    })
+  })
+
+  describe('when optimiser is sigkilled', function() {
+    it('should not produce an error', function(done) {
+      const error = new FailedCommandError('', 'SIGKILL', '', '')
+      SafeExec.promises.rejects(error)
+      ImageOptimiser.compressPng(sourcePath, err => {
+        expect(err).not.to.exist
+        done()
+      })
+    })
+  })
+})

services/filestore/test/unit/js/KeybuilderTests.js

@@ -0,0 +1,42 @@
+const SandboxedModule = require('sandboxed-module')
+
+const modulePath = '../../../app/js/KeyBuilder.js'
+
+describe('LocalFileWriter', function() {
+  let KeyBuilder
+  const key = 'wombat/potato'
+
+  beforeEach(function() {
+    KeyBuilder = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        }
+      }
+    })
+  })
+
+  describe('cachedKey', function() {
+    it('should add the format to the key', function() {
+      const opts = { format: 'png' }
+      const newKey = KeyBuilder.addCachingToKey(key, opts)
+      newKey.should.equal(`${key}-converted-cache/format-png`)
+    })
+
+    it('should add the style to the key', function() {
+      const opts = { style: 'thumbnail' }
+      const newKey = KeyBuilder.addCachingToKey(key, opts)
+      newKey.should.equal(`${key}-converted-cache/style-thumbnail`)
+    })
+
+    it('should add format first, then style', function() {
+      const opts = {
+        style: 'thumbnail',
+        format: 'png'
+      }
+      const newKey = KeyBuilder.addCachingToKey(key, opts)
+      newKey.should.equal(`${key}-converted-cache/format-png-style-thumbnail`)
+    })
+  })
+})

services/filestore/test/unit/js/LocalFileWriterTests.js

@@ -0,0 +1,111 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const modulePath = '../../../app/js/LocalFileWriter.js'
+const SandboxedModule = require('sandboxed-module')
+chai.use(require('sinon-chai'))
+
+describe('LocalFileWriter', function() {
+  const writeStream = 'writeStream'
+  const readStream = 'readStream'
+  const settings = { path: { uploadFolder: '/uploads' } }
+  const fsPath = '/uploads/wombat'
+  const filename = 'wombat'
+  let stream, fs, LocalFileWriter
+
+  beforeEach(function() {
+    fs = {
+      createWriteStream: sinon.stub().returns(writeStream),
+      unlink: sinon.stub().yields()
+    }
+    stream = {
+      pipeline: sinon.stub().yields()
+    }
+
+    LocalFileWriter = SandboxedModule.require(modulePath, {
+      requires: {
+        fs,
+        stream,
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        },
+        'settings-sharelatex': settings,
+        'metrics-sharelatex': {
+          inc: sinon.stub(),
+          Timer: sinon.stub().returns({ done: sinon.stub() })
+        }
+      }
+    })
+  })
+
+  describe('writeStream', function() {
+    it('writes the stream to the upload folder', function(done) {
+      LocalFileWriter.writeStream(readStream, filename, (err, path) => {
+        expect(err).not.to.exist
+        expect(fs.createWriteStream).to.have.been.calledWith(fsPath)
+        expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
+        expect(path).to.equal(fsPath)
+        done()
+      })
+    })
+
+    describe('when there is an error', function() {
+      const error = new Error('not enough ketchup')
+      beforeEach(function() {
+        stream.pipeline.yields(error)
+      })
+
+      it('should wrap the error', function() {
+        LocalFileWriter.writeStream(readStream, filename, err => {
+          expect(err).to.exist
+          expect(err.cause).to.equal(error)
+        })
+      })
+
+      it('should delete the temporary file', function() {
+        LocalFileWriter.writeStream(readStream, filename, () => {
+          expect(fs.unlink).to.have.been.calledWith(fsPath)
+        })
+      })
+    })
+  })
+
+  describe('deleteFile', function() {
+    it('should unlink the file', function(done) {
+      LocalFileWriter.deleteFile(fsPath, err => {
+        expect(err).not.to.exist
+        expect(fs.unlink).to.have.been.calledWith(fsPath)
+        done()
+      })
+    })
+
+    it('should not call unlink with an empty path', function(done) {
+      LocalFileWriter.deleteFile('', err => {
+        expect(err).not.to.exist
+        expect(fs.unlink).not.to.have.been.called
+        done()
+      })
+    })
+
+    it('should not throw a error if the file does not exist', function(done) {
+      const error = new Error('file not found')
+      error.code = 'ENOENT'
+      fs.unlink = sinon.stub().yields(error)
+      LocalFileWriter.deleteFile(fsPath, err => {
+        expect(err).not.to.exist
+        done()
+      })
+    })
+
+    it('should wrap the error', function(done) {
+      const error = new Error('failed to reticulate splines')
+      fs.unlink = sinon.stub().yields(error)
+      LocalFileWriter.deleteFile(fsPath, err => {
+        expect(err).to.exist
+        expect(err.cause).to.equal(error)
+        done()
+      })
+    })
+  })
+})

services/filestore/test/unit/js/PersistorManagerTests.js

@@ -0,0 +1,82 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const SandboxedModule = require('sandboxed-module')
+
+const modulePath = '../../../app/js/PersistorManager.js'
+
+describe('PersistorManager', function() {
+  let PersistorManager,
+    FSPersistorManager,
+    S3PersistorManager,
+    settings,
+    requires
+
+  beforeEach(function() {
+    FSPersistorManager = {
+      wrappedMethod: sinon.stub().returns('FSPersistorManager')
+    }
+    S3PersistorManager = {
+      wrappedMethod: sinon.stub().returns('S3PersistorManager')
+    }
+
+    settings = {
+      filestore: {}
+    }
+
+    requires = {
+      './S3PersistorManager': S3PersistorManager,
+      './FSPersistorManager': FSPersistorManager,
+      'settings-sharelatex': settings,
+      'logger-sharelatex': {
+        log() {},
+        err() {}
+      }
+    }
+  })
+
+  it('should implement the S3 wrapped method when S3 is configured', function() {
+    settings.filestore.backend = 's3'
+    PersistorManager = SandboxedModule.require(modulePath, { requires })
+
+    expect(PersistorManager).to.respondTo('wrappedMethod')
+    expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager')
+  })
+
+  it("should implement the S3 wrapped method when 'aws-sdk' is configured", function() {
+    settings.filestore.backend = 'aws-sdk'
+    PersistorManager = SandboxedModule.require(modulePath, { requires })
+
+    expect(PersistorManager).to.respondTo('wrappedMethod')
+    expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager')
+  })
+
+  it('should implement the FS wrapped method when FS is configured', function() {
+    settings.filestore.backend = 'fs'
+    PersistorManager = SandboxedModule.require(modulePath, { requires })
+
+    expect(PersistorManager).to.respondTo('wrappedMethod')
+    expect(PersistorManager.wrappedMethod()).to.equal('FSPersistorManager')
+  })
+
+  it('should throw an error when the backend is not configured', function() {
+    try {
+      SandboxedModule.require(modulePath, { requires })
+    } catch (err) {
+      expect(err.message).to.equal('no backend specified - config incomplete')
+      return
+    }
+    expect('should have caught an error').not.to.exist
+  })
+
+  it('should throw an error when the backend is unknown', function() {
+    settings.filestore.backend = 'magic'
+    try {
+      SandboxedModule.require(modulePath, { requires })
+    } catch (err) {
+      expect(err.message).to.equal('unknown filestore backend: magic')
+      return
+    }
+    expect('should have caught an error').not.to.exist
+  })
+})

services/filestore/test/unit/js/S3PersistorManagerTests.js

@@ -0,0 +1,801 @@
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const modulePath = '../../../app/js/S3PersistorManager.js'
+const SandboxedModule = require('sandboxed-module')
+
+const Errors = require('../../../app/js/Errors')
+
+describe('S3PersistorManagerTests', function() {
+  const defaultS3Key = 'frog'
+  const defaultS3Secret = 'prince'
+  const defaultS3Credentials = {
+    credentials: {
+      accessKeyId: defaultS3Key,
+      secretAccessKey: defaultS3Secret
+    }
+  }
+  const filename = '/wombat/potato.tex'
+  const bucket = 'womBucket'
+  const key = 'monKey'
+  const destKey = 'donKey'
+  const objectSize = 5555
+  const genericError = new Error('guru meditation error')
+  const files = [
+    { Key: 'llama', Size: 11 },
+    { Key: 'hippo', Size: 22 }
+  ]
+  const filesSize = 33
+
+  let Metrics,
+    S3,
+    Fs,
+    Meter,
+    MeteredStream,
+    ReadStream,
+    S3PersistorManager,
+    S3Client,
+    S3ReadStream,
+    S3NotFoundError,
+    FileNotFoundError,
+    EmptyPromise,
+    settings
+
+  beforeEach(function() {
+    settings = {
+      filestore: {
+        backend: 's3',
+        s3: {
+          secret: defaultS3Secret,
+          key: defaultS3Key
+        },
+        stores: {
+          user_files: 'sl_user_files'
+        }
+      }
+    }
+
+    EmptyPromise = {
+      promise: sinon.stub().resolves()
+    }
+
+    Metrics = {
+      count: sinon.stub()
+    }
+
+    ReadStream = {
+      pipe: sinon.stub().returns('readStream')
+    }
+
+    FileNotFoundError = new Error('File not found')
+    FileNotFoundError.code = 'ENOENT'
+
+    Fs = {
+      createReadStream: sinon.stub().returns(ReadStream)
+    }
+
+    MeteredStream = {
+      on: sinon.stub(),
+      bytes: objectSize
+    }
+    MeteredStream.on.withArgs('finish').yields()
+    Meter = sinon.stub().returns(MeteredStream)
+
+    S3NotFoundError = new Error('not found')
+    S3NotFoundError.code = 'NoSuchKey'
+
+    S3ReadStream = {
+      on: sinon.stub(),
+      pipe: sinon.stub().returns('s3Stream'),
+      removeListener: sinon.stub()
+    }
+    S3ReadStream.on.withArgs('readable').yields()
+    S3Client = {
+      getObject: sinon.stub().returns({
+        createReadStream: sinon.stub().returns(S3ReadStream)
+      }),
+      headObject: sinon.stub().returns({
+        promise: sinon.stub().resolves({
+          ContentLength: objectSize
+        })
+      }),
+      listObjects: sinon.stub().returns({
+        promise: sinon.stub().resolves({
+          Contents: files
+        })
+      }),
+      upload: sinon.stub().returns(EmptyPromise),
+      copyObject: sinon.stub().returns(EmptyPromise),
+      deleteObject: sinon.stub().returns(EmptyPromise),
+      deleteObjects: sinon.stub().returns(EmptyPromise)
+    }
+    S3 = sinon.stub().returns(S3Client)
+
+    S3PersistorManager = SandboxedModule.require(modulePath, {
+      requires: {
+        'aws-sdk/clients/s3': S3,
+        'settings-sharelatex': settings,
+        './Errors': Errors,
+        fs: Fs,
+        'stream-meter': Meter,
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        },
+        'metrics-sharelatex': Metrics
+      },
+      globals: { console }
+    })
+  })
+
+  describe('getFileStream', function() {
+    describe('when called with valid parameters', function() {
+      let stream
+
+      beforeEach(async function() {
+        stream = await S3PersistorManager.promises.getFileStream(bucket, key)
+      })
+
+      it('returns a stream', function() {
+        expect(stream).to.equal('s3Stream')
+      })
+
+      it('sets the AWS client up with credentials from settings', function() {
+        expect(S3).to.have.been.calledWith(defaultS3Credentials)
+      })
+
+      it('fetches the right key from the right bucket', function() {
+        expect(S3Client.getObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+
+      it('pipes the stream through the meter', function() {
+        expect(S3ReadStream.pipe).to.have.been.calledWith(MeteredStream)
+      })
+
+      it('records an ingress metric', function() {
+        expect(Metrics.count).to.have.been.calledWith('s3.ingress', objectSize)
+      })
+    })
+
+    describe('when called with a byte range', function() {
+      let stream
+
+      beforeEach(async function() {
+        stream = await S3PersistorManager.promises.getFileStream(bucket, key, {
+          start: 5,
+          end: 10
+        })
+      })
+
+      it('returns a stream', function() {
+        expect(stream).to.equal('s3Stream')
+      })
+
+      it('passes the byte range on to S3', function() {
+        expect(S3Client.getObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key,
+          Range: 'bytes=5-10'
+        })
+      })
+    })
+
+    describe('when there are alternative credentials', function() {
+      let stream
+      const alternativeSecret = 'giraffe'
+      const alternativeKey = 'hippo'
+      const alternativeS3Credentials = {
+        credentials: {
+          accessKeyId: alternativeKey,
+          secretAccessKey: alternativeSecret
+        }
+      }
+
+      beforeEach(async function() {
+        settings.filestore.s3BucketCreds = {}
+        settings.filestore.s3BucketCreds[bucket] = {
+          auth_key: alternativeKey,
+          auth_secret: alternativeSecret
+        }
+
+        stream = await S3PersistorManager.promises.getFileStream(bucket, key)
+      })
+
+      it('returns a stream', function() {
+        expect(stream).to.equal('s3Stream')
+      })
+
+      it('sets the AWS client up with the alternative credentials', function() {
+        expect(S3).to.have.been.calledWith(alternativeS3Credentials)
+      })
+
+      it('fetches the right key from the right bucket', function() {
+        expect(S3Client.getObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+
+      it('caches the credentials', async function() {
+        stream = await S3PersistorManager.promises.getFileStream(bucket, key)
+
+        expect(S3).to.have.been.calledOnceWith(alternativeS3Credentials)
+      })
+
+      it('uses the default credentials for an unknown bucket', async function() {
+        stream = await S3PersistorManager.promises.getFileStream(
+          'anotherBucket',
+          key
+        )
+
+        expect(S3).to.have.been.calledTwice
+        expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials)
+        expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials)
+      })
+
+      it('caches the default credentials', async function() {
+        stream = await S3PersistorManager.promises.getFileStream(
+          'anotherBucket',
+          key
+        )
+        stream = await S3PersistorManager.promises.getFileStream(
+          'anotherBucket',
+          key
+        )
+
+        expect(S3).to.have.been.calledTwice
+        expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials)
+        expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials)
+      })
+
+      it('throws an error if there are no credentials for the bucket', async function() {
+        delete settings.filestore.s3.key
+        delete settings.filestore.s3.secret
+
+        await expect(
+          S3PersistorManager.promises.getFileStream('anotherBucket', key)
+        ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.SettingsError)
+      })
+    })
+
+    describe("when the file doesn't exist", function() {
+      let error, stream
+
+      beforeEach(async function() {
+        S3ReadStream.on = sinon.stub()
+        S3ReadStream.on.withArgs('error').yields(S3NotFoundError)
+        try {
+          stream = await S3PersistorManager.promises.getFileStream(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('does not return a stream', function() {
+        expect(stream).not.to.exist
+      })
+
+      it('throws a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('wraps the error from S3', function() {
+        expect(error.cause).to.equal(S3NotFoundError)
+      })
+
+      it('stores the bucket and key in the error', function() {
+        expect(error.info).to.deep.equal({ Bucket: bucket, Key: key })
+      })
+    })
+
+    describe('when S3 encounters an unkown error', function() {
+      let error, stream
+
+      beforeEach(async function() {
+        S3ReadStream.on = sinon.stub()
+        S3ReadStream.on.withArgs('error').yields(genericError)
+        try {
+          stream = await S3PersistorManager.promises.getFileStream(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('does not return a stream', function() {
+        expect(stream).not.to.exist
+      })
+
+      it('throws a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('wraps the error from S3', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+
+      it('stores the bucket and key in the error', function() {
+        expect(error.info).to.deep.equal({ Bucket: bucket, Key: key })
+      })
+    })
+  })
+
+  describe('getFileSize', function() {
+    describe('when called with valid parameters', function() {
+      let size
+
+      beforeEach(async function() {
+        size = await S3PersistorManager.promises.getFileSize(bucket, key)
+      })
+
+      it('should return the object size', function() {
+        expect(size).to.equal(objectSize)
+      })
+
+      it('should pass the bucket and key to S3', function() {
+        expect(S3Client.headObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+    })
+
+    describe('when the object is not found', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.headObject = sinon.stub().returns({
+          promise: sinon.stub().rejects(S3NotFoundError)
+        })
+        try {
+          await S3PersistorManager.promises.getFileSize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should return a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(S3NotFoundError)
+      })
+    })
+
+    describe('when S3 returns an error', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.headObject = sinon.stub().returns({
+          promise: sinon.stub().rejects(genericError)
+        })
+        try {
+          await S3PersistorManager.promises.getFileSize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should return a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('sendStream', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return S3PersistorManager.promises.sendStream(bucket, key, ReadStream)
+      })
+
+      it('should upload the stream', function() {
+        expect(S3Client.upload).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key,
+          Body: 'readStream'
+        })
+      })
+
+      it('should meter the stream', function() {
+        expect(ReadStream.pipe).to.have.been.calledWith(MeteredStream)
+      })
+
+      it('should record an egress metric', function() {
+        expect(Metrics.count).to.have.been.calledWith('s3.egress', objectSize)
+      })
+    })
+
+    describe('when the upload fails', function() {
+      let error
+      beforeEach(async function() {
+        S3Client.upload = sinon.stub().returns({
+          promise: sinon.stub().rejects(genericError)
+        })
+        try {
+          await S3PersistorManager.promises.sendStream(bucket, key, ReadStream)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('throws a WriteError', function() {
+        expect(error).to.be.an.instanceOf(Errors.WriteError)
+      })
+    })
+  })
+
+  describe('sendFile', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return S3PersistorManager.promises.sendFile(bucket, key, filename)
+      })
+
+      it('should create a read stream for the file', function() {
+        expect(Fs.createReadStream).to.have.been.calledWith(filename)
+      })
+
+      it('should upload the stream', function() {
+        expect(S3Client.upload).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key,
+          Body: 'readStream'
+        })
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let error
+
+      beforeEach(async function() {
+        Fs.createReadStream = sinon.stub().throws(FileNotFoundError)
+        try {
+          await S3PersistorManager.promises.sendFile(bucket, key, filename)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('returns a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+
+      it('wraps the error', function() {
+        expect(error.cause).to.equal(FileNotFoundError)
+      })
+    })
+
+    describe('when reading the file throws an error', function() {
+      let error
+
+      beforeEach(async function() {
+        Fs.createReadStream = sinon.stub().throws(genericError)
+        try {
+          await S3PersistorManager.promises.sendFile(bucket, key, filename)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('returns a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('wraps the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('copyFile', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return S3PersistorManager.promises.copyFile(bucket, key, destKey)
+      })
+
+      it('should copy the object', function() {
+        expect(S3Client.copyObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: destKey,
+          CopySource: `${bucket}/${key}`
+        })
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.copyObject = sinon.stub().returns({
+          promise: sinon.stub().rejects(S3NotFoundError)
+        })
+        try {
+          await S3PersistorManager.promises.copyFile(bucket, key, destKey)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should throw a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+    })
+  })
+
+  describe('deleteFile', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return S3PersistorManager.promises.deleteFile(bucket, key)
+      })
+
+      it('should delete the object', function() {
+        expect(S3Client.deleteObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.deleteObject = sinon.stub().returns({
+          promise: sinon.stub().rejects(S3NotFoundError)
+        })
+        try {
+          await S3PersistorManager.promises.deleteFile(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should throw a NotFoundError', function() {
+        expect(error).to.be.an.instanceOf(Errors.NotFoundError)
+      })
+    })
+  })
+
+  describe('deleteDirectory', function() {
+    describe('with valid parameters', function() {
+      beforeEach(async function() {
+        return S3PersistorManager.promises.deleteDirectory(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(S3Client.listObjects).to.have.been.calledWith({
+          Bucket: bucket,
+          Prefix: key
+        })
+      })
+
+      it('should delete the objects using their keys', function() {
+        expect(S3Client.deleteObjects).to.have.been.calledWith({
+          Bucket: bucket,
+          Delete: {
+            Objects: [{ Key: 'llama' }, { Key: 'hippo' }],
+            Quiet: true
+          }
+        })
+      })
+    })
+
+    describe('when there are no files', function() {
+      beforeEach(async function() {
+        S3Client.listObjects = sinon
+          .stub()
+          .returns({ promise: sinon.stub().resolves({ Contents: [] }) })
+        return S3PersistorManager.promises.deleteDirectory(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(S3Client.listObjects).to.have.been.calledWith({
+          Bucket: bucket,
+          Prefix: key
+        })
+      })
+
+      it('should not try to delete any objects', function() {
+        expect(S3Client.deleteObjects).not.to.have.been.called
+      })
+    })
+
+    describe('when there is an error listing the objects', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.listObjects = sinon
+          .stub()
+          .returns({ promise: sinon.stub().rejects(genericError) })
+        try {
+          await S3PersistorManager.promises.deleteDirectory(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+
+      it('should not try to delete any objects', function() {
+        expect(S3Client.deleteObjects).not.to.have.been.called
+      })
+    })
+
+    describe('when there is an error deleting the objects', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.deleteObjects = sinon
+          .stub()
+          .returns({ promise: sinon.stub().rejects(genericError) })
+        try {
+          await S3PersistorManager.promises.deleteDirectory(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a WriteError', function() {
+        expect(error).to.be.an.instanceOf(Errors.WriteError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('directorySize', function() {
+    describe('with valid parameters', function() {
+      let size
+
+      beforeEach(async function() {
+        size = await S3PersistorManager.promises.directorySize(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(S3Client.listObjects).to.have.been.calledWith({
+          Bucket: bucket,
+          Prefix: key
+        })
+      })
+
+      it('should return the directory size', function() {
+        expect(size).to.equal(filesSize)
+      })
+    })
+
+    describe('when there are no files', function() {
+      let size
+
+      beforeEach(async function() {
+        S3Client.listObjects = sinon
+          .stub()
+          .returns({ promise: sinon.stub().resolves({ Contents: [] }) })
+        size = await S3PersistorManager.promises.directorySize(bucket, key)
+      })
+
+      it('should list the objects in the directory', function() {
+        expect(S3Client.listObjects).to.have.been.calledWith({
+          Bucket: bucket,
+          Prefix: key
+        })
+      })
+
+      it('should return zero', function() {
+        expect(size).to.equal(0)
+      })
+    })
+
+    describe('when there is an error listing the objects', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.listObjects = sinon
+          .stub()
+          .returns({ promise: sinon.stub().rejects(genericError) })
+        try {
+          await S3PersistorManager.promises.directorySize(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the error', function() {
+        expect(error.cause).to.equal(genericError)
+      })
+    })
+  })
+
+  describe('checkIfFileExists', function() {
+    describe('when the file exists', function() {
+      let exists
+
+      beforeEach(async function() {
+        exists = await S3PersistorManager.promises.checkIfFileExists(
+          bucket,
+          key
+        )
+      })
+
+      it('should get the object header', function() {
+        expect(S3Client.headObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+
+      it('should return that the file exists', function() {
+        expect(exists).to.equal(true)
+      })
+    })
+
+    describe('when the file does not exist', function() {
+      let exists
+
+      beforeEach(async function() {
+        S3Client.headObject = sinon
+          .stub()
+          .returns({ promise: sinon.stub().rejects(S3NotFoundError) })
+        exists = await S3PersistorManager.promises.checkIfFileExists(
+          bucket,
+          key
+        )
+      })
+
+      it('should get the object header', function() {
+        expect(S3Client.headObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+
+      it('should return that the file does not exist', function() {
+        expect(exists).to.equal(false)
+      })
+    })
+
+    describe('when there is an error', function() {
+      let error
+
+      beforeEach(async function() {
+        S3Client.headObject = sinon
+          .stub()
+          .returns({ promise: sinon.stub().rejects(genericError) })
+        try {
+          await S3PersistorManager.promises.checkIfFileExists(bucket, key)
+        } catch (err) {
+          error = err
+        }
+      })
+
+      it('should generate a ReadError', function() {
+        expect(error).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should wrap the upstream ReadError', function() {
+        expect(error.cause).to.be.an.instanceOf(Errors.ReadError)
+      })
+
+      it('should eventually wrap the error', function() {
+        expect(error.cause.cause).to.equal(genericError)
+      })
+    })
+  })
+})

services/filestore/test/unit/js/SafeExecTests.js

@@ -0,0 +1,109 @@
+const chai = require('chai')
+const should = chai.should()
+const { expect } = chai
+const modulePath = '../../../app/js/SafeExec'
+const SandboxedModule = require('sandboxed-module')
+
+describe('SafeExec', function() {
+  let settings, options, safeExec
+
+  beforeEach(function() {
+    settings = { enableConversions: true }
+    options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }
+
+    safeExec = SandboxedModule.require(modulePath, {
+      requires: {
+        'logger-sharelatex': {
+          log() {},
+          err() {}
+        },
+        'settings-sharelatex': settings
+      }
+    })
+  })
+
+  describe('safeExec', function() {
+    it('should execute a valid command', function(done) {
+      safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => {
+        stdout.should.equal('hello\n')
+        stderr.should.equal('')
+        should.not.exist(err)
+        done()
+      })
+    })
+
+    it('should error when conversions are disabled', function(done) {
+      settings.enableConversions = false
+      safeExec(['/bin/echo', 'hello'], options, err => {
+        expect(err).to.exist
+        done()
+      })
+    })
+
+    it('should execute a command with non-zero exit status', function(done) {
+      safeExec(['/usr/bin/env', 'false'], options, err => {
+        expect(err).to.exist
+        expect(err.name).to.equal('FailedCommandError')
+        expect(err.code).to.equal(1)
+        expect(err.stdout).to.equal('')
+        expect(err.stderr).to.equal('')
+        done()
+      })
+    })
+
+    it('should handle an invalid command', function(done) {
+      safeExec(['/bin/foobar'], options, err => {
+        err.code.should.equal('ENOENT')
+        done()
+      })
+    })
+
+    it('should handle a command that runs too long', function(done) {
+      safeExec(
+        ['/bin/sleep', '10'],
+        { timeout: 500, killSignal: 'SIGTERM' },
+        err => {
+          expect(err).to.exist
+          expect(err.name).to.equal('FailedCommandError')
+          expect(err.code).to.equal('SIGTERM')
+          done()
+        }
+      )
+    })
+  })
+
+  describe('as a promise', function() {
+    beforeEach(function() {
+      safeExec = safeExec.promises
+    })
+
+    it('should execute a valid command', async function() {
+      const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options)
+
+      stdout.should.equal('hello\n')
+      stderr.should.equal('')
+    })
+
+    it('should throw a ConversionsDisabledError when appropriate', async function() {
+      settings.enableConversions = false
+      try {
+        await safeExec(['/bin/echo', 'hello'], options)
+      } catch (err) {
+        expect(err.name).to.equal('ConversionsDisabledError')
+        return
+      }
+      expect('method did not throw an error').not.to.exist
+    })
+
+    it('should throw a FailedCommandError when appropriate', async function() {
+      try {
+        await safeExec(['/usr/bin/env', 'false'], options)
+      } catch (err) {
+        expect(err.name).to.equal('FailedCommandError')
+        expect(err.code).to.equal(1)
+        return
+      }
+      expect('method did not throw an error').not.to.exist
+    })
+  })
+})

services/filestore/test/unit/js/SettingsTests.js

@@ -0,0 +1,19 @@
+const chai = require('chai')
+const { expect } = chai
+
+describe('Settings', function() {
+  describe('s3', function() {
+    it('should use JSONified env var if present', function() {
+      const s3Settings = {
+        bucket1: {
+          auth_key: 'bucket1_key',
+          auth_secret: 'bucket1_secret'
+        }
+      }
+      process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3Settings)
+
+      const settings = require('settings-sharelatex')
+      expect(settings.filestore.s3BucketCreds).to.deep.equal(s3Settings)
+    })
+  })
+})