diff --git a/services/filestore/.eslintrc b/services/filestore/.eslintrc index 2e945d6ffb..76dad1561d 100644 --- a/services/filestore/.eslintrc +++ b/services/filestore/.eslintrc @@ -8,7 +8,7 @@ "prettier/standard" ], "parserOptions": { - "ecmaVersion": 2017 + "ecmaVersion": 2018 }, "plugins": [ "mocha", diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index d2ba18c328..6aa91a65f3 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -1,12 +1,9 @@ const OError = require('@overleaf/o-error') +const { Errors } = require('@overleaf/object-persistor') -class NotFoundError extends OError {} -class WriteError extends OError {} -class ReadError extends OError {} class HealthCheckError extends OError {} class ConversionsDisabledError extends OError {} class ConversionError extends OError {} -class SettingsError extends OError {} class TimeoutError extends OError {} class InvalidParametersError extends OError {} @@ -23,14 +20,11 @@ class FailedCommandError extends OError { } module.exports = { - NotFoundError, FailedCommandError, ConversionsDisabledError, - WriteError, - ReadError, ConversionError, HealthCheckError, - SettingsError, TimeoutError, - InvalidParametersError + InvalidParametersError, + ...Errors } diff --git a/services/filestore/app/js/FSPersistor.js b/services/filestore/app/js/FSPersistor.js deleted file mode 100644 index 60ee0f4053..0000000000 --- a/services/filestore/app/js/FSPersistor.js +++ /dev/null @@ -1,247 +0,0 @@ -const fs = require('fs') -const glob = require('glob') -const path = require('path') -const Stream = require('stream') -const { promisify, callbackify } = require('util') - -const LocalFileWriter = require('./LocalFileWriter').promises -const { NotFoundError, ReadError, WriteError } = require('./Errors') -const PersistorHelper = require('./PersistorHelper') - -const pipeline = promisify(Stream.pipeline) -const fsUnlink = promisify(fs.unlink) -const fsOpen = promisify(fs.open) -const fsStat = promisify(fs.stat) -const fsGlob = promisify(glob) - -const filterName = key => key.replace(/\//g, '_') - -async function sendFile(location, target, source) { - const filteredTarget = filterName(target) - - // actually copy the file (instead of moving it) to maintain consistent behaviour - // between the different implementations - try { - const sourceStream = fs.createReadStream(source) - const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) - await pipeline(sourceStream, targetStream) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to copy the specified file', - { location, target, source }, - WriteError - ) - } -} - -async function sendStream(location, target, sourceStream, sourceMd5) { - const fsPath = await LocalFileWriter.writeStream(sourceStream) - if (!sourceMd5) { - sourceMd5 = await _getFileMd5HashForPath(fsPath) - } - - try { - await sendFile(location, target, fsPath) - const destMd5 = await getFileMd5Hash(location, target) - if (sourceMd5 !== destMd5) { - await LocalFileWriter.deleteFile(`${location}/${filterName(target)}`) - throw new WriteError('md5 hash mismatch', { - sourceMd5, - destMd5, - location, - target - }) - } - } finally { - await LocalFileWriter.deleteFile(fsPath) - } -} - -// opts may be {start: Number, end: Number} -async function getFileStream(location, name, opts) { - const filteredName = filterName(name) - - try { - opts.fd = await fsOpen(`${location}/${filteredName}`, 'r') - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to open file for streaming', - { location, filteredName, opts }, - ReadError - ) - } - - return fs.createReadStream(null, opts) -} - -async function getRedirectUrl() { - // not implemented - return null -} - -async function getFileSize(location, filename) { - const fullPath = path.join(location, filterName(filename)) - - try { - const stat = await fsStat(fullPath) - return stat.size - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to stat file', - { location, filename }, - ReadError - ) - } -} - -async function getFileMd5Hash(location, filename) { - const fullPath = path.join(location, filterName(filename)) - try { - return await _getFileMd5HashForPath(fullPath) - } catch (err) { - throw new ReadError('unable to get md5 hash from file', { - location, - filename - }).withCause(err) - } -} - -async function copyFile(location, fromName, toName) { - const filteredFromName = filterName(fromName) - const filteredToName = filterName(toName) - - try { - const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) - const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) - await pipeline(sourceStream, targetStream) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to copy file', - { location, filteredFromName, filteredToName }, - WriteError - ) - } -} - -async function deleteFile(location, name) { - const filteredName = filterName(name) - try { - await fsUnlink(`${location}/${filteredName}`) - } catch (err) { - const wrappedError = PersistorHelper.wrapError( - err, - 'failed to delete file', - { location, filteredName }, - WriteError - ) - if (!(wrappedError instanceof NotFoundError)) { - // S3 doesn't give us a 404 when a file wasn't there to be deleted, so we - // should be consistent here as well - throw wrappedError - } - } -} - -// this is only called internally for clean-up by `FileHandler` and isn't part of the external API -async function deleteDirectory(location, name) { - const filteredName = filterName(name.replace(/\/$/, '')) - - try { - await Promise.all( - (await fsGlob(`${location}/${filteredName}*`)).map(file => fsUnlink(file)) - ) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to delete directory', - { location, filteredName }, - WriteError - ) - } -} - -async function checkIfFileExists(location, name) { - const filteredName = filterName(name) - try { - const stat = await fsStat(`${location}/${filteredName}`) - return !!stat - } catch (err) { - if (err.code === 'ENOENT') { - return false - } - throw PersistorHelper.wrapError( - err, - 'failed to stat file', - { location, filteredName }, - ReadError - ) - } -} - -// note, does not recurse into subdirectories, as we use a flattened directory structure -async function directorySize(location, name) { - const filteredName = filterName(name.replace(/\/$/, '')) - let size = 0 - - try { - const files = await fsGlob(`${location}/${filteredName}_*`) - for (const file of files) { - try { - const stat = await fsStat(file) - if (stat.isFile()) { - size += stat.size - } - } catch (err) { - // ignore files that may have just been deleted - if (err.code !== 'ENOENT') { - throw err - } - } - } - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to get directory size', - { location, name }, - ReadError - ) - } - - return size -} - -module.exports = { - sendFile: callbackify(sendFile), - sendStream: callbackify(sendStream), - getFileStream: callbackify(getFileStream), - getRedirectUrl: callbackify(getRedirectUrl), - getFileSize: callbackify(getFileSize), - getFileMd5Hash: callbackify(getFileMd5Hash), - copyFile: callbackify(copyFile), - deleteFile: callbackify(deleteFile), - deleteDirectory: callbackify(deleteDirectory), - checkIfFileExists: callbackify(checkIfFileExists), - directorySize: callbackify(directorySize), - promises: { - sendFile, - sendStream, - getFileStream, - getRedirectUrl, - getFileSize, - getFileMd5Hash, - copyFile, - deleteFile, - deleteDirectory, - checkIfFileExists, - directorySize - } -} - -async function _getFileMd5HashForPath(fullPath) { - const stream = fs.createReadStream(fullPath) - return PersistorHelper.calculateStreamMd5(stream) -} diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index 9e978c6a8b..5d1f0aa5b1 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -75,12 +75,11 @@ function getFile(req, res, next) { res.end() } else if (err) { next( - new Errors.ReadError('error transferring stream', { - bucket, - key, - format, - style - }).withCause(err) + new Errors.ReadError( + 'error transferring stream', + { bucket, key, format, style }, + err + ) ) } }) @@ -139,23 +138,17 @@ function copyFile(req, res, next) { }) req.requestLogger.setMessage('copying file') - PersistorManager.copyFile( - bucket, - `${oldProjectId}/${oldFileId}`, - key, - function(err) { + PersistorManager.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key) + .then(() => res.sendStatus(200)) + .catch(err => { if (err) { if (err instanceof Errors.NotFoundError) { res.sendStatus(404) } else { next(err) } - return } - - res.sendStatus(200) - } - ) + }) } function deleteFile(req, res, next) { diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index aec8e3bb3a..3258fa62fd 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -86,12 +86,11 @@ async function _convert(sourcePath, requestedFormat, command) { timeout: FOURTY_SECONDS }) } catch (err) { - throw new ConversionError('something went wrong converting file', { - stderr: err.stderr, - sourcePath, - requestedFormat, - destPath - }).withCause(err) + throw new ConversionError( + 'something went wrong converting file', + { stderr: err.stderr, sourcePath, requestedFormat, destPath }, + err + ) } timer.done() diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js index e3e93b33dd..6ac25a3c9e 100644 --- a/services/filestore/app/js/FileHandler.js +++ b/services/filestore/app/js/FileHandler.js @@ -37,9 +37,9 @@ async function insertFile(bucket, key, stream) { }) } if (Settings.enableConversions) { - await PersistorManager.promises.deleteDirectory(bucket, convertedKey) + await PersistorManager.deleteDirectory(bucket, convertedKey) } - await PersistorManager.promises.sendStream(bucket, key, stream) + await PersistorManager.sendStream(bucket, key, stream) } async function deleteFile(bucket, key) { @@ -51,9 +51,9 @@ async function deleteFile(bucket, key) { convertedKey }) } - const jobs = [PersistorManager.promises.deleteFile(bucket, key)] + const jobs = [PersistorManager.deleteObject(bucket, key)] if (Settings.enableConversions) { - jobs.push(PersistorManager.promises.deleteDirectory(bucket, convertedKey)) + jobs.push(PersistorManager.deleteDirectory(bucket, convertedKey)) } await Promise.all(jobs) } @@ -65,13 +65,13 @@ async function deleteProject(bucket, key) { key }) } - await PersistorManager.promises.deleteDirectory(bucket, key) + await PersistorManager.deleteDirectory(bucket, key) } async function getFile(bucket, key, opts) { opts = opts || {} if (!opts.format && !opts.style) { - return PersistorManager.promises.getFileStream(bucket, key, opts) + return PersistorManager.getObjectStream(bucket, key, opts) } else { return _getConvertedFile(bucket, key, opts) } @@ -89,28 +89,28 @@ async function getRedirectUrl(bucket, key, opts) { Object.values(Settings.filestore.stores).includes(bucket) && Settings.filestore.allowRedirects ) { - return PersistorManager.promises.getRedirectUrl(bucket, key) + return PersistorManager.getRedirectUrl(bucket, key) } return null } async function getFileSize(bucket, key) { - return PersistorManager.promises.getFileSize(bucket, key) + return PersistorManager.getObjectSize(bucket, key) } async function getDirectorySize(bucket, projectId) { - return PersistorManager.promises.directorySize(bucket, projectId) + return PersistorManager.directorySize(bucket, projectId) } async function _getConvertedFile(bucket, key, opts) { const convertedKey = KeyBuilder.addCachingToKey(key, opts) - const exists = await PersistorManager.promises.checkIfFileExists( + const exists = await PersistorManager.checkIfObjectExists( bucket, convertedKey ) if (exists) { - return PersistorManager.promises.getFileStream(bucket, convertedKey, opts) + return PersistorManager.getObjectStream(bucket, convertedKey, opts) } else { return _getConvertedFileAndCache(bucket, key, convertedKey, opts) } @@ -121,19 +121,14 @@ async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) { try { convertedFsPath = await _convertFile(bucket, key, opts) await ImageOptimiser.promises.compressPng(convertedFsPath) - await PersistorManager.promises.sendFile( - bucket, - convertedKey, - convertedFsPath - ) + await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath) } catch (err) { LocalFileWriter.deleteFile(convertedFsPath, () => {}) - throw new ConversionError('failed to convert file', { - opts, - bucket, - key, - convertedKey - }).withCause(err) + throw new ConversionError( + 'failed to convert file', + { opts, bucket, key, convertedKey }, + err + ) } // Send back the converted file from the local copy to avoid problems // with the file not being present in S3 yet. As described in the @@ -159,11 +154,11 @@ async function _convertFile(bucket, originalKey, opts) { try { originalFsPath = await _writeFileToDisk(bucket, originalKey, opts) } catch (err) { - throw new ConversionError('unable to write file to disk', { - bucket, - originalKey, - opts - }).withCause(err) + throw new ConversionError( + 'unable to write file to disk', + { bucket, originalKey, opts }, + err + ) } let promise @@ -184,21 +179,17 @@ async function _convertFile(bucket, originalKey, opts) { try { destPath = await promise } catch (err) { - throw new ConversionError('error converting file', { - bucket, - originalKey, - opts - }).withCause(err) + throw new ConversionError( + 'error converting file', + { bucket, originalKey, opts }, + err + ) } LocalFileWriter.deleteFile(originalFsPath, function() {}) return destPath } async function _writeFileToDisk(bucket, key, opts) { - const fileStream = await PersistorManager.promises.getFileStream( - bucket, - key, - opts - ) + const fileStream = await PersistorManager.getObjectStream(bucket, key, opts) return LocalFileWriter.promises.writeStream(fileStream, key) } diff --git a/services/filestore/app/js/GcsPersistor.js b/services/filestore/app/js/GcsPersistor.js deleted file mode 100644 index 05425abee5..0000000000 --- a/services/filestore/app/js/GcsPersistor.js +++ /dev/null @@ -1,305 +0,0 @@ -const settings = require('settings-sharelatex') -const fs = require('fs') -const { promisify } = require('util') -const Stream = require('stream') -const { Storage } = require('@google-cloud/storage') -const { callbackify } = require('util') -const { WriteError, ReadError, NotFoundError } = require('./Errors') -const asyncPool = require('tiny-async-pool') -const PersistorHelper = require('./PersistorHelper') - -const pipeline = promisify(Stream.pipeline) - -// endpoint settings will be null by default except for tests -// that's OK - GCS uses the locally-configured service account by default -const storage = new Storage(settings.filestore.gcs.endpoint) -// workaround for broken uploads with custom endpoints: -// https://github.com/googleapis/nodejs-storage/issues/898 -if ( - settings.filestore.gcs.endpoint && - settings.filestore.gcs.endpoint.apiEndpoint -) { - storage.interceptors.push({ - request: function(reqOpts) { - const url = new URL(reqOpts.uri) - url.host = settings.filestore.gcs.endpoint.apiEndpoint - if (settings.filestore.gcs.endpoint.apiScheme) { - url.protocol = settings.filestore.gcs.endpoint.apiScheme - } - reqOpts.uri = url.toString() - return reqOpts - } - }) -} - -const GcsPersistor = { - sendFile: callbackify(sendFile), - sendStream: callbackify(sendStream), - getFileStream: callbackify(getFileStream), - getRedirectUrl: callbackify(getRedirectUrl), - getFileMd5Hash: callbackify(getFileMd5Hash), - deleteDirectory: callbackify(deleteDirectory), - getFileSize: callbackify(getFileSize), - deleteFile: callbackify(deleteFile), - copyFile: callbackify(copyFile), - checkIfFileExists: callbackify(checkIfFileExists), - directorySize: callbackify(directorySize), - promises: { - sendFile, - sendStream, - getFileStream, - getRedirectUrl, - getFileMd5Hash, - deleteDirectory, - getFileSize, - deleteFile, - copyFile, - checkIfFileExists, - directorySize - } -} - -module.exports = GcsPersistor - -async function sendFile(bucketName, key, fsPath) { - return sendStream(bucketName, key, fs.createReadStream(fsPath)) -} - -async function sendStream(bucketName, key, readStream, sourceMd5) { - try { - // egress from us to gcs - const observeOptions = { metric: 'gcs.egress' } - - if (!sourceMd5) { - // if there is no supplied md5 hash, we calculate the hash as the data passes through - observeOptions.hash = 'md5' - } - - const observer = new PersistorHelper.ObserverStream(observeOptions) - - const writeOptions = { - // disabling of resumable uploads is recommended by Google: - resumable: false - } - - if (sourceMd5) { - writeOptions.validation = 'md5' - writeOptions.metadata = { - md5Hash: PersistorHelper.hexToBase64(sourceMd5) - } - } - - const uploadStream = storage - .bucket(bucketName) - .file(key) - .createWriteStream(writeOptions) - - await pipeline(readStream, observer, uploadStream) - - // if we didn't have an md5 hash, we should compare our computed one with Google's - // as we couldn't tell GCS about it beforehand - if (!sourceMd5) { - sourceMd5 = observer.getHash() - // throws on mismatch - await PersistorHelper.verifyMd5(GcsPersistor, bucketName, key, sourceMd5) - } - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'upload to GCS failed', - { bucketName, key }, - WriteError - ) - } -} - -async function getFileStream(bucketName, key, _opts = {}) { - const opts = Object.assign({}, _opts) - const stream = storage - .bucket(bucketName) - .file(key) - .createReadStream(opts) - - // ingress to us from gcs - const observer = new PersistorHelper.ObserverStream({ - metric: 'gcs.ingress' - }) - - try { - // wait for the pipeline to be ready, to catch non-200s - await PersistorHelper.getReadyPipeline(stream, observer) - return observer - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error reading file from GCS', - { bucketName, key, opts }, - ReadError - ) - } -} - -async function getRedirectUrl(bucketName, key) { - try { - const [url] = await storage - .bucket(bucketName) - .file(key) - .getSignedUrl({ - action: 'read', - expires: new Date().getTime() + settings.filestore.signedUrlExpiryInMs - }) - return url - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error generating signed url for GCS file', - { bucketName, key }, - ReadError - ) - } -} - -async function getFileSize(bucketName, key) { - try { - const [metadata] = await storage - .bucket(bucketName) - .file(key) - .getMetadata() - return metadata.size - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error getting size of GCS object', - { bucketName, key }, - ReadError - ) - } -} - -async function getFileMd5Hash(bucketName, key) { - try { - const [metadata] = await storage - .bucket(bucketName) - .file(key) - .getMetadata() - return PersistorHelper.base64ToHex(metadata.md5Hash) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error getting hash of GCS object', - { bucketName, key }, - ReadError - ) - } -} - -async function deleteFile(bucketName, key) { - try { - const file = storage.bucket(bucketName).file(key) - - if (settings.filestore.gcs.deletedBucketSuffix) { - await file.copy( - storage - .bucket(`${bucketName}${settings.filestore.gcs.deletedBucketSuffix}`) - .file(`${key}-${new Date().toISOString()}`) - ) - } - if (settings.filestore.gcs.unlockBeforeDelete) { - await file.setMetadata({ eventBasedHold: false }) - } - await file.delete() - } catch (err) { - const error = PersistorHelper.wrapError( - err, - 'error deleting GCS object', - { bucketName, key }, - WriteError - ) - if (!(error instanceof NotFoundError)) { - throw error - } - } -} - -async function deleteDirectory(bucketName, key) { - try { - const [files] = await storage - .bucket(bucketName) - .getFiles({ directory: key }) - - await asyncPool( - settings.filestore.gcs.deleteConcurrency, - files, - async file => { - await deleteFile(bucketName, file.name) - } - ) - } catch (err) { - const error = PersistorHelper.wrapError( - err, - 'failed to delete directory in GCS', - { bucketName, key }, - WriteError - ) - if (error instanceof NotFoundError) { - return - } - throw error - } -} - -async function directorySize(bucketName, key) { - let files - - try { - const [response] = await storage - .bucket(bucketName) - .getFiles({ directory: key }) - files = response - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to list objects in GCS', - { bucketName, key }, - ReadError - ) - } - - return files.reduce((acc, file) => Number(file.metadata.size) + acc, 0) -} - -async function checkIfFileExists(bucketName, key) { - try { - const [response] = await storage - .bucket(bucketName) - .file(key) - .exists() - return response - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error checking if file exists in GCS', - { bucketName, key }, - ReadError - ) - } -} - -async function copyFile(bucketName, sourceKey, destKey) { - try { - const src = storage.bucket(bucketName).file(sourceKey) - const dest = storage.bucket(bucketName).file(destKey) - await src.copy(dest) - } catch (err) { - // fake-gcs-server has a bug that returns an invalid response when the file does not exist - if (err.message === 'Cannot parse response as JSON: not found\n') { - err.code = 404 - } - throw PersistorHelper.wrapError( - err, - 'failed to copy file in GCS', - { bucketName, sourceKey, destKey }, - WriteError - ) - } -} diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index 0a4b10387e..4ee7534307 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -31,7 +31,7 @@ async function checkCanGetFiles() { try { await pipeline(sourceStream, buffer) } catch (err) { - throw new HealthCheckError('failed to get health-check file').withCause(err) + throw new HealthCheckError('failed to get health-check file', {}, err) } if (!buffer.size()) { diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js index 015f12fe33..6338dcecb0 100644 --- a/services/filestore/app/js/LocalFileWriter.js +++ b/services/filestore/app/js/LocalFileWriter.js @@ -30,9 +30,7 @@ async function writeStream(stream, key) { } catch (err) { await deleteFile(fsPath) - throw new WriteError('problem writing file locally', { - fsPath - }).withCause(err) + throw new WriteError('problem writing file locally', { fsPath }, err) } } @@ -44,7 +42,7 @@ async function deleteFile(fsPath) { await promisify(fs.unlink)(fsPath) } catch (err) { if (err.code !== 'ENOENT') { - throw new WriteError('failed to delete file', { fsPath }).withCause(err) + throw new WriteError('failed to delete file', { fsPath }, err) } } } diff --git a/services/filestore/app/js/MigrationPersistor.js b/services/filestore/app/js/MigrationPersistor.js deleted file mode 100644 index 2a9fe5d2a0..0000000000 --- a/services/filestore/app/js/MigrationPersistor.js +++ /dev/null @@ -1,230 +0,0 @@ -const metrics = require('metrics-sharelatex') -const Settings = require('settings-sharelatex') -const logger = require('logger-sharelatex') -const Stream = require('stream') -const { callbackify, promisify } = require('util') -const { NotFoundError, WriteError } = require('./Errors') - -const pipeline = promisify(Stream.pipeline) - -// Persistor that wraps two other persistors. Talks to the 'primary' by default, -// but will fall back to an older persistor in the case of a not-found error. -// If `Settings.filestore.fallback.copyOnMiss` is set, this will copy files from the fallback -// to the primary, in the event that they are missing. -// -// It is unlikely that the bucket/location name will be the same on the fallback -// as the primary. The bucket names should be overridden in `Settings.filestore.fallback.buckets` -// e.g. -// Settings.filestore.fallback.buckets = { -// myBucketOnS3: 'myBucketOnGCS' -// } - -module.exports = function(primary, fallback) { - function _wrapMethodOnBothPersistors(method) { - return async function(bucket, key, ...moreArgs) { - const fallbackBucket = _getFallbackBucket(bucket) - - await Promise.all([ - primary.promises[method](bucket, key, ...moreArgs), - fallback.promises[method](fallbackBucket, key, ...moreArgs) - ]) - } - } - - async function getFileStreamWithFallback(bucket, key, opts) { - const shouldCopy = - Settings.filestore.fallback.copyOnMiss && !opts.start && !opts.end - - try { - return await primary.promises.getFileStream(bucket, key, opts) - } catch (err) { - if (err instanceof NotFoundError) { - const fallbackBucket = _getFallbackBucket(bucket) - const fallbackStream = await fallback.promises.getFileStream( - fallbackBucket, - key, - opts - ) - // tee the stream to the client, and as a copy to the primary (if necessary) - // start listening on both straight away so that we don't consume bytes - // in one place before the other - const returnStream = new Stream.PassThrough() - pipeline(fallbackStream, returnStream) - - if (shouldCopy) { - const copyStream = new Stream.PassThrough() - pipeline(fallbackStream, copyStream) - - _copyStreamFromFallbackAndVerify( - copyStream, - fallbackBucket, - bucket, - key, - key - ).catch(() => { - // swallow errors, as this runs in the background and will log a warning - }) - } - return returnStream - } - throw err - } - } - - async function copyFileWithFallback(bucket, sourceKey, destKey) { - try { - return await primary.promises.copyFile(bucket, sourceKey, destKey) - } catch (err) { - if (err instanceof NotFoundError) { - const fallbackBucket = _getFallbackBucket(bucket) - const fallbackStream = await fallback.promises.getFileStream( - fallbackBucket, - sourceKey, - {} - ) - - const copyStream = new Stream.PassThrough() - pipeline(fallbackStream, copyStream) - - if (Settings.filestore.fallback.copyOnMiss) { - const missStream = new Stream.PassThrough() - pipeline(fallbackStream, missStream) - - // copy from sourceKey -> sourceKey - _copyStreamFromFallbackAndVerify( - missStream, - fallbackBucket, - bucket, - sourceKey, - sourceKey - ).then(() => { - // swallow errors, as this runs in the background and will log a warning - }) - } - // copy from sourceKey -> destKey - return _copyStreamFromFallbackAndVerify( - copyStream, - fallbackBucket, - bucket, - sourceKey, - destKey - ) - } - throw err - } - } - - function _getFallbackBucket(bucket) { - return Settings.filestore.fallback.buckets[bucket] || bucket - } - - function _wrapFallbackMethod(method) { - return async function(bucket, key, ...moreArgs) { - try { - return await primary.promises[method](bucket, key, ...moreArgs) - } catch (err) { - if (err instanceof NotFoundError) { - const fallbackBucket = _getFallbackBucket(bucket) - if (Settings.filestore.fallback.copyOnMiss) { - const fallbackStream = await fallback.promises.getFileStream( - fallbackBucket, - key, - {} - ) - // run in background - _copyStreamFromFallbackAndVerify( - fallbackStream, - fallbackBucket, - bucket, - key, - key - ).catch(err => { - logger.warn({ err }, 'failed to copy file from fallback') - }) - } - return fallback.promises[method](fallbackBucket, key, ...moreArgs) - } - throw err - } - } - } - - async function _copyStreamFromFallbackAndVerify( - stream, - sourceBucket, - destBucket, - sourceKey, - destKey - ) { - try { - let sourceMd5 - try { - sourceMd5 = await fallback.promises.getFileMd5Hash( - sourceBucket, - sourceKey - ) - } catch (err) { - logger.warn(err, 'error getting md5 hash from fallback persistor') - } - - await primary.promises.sendStream(destBucket, destKey, stream, sourceMd5) - } catch (err) { - const error = new WriteError( - 'unable to copy file to destination persistor', - { - sourceBucket, - destBucket, - sourceKey, - destKey - } - ).withCause(err) - metrics.inc('fallback.copy.failure') - - try { - await primary.promises.deleteFile(destBucket, destKey) - } catch (err) { - error.info.cleanupError = new WriteError( - 'unable to clean up destination copy artifact', - { - destBucket, - destKey - } - ).withCause(err) - } - - logger.warn({ error }, 'failed to copy file from fallback') - throw error - } - } - - return { - primaryPersistor: primary, - fallbackPersistor: fallback, - sendFile: primary.sendFile, - sendStream: primary.sendStream, - getFileStream: callbackify(getFileStreamWithFallback), - getRedirectUrl: primary.getRedirectUrl, - getFileMd5Hash: callbackify(_wrapFallbackMethod('getFileMd5Hash')), - deleteDirectory: callbackify( - _wrapMethodOnBothPersistors('deleteDirectory') - ), - getFileSize: callbackify(_wrapFallbackMethod('getFileSize')), - deleteFile: callbackify(_wrapMethodOnBothPersistors('deleteFile')), - copyFile: callbackify(copyFileWithFallback), - checkIfFileExists: callbackify(_wrapFallbackMethod('checkIfFileExists')), - directorySize: callbackify(_wrapFallbackMethod('directorySize')), - promises: { - sendFile: primary.promises.sendFile, - sendStream: primary.promises.sendStream, - getFileStream: getFileStreamWithFallback, - getRedirectUrl: primary.promises.getRedirectUrl, - getFileMd5Hash: _wrapFallbackMethod('getFileMd5Hash'), - deleteDirectory: _wrapMethodOnBothPersistors('deleteDirectory'), - getFileSize: _wrapFallbackMethod('getFileSize'), - deleteFile: _wrapMethodOnBothPersistors('deleteFile'), - copyFile: copyFileWithFallback, - checkIfFileExists: _wrapFallbackMethod('checkIfFileExists'), - directorySize: _wrapFallbackMethod('directorySize') - } - } -} diff --git a/services/filestore/app/js/PersistorHelper.js b/services/filestore/app/js/PersistorHelper.js deleted file mode 100644 index 1a836a2b09..0000000000 --- a/services/filestore/app/js/PersistorHelper.js +++ /dev/null @@ -1,177 +0,0 @@ -const crypto = require('crypto') -const Stream = require('stream') -const logger = require('logger-sharelatex') -const metrics = require('metrics-sharelatex') -const { WriteError, ReadError, NotFoundError } = require('./Errors') -const { promisify } = require('util') - -const pipeline = promisify(Stream.pipeline) - -// Observes data that passes through and computes some metadata for it -// - specifically, it computes the number of bytes transferred, and optionally -// computes a cryptographic hash based on the 'hash' option. e.g., pass -// { hash: 'md5' } to compute the md5 hash of the stream -// - if 'metric' is supplied as an option, this metric will be incremented by -// the number of bytes transferred -class ObserverStream extends Stream.Transform { - constructor(options) { - options.autoDestroy = true - super(options) - - this.bytes = 0 - - if (options.hash) { - this.hash = crypto.createHash(options.hash) - } - - if (options.metric) { - const onEnd = () => { - metrics.count(options.metric, this.bytes) - } - this.once('error', onEnd) - this.once('end', onEnd) - } - } - - _transform(chunk, encoding, done) { - if (this.hash) { - this.hash.update(chunk) - } - this.bytes += chunk.length - this.push(chunk) - done() - } - - getHash() { - return this.hash && this.hash.digest('hex') - } -} - -module.exports = { - ObserverStream, - calculateStreamMd5, - verifyMd5, - getReadyPipeline, - wrapError, - hexToBase64, - base64ToHex -} - -// returns a promise which resolves with the md5 hash of the stream -// - consumes the stream -function calculateStreamMd5(stream) { - const hash = crypto.createHash('md5') - hash.setEncoding('hex') - - return pipeline(stream, hash).then(() => hash.read()) -} - -// verifies the md5 hash of a file against the supplied md5 or the one stored in -// storage if not supplied - deletes the new file if the md5 does not match and -// throws an error -async function verifyMd5(persistor, bucket, key, sourceMd5, destMd5 = null) { - if (!destMd5) { - destMd5 = await persistor.promises.getFileMd5Hash(bucket, key) - } - - if (sourceMd5 !== destMd5) { - try { - await persistor.promises.deleteFile(bucket, key) - } catch (err) { - logger.warn(err, 'error deleting file for invalid upload') - } - - throw new WriteError('source and destination hashes do not match', { - sourceMd5, - destMd5, - bucket, - key - }) - } -} - -// resolves when a stream is 'readable', or rejects if the stream throws an error -// before that happens - this lets us handle protocol-level errors before trying -// to read them -function getReadyPipeline(...streams) { - return new Promise((resolve, reject) => { - const lastStream = streams.slice(-1)[0] - - // in case of error or stream close, we must ensure that we drain the - // previous stream so that it can clean up its socket (if it has one) - const drainPreviousStream = function(previousStream) { - // this stream is no longer reliable, so don't pipe anything more into it - previousStream.unpipe(this) - previousStream.resume() - } - - // handler to resolve when either: - // - an error happens, or - // - the last stream in the chain is readable - // for example, in the case of a 4xx error an error will occur and the - // streams will not become readable - const handler = function(err) { - // remove handler from all streams because we don't want to do this on - // later errors - lastStream.removeListener('readable', handler) - for (const stream of streams) { - stream.removeListener('error', handler) - } - - // return control to the caller - if (err) { - reject( - wrapError(err, 'error before stream became ready', {}, ReadError) - ) - } else { - resolve(lastStream) - } - } - - // ensure the handler fires when the last strem becomes readable - lastStream.on('readable', handler) - - for (const stream of streams) { - // when a stream receives a pipe, set up the drain handler to drain the - // connection if an error occurs or the stream is closed - stream.on('pipe', previousStream => { - stream.on('error', x => { - drainPreviousStream(previousStream) - }) - stream.on('close', () => { - drainPreviousStream(previousStream) - }) - }) - // add the handler function to resolve this method on error if we can't - // set up the pipeline - stream.on('error', handler) - } - - // begin the pipeline - for (let index = 0; index < streams.length - 1; index++) { - streams[index].pipe(streams[index + 1]) - } - }) -} - -function wrapError(error, message, params, ErrorType) { - if ( - error instanceof NotFoundError || - ['NoSuchKey', 'NotFound', 404, 'AccessDenied', 'ENOENT'].includes( - error.code - ) || - (error.response && error.response.statusCode === 404) - ) { - return new NotFoundError('no such file', params).withCause(error) - } else { - return new ErrorType(message, params).withCause(error) - } -} - -function base64ToHex(base64) { - return Buffer.from(base64, 'base64').toString('hex') -} - -function hexToBase64(hex) { - return Buffer.from(hex, 'hex').toString('base64') -} diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index d26ab77a92..c17e251ce9 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -1,39 +1,10 @@ const settings = require('settings-sharelatex') -const logger = require('logger-sharelatex') -logger.log( - { - backend: settings.filestore.backend, - fallback: settings.filestore.fallback && settings.filestore.fallback.backend - }, - 'Loading backend' -) -if (!settings.filestore.backend) { - throw new Error('no backend specified - config incomplete') -} +const persistorSettings = settings.filestore +persistorSettings.Metrics = require('metrics-sharelatex') +persistorSettings.paths = settings.path -function getPersistor(backend) { - switch (backend) { - case 'aws-sdk': - case 's3': - return require('./S3Persistor') - case 'fs': - return require('./FSPersistor') - case 'gcs': - return require('./GcsPersistor') - default: - throw new Error(`unknown filestore backend: ${backend}`) - } -} - -let persistor = getPersistor(settings.filestore.backend) - -if (settings.filestore.fallback && settings.filestore.fallback.backend) { - const migrationPersistor = require('./MigrationPersistor') - persistor = migrationPersistor( - persistor, - getPersistor(settings.filestore.fallback.backend) - ) -} +const ObjectPersistor = require('@overleaf/object-persistor') +const persistor = ObjectPersistor(persistorSettings) module.exports = persistor diff --git a/services/filestore/app/js/S3Persistor.js b/services/filestore/app/js/S3Persistor.js deleted file mode 100644 index f0df46f10d..0000000000 --- a/services/filestore/app/js/S3Persistor.js +++ /dev/null @@ -1,367 +0,0 @@ -const http = require('http') -const https = require('https') -http.globalAgent.maxSockets = 300 -https.globalAgent.maxSockets = 300 - -const settings = require('settings-sharelatex') - -const PersistorHelper = require('./PersistorHelper') - -const fs = require('fs') -const S3 = require('aws-sdk/clients/s3') -const { URL } = require('url') -const Stream = require('stream') -const { promisify, callbackify } = require('util') -const { - WriteError, - ReadError, - NotFoundError, - SettingsError -} = require('./Errors') -const pipeline = promisify(Stream.pipeline) - -const S3Persistor = { - sendFile: callbackify(sendFile), - sendStream: callbackify(sendStream), - getFileStream: callbackify(getFileStream), - getRedirectUrl: callbackify(getRedirectUrl), - getFileMd5Hash: callbackify(getFileMd5Hash), - deleteDirectory: callbackify(deleteDirectory), - getFileSize: callbackify(getFileSize), - deleteFile: callbackify(deleteFile), - copyFile: callbackify(copyFile), - checkIfFileExists: callbackify(checkIfFileExists), - directorySize: callbackify(directorySize), - promises: { - sendFile, - sendStream, - getFileStream, - getRedirectUrl, - getFileMd5Hash, - deleteDirectory, - getFileSize, - deleteFile, - copyFile, - checkIfFileExists, - directorySize - } -} - -module.exports = S3Persistor - -async function sendFile(bucketName, key, fsPath) { - return sendStream(bucketName, key, fs.createReadStream(fsPath)) -} - -async function sendStream(bucketName, key, readStream, sourceMd5) { - try { - // egress from us to S3 - const observeOptions = { metric: 's3.egress' } - let b64Hash - - if (sourceMd5) { - b64Hash = PersistorHelper.hexToBase64(sourceMd5) - } else { - // if there is no supplied md5 hash, we calculate the hash as the data passes through - observeOptions.hash = 'md5' - } - - const observer = new PersistorHelper.ObserverStream(observeOptions) - pipeline(readStream, observer) - - // if we have an md5 hash, pass this to S3 to verify the upload - const uploadOptions = { - Bucket: bucketName, - Key: key, - Body: observer - } - if (b64Hash) { - uploadOptions.ContentMD5 = b64Hash - } - - const response = await _getClientForBucket(bucketName) - .upload(uploadOptions, { partSize: settings.filestore.s3.partSize }) - .promise() - let destMd5 = _md5FromResponse(response) - if (!destMd5) { - // the eTag isn't in md5 format so we need to calculate it ourselves - const verifyStream = await getFileStream( - response.Bucket, - response.Key, - {} - ) - destMd5 = await PersistorHelper.calculateStreamMd5(verifyStream) - } - - // if we didn't have an md5 hash, we should compare our computed one with S3's - // as we couldn't tell S3 about it beforehand - if (!sourceMd5) { - sourceMd5 = observer.getHash() - // throws on mismatch - await PersistorHelper.verifyMd5( - S3Persistor, - bucketName, - key, - sourceMd5, - destMd5 - ) - } - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'upload to S3 failed', - { bucketName, key }, - WriteError - ) - } -} - -async function getFileStream(bucketName, key, opts) { - opts = opts || {} - - const params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - params.Range = `bytes=${opts.start}-${opts.end}` - } - - const stream = _getClientForBucket(bucketName) - .getObject(params) - .createReadStream() - - // ingress from S3 to us - const observer = new PersistorHelper.ObserverStream({ metric: 's3.ingress' }) - - try { - // wait for the pipeline to be ready, to catch non-200s - await PersistorHelper.getReadyPipeline(stream, observer) - return observer - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error reading file from S3', - { bucketName, key, opts }, - ReadError - ) - } -} - -async function getRedirectUrl() { - // not implemented - return null -} - -async function deleteDirectory(bucketName, key) { - let response - - try { - response = await _getClientForBucket(bucketName) - .listObjects({ Bucket: bucketName, Prefix: key }) - .promise() - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to list objects in S3', - { bucketName, key }, - ReadError - ) - } - - const objects = response.Contents.map(item => ({ Key: item.Key })) - if (objects.length) { - try { - await _getClientForBucket(bucketName) - .deleteObjects({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true - } - }) - .promise() - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to delete objects in S3', - { bucketName, key }, - WriteError - ) - } - } -} - -async function getFileSize(bucketName, key) { - try { - const response = await _getClientForBucket(bucketName) - .headObject({ Bucket: bucketName, Key: key }) - .promise() - return response.ContentLength - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error getting size of s3 object', - { bucketName, key }, - ReadError - ) - } -} - -async function getFileMd5Hash(bucketName, key) { - try { - const response = await _getClientForBucket(bucketName) - .headObject({ Bucket: bucketName, Key: key }) - .promise() - return _md5FromResponse(response) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error getting hash of s3 object', - { bucketName, key }, - ReadError - ) - } -} - -async function deleteFile(bucketName, key) { - try { - await _getClientForBucket(bucketName) - .deleteObject({ Bucket: bucketName, Key: key }) - .promise() - } catch (err) { - // s3 does not give us a NotFoundError here - throw PersistorHelper.wrapError( - err, - 'failed to delete file in S3', - { bucketName, key }, - WriteError - ) - } -} - -async function copyFile(bucketName, sourceKey, destKey) { - const params = { - Bucket: bucketName, - Key: destKey, - CopySource: `${bucketName}/${sourceKey}` - } - try { - await _getClientForBucket(bucketName) - .copyObject(params) - .promise() - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'failed to copy file in S3', - params, - WriteError - ) - } -} - -async function checkIfFileExists(bucketName, key) { - try { - await getFileSize(bucketName, key) - return true - } catch (err) { - if (err instanceof NotFoundError) { - return false - } - throw PersistorHelper.wrapError( - err, - 'error checking whether S3 object exists', - { bucketName, key }, - ReadError - ) - } -} - -async function directorySize(bucketName, key) { - try { - const response = await _getClientForBucket(bucketName) - .listObjects({ Bucket: bucketName, Prefix: key }) - .promise() - - return response.Contents.reduce((acc, item) => item.Size + acc, 0) - } catch (err) { - throw PersistorHelper.wrapError( - err, - 'error getting directory size in S3', - { bucketName, key }, - ReadError - ) - } -} - -const _clients = new Map() -let _defaultClient - -function _getClientForBucket(bucket) { - if (_clients[bucket]) { - return _clients[bucket] - } - - if ( - settings.filestore.s3BucketCreds && - settings.filestore.s3BucketCreds[bucket] - ) { - _clients[bucket] = new S3( - _buildClientOptions(settings.filestore.s3BucketCreds[bucket]) - ) - return _clients[bucket] - } - - // no specific credentials for the bucket - if (_defaultClient) { - return _defaultClient - } - - if (settings.filestore.s3.key) { - _defaultClient = new S3(_buildClientOptions()) - return _defaultClient - } - - throw new SettingsError( - 'no bucket-specific or default credentials provided', - { bucket } - ) -} - -function _buildClientOptions(bucketCredentials) { - const options = {} - - if (bucketCredentials) { - options.credentials = { - accessKeyId: bucketCredentials.auth_key, - secretAccessKey: bucketCredentials.auth_secret - } - } else { - options.credentials = { - accessKeyId: settings.filestore.s3.key, - secretAccessKey: settings.filestore.s3.secret - } - } - - if (settings.filestore.s3.endpoint) { - const endpoint = new URL(settings.filestore.s3.endpoint) - options.endpoint = settings.filestore.s3.endpoint - options.sslEnabled = endpoint.protocol === 'https' - } - - // path-style access is only used for acceptance tests - if (settings.filestore.s3.pathStyle) { - options.s3ForcePathStyle = true - } - - return options -} - -function _md5FromResponse(response) { - const md5 = (response.ETag || '').replace(/[ "]/g, '') - if (!md5.match(/^[a-f0-9]{32}$/)) { - return null - } - - return md5 -} diff --git a/services/filestore/package-lock.json b/services/filestore/package-lock.json index d740b46ff6..a7a0daeb90 100644 --- a/services/filestore/package-lock.json +++ b/services/filestore/package-lock.json @@ -314,9 +314,9 @@ } }, "mime": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "ms": { "version": "2.1.2", @@ -402,9 +402,9 @@ } }, "google-auth-library": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.2.tgz", - "integrity": "sha512-o/F/GiOPzDc49v5/6vfrEz3gRXvES49qGP84rrl3SO0efJA/M52hFwv2ozd1EC1TPrLj75Moj3iPgKGuGs6smA==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.3.tgz", + "integrity": "sha512-2Np6ojPmaJGXHSMsBhtTQEKfSMdLc8hefoihv7N2cwEr8E5bq39fhoat6TcXHwa0XoGO5Guh9sp3nxHFPmibMw==", "requires": { "arrify": "^2.0.0", "base64-js": "^1.3.0", @@ -503,6 +503,11 @@ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", "integrity": "sha512-QzB0/IMvB0eFxFK7Eqh+bfC8NLv3E9ScjWQrPOk6GgfNroxcVITdTlT8NRsRrcp5+QQJVPLkRqKG0PUdaWXmHw==" }, + "@types/node": { + "version": "10.17.26", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.26.tgz", + "integrity": "sha512-myMwkO2Cr82kirHY8uknNRHEVtn0wV3DTQfkrjx17jmkstDRZ24gNUdl8AHXVyVclTYI/bNjgTPTAWvWLqXqkw==" + }, "agent-base": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", @@ -620,9 +625,9 @@ } }, "mime": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "ms": { "version": "2.1.2", @@ -634,6 +639,26 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" }, + "protobufjs": { + "version": "6.8.9", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.9.tgz", + "integrity": "sha512-j2JlRdUeL/f4Z6x4aU4gj9I2LECglC+5qR2TrWb193Tla1qfdaNQTZ8I27Pt7K0Ajmvjjpft7O3KWTGciz4gpw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -881,9 +906,9 @@ } }, "mime": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "ms": { "version": "2.1.2", @@ -939,6 +964,13 @@ "semver": "^6.0.0", "shimmer": "^1.2.0", "uuid": "^3.2.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } } }, "@opencensus/propagation-stackdriver": { @@ -949,6 +981,13 @@ "@opencensus/core": "^0.0.20", "hex2dec": "^1.0.1", "uuid": "^3.2.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } } }, "@overleaf/o-error": { @@ -956,6 +995,210 @@ "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.0.0.tgz", "integrity": "sha512-LsM2s6Iy9G97ktPo0ys4VxtI/m3ahc1ZHwjo5XnhXtjeIkkkVAehsrcRRoV/yWepPjymB0oZonhcfojpjYR/tg==" }, + "@overleaf/object-persistor": { + "version": "git+https://github.com/overleaf/object-persistor.git#923c26a04dfeb9e79caaa5133394678253b5d006", + "from": "git+https://github.com/overleaf/object-persistor.git", + "requires": { + "@google-cloud/storage": "^5.1.1", + "@overleaf/o-error": "^3.0.0", + "aws-sdk": "^2.710.0", + "fast-crc32c": "^2.0.0", + "glob": "^7.1.6", + "logger-sharelatex": "^2.1.1", + "node-uuid": "^1.4.8", + "range-parser": "^1.2.1", + "tiny-async-pool": "^1.1.0" + }, + "dependencies": { + "@google-cloud/common": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-3.3.1.tgz", + "integrity": "sha512-bJamcNvZ2j5xS01uFBT1GqfHIKrtwpyUhIU/Xn3uwMZkK/t6JA3mlID0wuZlo7XjbjFSRT2iLBEmDWv9T2hP8g==", + "requires": { + "@google-cloud/projectify": "^2.0.0", + "@google-cloud/promisify": "^2.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^6.0.0", + "retry-request": "^4.1.1", + "teeny-request": "^7.0.0" + }, + "dependencies": { + "duplexify": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", + "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + } + } + }, + "@google-cloud/paginator": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.1.tgz", + "integrity": "sha512-ykqRmHRg6rcIZTE+JjUMNBKOQ8uvmbVrhY//lTxZgf5QBPbZW3PoN7VK+D43yCaRJJjRmmWsaG5YdxLR6h0n0A==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, + "@google-cloud/projectify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.0.0.tgz", + "integrity": "sha512-7wZ+m4N3Imtb5afOPfqNFyj9cKrlfVQ+t5YRxLS7tUpn8Pn/i7QuVubZRTXllaWjO4T5t/gm/r2x7oy5ajjvFQ==" + }, + "@google-cloud/promisify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.1.tgz", + "integrity": "sha512-82EQzwrNauw1fkbUSr3f+50Bcq7g4h0XvLOk8C5e9ABkXYHei7ZPi9tiMMD7Vh3SfcdH97d1ibJ3KBWp2o1J+w==" + }, + "@google-cloud/storage": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.1.1.tgz", + "integrity": "sha512-w/64V+eJl+vpYUXT15sBcO8pX0KTmb9Ni2ZNuQQ8HmyhAbEA3//G8JFaLPCXGBWO2/b0OQZytUT6q2wII9a9aQ==", + "requires": { + "@google-cloud/common": "^3.0.0", + "@google-cloud/paginator": "^3.0.0", + "@google-cloud/promisify": "^2.0.0", + "arrify": "^2.0.0", + "compressible": "^2.0.12", + "concat-stream": "^2.0.0", + "date-and-time": "^0.13.0", + "duplexify": "^3.5.0", + "extend": "^3.0.2", + "gaxios": "^3.0.0", + "gcs-resumable-upload": "^3.0.0", + "hash-stream-validation": "^0.2.2", + "mime": "^2.2.0", + "mime-types": "^2.0.8", + "onetime": "^5.1.0", + "p-limit": "^3.0.1", + "pumpify": "^2.0.0", + "readable-stream": "^3.4.0", + "snakeize": "^0.1.0", + "stream-events": "^1.0.1", + "through2": "^3.0.0", + "xdg-basedir": "^4.0.0" + } + }, + "gaxios": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.0.3.tgz", + "integrity": "sha512-PkzQludeIFhd535/yucALT/Wxyj/y2zLyrMwPcJmnLHDugmV49NvAi/vb+VUq/eWztATZCNcb8ue+ywPG+oLuw==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.1.0.tgz", + "integrity": "sha512-r57SV28+olVsflPlKyVig3Muo/VDlcsObMtvDGOEtEJXj+DDE8bEl0coIkXh//hbkSDTvo+f5lbihZOndYXQQQ==", + "requires": { + "gaxios": "^3.0.0", + "json-bigint": "^0.3.0" + } + }, + "gcs-resumable-upload": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.0.tgz", + "integrity": "sha512-gB8xH6EjYCv9lfBEL4FK5+AMgTY0feYoNHAYOV5nCuOrDPhy5MOiyJE8WosgxhbKBPS361H7fkwv6CTufEh9bg==", + "requires": { + "abort-controller": "^3.0.0", + "configstore": "^5.0.0", + "extend": "^3.0.2", + "gaxios": "^3.0.0", + "google-auth-library": "^6.0.0", + "pumpify": "^2.0.0", + "stream-events": "^1.0.4" + } + }, + "google-auth-library": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.3.tgz", + "integrity": "sha512-2Np6ojPmaJGXHSMsBhtTQEKfSMdLc8hefoihv7N2cwEr8E5bq39fhoat6TcXHwa0XoGO5Guh9sp3nxHFPmibMw==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^3.0.0", + "gcp-metadata": "^4.1.0", + "gtoken": "^5.0.0", + "jws": "^4.0.0", + "lru-cache": "^5.0.0" + } + }, + "google-p12-pem": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.1.tgz", + "integrity": "sha512-VlQgtozgNVVVcYTXS36eQz4PXPt9gIPqLOhHN0QiV6W6h4qSCNVKPtKC5INtJsaHHF2r7+nOIa26MJeJMTaZEQ==", + "requires": { + "node-forge": "^0.9.0" + } + }, + "gtoken": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.1.tgz", + "integrity": "sha512-33w4FNDkUcyIOq/TqyC+drnKdI4PdXmWp9lZzssyEQKuvu9ZFN3KttaSnDKo52U3E51oujVGop93mKxmqO8HHg==", + "requires": { + "gaxios": "^3.0.0", + "google-p12-pem": "^3.0.0", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, + "mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "p-limit": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.0.1.tgz", + "integrity": "sha512-mw/p92EyOzl2MhauKodw54Rx5ZK4624rNfgNaBguFZkHzyUG9WsDzFF5/yQVEJinbJDdP4jEfMN+uBquiGnaLg==", + "requires": { + "p-try": "^2.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "teeny-request": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-7.0.0.tgz", + "integrity": "sha512-kWD3sdGmIix6w7c8ZdVKxWq+3YwVPGWz+Mq0wRZXayEKY/YHb63b8uphfBzcFDmyq8frD9+UTc3wLyOhltRbtg==", + "requires": { + "http-proxy-agent": "^4.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.2.0", + "stream-events": "^1.0.5", + "uuid": "^8.0.0" + } + }, + "uuid": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.2.0.tgz", + "integrity": "sha512-CYpGiFTUrmI6OBMkAdjSDM0k5h8SkkiTP4WAjQgDgNB1S3Ou9VBEvr6q0Kv2H1mMk7IWfxYGpMH5sd5AvcIV2Q==" + } + } + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1016,9 +1259,9 @@ "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" }, "@sinonjs/commons": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz", - "integrity": "sha512-Debi3Baff1Qu1Unc3mjJ96MgpbwTn43S1+9yJ0llWygPwDNu2aaWBD6yc9y/Z8XDRNhx7U+u2UDg2OGQXkclUQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.0.tgz", + "integrity": "sha512-wEj54PfsZ5jGSwMX68G8ZXFawcSglQSXqCftWX3ec8MDUzQdHgcKvw97awHbY0efQEL5iKUOAmmVtoYgmrSG4Q==", "dev": true, "requires": { "type-detect": "4.0.8" @@ -1098,9 +1341,15 @@ } }, "@types/json-schema": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", - "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", + "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "dev": true + }, + "@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, "@types/long": { @@ -1109,14 +1358,14 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" }, "@types/node": { - "version": "10.17.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.16.tgz", - "integrity": "sha512-A4283YSA1OmnIivcpy/4nN86YlnKRiQp8PYwI2KdPCONEBN093QTb0gCtERtkLyVNGKKIGazTZ2nAmVzQU51zA==" + "version": "13.13.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.13.tgz", + "integrity": "sha512-UfvBE9oRCAJVzfR+3eWm/sdLFe/qroAPEXP3GPJ1SehQiEVgZT6NQZWYbPMiJ3UdcKM06v4j+S1lTcdWCmw+3g==" }, "@types/request": { - "version": "2.48.4", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.4.tgz", - "integrity": "sha512-W1t1MTKYR8PxICH+A4HgEIPuAC3sbljoEVfyZbeFJJDbr30guDspJri2XOaM2E+Un7ZjrihaDi7cf6fPa2tbgw==", + "version": "2.48.5", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.5.tgz", + "integrity": "sha512-/LO7xRVnL3DxJ1WkPGDQrp4VTV1reX9RkC85mJ+Qzykj2Bdw+mG15aAfDahc76HtknjzE16SX/Yddn6MxVbmGQ==", "requires": { "@types/caseless": "*", "@types/node": "*", @@ -1142,9 +1391,9 @@ "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==" }, "@types/tough-cookie": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.6.tgz", - "integrity": "sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==" }, "@typescript-eslint/experimental-utils": { "version": "1.13.0", @@ -1396,11 +1645,11 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "aws-sdk": { - "version": "2.648.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.648.0.tgz", - "integrity": "sha512-b+PdZmCFvZBisqXEH68jO4xB30LrDHQMWrEX6MJoZaOlxPJfpOqRFUH3zsiAXF5Q2jTdjYLtS5bs3vcIwRzi3Q==", + "version": "2.710.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.710.0.tgz", + "integrity": "sha512-GQTPH0DzJMpvvtZ3VO+grkKVdL/nqjWsIfcVf1c3oedvEjW24wSXQEs6KWAGbpG2jbHsYKH7kZ4XXuq428LVAw==", "requires": { - "buffer": "4.9.1", + "buffer": "4.9.2", "events": "1.1.1", "ieee754": "1.1.13", "jmespath": "0.15.0", @@ -1475,9 +1724,9 @@ "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" }, "binary-extensions": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", - "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", "dev": true }, "bindings": { @@ -1557,9 +1806,9 @@ "dev": true }, "buffer": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha512-DNK4ruAqtyHaN8Zne7PkBTO+dD1Lr0YfTduMqlIyjvQIoztBkUxrvL+hKeLW8NXFKHOq/2upkxuoS9znQ9bW9A==", + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4", @@ -1582,13 +1831,13 @@ "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==" }, "bunyan": { - "version": "1.8.12", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", - "integrity": "sha512-dmDUbGHeGcvCDLRFOscZkwx1ZO/aFz3bJOCi5nCgzdhFGPxwK+y5AcDBnqagNGlJZ7lje/l6JUEz9mQcutttdg==", + "version": "1.8.14", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz", + "integrity": "sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg==", "dev": true, "requires": { "dtrace-provider": "~0.8", - "moment": "^2.10.6", + "moment": "^2.19.3", "mv": "~2", "safe-json-stringify": "~1" } @@ -2113,22 +2362,22 @@ } }, "es-abstract": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", - "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1", - "is-callable": "^1.1.5", - "is-regex": "^1.0.5", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", "object-inspect": "^1.7.0", "object-keys": "^1.1.1", "object.assign": "^4.1.0", - "string.prototype.trimleft": "^2.1.1", - "string.prototype.trimright": "^2.1.1" + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" } }, "es-to-primitive": { @@ -2244,9 +2493,9 @@ } }, "eslint-config-prettier": { - "version": "6.10.1", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.10.1.tgz", - "integrity": "sha512-svTy6zh1ecQojvpbJSgH3aei/Rt7C6i090l5f2WQ4aB05lYHeZIR1qL4wZyyILTbtmnbHP5Yn8MrsOJMGa8RkQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.11.0.tgz", + "integrity": "sha512-oB8cpLWSAjOVFEJhhyMZh6NOEOtBVziaqdDQ86+qhDHFbZXoRTM7pNSvFRfW/W/L/LrQ38C99J5CGuRBBzBsdA==", "dev": true, "requires": { "get-stdin": "^6.0.0" @@ -2259,9 +2508,9 @@ "dev": true }, "eslint-import-resolver-node": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.3.tgz", - "integrity": "sha512-b8crLDo0M5RSe5YG8Pu2DYBj71tSB6OvXkfzwbJU2w7y8P4/yo0MyF8jU26IEuEuHF2K5/gcAJE3LhQGqBBbVg==", + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz", + "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==", "dev": true, "requires": { "debug": "^2.6.9", @@ -2285,9 +2534,9 @@ "dev": true }, "eslint-plugin-chai-friendly": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.5.0.tgz", - "integrity": "sha512-Pxe6z8C9fP0pn2X2nGFU/b3GBOCM/5FVus1hsMwJsXP3R7RiXFl7g0ksJbsc0GxiLyidTW4mEFk77qsNn7Tk7g==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.6.0.tgz", + "integrity": "sha512-Uvvv1gkbRGp/qfN15B0kQyQWg+oFA8buDSqrwmW3egNSk/FpqH2MjQqKOuKwmEL6w4QIQrIjDp+gg6kGGmD3oQ==", "dev": true }, "eslint-plugin-es": { @@ -2318,23 +2567,24 @@ } }, "eslint-plugin-import": { - "version": "2.20.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.20.2.tgz", - "integrity": "sha512-FObidqpXrR8OnCh4iNsxy+WACztJLXAHBO5hK79T1Hc77PgQZkyDGA5Ag9xAvRpglvLNxhH/zSmZ70/pZ31dHg==", + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.0.tgz", + "integrity": "sha512-66Fpf1Ln6aIS5Gr/55ts19eUuoDhAbZgnr6UxK5hbDx6l/QgQgx61AePq+BV4PP2uXQFClgMVzep5zZ94qqsxg==", "dev": true, "requires": { - "array-includes": "^3.0.3", - "array.prototype.flat": "^1.2.1", + "array-includes": "^3.1.1", + "array.prototype.flat": "^1.2.3", "contains-path": "^0.1.0", "debug": "^2.6.9", "doctrine": "1.5.0", - "eslint-import-resolver-node": "^0.3.2", - "eslint-module-utils": "^2.4.1", + "eslint-import-resolver-node": "^0.3.3", + "eslint-module-utils": "^2.6.0", "has": "^1.0.3", "minimatch": "^3.0.4", - "object.values": "^1.1.0", + "object.values": "^1.1.1", "read-pkg-up": "^2.0.0", - "resolve": "^1.12.0" + "resolve": "^1.17.0", + "tsconfig-paths": "^3.9.0" }, "dependencies": { "doctrine": { @@ -2346,6 +2596,15 @@ "esutils": "^2.0.2", "isarray": "^1.0.0" } + }, + "resolve": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", + "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } } } }, @@ -2402,9 +2661,9 @@ } }, "eslint-plugin-prettier": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.2.tgz", - "integrity": "sha512-GlolCC9y3XZfv3RQfwGew7NnuFDKsfI4lbvRK+PIIo23SFH+LemGs4cKwzAaRa+Mdb+lQO/STaIayno8T5sJJA==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.4.tgz", + "integrity": "sha512-jZDa8z76klRqo+TdGDTFJSavwbnWK2ZpqGKNZ+VvweMW516pDUMmQ2koXvxEE4JhzNvTv+radye/bWGBmA6jmg==", "dev": true, "requires": { "prettier-linter-helpers": "^1.0.0" @@ -2519,6 +2778,13 @@ "requires": { "d64": "^1.0.0", "uuid": "^3.0.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } } }, "events": { @@ -2765,9 +3031,9 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "fsevents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", - "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", "dev": true, "optional": true }, @@ -2907,33 +3173,6 @@ "retry-request": "^4.0.0", "semver": "^6.0.0", "walkdir": "^0.4.0" - }, - "dependencies": { - "@types/node": { - "version": "13.13.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.12.tgz", - "integrity": "sha512-zWz/8NEPxoXNT9YyF2osqyA9WjssZukYpgI4UYZpOjcyqwIUqWGkcCionaEb9Ki+FULyPyvNFpg/329Kd2/pbw==" - }, - "protobufjs": { - "version": "6.9.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.9.0.tgz", - "integrity": "sha512-LlGVfEWDXoI/STstRDdZZKb/qusoAWUnmLg9R8OLSO473mBLWHowx8clbX5/+mKDEI+v7GzjoK9tRPZMMcoTrg==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": "^13.7.0", - "long": "^4.0.0" - } - } } }, "google-p12-pem": { @@ -3238,9 +3477,9 @@ "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" }, "is-callable": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", "dev": true }, "is-date-object": { @@ -3288,12 +3527,12 @@ "dev": true }, "is-regex": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", - "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", "dev": true, "requires": { - "has": "^1.0.3" + "has-symbols": "^1.0.1" } }, "is-stream": { @@ -3403,6 +3642,15 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -3569,11 +3817,6 @@ "mv": "~2", "safe-json-stringify": "~1" } - }, - "yn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", - "integrity": "sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg==" } } }, @@ -3756,9 +3999,9 @@ "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "metrics-sharelatex": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.6.2.tgz", - "integrity": "sha512-bOLfkSCexiPgB96hdXhoOWyvvrwscgjeZPEqdcJ7BTGxY59anzvymNf5hTGJ1RtS4sblDKxITw3L5a+gYKhRYQ==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.7.0.tgz", + "integrity": "sha512-fv7TsM2pVhY/VuHqP5p9Ip2BhHfGPrP0e2TuZGCMK9/J83ZuGCrxkpaU3OOz31CWcmcQ5am4Du70EZAaIdA/2A==", "requires": { "@google-cloud/debug-agent": "^3.0.0", "@google-cloud/profiler": "^0.2.3", @@ -3768,6 +4011,13 @@ "prom-client": "^11.1.3", "underscore": "~1.6.0", "yn": "^3.1.1" + }, + "dependencies": { + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" + } } }, "mime": { @@ -3815,9 +4065,9 @@ } }, "mocha": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.1.1.tgz", - "integrity": "sha512-3qQsu3ijNS3GkWcccT5Zw0hf/rWvu1fTN9sPvEd81hlwsr30GX2GcDSSoBxo24IR8FelmrAydGC6/1J5QQP4WA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.2.0.tgz", + "integrity": "sha512-O9CIypScywTVpNaRrCAgoUnJgozpIofjKUYmJhiCIJMiuYnLI6otcb1/kpW9/n/tJODHGZ7i8aLQoDVsMtOKQQ==", "dev": true, "requires": { "ansi-colors": "3.2.3", @@ -3833,7 +4083,7 @@ "js-yaml": "3.13.1", "log-symbols": "3.0.0", "minimatch": "3.0.4", - "mkdirp": "0.5.3", + "mkdirp": "0.5.5", "ms": "2.1.1", "node-environment-flags": "1.0.6", "object.assign": "4.1.0", @@ -3900,21 +4150,6 @@ "path-exists": "^3.0.0" } }, - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - }, - "mkdirp": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", - "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } - }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -3973,16 +4208,6 @@ "y18n": "^4.0.0", "yargs-parser": "^13.1.2" } - }, - "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } } } }, @@ -3998,13 +4223,13 @@ "optional": true }, "mongodb": { - "version": "3.5.5", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.5.tgz", - "integrity": "sha512-GCjDxR3UOltDq00Zcpzql6dQo1sVry60OXJY3TDmFc2SWFY6c8Gn1Ardidc5jDirvJrx2GC3knGOImKphbSL3A==", + "version": "3.5.9", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.9.tgz", + "integrity": "sha512-vXHBY1CsGYcEPoVWhwgxIBeWqP3dSu9RuRDsoLRPTITrcrgm1f0Ubu1xqF9ozMwv53agmEiZm0YGo+7WL3Nbug==", "dev": true, "requires": { "bl": "^2.2.0", - "bson": "^1.1.1", + "bson": "^1.1.4", "denque": "^1.4.1", "require_optional": "^1.0.1", "safe-buffer": "^5.1.2", @@ -4086,9 +4311,9 @@ "dev": true }, "nise": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/nise/-/nise-4.0.3.tgz", - "integrity": "sha512-EGlhjm7/4KvmmE6B/UFsKh7eHykRl9VH+au8dduHLCyWUO/hr7+N+WtTvDUwc9zHuM1IaIJs/0lQ6Ag1jDkQSg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/nise/-/nise-4.0.4.tgz", + "integrity": "sha512-bTTRUNlemx6deJa+ZyoCUTRvH3liK5+N6VQZ4NIw90AgDXY6iPnsqplNFf6STcj+ePk0H/xqxnP75Lr0J0Fq3A==", "dev": true, "requires": { "@sinonjs/commons": "^1.7.0", @@ -4186,9 +4411,9 @@ "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "object-inspect": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", "dev": true }, "object-keys": { @@ -4324,9 +4549,9 @@ } }, "parse-duration": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.2.tgz", - "integrity": "sha512-0qfMZyjOUFBeEIvJ5EayfXJqaEXxQ+Oj2b7tWJM3hvEXvXsYCk05EDVI23oYnEw2NaFYUWdABEVPBvBMh8L/pA==" + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.3.tgz", + "integrity": "sha512-hMOZHfUmjxO5hMKn7Eft+ckP2M4nV4yzauLXiw3PndpkASnx5r8pDAMcOAiqxoemqWjMWmz4fOHQM6n6WwETXw==" }, "parse-json": { "version": "2.2.0", @@ -4441,9 +4666,9 @@ "dev": true }, "prettier-eslint": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.1.tgz", - "integrity": "sha512-KZT65QTosSAqBBqmrC+RpXbsMRe7Os2YSR9cAfFbDlyPAopzA/S5bioiZ3rpziNQNSJaOxmtXSx07EQ+o2Dlug==", + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.2.tgz", + "integrity": "sha512-u6EQqxUhaGfra9gy9shcR7MT7r/2twwEfRGy1tfzyaJvLQwSg34M9IU5HuF7FsLW2QUgr5VIUc56EPWibw1pdw==", "dev": true, "requires": { "@typescript-eslint/parser": "^1.10.2", @@ -5031,9 +5256,9 @@ } }, "protobufjs": { - "version": "6.8.8", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", - "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.9.0.tgz", + "integrity": "sha512-LlGVfEWDXoI/STstRDdZZKb/qusoAWUnmLg9R8OLSO473mBLWHowx8clbX5/+mKDEI+v7GzjoK9tRPZMMcoTrg==", "requires": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", @@ -5045,8 +5270,8 @@ "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.0", - "@types/node": "^10.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", "long": "^4.0.0" } }, @@ -5460,9 +5685,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sandboxed-module": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz", - "integrity": "sha512-wXiA6ULoGjCDwjn6evQF/Qi+oe77P+aCxizUktLBBKdqNbTxwec4GySJcS+O7iZFhme2ex04m+14KgknKKqFsw==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.4.tgz", + "integrity": "sha512-AwEPOdO8mg/wJjr876yCHP2DHqVN0MaggEXhp6IIf3bcI5cYoQl9QrrCHSrvToHjvdEiS5x4TVZRgjD2bEmNTA==", "dev": true, "requires": { "require-like": "0.1.2", @@ -5572,13 +5797,13 @@ "integrity": "sha512-meQNNykwecVxdu1RlYMKpQx4+wefIYpmxi6gexo/KAbwquJrBUrBmKYJrE8KFkVQAAVWEnwNdu21PgrD77J3xA==" }, "sinon": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.1.tgz", - "integrity": "sha512-iTTyiQo5T94jrOx7X7QLBZyucUJ2WvL9J13+96HMfm2CGoJYbIPqRfl6wgNcqmzk0DI28jeGx5bUTXizkrqBmg==", + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.2.tgz", + "integrity": "sha512-0uF8Q/QHkizNUmbK3LRFqx5cpTttEVXudywY9Uwzy8bTfZUhljZ7ARzSxnRHWYWtVTeh4Cw+tTb3iU21FQVO9A==", "dev": true, "requires": { - "@sinonjs/commons": "^1.7.0", - "@sinonjs/fake-timers": "^6.0.0", + "@sinonjs/commons": "^1.7.2", + "@sinonjs/fake-timers": "^6.0.1", "@sinonjs/formatio": "^5.0.1", "@sinonjs/samsam": "^5.0.3", "diff": "^4.0.2", @@ -5664,9 +5889,9 @@ } }, "spdx-correct": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", - "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", + "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", "dev": true, "requires": { "spdx-expression-parse": "^3.0.0", @@ -5674,15 +5899,15 @@ } }, "spdx-exceptions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", - "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==", "dev": true }, "spdx-expression-parse": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -5803,41 +6028,19 @@ } }, "string.prototype.trimend": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.0.tgz", - "integrity": "sha512-EEJnGqa/xNfIg05SxiPSqRS7S9qwDhYts1TSLR1BQfYUfPe1stofgGKvwERK9+9yf+PpfBMlpBaCHucXGPQfUA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5" } }, - "string.prototype.trimleft": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", - "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimstart": "^1.0.0" - } - }, - "string.prototype.trimright": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", - "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimend": "^1.0.0" - } - }, "string.prototype.trimstart": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.0.tgz", - "integrity": "sha512-iCP8g01NFYiiBOnwG1Xc3WZLyoo+RuBymwIlWncShXDDJYWN6DbnM3odslBJdgCdRlq94B5s63NWAZlcn2CS4w==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", "dev": true, "requires": { "define-properties": "^1.1.3", @@ -6071,6 +6274,18 @@ } } }, + "tsconfig-paths": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", + "integrity": "sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==", + "dev": true, + "requires": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.0", + "strip-bom": "^3.0.0" + } + }, "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", @@ -6133,9 +6348,9 @@ } }, "typescript": { - "version": "3.8.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.2.tgz", - "integrity": "sha512-EgOVgL/4xfVrCMbhYKUQTdF37SQn4Iw73H5BgCrF1Abdun7Kwy/QZsE/ssAy0y4LxBbvua3PIbFsbRczWWnDdQ==", + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.6.tgz", + "integrity": "sha512-Pspx3oKAPJtjNwE92YS05HQoY7z2SFyOpHo9MqJor3BXAGNaPUs83CuVp9VISFkSjyRfiTpmKuAYGJB7S7hOxw==", "dev": true }, "underscore": { @@ -6190,11 +6405,6 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - }, "v8-compile-cache": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", @@ -6241,9 +6451,9 @@ }, "dependencies": { "acorn": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", - "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", "dev": true }, "acorn-jsx": { @@ -6562,9 +6772,9 @@ } }, "yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", + "integrity": "sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg==" } } } diff --git a/services/filestore/package.json b/services/filestore/package.json index f8f468cdde..8efec070aa 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -13,7 +13,7 @@ "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "start": "node $NODE_APP_OPTIONS app.js", "nodemon": "nodemon --config nodemon.json", - "lint": "node_modules/.bin/eslint .", + "lint": "node_modules/.bin/eslint app test *.js", "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", @@ -22,45 +22,46 @@ "dependencies": { "@google-cloud/storage": "^4.7.0", "@overleaf/o-error": "^3.0.0", - "aws-sdk": "^2.648.0", + "@overleaf/object-persistor": "git+https://github.com/overleaf/object-persistor.git", + "aws-sdk": "^2.710.0", "body-parser": "^1.19.0", "express": "^4.17.1", "fast-crc32c": "^2.0.0", "glob": "^7.1.6", "lodash.once": "^4.1.1", - "logger-sharelatex": "2.1.1", - "metrics-sharelatex": "^2.6.2", + "logger-sharelatex": "^2.1.1", + "metrics-sharelatex": "^2.7.0", "node-uuid": "~1.4.8", "range-parser": "^1.2.1", "request": "^2.88.2", "request-promise-native": "^1.0.8", "settings-sharelatex": "^1.1.0", - "stream-buffers": "~0.2.5", + "stream-buffers": "~0.2.6", "tiny-async-pool": "^1.1.0" }, "devDependencies": { "babel-eslint": "^10.1.0", - "bunyan": "^1.8.12", + "bunyan": "^1.8.14", "chai": "4.2.0", "chai-as-promised": "^7.1.1", "disrequire": "^1.1.0", "eslint": "^6.8.0", - "eslint-config-prettier": "^6.10.0", - "eslint-config-standard": "^14.1.0", + "eslint-config-prettier": "^6.11.0", + "eslint-config-standard": "^14.1.1", "eslint-plugin-chai-expect": "^2.1.0", - "eslint-plugin-chai-friendly": "^0.5.0", - "eslint-plugin-import": "^2.20.1", + "eslint-plugin-chai-friendly": "^0.6.0", + "eslint-plugin-import": "^2.22.0", "eslint-plugin-mocha": "^6.3.0", - "eslint-plugin-node": "^11.0.0", - "eslint-plugin-prettier": "^3.1.2", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-prettier": "^3.1.4", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", - "mocha": "7.1.1", - "mongodb": "^3.5.5", - "prettier-eslint": "^9.0.1", + "mocha": "7.2.0", + "mongodb": "^3.5.9", + "prettier-eslint": "^9.0.2", "prettier-eslint-cli": "^5.0.0", - "sandboxed-module": "2.0.3", - "sinon": "9.0.1", + "sandboxed-module": "2.0.4", + "sinon": "9.0.2", "sinon-chai": "^3.5.0", "streamifier": "^0.1.1", "timekeeper": "^2.2.0" diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 6bc4f32719..076198a38b 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -114,6 +114,7 @@ class FilestoreApp { disrequire(Path.resolve(__dirname, '../../../app/js', file)) }) disrequire(Path.resolve(__dirname, '../../../app')) + disrequire('@overleaf/object-persistor') return require('../../../app') } diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 8382a48de5..6f7503410f 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -590,7 +590,7 @@ describe('Filestore', function() { describe('when copyOnMiss is disabled', function() { beforeEach(function() { - Settings.filestore.fallback.copyOnMiss = false + app.persistor.settings.copyOnMiss = false }) it('should fetch the file', async function() { @@ -611,7 +611,7 @@ describe('Filestore', function() { describe('when copyOnMiss is enabled', function() { beforeEach(function() { - Settings.filestore.fallback.copyOnMiss = true + app.persistor.settings.copyOnMiss = true }) it('should fetch the file', async function() { @@ -656,7 +656,7 @@ describe('Filestore', function() { describe('when copyOnMiss is false', function() { beforeEach(async function() { - Settings.filestore.fallback.copyOnMiss = false + app.persistor.settings.copyOnMiss = false const response = await rp(opts) expect(response.statusCode).to.equal(200) @@ -702,7 +702,7 @@ describe('Filestore', function() { describe('when copyOnMiss is true', function() { beforeEach(async function() { - Settings.filestore.fallback.copyOnMiss = true + app.persistor.settings.copyOnMiss = true const response = await rp(opts) expect(response.statusCode).to.equal(200) diff --git a/services/filestore/test/acceptance/js/TestHelper.js b/services/filestore/test/acceptance/js/TestHelper.js index df57303de1..935a8b50e7 100644 --- a/services/filestore/test/acceptance/js/TestHelper.js +++ b/services/filestore/test/acceptance/js/TestHelper.js @@ -34,11 +34,11 @@ function streamToString(stream) { async function uploadStringToPersistor(persistor, bucket, key, content) { const fileStream = streamifier.createReadStream(content) - await persistor.promises.sendStream(bucket, key, fileStream) + await persistor.sendStream(bucket, key, fileStream) } async function getStringFromPersistor(persistor, bucket, key) { - const stream = await persistor.promises.getFileStream(bucket, key, {}) + const stream = await persistor.getObjectStream(bucket, key, {}) return streamToString(stream) } diff --git a/services/filestore/test/unit/js/FSPersistorTests.js b/services/filestore/test/unit/js/FSPersistorTests.js deleted file mode 100644 index c8255987ba..0000000000 --- a/services/filestore/test/unit/js/FSPersistorTests.js +++ /dev/null @@ -1,353 +0,0 @@ -const sinon = require('sinon') -const chai = require('chai') -const { expect } = chai -const SandboxedModule = require('sandboxed-module') -const Errors = require('../../../app/js/Errors') -const StreamModule = require('stream') - -chai.use(require('sinon-chai')) -chai.use(require('chai-as-promised')) - -const modulePath = '../../../app/js/FSPersistor.js' - -describe('FSPersistorTests', function() { - const stat = { size: 4, isFile: sinon.stub().returns(true) } - const fd = 1234 - const writeStream = 'writeStream' - const remoteStream = 'remoteStream' - const tempFile = '/tmp/potato.txt' - const location = '/foo' - const error = new Error('guru meditation error') - const md5 = 'ffffffff' - - const files = ['animals/wombat.tex', 'vegetables/potato.tex'] - const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`] - const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex'] - let fs, stream, LocalFileWriter, FSPersistor, glob, readStream, crypto, Hash - - beforeEach(function() { - readStream = { - name: 'readStream', - on: sinon.stub().yields(), - pipe: sinon.stub() - } - fs = { - createReadStream: sinon.stub().returns(readStream), - createWriteStream: sinon.stub().returns(writeStream), - unlink: sinon.stub().yields(), - open: sinon.stub().yields(null, fd), - stat: sinon.stub().yields(null, stat) - } - glob = sinon.stub().yields(null, globs) - stream = { - pipeline: sinon.stub().yields(), - Transform: StreamModule.Transform - } - LocalFileWriter = { - promises: { - writeStream: sinon.stub().resolves(tempFile), - deleteFile: sinon.stub().resolves() - } - } - Hash = { - end: sinon.stub(), - read: sinon.stub().returns(md5), - digest: sinon.stub().returns(md5), - setEncoding: sinon.stub() - } - crypto = { - createHash: sinon.stub().returns(Hash) - } - FSPersistor = SandboxedModule.require(modulePath, { - requires: { - './LocalFileWriter': LocalFileWriter, - './Errors': Errors, - fs, - glob, - stream, - crypto, - // imported by PersistorHelper but otherwise unused here - 'logger-sharelatex': {}, - 'metrics-sharelatex': {} - }, - globals: { console } - }) - }) - - describe('sendFile', function() { - const localFilesystemPath = '/path/to/local/file' - it('should copy the file', async function() { - await FSPersistor.promises.sendFile( - location, - files[0], - localFilesystemPath - ) - expect(fs.createReadStream).to.have.been.calledWith(localFilesystemPath) - expect(fs.createWriteStream).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) - }) - - it('should return an error if the file cannot be stored', async function() { - stream.pipeline.yields(error) - await expect( - FSPersistor.promises.sendFile(location, files[0], localFilesystemPath) - ).to.eventually.be.rejected.and.have.property('cause', error) - }) - }) - - describe('sendStream', function() { - it('should send the stream to LocalFileWriter', async function() { - await FSPersistor.promises.sendStream(location, files[0], remoteStream) - expect(LocalFileWriter.promises.writeStream).to.have.been.calledWith( - remoteStream - ) - }) - - it('should delete the temporary file', async function() { - await FSPersistor.promises.sendStream(location, files[0], remoteStream) - expect(LocalFileWriter.promises.deleteFile).to.have.been.calledWith( - tempFile - ) - }) - - it('should return the error from LocalFileWriter', async function() { - LocalFileWriter.promises.writeStream.rejects(error) - await expect( - FSPersistor.promises.sendStream(location, files[0], remoteStream) - ).to.eventually.be.rejectedWith(error) - }) - - it('should send the temporary file to the filestore', async function() { - await FSPersistor.promises.sendStream(location, files[0], remoteStream) - expect(fs.createReadStream).to.have.been.calledWith(tempFile) - }) - - describe('when the md5 hash does not match', function() { - it('should return a write error', async function() { - await expect( - FSPersistor.promises.sendStream( - location, - files[0], - remoteStream, - '00000000' - ) - ) - .to.eventually.be.rejected.and.be.an.instanceOf(Errors.WriteError) - .and.have.property('message', 'md5 hash mismatch') - }) - - it('deletes the copied file', async function() { - try { - await FSPersistor.promises.sendStream( - location, - files[0], - remoteStream, - '00000000' - ) - } catch (_) {} - expect(LocalFileWriter.promises.deleteFile).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - }) - }) - }) - - describe('getFileStream', function() { - it('should use correct file location', async function() { - await FSPersistor.promises.getFileStream(location, files[0], {}) - expect(fs.open).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - }) - - it('should pass the options to createReadStream', async function() { - await FSPersistor.promises.getFileStream(location, files[0], { - start: 0, - end: 8 - }) - expect(fs.createReadStream).to.have.been.calledWith(null, { - start: 0, - end: 8, - fd - }) - }) - - it('should give a NotFoundError if the file does not exist', async function() { - const err = new Error() - err.code = 'ENOENT' - fs.open.yields(err) - - await expect(FSPersistor.promises.getFileStream(location, files[0], {})) - .to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) - .and.have.property('cause', err) - }) - - it('should wrap any other error', async function() { - fs.open.yields(error) - await expect(FSPersistor.promises.getFileStream(location, files[0], {})) - .to.eventually.be.rejectedWith('failed to open file for streaming') - .and.be.an.instanceOf(Errors.ReadError) - .and.have.property('cause', error) - }) - }) - - describe('getFileSize', function() { - const badFilename = 'neenaw.tex' - const size = 65536 - const noentError = new Error('not found') - noentError.code = 'ENOENT' - - beforeEach(function() { - fs.stat - .yields(error) - .withArgs(`${location}/${filteredFilenames[0]}`) - .yields(null, { size }) - .withArgs(`${location}/${badFilename}`) - .yields(noentError) - }) - - it('should return the file size', async function() { - expect( - await FSPersistor.promises.getFileSize(location, files[0]) - ).to.equal(size) - }) - - it('should throw a NotFoundError if the file does not exist', async function() { - await expect( - FSPersistor.promises.getFileSize(location, badFilename) - ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) - }) - - it('should wrap any other error', async function() { - await expect(FSPersistor.promises.getFileSize(location, 'raccoon')) - .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) - .and.have.property('cause', error) - }) - }) - - describe('copyFile', function() { - it('Should open the source for reading', async function() { - await FSPersistor.promises.copyFile(location, files[0], files[1]) - expect(fs.createReadStream).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - }) - - it('Should open the target for writing', async function() { - await FSPersistor.promises.copyFile(location, files[0], files[1]) - expect(fs.createWriteStream).to.have.been.calledWith( - `${location}/${filteredFilenames[1]}` - ) - }) - - it('Should pipe the source to the target', async function() { - await FSPersistor.promises.copyFile(location, files[0], files[1]) - expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) - }) - }) - - describe('deleteFile', function() { - it('Should call unlink with correct options', async function() { - await FSPersistor.promises.deleteFile(location, files[0]) - expect(fs.unlink).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - }) - - it('Should propagate the error', async function() { - fs.unlink.yields(error) - await expect( - FSPersistor.promises.deleteFile(location, files[0]) - ).to.eventually.be.rejected.and.have.property('cause', error) - }) - }) - - describe('deleteDirectory', function() { - it('Should call glob with correct options', async function() { - await FSPersistor.promises.deleteDirectory(location, files[0]) - expect(glob).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}*` - ) - }) - - it('Should call unlink on the returned files', async function() { - await FSPersistor.promises.deleteDirectory(location, files[0]) - for (const filename of globs) { - expect(fs.unlink).to.have.been.calledWith(filename) - } - }) - - it('Should propagate the error', async function() { - glob.yields(error) - await expect( - FSPersistor.promises.deleteDirectory(location, files[0]) - ).to.eventually.be.rejected.and.have.property('cause', error) - }) - }) - - describe('checkIfFileExists', function() { - const badFilename = 'pototo' - const noentError = new Error('not found') - noentError.code = 'ENOENT' - - beforeEach(function() { - fs.stat - .yields(error) - .withArgs(`${location}/${filteredFilenames[0]}`) - .yields(null, {}) - .withArgs(`${location}/${badFilename}`) - .yields(noentError) - }) - - it('Should call stat with correct options', async function() { - await FSPersistor.promises.checkIfFileExists(location, files[0]) - expect(fs.stat).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}` - ) - }) - - it('Should return true for existing files', async function() { - expect( - await FSPersistor.promises.checkIfFileExists(location, files[0]) - ).to.equal(true) - }) - - it('Should return false for non-existing files', async function() { - expect( - await FSPersistor.promises.checkIfFileExists(location, badFilename) - ).to.equal(false) - }) - - it('should wrap the error if there is a problem', async function() { - await expect(FSPersistor.promises.checkIfFileExists(location, 'llama')) - .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) - .and.have.property('cause', error) - }) - }) - - describe('directorySize', function() { - it('should wrap the error', async function() { - glob.yields(error) - await expect(FSPersistor.promises.directorySize(location, files[0])) - .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) - .and.include({ cause: error }) - .and.have.property('info') - .which.includes({ location, name: files[0] }) - }) - - it('should filter the directory name', async function() { - await FSPersistor.promises.directorySize(location, files[0]) - expect(glob).to.have.been.calledWith( - `${location}/${filteredFilenames[0]}_*` - ) - }) - - it('should sum directory files size', async function() { - expect( - await FSPersistor.promises.directorySize(location, files[0]) - ).to.equal(stat.size * files.length) - }) - }) -}) diff --git a/services/filestore/test/unit/js/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js index cd1d19ef02..fe446bd478 100644 --- a/services/filestore/test/unit/js/FileControllerTests.js +++ b/services/filestore/test/unit/js/FileControllerTests.js @@ -32,8 +32,8 @@ describe('FileController', function() { beforeEach(function() { PersistorManager = { sendStream: sinon.stub().yields(), - copyFile: sinon.stub().yields(), - deleteFile: sinon.stub().yields() + copyObject: sinon.stub().resolves(), + deleteObject: sinon.stub().yields() } FileHandler = { @@ -216,7 +216,9 @@ describe('FileController', function() { }) it('should return a 404 is the file is not found', function(done) { - FileHandler.getFileSize.yields(new Errors.NotFoundError()) + FileHandler.getFileSize.yields( + new Errors.NotFoundError({ message: 'not found', info: {} }) + ) res.sendStatus = code => { expect(code).to.equal(404) @@ -262,7 +264,7 @@ describe('FileController', function() { it('should send bucket name and both keys to PersistorManager', function(done) { res.sendStatus = code => { code.should.equal(200) - expect(PersistorManager.copyFile).to.have.been.calledWith( + expect(PersistorManager.copyObject).to.have.been.calledWith( bucket, oldKey, key @@ -273,7 +275,9 @@ describe('FileController', function() { }) it('should send a 404 if the original file was not found', function(done) { - PersistorManager.copyFile.yields(new Errors.NotFoundError()) + PersistorManager.copyObject.rejects( + new Errors.NotFoundError({ message: 'not found', info: {} }) + ) res.sendStatus = code => { code.should.equal(404) done() @@ -281,10 +285,12 @@ describe('FileController', function() { FileController.copyFile(req, res, next) }) - it('should send an error if there was an error', function() { - PersistorManager.copyFile.yields(error) - FileController.copyFile(req, res, next) - expect(next).to.have.been.calledWith(error) + it('should send an error if there was an error', function(done) { + PersistorManager.copyObject.rejects(error) + FileController.copyFile(req, res, err => { + expect(err).to.equal(error) + done() + }) }) }) diff --git a/services/filestore/test/unit/js/FileConverterTests.js b/services/filestore/test/unit/js/FileConverterTests.js index 72d6413417..671544098e 100644 --- a/services/filestore/test/unit/js/FileConverterTests.js +++ b/services/filestore/test/unit/js/FileConverterTests.js @@ -2,6 +2,7 @@ const sinon = require('sinon') const chai = require('chai') const { expect } = chai const SandboxedModule = require('sandboxed-module') +const { Errors } = require('@overleaf/object-persistor') const modulePath = '../../../app/js/FileConverter.js' @@ -22,6 +23,8 @@ describe('FileConverter', function() { promises: sinon.stub().resolves(destPath) } + const ObjectPersistor = { Errors } + FileConverter = SandboxedModule.require(modulePath, { requires: { './SafeExec': SafeExec, @@ -29,7 +32,8 @@ describe('FileConverter', function() { inc: sinon.stub(), Timer: sinon.stub().returns({ done: sinon.stub() }) }, - 'settings-sharelatex': Settings + 'settings-sharelatex': Settings, + '@overleaf/object-persistor': ObjectPersistor } }) }) diff --git a/services/filestore/test/unit/js/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js index 22fe8d1503..ecce74d870 100644 --- a/services/filestore/test/unit/js/FileHandlerTests.js +++ b/services/filestore/test/unit/js/FileHandlerTests.js @@ -4,6 +4,7 @@ const { expect } = chai const modulePath = '../../../app/js/FileHandler.js' const SandboxedModule = require('sandboxed-module') const { ObjectId } = require('mongodb') +const { Errors } = require('@overleaf/object-persistor') chai.use(require('sinon-chai')) chai.use(require('chai-as-promised')) @@ -32,17 +33,15 @@ describe('FileHandler', function() { beforeEach(function() { PersistorManager = { - promises: { - getFileStream: sinon.stub().resolves(sourceStream), - getRedirectUrl: sinon.stub().resolves(redirectUrl), - checkIfFileExists: sinon.stub().resolves(), - deleteFile: sinon.stub().resolves(), - deleteDirectory: sinon.stub().resolves(), - sendStream: sinon.stub().resolves(), - insertFile: sinon.stub().resolves(), - sendFile: sinon.stub().resolves(), - directorySize: sinon.stub().resolves() - } + getObjectStream: sinon.stub().resolves(sourceStream), + getRedirectUrl: sinon.stub().resolves(redirectUrl), + checkIfObjectExists: sinon.stub().resolves(), + deleteObject: sinon.stub().resolves(), + deleteDirectory: sinon.stub().resolves(), + sendStream: sinon.stub().resolves(), + insertFile: sinon.stub().resolves(), + sendFile: sinon.stub().resolves(), + directorySize: sinon.stub().resolves() } LocalFileWriter = { // the callback style is used for detached cleanup calls @@ -73,6 +72,8 @@ describe('FileHandler', function() { createReadStream: sinon.stub().returns(readStream) } + const ObjectPersistor = { Errors } + FileHandler = SandboxedModule.require(modulePath, { requires: { './PersistorManager': PersistorManager, @@ -81,6 +82,7 @@ describe('FileHandler', function() { './KeyBuilder': KeyBuilder, './ImageOptimiser': ImageOptimiser, 'settings-sharelatex': Settings, + '@overleaf/object-persistor': ObjectPersistor, fs: fs }, globals: { console } @@ -93,7 +95,7 @@ describe('FileHandler', function() { it('should send file to the filestore', function(done) { FileHandler.insertFile(bucket, key, stream, err => { expect(err).not.to.exist - expect(PersistorManager.promises.sendStream).to.have.been.calledWith( + expect(PersistorManager.sendStream).to.have.been.calledWith( bucket, key, stream @@ -105,8 +107,7 @@ describe('FileHandler', function() { it('should not make a delete request for the convertedKey folder', function(done) { FileHandler.insertFile(bucket, key, stream, err => { expect(err).not.to.exist - expect(PersistorManager.promises.deleteDirectory).not.to.have.been - .called + expect(PersistorManager.deleteDirectory).not.to.have.been.called done() }) }) @@ -137,9 +138,10 @@ describe('FileHandler', function() { it('should delete the convertedKey folder', function(done) { FileHandler.insertFile(bucket, key, stream, err => { expect(err).not.to.exist - expect( - PersistorManager.promises.deleteDirectory - ).to.have.been.calledWith(bucket, convertedFolderKey) + expect(PersistorManager.deleteDirectory).to.have.been.calledWith( + bucket, + convertedFolderKey + ) done() }) }) @@ -150,7 +152,7 @@ describe('FileHandler', function() { it('should tell the filestore manager to delete the file', function(done) { FileHandler.deleteFile(bucket, key, err => { expect(err).not.to.exist - expect(PersistorManager.promises.deleteFile).to.have.been.calledWith( + expect(PersistorManager.deleteObject).to.have.been.calledWith( bucket, key ) @@ -161,8 +163,7 @@ describe('FileHandler', function() { it('should not tell the filestore manager to delete the cached folder', function(done) { FileHandler.deleteFile(bucket, key, err => { expect(err).not.to.exist - expect(PersistorManager.promises.deleteDirectory).not.to.have.been - .called + expect(PersistorManager.deleteDirectory).not.to.have.been.called done() }) }) @@ -193,9 +194,10 @@ describe('FileHandler', function() { it('should delete the convertedKey folder', function(done) { FileHandler.deleteFile(bucket, key, err => { expect(err).not.to.exist - expect( - PersistorManager.promises.deleteDirectory - ).to.have.been.calledWith(bucket, convertedFolderKey) + expect(PersistorManager.deleteDirectory).to.have.been.calledWith( + bucket, + convertedFolderKey + ) done() }) }) @@ -206,9 +208,10 @@ describe('FileHandler', function() { it('should tell the filestore manager to delete the folder', function(done) { FileHandler.deleteProject(bucket, projectKey, err => { expect(err).not.to.exist - expect( - PersistorManager.promises.deleteDirectory - ).to.have.been.calledWith(bucket, projectKey) + expect(PersistorManager.deleteDirectory).to.have.been.calledWith( + bucket, + projectKey + ) done() }) }) @@ -234,7 +237,7 @@ describe('FileHandler', function() { const options = { start: 0, end: 8 } FileHandler.getFile(bucket, key, options, err => { expect(err).not.to.exist - expect(PersistorManager.promises.getFileStream).to.have.been.calledWith( + expect(PersistorManager.getObjectStream).to.have.been.calledWith( bucket, key, options @@ -265,17 +268,16 @@ describe('FileHandler', function() { it('should return the the converted stream', function() { expect(result.err).not.to.exist expect(result.stream).to.equal(readStream) - expect( - PersistorManager.promises.getFileStream - ).to.have.been.calledWith(bucket, key) + expect(PersistorManager.getObjectStream).to.have.been.calledWith( + bucket, + key + ) }) }) describe('when the file is cached', function() { beforeEach(function(done) { - PersistorManager.promises.checkIfFileExists = sinon - .stub() - .resolves(true) + PersistorManager.checkIfObjectExists = sinon.stub().resolves(true) FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => { result = { err, stream } done() @@ -293,9 +295,10 @@ describe('FileHandler', function() { it('should return the cached stream', function() { expect(result.err).not.to.exist expect(result.stream).to.equal(sourceStream) - expect( - PersistorManager.promises.getFileStream - ).to.have.been.calledWith(bucket, convertedKey) + expect(PersistorManager.getObjectStream).to.have.been.calledWith( + bucket, + convertedKey + ) }) }) }) @@ -341,9 +344,10 @@ describe('FileHandler', function() { it('should call the persistor to get a redirect url', function(done) { FileHandler.getRedirectUrl(bucket, key, () => { - expect( - PersistorManager.promises.getRedirectUrl - ).to.have.been.calledWith(bucket, key) + expect(PersistorManager.getRedirectUrl).to.have.been.calledWith( + bucket, + key + ) done() }) }) @@ -383,7 +387,7 @@ describe('FileHandler', function() { it('should call the filestore manager to get directory size', function(done) { FileHandler.getDirectorySize(bucket, key, err => { expect(err).not.to.exist - expect(PersistorManager.promises.directorySize).to.have.been.calledWith( + expect(PersistorManager.directorySize).to.have.been.calledWith( bucket, key ) diff --git a/services/filestore/test/unit/js/GcsPersistorTests.js b/services/filestore/test/unit/js/GcsPersistorTests.js deleted file mode 100644 index 027a63298b..0000000000 --- a/services/filestore/test/unit/js/GcsPersistorTests.js +++ /dev/null @@ -1,683 +0,0 @@ -const sinon = require('sinon') -const chai = require('chai') -const { expect } = chai -const modulePath = '../../../app/js/GcsPersistor.js' -const SandboxedModule = require('sandboxed-module') -const { ObjectId } = require('mongodb') -const asyncPool = require('tiny-async-pool') - -const Errors = require('../../../app/js/Errors') - -describe('GcsPersistorTests', function() { - const filename = '/wombat/potato.tex' - const bucket = 'womBucket' - const key = 'monKey' - const destKey = 'donKey' - const genericError = new Error('guru meditation error') - const filesSize = 33 - const md5 = 'ffffffff00000000ffffffff00000000' - const WriteStream = 'writeStream' - const redirectUrl = 'https://wombat.potato/giraffe' - - let Metrics, - Logger, - Transform, - Storage, - Fs, - GcsNotFoundError, - ReadStream, - Stream, - GcsBucket, - GcsFile, - GcsPersistor, - FileNotFoundError, - Hash, - settings, - crypto, - files - - beforeEach(function() { - settings = { - filestore: { - backend: 'gcs', - stores: { - user_files: 'user_files' - }, - gcs: { - directoryKeyRegex: /^[0-9a-fA-F]{24}\/[0-9a-fA-F]{24}/ - } - } - } - - files = [ - { - metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' }, - delete: sinon.stub() - }, - { - metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' }, - delete: sinon.stub() - } - ] - - ReadStream = { - pipe: sinon.stub().returns('readStream'), - on: sinon.stub(), - removeListener: sinon.stub() - } - ReadStream.on.withArgs('end').yields() - ReadStream.on.withArgs('pipe').yields({ - unpipe: sinon.stub(), - resume: sinon.stub(), - on: sinon.stub() - }) - - Transform = class { - on(event, callback) { - if (event === 'readable') { - callback() - } - } - - once() {} - removeListener() {} - } - - Stream = { - pipeline: sinon.stub().yields(), - Transform: Transform - } - - Metrics = { - count: sinon.stub() - } - - GcsFile = { - delete: sinon.stub().resolves(), - createReadStream: sinon.stub().returns(ReadStream), - getMetadata: sinon.stub().resolves([files[0].metadata]), - createWriteStream: sinon.stub().returns(WriteStream), - copy: sinon.stub().resolves(), - exists: sinon.stub().resolves([true]), - getSignedUrl: sinon.stub().resolves([redirectUrl]) - } - - GcsBucket = { - file: sinon.stub().returns(GcsFile), - getFiles: sinon.stub().resolves([files]) - } - - Storage = class { - constructor() { - this.interceptors = [] - } - } - Storage.prototype.bucket = sinon.stub().returns(GcsBucket) - - GcsNotFoundError = new Error('File not found') - GcsNotFoundError.code = 404 - - Fs = { - createReadStream: sinon.stub().returns(ReadStream) - } - - FileNotFoundError = new Error('File not found') - FileNotFoundError.code = 'ENOENT' - - Hash = { - end: sinon.stub(), - read: sinon.stub().returns(md5), - digest: sinon.stub().returns(md5), - setEncoding: sinon.stub() - } - crypto = { - createHash: sinon.stub().returns(Hash) - } - - Logger = { - warn: sinon.stub() - } - - GcsPersistor = SandboxedModule.require(modulePath, { - requires: { - '@google-cloud/storage': { Storage }, - 'settings-sharelatex': settings, - 'logger-sharelatex': Logger, - 'tiny-async-pool': asyncPool, - './Errors': Errors, - fs: Fs, - stream: Stream, - 'metrics-sharelatex': Metrics, - crypto - }, - globals: { console, Buffer } - }) - }) - - describe('getFileStream', function() { - describe('when called with valid parameters', function() { - let stream - - beforeEach(async function() { - stream = await GcsPersistor.promises.getFileStream(bucket, key) - }) - - it('returns a metered stream', function() { - expect(stream).to.be.instanceOf(Transform) - }) - - it('fetches the right key from the right bucket', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.createReadStream).to.have.been.called - }) - - it('pipes the stream through the meter', function() { - expect(ReadStream.pipe).to.have.been.calledWith( - sinon.match.instanceOf(Transform) - ) - }) - }) - - describe('when called with a byte range', function() { - let stream - - beforeEach(async function() { - stream = await GcsPersistor.promises.getFileStream(bucket, key, { - start: 5, - end: 10 - }) - }) - - it('returns a metered stream', function() { - expect(stream).to.be.instanceOf(Transform) - }) - - it('passes the byte range on to GCS', function() { - expect(GcsFile.createReadStream).to.have.been.calledWith({ - start: 5, - end: 10 - }) - }) - }) - - describe("when the file doesn't exist", function() { - let error, stream - - beforeEach(async function() { - Transform.prototype.on = sinon.stub() - ReadStream.on.withArgs('error').yields(GcsNotFoundError) - try { - stream = await GcsPersistor.promises.getFileStream(bucket, key) - } catch (e) { - error = e - } - }) - - it('does not return a stream', function() { - expect(stream).not.to.exist - }) - - it('throws a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - - it('wraps the error', function() { - expect(error.cause).to.exist - }) - - it('stores the bucket and key in the error', function() { - expect(error.info).to.include({ bucketName: bucket, key: key }) - }) - }) - - describe('when Gcs encounters an unkown error', function() { - let error, stream - - beforeEach(async function() { - Transform.prototype.on = sinon.stub() - ReadStream.on.withArgs('error').yields(genericError) - try { - stream = await GcsPersistor.promises.getFileStream(bucket, key) - } catch (err) { - error = err - } - }) - - it('does not return a stream', function() { - expect(stream).not.to.exist - }) - - it('throws a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('wraps the error', function() { - expect(error.cause).to.exist - }) - - it('stores the bucket and key in the error', function() { - expect(error.info).to.include({ bucketName: bucket, key: key }) - }) - }) - }) - - describe('getFile', function() { - let signedUrl - - beforeEach(async function() { - signedUrl = await GcsPersistor.promises.getRedirectUrl(bucket, key) - }) - - it('should request a signed URL', function() { - expect(GcsFile.getSignedUrl).to.have.been.called - }) - - it('should return the url', function() { - expect(signedUrl).to.equal(redirectUrl) - }) - }) - - describe('getFileSize', function() { - describe('when called with valid parameters', function() { - let size - - beforeEach(async function() { - size = await GcsPersistor.promises.getFileSize(bucket, key) - }) - - it('should return the object size', function() { - expect(size).to.equal(files[0].metadata.size) - }) - - it('should pass the bucket and key to GCS', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.getMetadata).to.have.been.called - }) - }) - - describe('when the object is not found', function() { - let error - - beforeEach(async function() { - GcsFile.getMetadata = sinon.stub().rejects(GcsNotFoundError) - try { - await GcsPersistor.promises.getFileSize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(GcsNotFoundError) - }) - }) - - describe('when GCS returns an error', function() { - let error - - beforeEach(async function() { - GcsFile.getMetadata = sinon.stub().rejects(genericError) - try { - await GcsPersistor.promises.getFileSize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('sendStream', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return GcsPersistor.promises.sendStream(bucket, key, ReadStream) - }) - - it('should upload the stream', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.createWriteStream).to.have.been.called - }) - - it('should not try to create a resumable upload', function() { - expect(GcsFile.createWriteStream).to.have.been.calledWith({ - resumable: false - }) - }) - - it('should meter the stream and pass it to GCS', function() { - expect(Stream.pipeline).to.have.been.calledWith( - ReadStream, - sinon.match.instanceOf(Transform), - WriteStream - ) - }) - - it('calculates the md5 hash of the file', function() { - expect(Hash.digest).to.have.been.called - }) - }) - - describe('when a hash is supplied', function() { - beforeEach(async function() { - return GcsPersistor.promises.sendStream( - bucket, - key, - ReadStream, - 'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb' - ) - }) - - it('should not calculate the md5 hash of the file', function() { - expect(Hash.digest).not.to.have.been.called - }) - - it('sends the hash in base64', function() { - expect(GcsFile.createWriteStream).to.have.been.calledWith({ - validation: 'md5', - metadata: { - md5Hash: 'qqqqqru7u7uqqqqqu7u7uw==' - }, - resumable: false - }) - }) - - it('does not fetch the md5 hash of the uploaded file', function() { - expect(GcsFile.getMetadata).not.to.have.been.called - }) - }) - - describe('when the upload fails', function() { - let error - beforeEach(async function() { - Stream.pipeline - .withArgs( - ReadStream, - sinon.match.instanceOf(Transform), - WriteStream, - sinon.match.any - ) - .yields(genericError) - try { - await GcsPersistor.promises.sendStream(bucket, key, ReadStream) - } catch (err) { - error = err - } - }) - - it('throws a WriteError', function() { - expect(error).to.be.an.instanceOf(Errors.WriteError) - }) - - it('wraps the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('sendFile', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return GcsPersistor.promises.sendFile(bucket, key, filename) - }) - - it('should create a read stream for the file', function() { - expect(Fs.createReadStream).to.have.been.calledWith(filename) - }) - - it('should create a write stream', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.createWriteStream).to.have.been.called - }) - - it('should upload the stream via the meter', function() { - expect(Stream.pipeline).to.have.been.calledWith( - ReadStream, - sinon.match.instanceOf(Transform), - WriteStream - ) - }) - }) - }) - - describe('copyFile', function() { - const destinationFile = 'destFile' - - beforeEach(function() { - GcsBucket.file.withArgs(destKey).returns(destinationFile) - }) - - describe('with valid parameters', function() { - beforeEach(async function() { - return GcsPersistor.promises.copyFile(bucket, key, destKey) - }) - - it('should copy the object', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.copy).to.have.been.calledWith(destinationFile) - }) - }) - - describe('when the file does not exist', function() { - let error - - beforeEach(async function() { - GcsFile.copy = sinon.stub().rejects(GcsNotFoundError) - try { - await GcsPersistor.promises.copyFile(bucket, key, destKey) - } catch (err) { - error = err - } - }) - - it('should throw a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - }) - }) - - describe('deleteFile', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return GcsPersistor.promises.deleteFile(bucket, key) - }) - - it('should delete the object', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.delete).to.have.been.called - }) - }) - - describe('when the file does not exist', function() { - let error - - beforeEach(async function() { - GcsFile.delete = sinon.stub().rejects(GcsNotFoundError) - try { - await GcsPersistor.promises.deleteFile(bucket, key) - } catch (err) { - error = err - } - }) - - it('should not throw an error', function() { - expect(error).not.to.exist - }) - }) - }) - - describe('deleteDirectory', function() { - const directoryName = `${ObjectId()}/${ObjectId()}` - describe('with valid parameters', function() { - beforeEach(async function() { - return GcsPersistor.promises.deleteDirectory(bucket, directoryName) - }) - - it('should list the objects in the directory', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.getFiles).to.have.been.calledWith({ - directory: directoryName - }) - }) - - it('should delete the files', function() { - expect(GcsFile.delete).to.have.been.calledTwice - }) - }) - - describe('when there is an error listing the objects', function() { - let error - - beforeEach(async function() { - GcsBucket.getFiles = sinon.stub().rejects(genericError) - try { - await GcsPersistor.promises.deleteDirectory(bucket, directoryName) - } catch (err) { - error = err - } - }) - - it('should generate a WriteError', function() { - expect(error).to.be.an.instanceOf(Errors.WriteError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('directorySize', function() { - describe('with valid parameters', function() { - let size - - beforeEach(async function() { - size = await GcsPersistor.promises.directorySize(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key }) - }) - - it('should return the directory size', function() { - expect(size).to.equal(filesSize) - }) - }) - - describe('when there are no files', function() { - let size - - beforeEach(async function() { - GcsBucket.getFiles.resolves([[]]) - size = await GcsPersistor.promises.directorySize(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key }) - }) - - it('should return zero', function() { - expect(size).to.equal(0) - }) - }) - - describe('when there is an error listing the objects', function() { - let error - - beforeEach(async function() { - GcsBucket.getFiles.rejects(genericError) - try { - await GcsPersistor.promises.directorySize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('checkIfFileExists', function() { - describe('when the file exists', function() { - let exists - - beforeEach(async function() { - exists = await GcsPersistor.promises.checkIfFileExists(bucket, key) - }) - - it('should ask the file if it exists', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.exists).to.have.been.called - }) - - it('should return that the file exists', function() { - expect(exists).to.equal(true) - }) - }) - - describe('when the file does not exist', function() { - let exists - - beforeEach(async function() { - GcsFile.exists = sinon.stub().resolves([false]) - exists = await GcsPersistor.promises.checkIfFileExists(bucket, key) - }) - - it('should get the object header', function() { - expect(Storage.prototype.bucket).to.have.been.calledWith(bucket) - expect(GcsBucket.file).to.have.been.calledWith(key) - expect(GcsFile.exists).to.have.been.called - }) - - it('should return that the file does not exist', function() { - expect(exists).to.equal(false) - }) - }) - - describe('when there is an error', function() { - let error - - beforeEach(async function() { - GcsFile.exists = sinon.stub().rejects(genericError) - try { - await GcsPersistor.promises.checkIfFileExists(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) -}) diff --git a/services/filestore/test/unit/js/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js index 5f5158f28a..0316b5d800 100644 --- a/services/filestore/test/unit/js/LocalFileWriterTests.js +++ b/services/filestore/test/unit/js/LocalFileWriterTests.js @@ -3,6 +3,7 @@ const chai = require('chai') const { expect } = chai const modulePath = '../../../app/js/LocalFileWriter.js' const SandboxedModule = require('sandboxed-module') +const { Errors } = require('@overleaf/object-persistor') chai.use(require('sinon-chai')) describe('LocalFileWriter', function() { @@ -22,6 +23,8 @@ describe('LocalFileWriter', function() { pipeline: sinon.stub().yields() } + const ObjectPersistor = { Errors } + LocalFileWriter = SandboxedModule.require(modulePath, { requires: { fs, @@ -30,7 +33,8 @@ describe('LocalFileWriter', function() { 'metrics-sharelatex': { inc: sinon.stub(), Timer: sinon.stub().returns({ done: sinon.stub() }) - } + }, + '@overleaf/object-persistor': ObjectPersistor } }) }) diff --git a/services/filestore/test/unit/js/MigrationPersistorTests.js b/services/filestore/test/unit/js/MigrationPersistorTests.js deleted file mode 100644 index db8401c78c..0000000000 --- a/services/filestore/test/unit/js/MigrationPersistorTests.js +++ /dev/null @@ -1,519 +0,0 @@ -const sinon = require('sinon') -const chai = require('chai') -const { expect } = chai -const modulePath = '../../../app/js/MigrationPersistor.js' -const SandboxedModule = require('sandboxed-module') - -const Errors = require('../../../app/js/Errors') - -// Not all methods are tested here, but a method with each type of wrapping has -// tests. Specifically, the following wrapping methods are tested here: -// getFileStream: _wrapFallbackMethod -// sendStream: forward-to-primary -// deleteFile: _wrapMethodOnBothPersistors -// copyFile: copyFileWithFallback - -describe('MigrationPersistorTests', function() { - const bucket = 'womBucket' - const fallbackBucket = 'bucKangaroo' - const key = 'monKey' - const destKey = 'donKey' - const genericError = new Error('guru meditation error') - const notFoundError = new Errors.NotFoundError('not found') - const size = 33 - const md5 = 'ffffffff' - - let Metrics, - Settings, - Logger, - Stream, - MigrationPersistor, - fileStream, - newPersistor - - beforeEach(function() { - fileStream = { - name: 'fileStream', - on: sinon - .stub() - .withArgs('end') - .yields(), - pipe: sinon.stub() - } - - newPersistor = function(hasFile) { - return { - promises: { - sendFile: sinon.stub().resolves(), - sendStream: sinon.stub().resolves(), - getFileStream: hasFile - ? sinon.stub().resolves(fileStream) - : sinon.stub().rejects(notFoundError), - deleteDirectory: sinon.stub().resolves(), - getFileSize: hasFile - ? sinon.stub().resolves(size) - : sinon.stub().rejects(notFoundError), - deleteFile: sinon.stub().resolves(), - copyFile: hasFile - ? sinon.stub().resolves() - : sinon.stub().rejects(notFoundError), - checkIfFileExists: sinon.stub().resolves(hasFile), - directorySize: hasFile - ? sinon.stub().resolves(size) - : sinon.stub().rejects(notFoundError), - getFileMd5Hash: hasFile - ? sinon.stub().resolves(md5) - : sinon.stub().rejects(notFoundError) - } - } - } - - Settings = { - filestore: { - fallback: { - buckets: { - [bucket]: fallbackBucket - } - } - } - } - - Metrics = { - inc: sinon.stub() - } - - Stream = { - pipeline: sinon.stub().yields(), - PassThrough: sinon.stub() - } - - Logger = { - warn: sinon.stub() - } - - MigrationPersistor = SandboxedModule.require(modulePath, { - requires: { - 'settings-sharelatex': Settings, - stream: Stream, - './Errors': Errors, - 'metrics-sharelatex': Metrics, - 'logger-sharelatex': Logger - }, - globals: { console } - }) - }) - - describe('getFileStream', function() { - const options = { wombat: 'potato' } - describe('when the primary persistor has the file', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor, response - beforeEach(async function() { - primaryPersistor = newPersistor(true) - fallbackPersistor = newPersistor(false) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - response = await migrationPersistor.promises.getFileStream( - bucket, - key, - options - ) - }) - - it('should return the file stream', function() { - expect(response).to.equal(fileStream) - }) - - it('should fetch the file from the primary persistor, with the correct options', function() { - expect( - primaryPersistor.promises.getFileStream - ).to.have.been.calledWithExactly(bucket, key, options) - }) - - it('should not query the fallback persistor', function() { - expect(fallbackPersistor.promises.getFileStream).not.to.have.been.called - }) - }) - - describe('when the fallback persistor has the file', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor, response - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(true) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - response = await migrationPersistor.promises.getFileStream( - bucket, - key, - options - ) - }) - - it('should return the file stream', function() { - expect(response).to.be.an.instanceOf(Stream.PassThrough) - }) - - it('should fetch the file from the primary persistor with the correct options', function() { - expect( - primaryPersistor.promises.getFileStream - ).to.have.been.calledWithExactly(bucket, key, options) - }) - - it('should fetch the file from the fallback persistor with the fallback bucket with the correct options', function() { - expect( - fallbackPersistor.promises.getFileStream - ).to.have.been.calledWithExactly(fallbackBucket, key, options) - }) - - it('should create one read stream', function() { - expect(fallbackPersistor.promises.getFileStream).to.have.been.calledOnce - }) - - it('should not send the file to the primary', function() { - expect(primaryPersistor.promises.sendStream).not.to.have.been.called - }) - }) - - describe('when the file should be copied to the primary', function() { - let primaryPersistor, - fallbackPersistor, - migrationPersistor, - returnedStream - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(true) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - Settings.filestore.fallback.copyOnMiss = true - returnedStream = await migrationPersistor.promises.getFileStream( - bucket, - key, - options - ) - }) - - it('should create one read stream', function() { - expect(fallbackPersistor.promises.getFileStream).to.have.been.calledOnce - }) - - it('should get the md5 hash from the source', function() { - expect( - fallbackPersistor.promises.getFileMd5Hash - ).to.have.been.calledWith(fallbackBucket, key) - }) - - it('should send a stream to the primary', function() { - expect( - primaryPersistor.promises.sendStream - ).to.have.been.calledWithExactly( - bucket, - key, - sinon.match.instanceOf(Stream.PassThrough), - md5 - ) - }) - - it('should send a stream to the client', function() { - expect(returnedStream).to.be.an.instanceOf(Stream.PassThrough) - }) - }) - - describe('when neither persistor has the file', function() { - it('rejects with a NotFoundError', async function() { - const migrationPersistor = MigrationPersistor( - newPersistor(false), - newPersistor(false) - ) - return expect( - migrationPersistor.promises.getFileStream(bucket, key) - ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) - }) - }) - - describe('when the primary persistor throws an unexpected error', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor, error - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(true) - primaryPersistor.promises.getFileStream = sinon - .stub() - .rejects(genericError) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - try { - await migrationPersistor.promises.getFileStream(bucket, key, options) - } catch (err) { - error = err - } - }) - - it('rejects with the error', function() { - expect(error).to.equal(genericError) - }) - - it('does not call the fallback', function() { - expect(fallbackPersistor.promises.getFileStream).not.to.have.been.called - }) - }) - - describe('when the fallback persistor throws an unexpected error', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor, error - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(false) - fallbackPersistor.promises.getFileStream = sinon - .stub() - .rejects(genericError) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - try { - await migrationPersistor.promises.getFileStream(bucket, key, options) - } catch (err) { - error = err - } - }) - - it('rejects with the error', function() { - expect(error).to.equal(genericError) - }) - - it('should have called the fallback', function() { - expect( - fallbackPersistor.promises.getFileStream - ).to.have.been.calledWith(fallbackBucket, key) - }) - }) - }) - - describe('sendStream', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor - beforeEach(function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(false) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - }) - - describe('when it works', function() { - beforeEach(async function() { - return migrationPersistor.promises.sendStream(bucket, key, fileStream) - }) - - it('should send the file to the primary persistor', function() { - expect( - primaryPersistor.promises.sendStream - ).to.have.been.calledWithExactly(bucket, key, fileStream) - }) - - it('should not send the file to the fallback persistor', function() { - expect(fallbackPersistor.promises.sendStream).not.to.have.been.called - }) - }) - - describe('when the primary persistor throws an error', function() { - it('returns the error', async function() { - primaryPersistor.promises.sendStream.rejects(notFoundError) - return expect( - migrationPersistor.promises.sendStream(bucket, key, fileStream) - ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) - }) - }) - }) - - describe('deleteFile', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor - beforeEach(function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(false) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - }) - - describe('when it works', function() { - beforeEach(async function() { - return migrationPersistor.promises.deleteFile(bucket, key) - }) - - it('should delete the file from the primary', function() { - expect( - primaryPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(bucket, key) - }) - - it('should delete the file from the fallback', function() { - expect( - fallbackPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(fallbackBucket, key) - }) - }) - - describe('when the primary persistor throws an error', function() { - let error - beforeEach(async function() { - primaryPersistor.promises.deleteFile.rejects(genericError) - try { - await migrationPersistor.promises.deleteFile(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return the error', function() { - expect(error).to.equal(genericError) - }) - - it('should delete the file from the primary', function() { - expect( - primaryPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(bucket, key) - }) - - it('should delete the file from the fallback', function() { - expect( - fallbackPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(fallbackBucket, key) - }) - }) - - describe('when the fallback persistor throws an error', function() { - let error - beforeEach(async function() { - fallbackPersistor.promises.deleteFile.rejects(genericError) - try { - await migrationPersistor.promises.deleteFile(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return the error', function() { - expect(error).to.equal(genericError) - }) - - it('should delete the file from the primary', function() { - expect( - primaryPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(bucket, key) - }) - - it('should delete the file from the fallback', function() { - expect( - fallbackPersistor.promises.deleteFile - ).to.have.been.calledWithExactly(fallbackBucket, key) - }) - }) - }) - - describe('copyFile', function() { - describe('when the file exists on the primary', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor - beforeEach(async function() { - primaryPersistor = newPersistor(true) - fallbackPersistor = newPersistor(false) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - return migrationPersistor.promises.copyFile(bucket, key, destKey) - }) - - it('should call copyFile to copy the file', function() { - expect( - primaryPersistor.promises.copyFile - ).to.have.been.calledWithExactly(bucket, key, destKey) - }) - - it('should not try to read from the fallback', function() { - expect(fallbackPersistor.promises.getFileStream).not.to.have.been.called - }) - }) - - describe('when the file does not exist on the primary', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(true) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - return migrationPersistor.promises.copyFile(bucket, key, destKey) - }) - - it('should call copyFile to copy the file', function() { - expect( - primaryPersistor.promises.copyFile - ).to.have.been.calledWithExactly(bucket, key, destKey) - }) - - it('should fetch the file from the fallback', function() { - expect( - fallbackPersistor.promises.getFileStream - ).not.to.have.been.calledWithExactly(fallbackBucket, key) - }) - - it('should get the md5 hash from the source', function() { - expect( - fallbackPersistor.promises.getFileMd5Hash - ).to.have.been.calledWith(fallbackBucket, key) - }) - - it('should send the file to the primary', function() { - expect( - primaryPersistor.promises.sendStream - ).to.have.been.calledWithExactly( - bucket, - destKey, - sinon.match.instanceOf(Stream.PassThrough), - md5 - ) - }) - }) - - describe('when the file does not exist on the fallback', function() { - let primaryPersistor, fallbackPersistor, migrationPersistor, error - beforeEach(async function() { - primaryPersistor = newPersistor(false) - fallbackPersistor = newPersistor(false) - migrationPersistor = MigrationPersistor( - primaryPersistor, - fallbackPersistor - ) - try { - await migrationPersistor.promises.copyFile(bucket, key, destKey) - } catch (err) { - error = err - } - }) - - it('should call copyFile to copy the file', function() { - expect( - primaryPersistor.promises.copyFile - ).to.have.been.calledWithExactly(bucket, key, destKey) - }) - - it('should fetch the file from the fallback', function() { - expect( - fallbackPersistor.promises.getFileStream - ).not.to.have.been.calledWithExactly(fallbackBucket, key) - }) - - it('should return a not-found error', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - }) - }) -}) diff --git a/services/filestore/test/unit/js/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js deleted file mode 100644 index cdc9de0f92..0000000000 --- a/services/filestore/test/unit/js/PersistorManagerTests.js +++ /dev/null @@ -1,78 +0,0 @@ -const sinon = require('sinon') -const chai = require('chai') -const { expect } = chai -const SandboxedModule = require('sandboxed-module') - -const modulePath = '../../../app/js/PersistorManager.js' - -describe('PersistorManager', function() { - let PersistorManager, FSPersistor, S3Persistor, settings, requires - - beforeEach(function() { - FSPersistor = { - wrappedMethod: sinon.stub().returns('FSPersistor') - } - S3Persistor = { - wrappedMethod: sinon.stub().returns('S3Persistor') - } - - settings = { - filestore: {} - } - - requires = { - './S3Persistor': S3Persistor, - './FSPersistor': FSPersistor, - 'settings-sharelatex': settings, - 'logger-sharelatex': { - log() {}, - err() {} - } - } - }) - - it('should implement the S3 wrapped method when S3 is configured', function() { - settings.filestore.backend = 's3' - PersistorManager = SandboxedModule.require(modulePath, { requires }) - - expect(PersistorManager).to.respondTo('wrappedMethod') - expect(PersistorManager.wrappedMethod()).to.equal('S3Persistor') - }) - - it("should implement the S3 wrapped method when 'aws-sdk' is configured", function() { - settings.filestore.backend = 'aws-sdk' - PersistorManager = SandboxedModule.require(modulePath, { requires }) - - expect(PersistorManager).to.respondTo('wrappedMethod') - expect(PersistorManager.wrappedMethod()).to.equal('S3Persistor') - }) - - it('should implement the FS wrapped method when FS is configured', function() { - settings.filestore.backend = 'fs' - PersistorManager = SandboxedModule.require(modulePath, { requires }) - - expect(PersistorManager).to.respondTo('wrappedMethod') - expect(PersistorManager.wrappedMethod()).to.equal('FSPersistor') - }) - - it('should throw an error when the backend is not configured', function() { - try { - SandboxedModule.require(modulePath, { requires }) - } catch (err) { - expect(err.message).to.equal('no backend specified - config incomplete') - return - } - expect('should have caught an error').not.to.exist - }) - - it('should throw an error when the backend is unknown', function() { - settings.filestore.backend = 'magic' - try { - SandboxedModule.require(modulePath, { requires }) - } catch (err) { - expect(err.message).to.equal('unknown filestore backend: magic') - return - } - expect('should have caught an error').not.to.exist - }) -}) diff --git a/services/filestore/test/unit/js/S3PersistorTests.js b/services/filestore/test/unit/js/S3PersistorTests.js deleted file mode 100644 index c236de25ef..0000000000 --- a/services/filestore/test/unit/js/S3PersistorTests.js +++ /dev/null @@ -1,865 +0,0 @@ -const sinon = require('sinon') -const chai = require('chai') -const { expect } = chai -const modulePath = '../../../app/js/S3Persistor.js' -const SandboxedModule = require('sandboxed-module') - -const Errors = require('../../../app/js/Errors') - -describe('S3PersistorTests', function() { - const defaultS3Key = 'frog' - const defaultS3Secret = 'prince' - const defaultS3Credentials = { - credentials: { - accessKeyId: defaultS3Key, - secretAccessKey: defaultS3Secret - } - } - const filename = '/wombat/potato.tex' - const bucket = 'womBucket' - const key = 'monKey' - const destKey = 'donKey' - const objectSize = 5555 - const genericError = new Error('guru meditation error') - const files = [ - { Key: 'llama', Size: 11 }, - { Key: 'hippo', Size: 22 } - ] - const filesSize = 33 - const md5 = 'ffffffff00000000ffffffff00000000' - - let Metrics, - Logger, - Transform, - S3, - Fs, - ReadStream, - Stream, - S3Persistor, - S3Client, - S3ReadStream, - S3NotFoundError, - S3AccessDeniedError, - FileNotFoundError, - EmptyPromise, - settings, - Hash, - crypto - - beforeEach(function() { - settings = { - filestore: { - backend: 's3', - s3: { - secret: defaultS3Secret, - key: defaultS3Key, - partSize: 100 * 1024 * 1024 - }, - stores: { - user_files: 'sl_user_files' - } - } - } - - Transform = class { - on(event, callback) { - if (event === 'readable') { - callback() - } - } - - once() {} - removeListener() {} - } - - Stream = { - pipeline: sinon.stub().yields(), - Transform: Transform - } - - EmptyPromise = { - promise: sinon.stub().resolves() - } - - Metrics = { - count: sinon.stub() - } - - ReadStream = { - pipe: sinon.stub().returns('readStream'), - on: sinon.stub(), - removeListener: sinon.stub() - } - ReadStream.on.withArgs('end').yields() - ReadStream.on.withArgs('pipe').yields({ - unpipe: sinon.stub(), - resume: sinon.stub() - }) - - FileNotFoundError = new Error('File not found') - FileNotFoundError.code = 'ENOENT' - - Fs = { - createReadStream: sinon.stub().returns(ReadStream) - } - - S3NotFoundError = new Error('not found') - S3NotFoundError.code = 'NoSuchKey' - - S3AccessDeniedError = new Error('access denied') - S3AccessDeniedError.code = 'AccessDenied' - - S3ReadStream = { - on: sinon.stub(), - pipe: sinon.stub(), - removeListener: sinon.stub() - } - S3ReadStream.on.withArgs('end').yields() - S3ReadStream.on.withArgs('pipe').yields({ - unpipe: sinon.stub(), - resume: sinon.stub() - }) - S3Client = { - getObject: sinon.stub().returns({ - createReadStream: sinon.stub().returns(S3ReadStream) - }), - headObject: sinon.stub().returns({ - promise: sinon.stub().resolves({ - ContentLength: objectSize, - ETag: md5 - }) - }), - listObjects: sinon.stub().returns({ - promise: sinon.stub().resolves({ - Contents: files - }) - }), - upload: sinon - .stub() - .returns({ promise: sinon.stub().resolves({ ETag: `"${md5}"` }) }), - copyObject: sinon.stub().returns(EmptyPromise), - deleteObject: sinon.stub().returns(EmptyPromise), - deleteObjects: sinon.stub().returns(EmptyPromise) - } - S3 = sinon.stub().returns(S3Client) - - Hash = { - end: sinon.stub(), - read: sinon.stub().returns(md5), - digest: sinon.stub().returns(md5), - setEncoding: sinon.stub() - } - crypto = { - createHash: sinon.stub().returns(Hash) - } - - Logger = { - warn: sinon.stub() - } - - S3Persistor = SandboxedModule.require(modulePath, { - requires: { - 'aws-sdk/clients/s3': S3, - 'settings-sharelatex': settings, - 'logger-sharelatex': Logger, - './Errors': Errors, - fs: Fs, - stream: Stream, - 'metrics-sharelatex': Metrics, - crypto - }, - globals: { console, Buffer } - }) - }) - - describe('getFileStream', function() { - describe('when called with valid parameters', function() { - let stream - - beforeEach(async function() { - stream = await S3Persistor.promises.getFileStream(bucket, key) - }) - - it('returns a metered stream', function() { - expect(stream).to.be.instanceOf(Transform) - }) - - it('sets the AWS client up with credentials from settings', function() { - expect(S3).to.have.been.calledWith(defaultS3Credentials) - }) - - it('fetches the right key from the right bucket', function() { - expect(S3Client.getObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - - it('pipes the stream through the meter', async function() { - expect(S3ReadStream.pipe).to.have.been.calledWith( - sinon.match.instanceOf(Transform) - ) - }) - }) - - describe('when called with a byte range', function() { - let stream - - beforeEach(async function() { - stream = await S3Persistor.promises.getFileStream(bucket, key, { - start: 5, - end: 10 - }) - }) - - it('returns a metered stream', function() { - expect(stream).to.be.instanceOf(Stream.Transform) - }) - - it('passes the byte range on to S3', function() { - expect(S3Client.getObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key, - Range: 'bytes=5-10' - }) - }) - }) - - describe('when there are alternative credentials', function() { - let stream - const alternativeSecret = 'giraffe' - const alternativeKey = 'hippo' - const alternativeS3Credentials = { - credentials: { - accessKeyId: alternativeKey, - secretAccessKey: alternativeSecret - } - } - - beforeEach(async function() { - settings.filestore.s3BucketCreds = {} - settings.filestore.s3BucketCreds[bucket] = { - auth_key: alternativeKey, - auth_secret: alternativeSecret - } - - stream = await S3Persistor.promises.getFileStream(bucket, key) - }) - - it('returns a metered stream', function() { - expect(stream).to.be.instanceOf(Stream.Transform) - }) - - it('sets the AWS client up with the alternative credentials', function() { - expect(S3).to.have.been.calledWith(alternativeS3Credentials) - }) - - it('fetches the right key from the right bucket', function() { - expect(S3Client.getObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - - it('caches the credentials', async function() { - stream = await S3Persistor.promises.getFileStream(bucket, key) - - expect(S3).to.have.been.calledOnceWith(alternativeS3Credentials) - }) - - it('uses the default credentials for an unknown bucket', async function() { - stream = await S3Persistor.promises.getFileStream('anotherBucket', key) - - expect(S3).to.have.been.calledTwice - expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials) - expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials) - }) - - it('caches the default credentials', async function() { - stream = await S3Persistor.promises.getFileStream('anotherBucket', key) - stream = await S3Persistor.promises.getFileStream('anotherBucket', key) - - expect(S3).to.have.been.calledTwice - expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials) - expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials) - }) - - it('throws an error if there are no credentials for the bucket', async function() { - delete settings.filestore.s3.key - delete settings.filestore.s3.secret - - await expect( - S3Persistor.promises.getFileStream('anotherBucket', key) - ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.SettingsError) - }) - }) - - describe("when the file doesn't exist", function() { - let error, stream - - beforeEach(async function() { - Transform.prototype.on = sinon.stub() - S3ReadStream.on.withArgs('error').yields(S3NotFoundError) - try { - stream = await S3Persistor.promises.getFileStream(bucket, key) - } catch (err) { - error = err - } - }) - - it('does not return a stream', function() { - expect(stream).not.to.exist - }) - - it('throws a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - - it('wraps the error', function() { - expect(error.cause).to.exist - }) - - it('stores the bucket and key in the error', function() { - expect(error.info).to.include({ bucketName: bucket, key: key }) - }) - }) - - describe('when access to the file is denied', function() { - let error, stream - - beforeEach(async function() { - Transform.prototype.on = sinon.stub() - S3ReadStream.on.withArgs('error').yields(S3AccessDeniedError) - try { - stream = await S3Persistor.promises.getFileStream(bucket, key) - } catch (err) { - error = err - } - }) - - it('does not return a stream', function() { - expect(stream).not.to.exist - }) - - it('throws a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - - it('wraps the error', function() { - expect(error.cause).to.exist - }) - - it('stores the bucket and key in the error', function() { - expect(error.info).to.include({ bucketName: bucket, key: key }) - }) - }) - - describe('when S3 encounters an unkown error', function() { - let error, stream - - beforeEach(async function() { - Transform.prototype.on = sinon.stub() - S3ReadStream.on.withArgs('error').yields(genericError) - try { - stream = await S3Persistor.promises.getFileStream(bucket, key) - } catch (err) { - error = err - } - }) - - it('does not return a stream', function() { - expect(stream).not.to.exist - }) - - it('throws a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('wraps the error', function() { - expect(error.cause).to.exist - }) - - it('stores the bucket and key in the error', function() { - expect(error.info).to.include({ bucketName: bucket, key: key }) - }) - }) - }) - - describe('getFileSize', function() { - describe('when called with valid parameters', function() { - let size - - beforeEach(async function() { - size = await S3Persistor.promises.getFileSize(bucket, key) - }) - - it('should return the object size', function() { - expect(size).to.equal(objectSize) - }) - - it('should pass the bucket and key to S3', function() { - expect(S3Client.headObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - }) - - describe('when the object is not found', function() { - let error - - beforeEach(async function() { - S3Client.headObject = sinon.stub().returns({ - promise: sinon.stub().rejects(S3NotFoundError) - }) - try { - await S3Persistor.promises.getFileSize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(S3NotFoundError) - }) - }) - - describe('when S3 returns an error', function() { - let error - - beforeEach(async function() { - S3Client.headObject = sinon.stub().returns({ - promise: sinon.stub().rejects(genericError) - }) - try { - await S3Persistor.promises.getFileSize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should return a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('sendStream', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return S3Persistor.promises.sendStream(bucket, key, ReadStream) - }) - - it('should upload the stream', function() { - expect(S3Client.upload).to.have.been.calledWith({ - Bucket: bucket, - Key: key, - Body: sinon.match.instanceOf(Stream.Transform) - }) - }) - - it('should upload files in a single part', function() { - expect(S3Client.upload).to.have.been.calledWith(sinon.match.any, { - partSize: 100 * 1024 * 1024 - }) - }) - - it('should meter the stream', function() { - expect(Stream.pipeline).to.have.been.calledWith( - ReadStream, - sinon.match.instanceOf(Stream.Transform) - ) - }) - - it('calculates the md5 hash of the file', function() { - expect(Hash.digest).to.have.been.called - }) - }) - - describe('when a hash is supploed', function() { - beforeEach(async function() { - return S3Persistor.promises.sendStream( - bucket, - key, - ReadStream, - 'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb' - ) - }) - - it('should not calculate the md5 hash of the file', function() { - expect(Hash.digest).not.to.have.been.called - }) - - it('sends the hash in base64', function() { - expect(S3Client.upload).to.have.been.calledWith({ - Bucket: bucket, - Key: key, - Body: sinon.match.instanceOf(Transform), - ContentMD5: 'qqqqqru7u7uqqqqqu7u7uw==' - }) - }) - - it('does not fetch the md5 hash of the uploaded file', function() { - expect(S3Client.headObject).not.to.have.been.called - }) - }) - - describe('when the upload fails', function() { - let error - beforeEach(async function() { - S3Client.upload = sinon.stub().returns({ - promise: sinon.stub().rejects(genericError) - }) - try { - await S3Persistor.promises.sendStream(bucket, key, ReadStream) - } catch (err) { - error = err - } - }) - - it('throws a WriteError', function() { - expect(error).to.be.an.instanceOf(Errors.WriteError) - }) - }) - - describe("when the etag isn't a valid md5 hash", function() { - beforeEach(async function() { - S3Client.upload = sinon.stub().returns({ - promise: sinon.stub().resolves({ - ETag: 'somethingthatisntanmd5', - Bucket: bucket, - Key: key - }) - }) - - await S3Persistor.promises.sendStream(bucket, key, ReadStream) - }) - - it('should re-fetch the file to verify it', function() { - expect(S3Client.getObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - - it('should meter the download', function() { - expect(S3ReadStream.pipe).to.have.been.calledWith( - sinon.match.instanceOf(Stream.Transform) - ) - }) - - it('should calculate the md5 hash from the file', function() { - expect(Hash.digest).to.have.been.called - }) - }) - }) - - describe('sendFile', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return S3Persistor.promises.sendFile(bucket, key, filename) - }) - - it('should create a read stream for the file', function() { - expect(Fs.createReadStream).to.have.been.calledWith(filename) - }) - - it('should upload the stream', function() { - expect(S3Client.upload).to.have.been.calledWith({ - Bucket: bucket, - Key: key, - Body: sinon.match.instanceOf(Transform) - }) - }) - }) - }) - - describe('copyFile', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return S3Persistor.promises.copyFile(bucket, key, destKey) - }) - - it('should copy the object', function() { - expect(S3Client.copyObject).to.have.been.calledWith({ - Bucket: bucket, - Key: destKey, - CopySource: `${bucket}/${key}` - }) - }) - }) - - describe('when the file does not exist', function() { - let error - - beforeEach(async function() { - S3Client.copyObject = sinon.stub().returns({ - promise: sinon.stub().rejects(S3NotFoundError) - }) - try { - await S3Persistor.promises.copyFile(bucket, key, destKey) - } catch (err) { - error = err - } - }) - - it('should throw a NotFoundError', function() { - expect(error).to.be.an.instanceOf(Errors.NotFoundError) - }) - }) - }) - - describe('deleteFile', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return S3Persistor.promises.deleteFile(bucket, key) - }) - - it('should delete the object', function() { - expect(S3Client.deleteObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - }) - }) - - describe('deleteDirectory', function() { - describe('with valid parameters', function() { - beforeEach(async function() { - return S3Persistor.promises.deleteDirectory(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(S3Client.listObjects).to.have.been.calledWith({ - Bucket: bucket, - Prefix: key - }) - }) - - it('should delete the objects using their keys', function() { - expect(S3Client.deleteObjects).to.have.been.calledWith({ - Bucket: bucket, - Delete: { - Objects: [{ Key: 'llama' }, { Key: 'hippo' }], - Quiet: true - } - }) - }) - }) - - describe('when there are no files', function() { - beforeEach(async function() { - S3Client.listObjects = sinon - .stub() - .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) - return S3Persistor.promises.deleteDirectory(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(S3Client.listObjects).to.have.been.calledWith({ - Bucket: bucket, - Prefix: key - }) - }) - - it('should not try to delete any objects', function() { - expect(S3Client.deleteObjects).not.to.have.been.called - }) - }) - - describe('when there is an error listing the objects', function() { - let error - - beforeEach(async function() { - S3Client.listObjects = sinon - .stub() - .returns({ promise: sinon.stub().rejects(genericError) }) - try { - await S3Persistor.promises.deleteDirectory(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - - it('should not try to delete any objects', function() { - expect(S3Client.deleteObjects).not.to.have.been.called - }) - }) - - describe('when there is an error deleting the objects', function() { - let error - - beforeEach(async function() { - S3Client.deleteObjects = sinon - .stub() - .returns({ promise: sinon.stub().rejects(genericError) }) - try { - await S3Persistor.promises.deleteDirectory(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a WriteError', function() { - expect(error).to.be.an.instanceOf(Errors.WriteError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('directorySize', function() { - describe('with valid parameters', function() { - let size - - beforeEach(async function() { - size = await S3Persistor.promises.directorySize(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(S3Client.listObjects).to.have.been.calledWith({ - Bucket: bucket, - Prefix: key - }) - }) - - it('should return the directory size', function() { - expect(size).to.equal(filesSize) - }) - }) - - describe('when there are no files', function() { - let size - - beforeEach(async function() { - S3Client.listObjects = sinon - .stub() - .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) - size = await S3Persistor.promises.directorySize(bucket, key) - }) - - it('should list the objects in the directory', function() { - expect(S3Client.listObjects).to.have.been.calledWith({ - Bucket: bucket, - Prefix: key - }) - }) - - it('should return zero', function() { - expect(size).to.equal(0) - }) - }) - - describe('when there is an error listing the objects', function() { - let error - - beforeEach(async function() { - S3Client.listObjects = sinon - .stub() - .returns({ promise: sinon.stub().rejects(genericError) }) - try { - await S3Persistor.promises.directorySize(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the error', function() { - expect(error.cause).to.equal(genericError) - }) - }) - }) - - describe('checkIfFileExists', function() { - describe('when the file exists', function() { - let exists - - beforeEach(async function() { - exists = await S3Persistor.promises.checkIfFileExists(bucket, key) - }) - - it('should get the object header', function() { - expect(S3Client.headObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - - it('should return that the file exists', function() { - expect(exists).to.equal(true) - }) - }) - - describe('when the file does not exist', function() { - let exists - - beforeEach(async function() { - S3Client.headObject = sinon - .stub() - .returns({ promise: sinon.stub().rejects(S3NotFoundError) }) - exists = await S3Persistor.promises.checkIfFileExists(bucket, key) - }) - - it('should get the object header', function() { - expect(S3Client.headObject).to.have.been.calledWith({ - Bucket: bucket, - Key: key - }) - }) - - it('should return that the file does not exist', function() { - expect(exists).to.equal(false) - }) - }) - - describe('when there is an error', function() { - let error - - beforeEach(async function() { - S3Client.headObject = sinon - .stub() - .returns({ promise: sinon.stub().rejects(genericError) }) - try { - await S3Persistor.promises.checkIfFileExists(bucket, key) - } catch (err) { - error = err - } - }) - - it('should generate a ReadError', function() { - expect(error).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should wrap the upstream ReadError', function() { - expect(error.cause).to.be.an.instanceOf(Errors.ReadError) - }) - - it('should eventually wrap the error', function() { - expect(error.cause.cause).to.equal(genericError) - }) - }) - }) -}) diff --git a/services/filestore/test/unit/js/SafeExecTests.js b/services/filestore/test/unit/js/SafeExecTests.js index 6b89c53c01..4d31f6f57f 100644 --- a/services/filestore/test/unit/js/SafeExecTests.js +++ b/services/filestore/test/unit/js/SafeExecTests.js @@ -2,6 +2,7 @@ const chai = require('chai') const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/SafeExec' +const { Errors } = require('@overleaf/object-persistor') const SandboxedModule = require('sandboxed-module') describe('SafeExec', function() { @@ -11,10 +12,13 @@ describe('SafeExec', function() { settings = { enableConversions: true } options = { timeout: 10 * 1000, killSignal: 'SIGTERM' } + const ObjectPersistor = { Errors } + safeExec = SandboxedModule.require(modulePath, { globals: { process }, requires: { - 'settings-sharelatex': settings + 'settings-sharelatex': settings, + '@overleaf/object-persistor': ObjectPersistor } }) })