Create new module from overleaf/filestore persistors (#1)
* Create new module from overleaf/filestore persistors * Convert persistors to ES6 classes with local settings * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Update .gitignore Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Switch to AGPL license * Paginate S3 list-object results * Remove S3 client caching * Clean up S3 md5-verification mechanism * Update README for recent changes * Update README.md Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Remove package-lock * Remove comment about FileHandler * Add directory marker to FSPersistor.deleteDirectory * Don't copy opts in GcsPersistor.getObjectStream * Use Date.now instead of getTime * Catch errors in migration persistor * Check that settings.buckets exists * Don't mutate options in ObserverStream constructor * Update src/PersistorHelper.js Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * Lint and format fixes Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
This commit is contained in:
Simon Detheridge
committed by
GitHub
GitHub
co-authored by
Eric Mc Sween
Eric Mc Sween
parent
b83ee27f83
commit
e92b75a2f8
@@ -0,0 +1,333 @@
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const Errors = require('../../src/Errors')
|
||||
const StreamModule = require('stream')
|
||||
|
||||
const modulePath = '../../src/FSPersistor.js'
|
||||
|
||||
describe('FSPersistorTests', function () {
|
||||
const stat = { size: 4, isFile: sinon.stub().returns(true) }
|
||||
const fd = 1234
|
||||
const writeStream = 'writeStream'
|
||||
const remoteStream = 'remoteStream'
|
||||
const location = '/foo'
|
||||
const error = new Error('guru meditation error')
|
||||
const md5 = 'ffffffff'
|
||||
|
||||
const files = ['animals/wombat.tex', 'vegetables/potato.tex']
|
||||
const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`]
|
||||
const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex']
|
||||
let fs, stream, FSPersistor, glob, readStream, crypto, Hash, uuid, tempFile
|
||||
|
||||
beforeEach(function () {
|
||||
const randomNumber = Math.random().toString()
|
||||
readStream = {
|
||||
name: 'readStream',
|
||||
on: sinon.stub().yields(),
|
||||
pipe: sinon.stub()
|
||||
}
|
||||
uuid = {
|
||||
v1: () => randomNumber
|
||||
}
|
||||
tempFile = `/tmp/${randomNumber}`
|
||||
fs = {
|
||||
createReadStream: sinon.stub().returns(readStream),
|
||||
createWriteStream: sinon.stub().returns(writeStream),
|
||||
unlink: sinon.stub().yields(),
|
||||
open: sinon.stub().yields(null, fd),
|
||||
stat: sinon.stub().yields(null, stat)
|
||||
}
|
||||
glob = sinon.stub().yields(null, globs)
|
||||
stream = {
|
||||
pipeline: sinon.stub().yields(),
|
||||
Transform: StreamModule.Transform
|
||||
}
|
||||
Hash = {
|
||||
end: sinon.stub(),
|
||||
read: sinon.stub().returns(md5),
|
||||
digest: sinon.stub().returns(md5),
|
||||
setEncoding: sinon.stub()
|
||||
}
|
||||
crypto = {
|
||||
createHash: sinon.stub().returns(Hash)
|
||||
}
|
||||
FSPersistor = new (SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
'./Errors': Errors,
|
||||
fs,
|
||||
glob,
|
||||
stream,
|
||||
crypto,
|
||||
'node-uuid': uuid,
|
||||
// imported by PersistorHelper but otherwise unused here
|
||||
'logger-sharelatex': {},
|
||||
'metrics-sharelatex': {}
|
||||
},
|
||||
globals: { console }
|
||||
}))({ paths: { uploadFolder: '/tmp' } })
|
||||
})
|
||||
|
||||
describe('sendFile', function () {
|
||||
const localFilesystemPath = '/path/to/local/file'
|
||||
it('should copy the file', async function () {
|
||||
await FSPersistor.sendFile(location, files[0], localFilesystemPath)
|
||||
expect(fs.createReadStream).to.have.been.calledWith(localFilesystemPath)
|
||||
expect(fs.createWriteStream).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
|
||||
})
|
||||
|
||||
it('should return an error if the file cannot be stored', async function () {
|
||||
stream.pipeline.yields(error)
|
||||
await expect(
|
||||
FSPersistor.sendFile(location, files[0], localFilesystemPath)
|
||||
).to.eventually.be.rejected.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function () {
|
||||
it('should write the stream to disk', async function () {
|
||||
await FSPersistor.sendStream(location, files[0], remoteStream)
|
||||
expect(stream.pipeline).to.have.been.calledWith(remoteStream, writeStream)
|
||||
})
|
||||
|
||||
it('should delete the temporary file', async function () {
|
||||
await FSPersistor.sendStream(location, files[0], remoteStream)
|
||||
expect(fs.unlink).to.have.been.calledWith(tempFile)
|
||||
})
|
||||
|
||||
it('should wrap the error from the filesystem', async function () {
|
||||
stream.pipeline.yields(error)
|
||||
await expect(FSPersistor.sendStream(location, files[0], remoteStream))
|
||||
.to.eventually.be.rejected.and.be.instanceOf(Errors.WriteError)
|
||||
.and.have.property('cause', error)
|
||||
})
|
||||
|
||||
it('should send the temporary file to the filestore', async function () {
|
||||
await FSPersistor.sendStream(location, files[0], remoteStream)
|
||||
expect(fs.createReadStream).to.have.been.calledWith(tempFile)
|
||||
})
|
||||
|
||||
describe('when the md5 hash does not match', function () {
|
||||
it('should return a write error', async function () {
|
||||
await expect(
|
||||
FSPersistor.sendStream(location, files[0], remoteStream, '00000000')
|
||||
)
|
||||
.to.eventually.be.rejected.and.be.an.instanceOf(Errors.WriteError)
|
||||
.and.have.property('message', 'md5 hash mismatch')
|
||||
})
|
||||
|
||||
it('deletes the copied file', async function () {
|
||||
try {
|
||||
await FSPersistor.sendStream(
|
||||
location,
|
||||
files[0],
|
||||
remoteStream,
|
||||
'00000000'
|
||||
)
|
||||
} catch (_) {}
|
||||
expect(fs.unlink).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectStream', function () {
|
||||
it('should use correct file location', async function () {
|
||||
await FSPersistor.getObjectStream(location, files[0], {})
|
||||
expect(fs.open).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
})
|
||||
|
||||
it('should pass the options to createReadStream', async function () {
|
||||
await FSPersistor.getObjectStream(location, files[0], {
|
||||
start: 0,
|
||||
end: 8
|
||||
})
|
||||
expect(fs.createReadStream).to.have.been.calledWith(null, {
|
||||
start: 0,
|
||||
end: 8,
|
||||
fd
|
||||
})
|
||||
})
|
||||
|
||||
it('should give a NotFoundError if the file does not exist', async function () {
|
||||
const err = new Error()
|
||||
err.code = 'ENOENT'
|
||||
fs.open.yields(err)
|
||||
|
||||
await expect(FSPersistor.getObjectStream(location, files[0], {}))
|
||||
.to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
.and.have.property('cause', err)
|
||||
})
|
||||
|
||||
it('should wrap any other error', async function () {
|
||||
fs.open.yields(error)
|
||||
await expect(FSPersistor.getObjectStream(location, files[0], {}))
|
||||
.to.eventually.be.rejectedWith('failed to open file for streaming')
|
||||
.and.be.an.instanceOf(Errors.ReadError)
|
||||
.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectSize', function () {
|
||||
const badFilename = 'neenaw.tex'
|
||||
const size = 65536
|
||||
const noentError = new Error('not found')
|
||||
noentError.code = 'ENOENT'
|
||||
|
||||
beforeEach(function () {
|
||||
fs.stat
|
||||
.yields(error)
|
||||
.withArgs(`${location}/${filteredFilenames[0]}`)
|
||||
.yields(null, { size })
|
||||
.withArgs(`${location}/${badFilename}`)
|
||||
.yields(noentError)
|
||||
})
|
||||
|
||||
it('should return the file size', async function () {
|
||||
expect(await FSPersistor.getObjectSize(location, files[0])).to.equal(size)
|
||||
})
|
||||
|
||||
it('should throw a NotFoundError if the file does not exist', async function () {
|
||||
await expect(
|
||||
FSPersistor.getObjectSize(location, badFilename)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('should wrap any other error', async function () {
|
||||
await expect(FSPersistor.getObjectSize(location, 'raccoon'))
|
||||
.to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
|
||||
.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyObject', function () {
|
||||
it('Should open the source for reading', async function () {
|
||||
await FSPersistor.copyObject(location, files[0], files[1])
|
||||
expect(fs.createReadStream).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
})
|
||||
|
||||
it('Should open the target for writing', async function () {
|
||||
await FSPersistor.copyObject(location, files[0], files[1])
|
||||
expect(fs.createWriteStream).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[1]}`
|
||||
)
|
||||
})
|
||||
|
||||
it('Should pipe the source to the target', async function () {
|
||||
await FSPersistor.copyObject(location, files[0], files[1])
|
||||
expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteObject', function () {
|
||||
it('Should call unlink with correct options', async function () {
|
||||
await FSPersistor.deleteObject(location, files[0])
|
||||
expect(fs.unlink).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
})
|
||||
|
||||
it('Should propagate the error', async function () {
|
||||
fs.unlink.yields(error)
|
||||
await expect(
|
||||
FSPersistor.deleteObject(location, files[0])
|
||||
).to.eventually.be.rejected.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteDirectory', function () {
|
||||
it('Should call glob with correct options', async function () {
|
||||
await FSPersistor.deleteDirectory(location, files[0])
|
||||
expect(glob).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}_*`
|
||||
)
|
||||
})
|
||||
|
||||
it('Should call unlink on the returned files', async function () {
|
||||
await FSPersistor.deleteDirectory(location, files[0])
|
||||
for (const filename of globs) {
|
||||
expect(fs.unlink).to.have.been.calledWith(filename)
|
||||
}
|
||||
})
|
||||
|
||||
it('Should propagate the error', async function () {
|
||||
glob.yields(error)
|
||||
await expect(
|
||||
FSPersistor.deleteDirectory(location, files[0])
|
||||
).to.eventually.be.rejected.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkIfObjectExists', function () {
|
||||
const badFilename = 'pototo'
|
||||
const noentError = new Error('not found')
|
||||
noentError.code = 'ENOENT'
|
||||
|
||||
beforeEach(function () {
|
||||
fs.stat
|
||||
.yields(error)
|
||||
.withArgs(`${location}/${filteredFilenames[0]}`)
|
||||
.yields(null, {})
|
||||
.withArgs(`${location}/${badFilename}`)
|
||||
.yields(noentError)
|
||||
})
|
||||
|
||||
it('Should call stat with correct options', async function () {
|
||||
await FSPersistor.checkIfObjectExists(location, files[0])
|
||||
expect(fs.stat).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}`
|
||||
)
|
||||
})
|
||||
|
||||
it('Should return true for existing files', async function () {
|
||||
expect(
|
||||
await FSPersistor.checkIfObjectExists(location, files[0])
|
||||
).to.equal(true)
|
||||
})
|
||||
|
||||
it('Should return false for non-existing files', async function () {
|
||||
expect(
|
||||
await FSPersistor.checkIfObjectExists(location, badFilename)
|
||||
).to.equal(false)
|
||||
})
|
||||
|
||||
it('should wrap the error if there is a problem', async function () {
|
||||
await expect(FSPersistor.checkIfObjectExists(location, 'llama'))
|
||||
.to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
|
||||
.and.have.property('cause', error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('directorySize', function () {
|
||||
it('should wrap the error', async function () {
|
||||
glob.yields(error)
|
||||
await expect(FSPersistor.directorySize(location, files[0]))
|
||||
.to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError)
|
||||
.and.include({ cause: error })
|
||||
.and.have.property('info')
|
||||
.which.includes({ location, name: files[0] })
|
||||
})
|
||||
|
||||
it('should filter the directory name', async function () {
|
||||
await FSPersistor.directorySize(location, files[0])
|
||||
expect(glob).to.have.been.calledWith(
|
||||
`${location}/${filteredFilenames[0]}_*`
|
||||
)
|
||||
})
|
||||
|
||||
it('should sum directory files size', async function () {
|
||||
expect(await FSPersistor.directorySize(location, files[0])).to.equal(
|
||||
stat.size * files.length
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,671 @@
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const modulePath = '../../src/GcsPersistor.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const asyncPool = require('tiny-async-pool')
|
||||
|
||||
const Errors = require('../../src/Errors')
|
||||
|
||||
describe('GcsPersistorTests', function () {
|
||||
const filename = '/wombat/potato.tex'
|
||||
const bucket = 'womBucket'
|
||||
const key = 'monKey'
|
||||
const destKey = 'donKey'
|
||||
const genericError = new Error('guru meditation error')
|
||||
const filesSize = 33
|
||||
const md5 = 'ffffffff00000000ffffffff00000000'
|
||||
const WriteStream = 'writeStream'
|
||||
const redirectUrl = 'https://wombat.potato/giraffe'
|
||||
|
||||
let Logger,
|
||||
Transform,
|
||||
Storage,
|
||||
Fs,
|
||||
GcsNotFoundError,
|
||||
ReadStream,
|
||||
Stream,
|
||||
GcsBucket,
|
||||
GcsFile,
|
||||
GcsPersistor,
|
||||
FileNotFoundError,
|
||||
Hash,
|
||||
Settings,
|
||||
crypto,
|
||||
files
|
||||
|
||||
beforeEach(function () {
|
||||
Settings = {
|
||||
directoryKeyRegex: /^[0-9a-fA-F]{24}\/[0-9a-fA-F]{24}/,
|
||||
Metrics: {
|
||||
count: sinon.stub()
|
||||
}
|
||||
}
|
||||
|
||||
files = [
|
||||
{
|
||||
metadata: { size: 11, md5Hash: '/////wAAAAD/////AAAAAA==' },
|
||||
delete: sinon.stub()
|
||||
},
|
||||
{
|
||||
metadata: { size: 22, md5Hash: '/////wAAAAD/////AAAAAA==' },
|
||||
delete: sinon.stub()
|
||||
}
|
||||
]
|
||||
|
||||
ReadStream = {
|
||||
pipe: sinon.stub().returns('readStream'),
|
||||
on: sinon.stub(),
|
||||
removeListener: sinon.stub()
|
||||
}
|
||||
ReadStream.on.withArgs('end').yields()
|
||||
ReadStream.on.withArgs('pipe').yields({
|
||||
unpipe: sinon.stub(),
|
||||
resume: sinon.stub(),
|
||||
on: sinon.stub()
|
||||
})
|
||||
|
||||
Transform = class {
|
||||
on(event, callback) {
|
||||
if (event === 'readable') {
|
||||
callback()
|
||||
}
|
||||
}
|
||||
|
||||
once() {}
|
||||
removeListener() {}
|
||||
}
|
||||
|
||||
Stream = {
|
||||
pipeline: sinon.stub().yields(),
|
||||
Transform: Transform
|
||||
}
|
||||
|
||||
GcsFile = {
|
||||
delete: sinon.stub().resolves(),
|
||||
createReadStream: sinon.stub().returns(ReadStream),
|
||||
getMetadata: sinon.stub().resolves([files[0].metadata]),
|
||||
createWriteStream: sinon.stub().returns(WriteStream),
|
||||
copy: sinon.stub().resolves(),
|
||||
exists: sinon.stub().resolves([true]),
|
||||
getSignedUrl: sinon.stub().resolves([redirectUrl])
|
||||
}
|
||||
|
||||
GcsBucket = {
|
||||
file: sinon.stub().returns(GcsFile),
|
||||
getFiles: sinon.stub().resolves([files])
|
||||
}
|
||||
|
||||
Storage = class {
|
||||
constructor() {
|
||||
this.interceptors = []
|
||||
}
|
||||
}
|
||||
Storage.prototype.bucket = sinon.stub().returns(GcsBucket)
|
||||
|
||||
GcsNotFoundError = new Error('File not found')
|
||||
GcsNotFoundError.code = 404
|
||||
|
||||
Fs = {
|
||||
createReadStream: sinon.stub().returns(ReadStream)
|
||||
}
|
||||
|
||||
FileNotFoundError = new Error('File not found')
|
||||
FileNotFoundError.code = 'ENOENT'
|
||||
|
||||
Hash = {
|
||||
end: sinon.stub(),
|
||||
read: sinon.stub().returns(md5),
|
||||
digest: sinon.stub().returns(md5),
|
||||
setEncoding: sinon.stub()
|
||||
}
|
||||
crypto = {
|
||||
createHash: sinon.stub().returns(Hash)
|
||||
}
|
||||
|
||||
Logger = {
|
||||
warn: sinon.stub()
|
||||
}
|
||||
|
||||
GcsPersistor = new (SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
'@google-cloud/storage': { Storage },
|
||||
'logger-sharelatex': Logger,
|
||||
'tiny-async-pool': asyncPool,
|
||||
'./Errors': Errors,
|
||||
fs: Fs,
|
||||
stream: Stream,
|
||||
crypto
|
||||
},
|
||||
globals: { console, Buffer }
|
||||
}))(Settings)
|
||||
})
|
||||
|
||||
describe('getObjectStream', function () {
|
||||
describe('when called with valid parameters', function () {
|
||||
let stream
|
||||
|
||||
beforeEach(async function () {
|
||||
stream = await GcsPersistor.getObjectStream(bucket, key)
|
||||
})
|
||||
|
||||
it('returns a metered stream', function () {
|
||||
expect(stream).to.be.instanceOf(Transform)
|
||||
})
|
||||
|
||||
it('fetches the right key from the right bucket', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.createReadStream).to.have.been.called
|
||||
})
|
||||
|
||||
it('pipes the stream through the meter', function () {
|
||||
expect(ReadStream.pipe).to.have.been.calledWith(
|
||||
sinon.match.instanceOf(Transform)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when called with a byte range', function () {
|
||||
let stream
|
||||
|
||||
beforeEach(async function () {
|
||||
stream = await GcsPersistor.getObjectStream(bucket, key, {
|
||||
start: 5,
|
||||
end: 10
|
||||
})
|
||||
})
|
||||
|
||||
it('returns a metered stream', function () {
|
||||
expect(stream).to.be.instanceOf(Transform)
|
||||
})
|
||||
|
||||
it('passes the byte range on to GCS', function () {
|
||||
expect(GcsFile.createReadStream).to.have.been.calledWith({
|
||||
start: 5,
|
||||
end: 10
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("when the file doesn't exist", function () {
|
||||
let error, stream
|
||||
|
||||
beforeEach(async function () {
|
||||
Transform.prototype.on = sinon.stub()
|
||||
ReadStream.on.withArgs('error').yields(GcsNotFoundError)
|
||||
try {
|
||||
stream = await GcsPersistor.getObjectStream(bucket, key)
|
||||
} catch (e) {
|
||||
error = e
|
||||
}
|
||||
})
|
||||
|
||||
it('does not return a stream', function () {
|
||||
expect(stream).not.to.exist
|
||||
})
|
||||
|
||||
it('throws a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.exist
|
||||
})
|
||||
|
||||
it('stores the bucket and key in the error', function () {
|
||||
expect(error.info).to.include({ bucketName: bucket, key: key })
|
||||
})
|
||||
})
|
||||
|
||||
describe('when Gcs encounters an unkown error', function () {
|
||||
let error, stream
|
||||
|
||||
beforeEach(async function () {
|
||||
Transform.prototype.on = sinon.stub()
|
||||
ReadStream.on.withArgs('error').yields(genericError)
|
||||
try {
|
||||
stream = await GcsPersistor.getObjectStream(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('does not return a stream', function () {
|
||||
expect(stream).not.to.exist
|
||||
})
|
||||
|
||||
it('throws a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.exist
|
||||
})
|
||||
|
||||
it('stores the bucket and key in the error', function () {
|
||||
expect(error.info).to.include({ bucketName: bucket, key: key })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFile', function () {
|
||||
let signedUrl
|
||||
|
||||
beforeEach(async function () {
|
||||
signedUrl = await GcsPersistor.getRedirectUrl(bucket, key)
|
||||
})
|
||||
|
||||
it('should request a signed URL', function () {
|
||||
expect(GcsFile.getSignedUrl).to.have.been.called
|
||||
})
|
||||
|
||||
it('should return the url', function () {
|
||||
expect(signedUrl).to.equal(redirectUrl)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectSize', function () {
|
||||
describe('when called with valid parameters', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
size = await GcsPersistor.getObjectSize(bucket, key)
|
||||
})
|
||||
|
||||
it('should return the object size', function () {
|
||||
expect(size).to.equal(files[0].metadata.size)
|
||||
})
|
||||
|
||||
it('should pass the bucket and key to GCS', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.getMetadata).to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the object is not found', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.getMetadata = sinon.stub().rejects(GcsNotFoundError)
|
||||
try {
|
||||
await GcsPersistor.getObjectSize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(GcsNotFoundError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when GCS returns an error', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.getMetadata = sinon.stub().rejects(genericError)
|
||||
try {
|
||||
await GcsPersistor.getObjectSize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.sendStream(bucket, key, ReadStream)
|
||||
})
|
||||
|
||||
it('should upload the stream', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.createWriteStream).to.have.been.called
|
||||
})
|
||||
|
||||
it('should not try to create a resumable upload', function () {
|
||||
expect(GcsFile.createWriteStream).to.have.been.calledWith({
|
||||
resumable: false
|
||||
})
|
||||
})
|
||||
|
||||
it('should meter the stream and pass it to GCS', function () {
|
||||
expect(Stream.pipeline).to.have.been.calledWith(
|
||||
ReadStream,
|
||||
sinon.match.instanceOf(Transform),
|
||||
WriteStream
|
||||
)
|
||||
})
|
||||
|
||||
it('calculates the md5 hash of the file', function () {
|
||||
expect(Hash.digest).to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when a hash is supplied', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.sendStream(
|
||||
bucket,
|
||||
key,
|
||||
ReadStream,
|
||||
'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb'
|
||||
)
|
||||
})
|
||||
|
||||
it('should not calculate the md5 hash of the file', function () {
|
||||
expect(Hash.digest).not.to.have.been.called
|
||||
})
|
||||
|
||||
it('sends the hash in base64', function () {
|
||||
expect(GcsFile.createWriteStream).to.have.been.calledWith({
|
||||
validation: 'md5',
|
||||
metadata: {
|
||||
md5Hash: 'qqqqqru7u7uqqqqqu7u7uw=='
|
||||
},
|
||||
resumable: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not fetch the md5 hash of the uploaded file', function () {
|
||||
expect(GcsFile.getMetadata).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the upload fails', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
Stream.pipeline
|
||||
.withArgs(
|
||||
ReadStream,
|
||||
sinon.match.instanceOf(Transform),
|
||||
WriteStream,
|
||||
sinon.match.any
|
||||
)
|
||||
.yields(genericError)
|
||||
try {
|
||||
await GcsPersistor.sendStream(bucket, key, ReadStream)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('throws a WriteError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.WriteError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendFile', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.sendFile(bucket, key, filename)
|
||||
})
|
||||
|
||||
it('should create a read stream for the file', function () {
|
||||
expect(Fs.createReadStream).to.have.been.calledWith(filename)
|
||||
})
|
||||
|
||||
it('should create a write stream', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.createWriteStream).to.have.been.called
|
||||
})
|
||||
|
||||
it('should upload the stream via the meter', function () {
|
||||
expect(Stream.pipeline).to.have.been.calledWith(
|
||||
ReadStream,
|
||||
sinon.match.instanceOf(Transform),
|
||||
WriteStream
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyObject', function () {
|
||||
const destinationFile = 'destFile'
|
||||
|
||||
beforeEach(function () {
|
||||
GcsBucket.file.withArgs(destKey).returns(destinationFile)
|
||||
})
|
||||
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should copy the object', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.copy).to.have.been.calledWith(destinationFile)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.copy = sinon.stub().rejects(GcsNotFoundError)
|
||||
try {
|
||||
await GcsPersistor.copyObject(bucket, key, destKey)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should throw a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteObject', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.deleteObject(bucket, key)
|
||||
})
|
||||
|
||||
it('should delete the object', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.delete).to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.delete = sinon.stub().rejects(GcsNotFoundError)
|
||||
try {
|
||||
await GcsPersistor.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should not throw an error', function () {
|
||||
expect(error).not.to.exist
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteDirectory', function () {
|
||||
const directoryName = `${ObjectId()}/${ObjectId()}`
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return GcsPersistor.deleteDirectory(bucket, directoryName)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.getFiles).to.have.been.calledWith({
|
||||
directory: directoryName
|
||||
})
|
||||
})
|
||||
|
||||
it('should delete the files', function () {
|
||||
expect(GcsFile.delete).to.have.been.calledTwice
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error listing the objects', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsBucket.getFiles = sinon.stub().rejects(genericError)
|
||||
try {
|
||||
await GcsPersistor.deleteDirectory(bucket, directoryName)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a WriteError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.WriteError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('directorySize', function () {
|
||||
describe('with valid parameters', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
size = await GcsPersistor.directorySize(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
|
||||
})
|
||||
|
||||
it('should return the directory size', function () {
|
||||
expect(size).to.equal(filesSize)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are no files', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsBucket.getFiles.resolves([[]])
|
||||
size = await GcsPersistor.directorySize(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.getFiles).to.have.been.calledWith({ directory: key })
|
||||
})
|
||||
|
||||
it('should return zero', function () {
|
||||
expect(size).to.equal(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error listing the objects', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsBucket.getFiles.rejects(genericError)
|
||||
try {
|
||||
await GcsPersistor.directorySize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkIfObjectExists', function () {
|
||||
describe('when the file exists', function () {
|
||||
let exists
|
||||
|
||||
beforeEach(async function () {
|
||||
exists = await GcsPersistor.checkIfObjectExists(bucket, key)
|
||||
})
|
||||
|
||||
it('should ask the file if it exists', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.exists).to.have.been.called
|
||||
})
|
||||
|
||||
it('should return that the file exists', function () {
|
||||
expect(exists).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function () {
|
||||
let exists
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.exists = sinon.stub().resolves([false])
|
||||
exists = await GcsPersistor.checkIfObjectExists(bucket, key)
|
||||
})
|
||||
|
||||
it('should get the object header', function () {
|
||||
expect(Storage.prototype.bucket).to.have.been.calledWith(bucket)
|
||||
expect(GcsBucket.file).to.have.been.calledWith(key)
|
||||
expect(GcsFile.exists).to.have.been.called
|
||||
})
|
||||
|
||||
it('should return that the file does not exist', function () {
|
||||
expect(exists).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
GcsFile.exists = sinon.stub().rejects(genericError)
|
||||
try {
|
||||
await GcsPersistor.checkIfObjectExists(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,522 @@
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const modulePath = '../../src/MigrationPersistor.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
const Errors = require('../../src/Errors')
|
||||
|
||||
// Not all methods are tested here, but a method with each type of wrapping has
|
||||
// tests. Specifically, the following wrapping methods are tested here:
|
||||
// getObjectStream: _wrapFallbackMethod
|
||||
// sendStream: forward-to-primary
|
||||
// deleteObject: _wrapMethodOnBothPersistors
|
||||
// copyObject: copyFileWithFallback
|
||||
|
||||
describe('MigrationPersistorTests', function () {
|
||||
const bucket = 'womBucket'
|
||||
const fallbackBucket = 'bucKangaroo'
|
||||
const key = 'monKey'
|
||||
const destKey = 'donKey'
|
||||
const genericError = new Error('guru meditation error')
|
||||
const notFoundError = new Errors.NotFoundError('not found')
|
||||
const size = 33
|
||||
const md5 = 'ffffffff'
|
||||
|
||||
let Settings, Logger, Stream, MigrationPersistor, fileStream, newPersistor
|
||||
|
||||
beforeEach(function () {
|
||||
fileStream = {
|
||||
name: 'fileStream',
|
||||
on: sinon.stub().withArgs('end').yields(),
|
||||
pipe: sinon.stub()
|
||||
}
|
||||
|
||||
newPersistor = function (hasFile) {
|
||||
return {
|
||||
sendFile: sinon.stub().resolves(),
|
||||
sendStream: sinon.stub().resolves(),
|
||||
getObjectStream: hasFile
|
||||
? sinon.stub().resolves(fileStream)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
deleteDirectory: sinon.stub().resolves(),
|
||||
getObjectSize: hasFile
|
||||
? sinon.stub().resolves(size)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
deleteObject: sinon.stub().resolves(),
|
||||
copyObject: hasFile
|
||||
? sinon.stub().resolves()
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
checkIfObjectExists: sinon.stub().resolves(hasFile),
|
||||
directorySize: hasFile
|
||||
? sinon.stub().resolves(size)
|
||||
: sinon.stub().rejects(notFoundError),
|
||||
getObjectMd5Hash: hasFile
|
||||
? sinon.stub().resolves(md5)
|
||||
: sinon.stub().rejects(notFoundError)
|
||||
}
|
||||
}
|
||||
|
||||
Settings = {
|
||||
buckets: {
|
||||
[bucket]: fallbackBucket
|
||||
}
|
||||
}
|
||||
|
||||
Stream = {
|
||||
pipeline: sinon.stub().yields(),
|
||||
PassThrough: sinon.stub()
|
||||
}
|
||||
|
||||
Logger = {
|
||||
warn: sinon.stub()
|
||||
}
|
||||
|
||||
MigrationPersistor = SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
stream: Stream,
|
||||
'./Errors': Errors,
|
||||
'logger-sharelatex': Logger
|
||||
},
|
||||
globals: { console }
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectStream', function () {
|
||||
const options = { wombat: 'potato' }
|
||||
describe('when the primary persistor has the file', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, response
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(true)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
response = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should return the file stream', function () {
|
||||
expect(response).to.equal(fileStream)
|
||||
})
|
||||
|
||||
it('should fetch the file from the primary persistor, with the correct options', function () {
|
||||
expect(primaryPersistor.getObjectStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should not query the fallback persistor', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor has the file', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, response
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
response = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should return the file stream', function () {
|
||||
expect(response).to.be.an.instanceOf(Stream.PassThrough)
|
||||
})
|
||||
|
||||
it('should fetch the file from the primary persistor with the correct options', function () {
|
||||
expect(primaryPersistor.getObjectStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback persistor with the fallback bucket with the correct options', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).to.have.been.calledWithExactly(fallbackBucket, key, options)
|
||||
})
|
||||
|
||||
it('should create one read stream', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should not send the file to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file should be copied to the primary', function () {
|
||||
let primaryPersistor,
|
||||
fallbackPersistor,
|
||||
migrationPersistor,
|
||||
returnedStream
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
Settings.copyOnMiss = true
|
||||
returnedStream = await migrationPersistor.getObjectStream(
|
||||
bucket,
|
||||
key,
|
||||
options
|
||||
)
|
||||
})
|
||||
|
||||
it('should create one read stream', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should get the md5 hash from the source', function () {
|
||||
expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should send a stream to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
sinon.match.instanceOf(Stream.PassThrough),
|
||||
md5
|
||||
)
|
||||
})
|
||||
|
||||
it('should send a stream to the client', function () {
|
||||
expect(returnedStream).to.be.an.instanceOf(Stream.PassThrough)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when neither persistor has the file', function () {
|
||||
it('rejects with a NotFoundError', async function () {
|
||||
const migrationPersistor = new MigrationPersistor(
|
||||
newPersistor(false),
|
||||
newPersistor(false),
|
||||
Settings
|
||||
)
|
||||
return expect(
|
||||
migrationPersistor.getObjectStream(bucket, key)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an unexpected error', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
primaryPersistor.getObjectStream = sinon.stub().rejects(genericError)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.getObjectStream(bucket, key, options)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects with the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('does not call the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor throws an unexpected error', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
fallbackPersistor.getObjectStream = sinon.stub().rejects(genericError)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.getObjectStream(bucket, key, options)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects with the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should have called the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
})
|
||||
|
||||
describe('when it works', function () {
|
||||
beforeEach(async function () {
|
||||
return migrationPersistor.sendStream(bucket, key, fileStream)
|
||||
})
|
||||
|
||||
it('should send the file to the primary persistor', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
fileStream
|
||||
)
|
||||
})
|
||||
|
||||
it('should not send the file to the fallback persistor', function () {
|
||||
expect(fallbackPersistor.sendStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an error', function () {
|
||||
it('returns the error', async function () {
|
||||
primaryPersistor.sendStream.rejects(notFoundError)
|
||||
return expect(
|
||||
migrationPersistor.sendStream(bucket, key, fileStream)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteObject', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
})
|
||||
|
||||
describe('when it works', function () {
|
||||
beforeEach(async function () {
|
||||
return migrationPersistor.deleteObject(bucket, key)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the primary persistor throws an error', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor.deleteObject.rejects(genericError)
|
||||
try {
|
||||
await migrationPersistor.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the fallback persistor throws an error', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
fallbackPersistor.deleteObject.rejects(genericError)
|
||||
try {
|
||||
await migrationPersistor.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return the error', function () {
|
||||
expect(error).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should delete the file from the primary', function () {
|
||||
expect(primaryPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file from the fallback', function () {
|
||||
expect(fallbackPersistor.deleteObject).to.have.been.calledWithExactly(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyObject', function () {
|
||||
describe('when the file exists on the primary', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(true)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
return migrationPersistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should not try to read from the fallback', function () {
|
||||
expect(fallbackPersistor.getObjectStream).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist on the primary', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(true)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
return migrationPersistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).not.to.have.been.calledWithExactly(fallbackBucket, key)
|
||||
})
|
||||
|
||||
it('should get the md5 hash from the source', function () {
|
||||
expect(fallbackPersistor.getObjectMd5Hash).to.have.been.calledWith(
|
||||
fallbackBucket,
|
||||
key
|
||||
)
|
||||
})
|
||||
|
||||
it('should send the file to the primary', function () {
|
||||
expect(primaryPersistor.sendStream).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
destKey,
|
||||
sinon.match.instanceOf(Stream.PassThrough),
|
||||
md5
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist on the fallback', function () {
|
||||
let primaryPersistor, fallbackPersistor, migrationPersistor, error
|
||||
beforeEach(async function () {
|
||||
primaryPersistor = newPersistor(false)
|
||||
fallbackPersistor = newPersistor(false)
|
||||
migrationPersistor = new MigrationPersistor(
|
||||
primaryPersistor,
|
||||
fallbackPersistor,
|
||||
Settings
|
||||
)
|
||||
try {
|
||||
await migrationPersistor.copyObject(bucket, key, destKey)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should call copyObject to copy the file', function () {
|
||||
expect(primaryPersistor.copyObject).to.have.been.calledWithExactly(
|
||||
bucket,
|
||||
key,
|
||||
destKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file from the fallback', function () {
|
||||
expect(
|
||||
fallbackPersistor.getObjectStream
|
||||
).not.to.have.been.calledWithExactly(fallbackBucket, key)
|
||||
})
|
||||
|
||||
it('should return a not-found error', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,82 @@
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
const modulePath = '../../src/PersistorFactory.js'
|
||||
|
||||
describe('PersistorManager', function () {
|
||||
let PersistorFactory, FSPersistor, S3Persistor, Settings, GcsPersistor
|
||||
|
||||
beforeEach(function () {
|
||||
FSPersistor = class {
|
||||
wrappedMethod() {
|
||||
return 'FSPersistor'
|
||||
}
|
||||
}
|
||||
S3Persistor = class {
|
||||
wrappedMethod() {
|
||||
return 'S3Persistor'
|
||||
}
|
||||
}
|
||||
GcsPersistor = class {
|
||||
wrappedMethod() {
|
||||
return 'GcsPersistor'
|
||||
}
|
||||
}
|
||||
|
||||
Settings = {}
|
||||
const requires = {
|
||||
'./GcsPersistor': GcsPersistor,
|
||||
'./S3Persistor': S3Persistor,
|
||||
'./FSPersistor': FSPersistor,
|
||||
'logger-sharelatex': {
|
||||
info() {},
|
||||
err() {}
|
||||
}
|
||||
}
|
||||
PersistorFactory = SandboxedModule.require(modulePath, { requires })
|
||||
})
|
||||
|
||||
it('should implement the S3 wrapped method when S3 is configured', function () {
|
||||
Settings.backend = 's3'
|
||||
|
||||
expect(PersistorFactory(Settings)).to.respondTo('wrappedMethod')
|
||||
expect(PersistorFactory(Settings).wrappedMethod()).to.equal('S3Persistor')
|
||||
})
|
||||
|
||||
it("should implement the S3 wrapped method when 'aws-sdk' is configured", function () {
|
||||
Settings.backend = 'aws-sdk'
|
||||
|
||||
expect(PersistorFactory(Settings)).to.respondTo('wrappedMethod')
|
||||
expect(PersistorFactory(Settings).wrappedMethod()).to.equal('S3Persistor')
|
||||
})
|
||||
|
||||
it('should implement the FS wrapped method when FS is configured', function () {
|
||||
Settings.backend = 'fs'
|
||||
|
||||
expect(PersistorFactory(Settings)).to.respondTo('wrappedMethod')
|
||||
expect(PersistorFactory(Settings).wrappedMethod()).to.equal('FSPersistor')
|
||||
})
|
||||
|
||||
it('should throw an error when the backend is not configured', function () {
|
||||
try {
|
||||
PersistorFactory(Settings)
|
||||
} catch (err) {
|
||||
expect(err.message).to.equal('no backend specified - config incomplete')
|
||||
return
|
||||
}
|
||||
expect('should have caught an error').not.to.exist
|
||||
})
|
||||
|
||||
it('should throw an error when the backend is unknown', function () {
|
||||
Settings.backend = 'magic'
|
||||
try {
|
||||
PersistorFactory(Settings)
|
||||
} catch (err) {
|
||||
expect(err.message).to.equal('unknown backend')
|
||||
expect(err.info.backend).to.equal('magic')
|
||||
return
|
||||
}
|
||||
expect('should have caught an error').not.to.exist
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,921 @@
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const modulePath = '../../src/S3Persistor.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
const Errors = require('../../src/Errors')
|
||||
|
||||
describe('S3PersistorTests', function () {
|
||||
const defaultS3Key = 'frog'
|
||||
const defaultS3Secret = 'prince'
|
||||
const defaultS3Credentials = {
|
||||
credentials: {
|
||||
accessKeyId: defaultS3Key,
|
||||
secretAccessKey: defaultS3Secret
|
||||
}
|
||||
}
|
||||
const filename = '/wombat/potato.tex'
|
||||
const bucket = 'womBucket'
|
||||
const key = 'monKey'
|
||||
const destKey = 'donKey'
|
||||
const objectSize = 5555
|
||||
const genericError = new Error('guru meditation error')
|
||||
const files = [
|
||||
{ Key: 'llama', Size: 11 },
|
||||
{ Key: 'hippo', Size: 22 }
|
||||
]
|
||||
const filesSize = 33
|
||||
const md5 = 'ffffffff00000000ffffffff00000000'
|
||||
|
||||
let Metrics,
|
||||
Logger,
|
||||
Transform,
|
||||
S3,
|
||||
Fs,
|
||||
ReadStream,
|
||||
Stream,
|
||||
S3Persistor,
|
||||
S3Client,
|
||||
S3ReadStream,
|
||||
S3NotFoundError,
|
||||
S3AccessDeniedError,
|
||||
FileNotFoundError,
|
||||
EmptyPromise,
|
||||
settings,
|
||||
Hash,
|
||||
crypto
|
||||
|
||||
beforeEach(function () {
|
||||
settings = {
|
||||
secret: defaultS3Secret,
|
||||
key: defaultS3Key,
|
||||
partSize: 100 * 1024 * 1024
|
||||
}
|
||||
|
||||
Transform = class {
|
||||
on(event, callback) {
|
||||
if (event === 'readable') {
|
||||
callback()
|
||||
}
|
||||
}
|
||||
|
||||
once() {}
|
||||
removeListener() {}
|
||||
}
|
||||
|
||||
Stream = {
|
||||
pipeline: sinon.stub().yields(),
|
||||
Transform: Transform
|
||||
}
|
||||
|
||||
EmptyPromise = {
|
||||
promise: sinon.stub().resolves()
|
||||
}
|
||||
|
||||
Metrics = {
|
||||
count: sinon.stub()
|
||||
}
|
||||
|
||||
ReadStream = {
|
||||
pipe: sinon.stub().returns('readStream'),
|
||||
on: sinon.stub(),
|
||||
removeListener: sinon.stub()
|
||||
}
|
||||
ReadStream.on.withArgs('end').yields()
|
||||
ReadStream.on.withArgs('pipe').yields({
|
||||
unpipe: sinon.stub(),
|
||||
resume: sinon.stub()
|
||||
})
|
||||
|
||||
FileNotFoundError = new Error('File not found')
|
||||
FileNotFoundError.code = 'ENOENT'
|
||||
|
||||
Fs = {
|
||||
createReadStream: sinon.stub().returns(ReadStream)
|
||||
}
|
||||
|
||||
S3NotFoundError = new Error('not found')
|
||||
S3NotFoundError.code = 'NoSuchKey'
|
||||
|
||||
S3AccessDeniedError = new Error('access denied')
|
||||
S3AccessDeniedError.code = 'AccessDenied'
|
||||
|
||||
S3ReadStream = {
|
||||
on: sinon.stub(),
|
||||
pipe: sinon.stub(),
|
||||
removeListener: sinon.stub()
|
||||
}
|
||||
S3ReadStream.on.withArgs('end').yields()
|
||||
S3ReadStream.on.withArgs('pipe').yields({
|
||||
unpipe: sinon.stub(),
|
||||
resume: sinon.stub()
|
||||
})
|
||||
S3Client = {
|
||||
getObject: sinon.stub().returns({
|
||||
createReadStream: sinon.stub().returns(S3ReadStream)
|
||||
}),
|
||||
headObject: sinon.stub().returns({
|
||||
promise: sinon.stub().resolves({
|
||||
ContentLength: objectSize,
|
||||
ETag: md5
|
||||
})
|
||||
}),
|
||||
listObjectsV2: sinon.stub().returns({
|
||||
promise: sinon.stub().resolves({
|
||||
Contents: files
|
||||
})
|
||||
}),
|
||||
upload: sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().resolves({ ETag: `"${md5}"` }) }),
|
||||
copyObject: sinon.stub().returns(EmptyPromise),
|
||||
deleteObject: sinon.stub().returns(EmptyPromise),
|
||||
deleteObjects: sinon.stub().returns(EmptyPromise)
|
||||
}
|
||||
S3 = sinon.stub().returns(S3Client)
|
||||
|
||||
Hash = {
|
||||
end: sinon.stub(),
|
||||
read: sinon.stub().returns(md5),
|
||||
setEncoding: sinon.stub()
|
||||
}
|
||||
crypto = {
|
||||
createHash: sinon.stub().returns(Hash)
|
||||
}
|
||||
|
||||
Logger = {
|
||||
warn: sinon.stub()
|
||||
}
|
||||
|
||||
S3Persistor = new (SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
'aws-sdk/clients/s3': S3,
|
||||
'logger-sharelatex': Logger,
|
||||
'./Errors': Errors,
|
||||
fs: Fs,
|
||||
stream: Stream,
|
||||
'metrics-sharelatex': Metrics,
|
||||
crypto
|
||||
},
|
||||
globals: { console, Buffer }
|
||||
}))(settings)
|
||||
})
|
||||
|
||||
describe('getObjectStream', function () {
|
||||
describe('when called with valid parameters', function () {
|
||||
let stream
|
||||
|
||||
beforeEach(async function () {
|
||||
stream = await S3Persistor.getObjectStream(bucket, key)
|
||||
})
|
||||
|
||||
it('returns a metered stream', function () {
|
||||
expect(stream).to.be.instanceOf(Transform)
|
||||
})
|
||||
|
||||
it('sets the AWS client up with credentials from settings', function () {
|
||||
expect(S3).to.have.been.calledWith(defaultS3Credentials)
|
||||
})
|
||||
|
||||
it('fetches the right key from the right bucket', function () {
|
||||
expect(S3Client.getObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('pipes the stream through the meter', async function () {
|
||||
expect(S3ReadStream.pipe).to.have.been.calledWith(
|
||||
sinon.match.instanceOf(Transform)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when called with a byte range', function () {
|
||||
let stream
|
||||
|
||||
beforeEach(async function () {
|
||||
stream = await S3Persistor.getObjectStream(bucket, key, {
|
||||
start: 5,
|
||||
end: 10
|
||||
})
|
||||
})
|
||||
|
||||
it('returns a metered stream', function () {
|
||||
expect(stream).to.be.instanceOf(Stream.Transform)
|
||||
})
|
||||
|
||||
it('passes the byte range on to S3', function () {
|
||||
expect(S3Client.getObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Range: 'bytes=5-10'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are alternative credentials', function () {
|
||||
let stream
|
||||
const alternativeSecret = 'giraffe'
|
||||
const alternativeKey = 'hippo'
|
||||
const alternativeS3Credentials = {
|
||||
credentials: {
|
||||
accessKeyId: alternativeKey,
|
||||
secretAccessKey: alternativeSecret
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(async function () {
|
||||
settings.bucketCreds = {}
|
||||
settings.bucketCreds[bucket] = {
|
||||
auth_key: alternativeKey,
|
||||
auth_secret: alternativeSecret
|
||||
}
|
||||
|
||||
stream = await S3Persistor.getObjectStream(bucket, key)
|
||||
})
|
||||
|
||||
it('returns a metered stream', function () {
|
||||
expect(stream).to.be.instanceOf(Stream.Transform)
|
||||
})
|
||||
|
||||
it('sets the AWS client up with the alternative credentials', function () {
|
||||
expect(S3).to.have.been.calledWith(alternativeS3Credentials)
|
||||
})
|
||||
|
||||
it('fetches the right key from the right bucket', function () {
|
||||
expect(S3Client.getObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('uses the default credentials for an unknown bucket', async function () {
|
||||
stream = await S3Persistor.getObjectStream('anotherBucket', key)
|
||||
|
||||
expect(S3).to.have.been.calledTwice
|
||||
expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials)
|
||||
expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials)
|
||||
})
|
||||
|
||||
it('throws an error if there are no credentials for the bucket', async function () {
|
||||
delete settings.key
|
||||
delete settings.secret
|
||||
|
||||
await expect(
|
||||
S3Persistor.getObjectStream('anotherBucket', key)
|
||||
).to.eventually.be.rejected.and.be.an.instanceOf(Errors.SettingsError)
|
||||
})
|
||||
})
|
||||
|
||||
describe("when the file doesn't exist", function () {
|
||||
let error, stream
|
||||
|
||||
beforeEach(async function () {
|
||||
Transform.prototype.on = sinon.stub()
|
||||
S3ReadStream.on.withArgs('error').yields(S3NotFoundError)
|
||||
try {
|
||||
stream = await S3Persistor.getObjectStream(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('does not return a stream', function () {
|
||||
expect(stream).not.to.exist
|
||||
})
|
||||
|
||||
it('throws a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.exist
|
||||
})
|
||||
|
||||
it('stores the bucket and key in the error', function () {
|
||||
expect(error.info).to.include({ bucketName: bucket, key: key })
|
||||
})
|
||||
})
|
||||
|
||||
describe('when access to the file is denied', function () {
|
||||
let error, stream
|
||||
|
||||
beforeEach(async function () {
|
||||
Transform.prototype.on = sinon.stub()
|
||||
S3ReadStream.on.withArgs('error').yields(S3AccessDeniedError)
|
||||
try {
|
||||
stream = await S3Persistor.getObjectStream(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('does not return a stream', function () {
|
||||
expect(stream).not.to.exist
|
||||
})
|
||||
|
||||
it('throws a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.exist
|
||||
})
|
||||
|
||||
it('stores the bucket and key in the error', function () {
|
||||
expect(error.info).to.include({ bucketName: bucket, key: key })
|
||||
})
|
||||
})
|
||||
|
||||
describe('when S3 encounters an unkown error', function () {
|
||||
let error, stream
|
||||
|
||||
beforeEach(async function () {
|
||||
Transform.prototype.on = sinon.stub()
|
||||
S3ReadStream.on.withArgs('error').yields(genericError)
|
||||
try {
|
||||
stream = await S3Persistor.getObjectStream(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('does not return a stream', function () {
|
||||
expect(stream).not.to.exist
|
||||
})
|
||||
|
||||
it('throws a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('wraps the error', function () {
|
||||
expect(error.cause).to.exist
|
||||
})
|
||||
|
||||
it('stores the bucket and key in the error', function () {
|
||||
expect(error.info).to.include({ bucketName: bucket, key: key })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectSize', function () {
|
||||
describe('when called with valid parameters', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
size = await S3Persistor.getObjectSize(bucket, key)
|
||||
})
|
||||
|
||||
it('should return the object size', function () {
|
||||
expect(size).to.equal(objectSize)
|
||||
})
|
||||
|
||||
it('should pass the bucket and key to S3', function () {
|
||||
expect(S3Client.headObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the object is not found', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.headObject = sinon.stub().returns({
|
||||
promise: sinon.stub().rejects(S3NotFoundError)
|
||||
})
|
||||
try {
|
||||
await S3Persistor.getObjectSize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(S3NotFoundError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when S3 returns an error', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.headObject = sinon.stub().returns({
|
||||
promise: sinon.stub().rejects(genericError)
|
||||
})
|
||||
try {
|
||||
await S3Persistor.getObjectSize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should return a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendStream', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.sendStream(bucket, key, ReadStream)
|
||||
})
|
||||
|
||||
it('should upload the stream', function () {
|
||||
expect(S3Client.upload).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: sinon.match.instanceOf(Stream.Transform)
|
||||
})
|
||||
})
|
||||
|
||||
it('should upload files in a single part', function () {
|
||||
expect(S3Client.upload).to.have.been.calledWith(sinon.match.any, {
|
||||
partSize: 100 * 1024 * 1024
|
||||
})
|
||||
})
|
||||
|
||||
it('should meter the stream', function () {
|
||||
expect(ReadStream.pipe).to.have.been.calledWith(
|
||||
sinon.match.instanceOf(Stream.Transform)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when a hash is supploed', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.sendStream(
|
||||
bucket,
|
||||
key,
|
||||
ReadStream,
|
||||
'aaaaaaaabbbbbbbbaaaaaaaabbbbbbbb'
|
||||
)
|
||||
})
|
||||
|
||||
it('sends the hash in base64', function () {
|
||||
expect(S3Client.upload).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: sinon.match.instanceOf(Transform),
|
||||
ContentMD5: 'qqqqqru7u7uqqqqqu7u7uw=='
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the upload fails', function () {
|
||||
let error
|
||||
beforeEach(async function () {
|
||||
S3Client.upload = sinon.stub().returns({
|
||||
promise: sinon.stub().rejects(genericError)
|
||||
})
|
||||
try {
|
||||
await S3Persistor.sendStream(bucket, key, ReadStream)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('throws a WriteError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.WriteError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sendFile', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.sendFile(bucket, key, filename)
|
||||
})
|
||||
|
||||
it('should create a read stream for the file', function () {
|
||||
expect(Fs.createReadStream).to.have.been.calledWith(filename)
|
||||
})
|
||||
|
||||
it('should upload the stream', function () {
|
||||
expect(S3Client.upload).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: sinon.match.instanceOf(Transform)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getObjectMd5Hash', function () {
|
||||
describe('when the etag is a valid md5 hash', function () {
|
||||
let hash
|
||||
beforeEach(async function () {
|
||||
hash = await S3Persistor.getObjectMd5Hash(bucket, key)
|
||||
})
|
||||
|
||||
it('should return the object hash', function () {
|
||||
expect(hash).to.equal(md5)
|
||||
})
|
||||
|
||||
it('should get the hash from the object metadata', function () {
|
||||
expect(S3Client.headObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should not download the object', function () {
|
||||
expect(S3Client.getObject).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe("when the etag isn't a valid md5 hash", function () {
|
||||
let hash
|
||||
beforeEach(async function () {
|
||||
S3Client.headObject = sinon.stub().returns({
|
||||
promise: sinon.stub().resolves({
|
||||
ETag: 'somethingthatisntanmd5',
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
hash = await S3Persistor.getObjectMd5Hash(bucket, key)
|
||||
})
|
||||
|
||||
it('should re-fetch the file to verify it', function () {
|
||||
expect(S3Client.getObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should meter the download', function () {
|
||||
expect(S3ReadStream.pipe).to.have.been.calledWith(
|
||||
sinon.match.instanceOf(Stream.Transform)
|
||||
)
|
||||
})
|
||||
|
||||
it('should calculate the md5 hash from the file', function () {
|
||||
expect(Hash.read).to.have.been.called
|
||||
})
|
||||
|
||||
it('should return the md5 hash', function () {
|
||||
expect(hash).to.equal(md5)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyObject', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.copyObject(bucket, key, destKey)
|
||||
})
|
||||
|
||||
it('should copy the object', function () {
|
||||
expect(S3Client.copyObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: destKey,
|
||||
CopySource: `${bucket}/${key}`
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.copyObject = sinon.stub().returns({
|
||||
promise: sinon.stub().rejects(S3NotFoundError)
|
||||
})
|
||||
try {
|
||||
await S3Persistor.copyObject(bucket, key, destKey)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should throw a NotFoundError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.NotFoundError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteObject', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.deleteObject(bucket, key)
|
||||
})
|
||||
|
||||
it('should delete the object', function () {
|
||||
expect(S3Client.deleteObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteDirectory', function () {
|
||||
describe('with valid parameters', function () {
|
||||
beforeEach(async function () {
|
||||
return S3Persistor.deleteDirectory(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(S3Client.listObjectsV2).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should delete the objects using their keys', function () {
|
||||
expect(S3Client.deleteObjects).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Delete: {
|
||||
Objects: [{ Key: 'llama' }, { Key: 'hippo' }],
|
||||
Quiet: true
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are no files', function () {
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2 = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().resolves({ Contents: [] }) })
|
||||
return S3Persistor.deleteDirectory(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(S3Client.listObjectsV2).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should not try to delete any objects', function () {
|
||||
expect(S3Client.deleteObjects).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are more files available', function () {
|
||||
const continuationToken = 'wombat'
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2.onCall(0).returns({
|
||||
promise: sinon.stub().resolves({
|
||||
Contents: files,
|
||||
IsTruncated: true,
|
||||
NextContinuationToken: continuationToken
|
||||
})
|
||||
})
|
||||
|
||||
return S3Persistor.deleteDirectory(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects a second time, with a continuation token', function () {
|
||||
expect(S3Client.listObjectsV2).to.be.calledTwice
|
||||
expect(S3Client.listObjectsV2).to.be.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
expect(S3Client.listObjectsV2).to.be.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
})
|
||||
|
||||
it('should delete both sets of files', function () {
|
||||
expect(S3Client.deleteObjects).to.have.been.calledTwice
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error listing the objects', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2 = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().rejects(genericError) })
|
||||
try {
|
||||
await S3Persistor.deleteDirectory(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
|
||||
it('should not try to delete any objects', function () {
|
||||
expect(S3Client.deleteObjects).not.to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error deleting the objects', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.deleteObjects = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().rejects(genericError) })
|
||||
try {
|
||||
await S3Persistor.deleteDirectory(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a WriteError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.WriteError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('directorySize', function () {
|
||||
describe('with valid parameters', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
size = await S3Persistor.directorySize(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(S3Client.listObjectsV2).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should return the directory size', function () {
|
||||
expect(size).to.equal(filesSize)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are no files', function () {
|
||||
let size
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2 = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().resolves({ Contents: [] }) })
|
||||
size = await S3Persistor.directorySize(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects in the directory', function () {
|
||||
expect(S3Client.listObjectsV2).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should return zero', function () {
|
||||
expect(size).to.equal(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are more files available', function () {
|
||||
const continuationToken = 'wombat'
|
||||
let size
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2.onCall(0).returns({
|
||||
promise: sinon.stub().resolves({
|
||||
Contents: files,
|
||||
IsTruncated: true,
|
||||
NextContinuationToken: continuationToken
|
||||
})
|
||||
})
|
||||
|
||||
size = await S3Persistor.directorySize(bucket, key)
|
||||
})
|
||||
|
||||
it('should list the objects a second time, with a continuation token', function () {
|
||||
expect(S3Client.listObjectsV2).to.be.calledTwice
|
||||
expect(S3Client.listObjectsV2).to.be.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key
|
||||
})
|
||||
expect(S3Client.listObjectsV2).to.be.calledWith({
|
||||
Bucket: bucket,
|
||||
Prefix: key,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
})
|
||||
|
||||
it('should return the size of both sets of files', function () {
|
||||
expect(size).to.equal(filesSize * 2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error listing the objects', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.listObjectsV2 = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().rejects(genericError) })
|
||||
try {
|
||||
await S3Persistor.directorySize(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the error', function () {
|
||||
expect(error.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkIfObjectExists', function () {
|
||||
describe('when the file exists', function () {
|
||||
let exists
|
||||
|
||||
beforeEach(async function () {
|
||||
exists = await S3Persistor.checkIfObjectExists(bucket, key)
|
||||
})
|
||||
|
||||
it('should get the object header', function () {
|
||||
expect(S3Client.headObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should return that the file exists', function () {
|
||||
expect(exists).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function () {
|
||||
let exists
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.headObject = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().rejects(S3NotFoundError) })
|
||||
exists = await S3Persistor.checkIfObjectExists(bucket, key)
|
||||
})
|
||||
|
||||
it('should get the object header', function () {
|
||||
expect(S3Client.headObject).to.have.been.calledWith({
|
||||
Bucket: bucket,
|
||||
Key: key
|
||||
})
|
||||
})
|
||||
|
||||
it('should return that the file does not exist', function () {
|
||||
expect(exists).to.equal(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there is an error', function () {
|
||||
let error
|
||||
|
||||
beforeEach(async function () {
|
||||
S3Client.headObject = sinon
|
||||
.stub()
|
||||
.returns({ promise: sinon.stub().rejects(genericError) })
|
||||
try {
|
||||
await S3Persistor.checkIfObjectExists(bucket, key)
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
})
|
||||
|
||||
it('should generate a ReadError', function () {
|
||||
expect(error).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should wrap the upstream ReadError', function () {
|
||||
expect(error.cause).to.be.an.instanceOf(Errors.ReadError)
|
||||
})
|
||||
|
||||
it('should eventually wrap the error', function () {
|
||||
expect(error.cause.cause).to.equal(genericError)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user