Add MigrationPersistor for sending 404 requests to a fallback persistor
This commit is contained in:
Simon Detheridge
@@ -11,6 +11,7 @@ const S3 = require('aws-sdk/clients/s3')
|
||||
const Stream = require('stream')
|
||||
const request = require('request')
|
||||
const { promisify } = require('util')
|
||||
const streamifier = require('streamifier')
|
||||
chai.use(require('chai-as-promised'))
|
||||
|
||||
const fsWriteFile = promisify(fs.writeFile)
|
||||
@@ -25,6 +26,19 @@ async function getMetric(filestoreUrl, metric) {
|
||||
return parseInt(found ? found[1] : 0) || 0
|
||||
}
|
||||
|
||||
if (!process.env.AWS_ACCESS_KEY_ID) {
|
||||
throw new Error('please provide credentials for the AWS S3 test server')
|
||||
}
|
||||
|
||||
function streamToString(stream) {
|
||||
const chunks = []
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', chunk => chunks.push(chunk))
|
||||
stream.on('error', reject)
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
|
||||
})
|
||||
}
|
||||
|
||||
// store settings for multiple backends, so that we can test each one.
|
||||
// fs will always be available - add others if they are configured
|
||||
const BackendSettings = {
|
||||
@@ -35,11 +49,8 @@ const BackendSettings = {
|
||||
public_files: Path.resolve(__dirname, '../../../public_files'),
|
||||
template_files: Path.resolve(__dirname, '../../../template_files')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.AWS_ACCESS_KEY_ID) {
|
||||
BackendSettings.S3Persistor = {
|
||||
},
|
||||
S3Persistor: {
|
||||
backend: 's3',
|
||||
s3: {
|
||||
key: process.env.AWS_ACCESS_KEY_ID,
|
||||
@@ -52,6 +63,62 @@ if (process.env.AWS_ACCESS_KEY_ID) {
|
||||
template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
|
||||
public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
|
||||
}
|
||||
},
|
||||
FallbackS3ToFSPersistor: {
|
||||
backend: 's3',
|
||||
s3: {
|
||||
key: process.env.AWS_ACCESS_KEY_ID,
|
||||
secret: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
endpoint: process.env.AWS_S3_ENDPOINT,
|
||||
pathStyle: true
|
||||
},
|
||||
stores: {
|
||||
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
|
||||
template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
|
||||
public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
|
||||
},
|
||||
fallback: {
|
||||
backend: 'fs',
|
||||
buckets: {
|
||||
[process.env.AWS_S3_USER_FILES_BUCKET_NAME]: Path.resolve(
|
||||
__dirname,
|
||||
'../../../user_files'
|
||||
),
|
||||
[process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME]: Path.resolve(
|
||||
__dirname,
|
||||
'../../../public_files'
|
||||
),
|
||||
[process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME]: Path.resolve(
|
||||
__dirname,
|
||||
'../../../template_files'
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
FallbackFSToS3Persistor: {
|
||||
backend: 'fs',
|
||||
s3: {
|
||||
key: process.env.AWS_ACCESS_KEY_ID,
|
||||
secret: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
endpoint: process.env.AWS_S3_ENDPOINT,
|
||||
pathStyle: true
|
||||
},
|
||||
stores: {
|
||||
user_files: Path.resolve(__dirname, '../../../user_files'),
|
||||
public_files: Path.resolve(__dirname, '../../../public_files'),
|
||||
template_files: Path.resolve(__dirname, '../../../template_files')
|
||||
},
|
||||
fallback: {
|
||||
backend: 's3',
|
||||
buckets: {
|
||||
[Path.resolve(__dirname, '../../../user_files')]: process.env
|
||||
.AWS_S3_USER_FILES_BUCKET_NAME,
|
||||
[Path.resolve(__dirname, '../../../public_files')]: process.env
|
||||
.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
|
||||
[Path.resolve(__dirname, '../../../template_files')]: process.env
|
||||
.AWS_S3_PUBLIC_FILES_BUCKET_NAME
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,23 +167,21 @@ describe('Filestore', function() {
|
||||
})
|
||||
|
||||
describe('with a file on the server', function() {
|
||||
let fileId, fileUrl
|
||||
let fileId, fileUrl, constantFileContent
|
||||
|
||||
const localFileReadPath =
|
||||
'/tmp/filestore_acceptance_tests_file_read.txt'
|
||||
const constantFileContent = [
|
||||
'hello world',
|
||||
`line 2 goes here ${Math.random()}`,
|
||||
'there are 3 lines in all'
|
||||
].join('\n')
|
||||
|
||||
before(async function() {
|
||||
await fsWriteFile(localFileReadPath, constantFileContent)
|
||||
})
|
||||
|
||||
beforeEach(async function() {
|
||||
fileId = Math.random()
|
||||
fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${directoryName}%2F${fileId}`
|
||||
constantFileContent = [
|
||||
'hello world',
|
||||
`line 2 goes here ${Math.random()}`,
|
||||
'there are 3 lines in all'
|
||||
].join('\n')
|
||||
|
||||
await fsWriteFile(localFileReadPath, constantFileContent)
|
||||
|
||||
const writeStream = request.post(fileUrl)
|
||||
const readStream = fs.createReadStream(localFileReadPath)
|
||||
@@ -177,7 +242,7 @@ describe('Filestore', function() {
|
||||
})
|
||||
|
||||
it('should be able to copy files', async function() {
|
||||
const newProjectID = 'acceptance_tests_copyied_project'
|
||||
const newProjectID = 'acceptance_tests_copied_project'
|
||||
const newFileId = Math.random()
|
||||
const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}`
|
||||
const opts = {
|
||||
@@ -198,6 +263,18 @@ describe('Filestore', function() {
|
||||
expect(response.body).to.equal(constantFileContent)
|
||||
})
|
||||
|
||||
it('should be able to overwrite the file', async function() {
|
||||
const newContent = `here is some different content, ${Math.random()}`
|
||||
const writeStream = request.post(fileUrl)
|
||||
const readStream = streamifier.createReadStream(newContent)
|
||||
// hack to consume the result to ensure the http request has been fully processed
|
||||
const resultStream = fs.createWriteStream('/dev/null')
|
||||
await pipeline(readStream, writeStream, resultStream)
|
||||
|
||||
const response = await rp.get(fileUrl)
|
||||
expect(response.body).to.equal(newContent)
|
||||
})
|
||||
|
||||
if (backend === 'S3Persistor') {
|
||||
it('should record an egress metric for the upload', async function() {
|
||||
const metric = await getMetric(filestoreUrl, 's3_egress')
|
||||
@@ -292,10 +369,10 @@ describe('Filestore', function() {
|
||||
|
||||
if (backend === 'S3Persistor') {
|
||||
describe('with a file in a specific bucket', function() {
|
||||
let constantFileContents, fileId, fileUrl, bucketName
|
||||
let constantFileContent, fileId, fileUrl, bucketName
|
||||
|
||||
beforeEach(async function() {
|
||||
constantFileContents = `This is a file in a different S3 bucket ${Math.random()}`
|
||||
constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
|
||||
fileId = Math.random().toString()
|
||||
bucketName = Math.random().toString()
|
||||
fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
|
||||
@@ -320,14 +397,302 @@ describe('Filestore', function() {
|
||||
.upload({
|
||||
Bucket: bucketName,
|
||||
Key: fileId,
|
||||
Body: constantFileContents
|
||||
Body: constantFileContent
|
||||
})
|
||||
.promise()
|
||||
})
|
||||
|
||||
it('should get the file from the specified bucket', async function() {
|
||||
const response = await rp.get(fileUrl)
|
||||
expect(response.body).to.equal(constantFileContents)
|
||||
expect(response.body).to.equal(constantFileContent)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
if (BackendSettings[backend].fallback) {
|
||||
describe('with a fallback', function() {
|
||||
async function uploadStringToPersistor(
|
||||
persistor,
|
||||
bucket,
|
||||
key,
|
||||
content
|
||||
) {
|
||||
const fileStream = streamifier.createReadStream(content)
|
||||
await persistor.promises.sendStream(bucket, key, fileStream)
|
||||
}
|
||||
|
||||
async function getStringFromPersistor(persistor, bucket, key) {
|
||||
const stream = await persistor.promises.getFileStream(
|
||||
bucket,
|
||||
key,
|
||||
{}
|
||||
)
|
||||
return streamToString(stream)
|
||||
}
|
||||
|
||||
async function expectPersistorToHaveFile(
|
||||
persistor,
|
||||
bucket,
|
||||
key,
|
||||
content
|
||||
) {
|
||||
const foundContent = await getStringFromPersistor(
|
||||
persistor,
|
||||
bucket,
|
||||
key
|
||||
)
|
||||
expect(foundContent).to.equal(content)
|
||||
}
|
||||
|
||||
async function expectPersistorNotToHaveFile(persistor, bucket, key) {
|
||||
await expect(
|
||||
getStringFromPersistor(persistor, bucket, key)
|
||||
).to.eventually.have.been.rejected.with.property(
|
||||
'name',
|
||||
'NotFoundError'
|
||||
)
|
||||
}
|
||||
|
||||
let constantFileContent,
|
||||
fileId,
|
||||
fileKey,
|
||||
fileUrl,
|
||||
bucket,
|
||||
fallbackBucket
|
||||
const projectId = 'acceptance_tests'
|
||||
|
||||
beforeEach(function() {
|
||||
constantFileContent = `This is yet more file content ${Math.random()}`
|
||||
fileId = Math.random().toString()
|
||||
fileKey = `${projectId}/${directoryName}/${fileId}`
|
||||
fileUrl = `${filestoreUrl}/project/${projectId}/file/${directoryName}%2F${fileId}`
|
||||
|
||||
bucket = Settings.filestore.stores.user_files
|
||||
fallbackBucket = Settings.filestore.fallback.buckets[bucket]
|
||||
})
|
||||
|
||||
describe('with a file in the fallback bucket', function() {
|
||||
beforeEach(async function() {
|
||||
await uploadStringToPersistor(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should not find file in the primary', async function() {
|
||||
await expectPersistorNotToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should find the file in the fallback', async function() {
|
||||
await expectPersistorToHaveFile(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should fetch the file', async function() {
|
||||
const res = await rp.get(fileUrl)
|
||||
expect(res.body).to.equal(constantFileContent)
|
||||
})
|
||||
|
||||
it('should not copy the file to the primary', async function() {
|
||||
await rp.get(fileUrl)
|
||||
|
||||
await expectPersistorNotToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey
|
||||
)
|
||||
})
|
||||
|
||||
describe('when copyOnMiss is enabled', function() {
|
||||
beforeEach(function() {
|
||||
Settings.filestore.fallback.copyOnMiss = true
|
||||
})
|
||||
|
||||
it('copies the file to the primary', async function() {
|
||||
await rp.get(fileUrl)
|
||||
// wait for the file to copy in the background
|
||||
await promisify(setTimeout)(1000)
|
||||
|
||||
await expectPersistorToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when copying a file', function() {
|
||||
let newFileId, newFileUrl, newFileKey
|
||||
const newProjectID = 'acceptance_tests_copied_project'
|
||||
|
||||
beforeEach(async function() {
|
||||
newFileId = Math.random()
|
||||
newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}`
|
||||
newFileKey = `${newProjectID}/${directoryName}/${newFileId}`
|
||||
|
||||
const opts = {
|
||||
method: 'put',
|
||||
uri: newFileUrl,
|
||||
json: {
|
||||
source: {
|
||||
project_id: 'acceptance_tests',
|
||||
file_id: `${directoryName}/${fileId}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const response = await rp(opts)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
})
|
||||
|
||||
it('should leave the old file in the old bucket', async function() {
|
||||
await expectPersistorToHaveFile(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should not create a new file in the old bucket', async function() {
|
||||
await expectPersistorNotToHaveFile(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
newFileKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should not copy the old file to the new bucket', async function() {
|
||||
await expectPersistorNotToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey
|
||||
)
|
||||
})
|
||||
|
||||
it('should create a new file in the new bucket', async function() {
|
||||
await expectPersistorToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
newFileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when sending a file', function() {
|
||||
beforeEach(async function() {
|
||||
const writeStream = request.post(fileUrl)
|
||||
const readStream = streamifier.createReadStream(
|
||||
constantFileContent
|
||||
)
|
||||
// hack to consume the result to ensure the http request has been fully processed
|
||||
const resultStream = fs.createWriteStream('/dev/null')
|
||||
await pipeline(readStream, writeStream, resultStream)
|
||||
})
|
||||
|
||||
it('should store the file on the primary', async function() {
|
||||
await expectPersistorToHaveFile(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should not store the file on the fallback', async function() {
|
||||
await expectPersistorNotToHaveFile(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
`acceptance_tests/${directoryName}/${fileId}`
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when deleting a file', function() {
|
||||
describe('when the file exists on the primary', function() {
|
||||
beforeEach(async function() {
|
||||
await uploadStringToPersistor(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file', async function() {
|
||||
const response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
await expect(
|
||||
rp.get(fileUrl)
|
||||
).to.eventually.be.rejected.and.have.property('statusCode', 404)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file exists on the fallback', function() {
|
||||
beforeEach(async function() {
|
||||
await uploadStringToPersistor(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the file', async function() {
|
||||
const response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
await expect(
|
||||
rp.get(fileUrl)
|
||||
).to.eventually.be.rejected.and.have.property('statusCode', 404)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file exists on both the primary and the fallback', function() {
|
||||
beforeEach(async function() {
|
||||
await uploadStringToPersistor(
|
||||
app.persistor.primaryPersistor,
|
||||
bucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
await uploadStringToPersistor(
|
||||
app.persistor.fallbackPersistor,
|
||||
fallbackBucket,
|
||||
fileKey,
|
||||
constantFileContent
|
||||
)
|
||||
})
|
||||
|
||||
it('should delete the files', async function() {
|
||||
const response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
await expect(
|
||||
rp.get(fileUrl)
|
||||
).to.eventually.be.rejected.and.have.property('statusCode', 404)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when the file does not exist', function() {
|
||||
it('should return return 204', async function() {
|
||||
// S3 doesn't give us a 404 when the object doesn't exist, so to stay
|
||||
// consistent we merrily return 204 ourselves here as well
|
||||
const response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user