[history-v1] make back_fill_file_hash_fix_up compatible with Server Pro (#27280)

* [history-v1] move MockFilestore into shared place

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [history-v1] make back_fill_file_hash_fix_up compatible with Server Pro

---------

Co-authored-by: Brian Gough <brian.gough@overleaf.com>
GitOrigin-RevId: 70ea57e1503031d9f14dcd60c4c110e746450587

services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs

@@ -9,15 +9,12 @@ import { Blob } from 'overleaf-editor-core'
 import {
   BlobStore,
   getStringLengthOfFile,
-  GLOBAL_BLOBS,
   makeBlobForFile,
 } from '../lib/blob_store/index.js'
 import { db } from '../lib/mongodb.js'
 import commandLineArgs from 'command-line-args'
 import readline from 'node:readline'
-import { _blobIsBackedUp, backupBlob } from '../lib/backupBlob.mjs'
 import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
-import filestorePersistor from '../lib/persistor.js'
 import { setTimeout } from 'node:timers/promises'
 
 // Silence warning.
@@ -52,12 +49,11 @@ ObjectId.cacheHexString = true
  */
 
 /**
- * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_DELETE_PERMISSION: boolean, FIX_MISSING_HASH: boolean, LOGS: string}}
+ * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}}
  */
 function parseArgs() {
   const args = commandLineArgs([
     { name: 'fixNotFound', type: String, defaultValue: 'true' },
-    { name: 'fixDeletePermission', type: String, defaultValue: 'true' },
     { name: 'fixHashMismatch', type: String, defaultValue: 'true' },
     { name: 'fixMissingHash', type: String, defaultValue: 'true' },
     { name: 'logs', type: String, defaultValue: '' },
@@ -74,20 +70,13 @@ function parseArgs() {
   }
   return {
     FIX_HASH_MISMATCH: boolVal('fixNotFound'),
-    FIX_DELETE_PERMISSION: boolVal('fixDeletePermission'),
     FIX_NOT_FOUND: boolVal('fixHashMismatch'),
     FIX_MISSING_HASH: boolVal('fixMissingHash'),
     LOGS: args.logs,
   }
 }
 
-const {
-  FIX_HASH_MISMATCH,
-  FIX_DELETE_PERMISSION,
-  FIX_NOT_FOUND,
-  FIX_MISSING_HASH,
-  LOGS,
-} = parseArgs()
+const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs()
 if (!LOGS) {
   throw new Error('--logs parameter missing')
 }
@@ -105,6 +94,37 @@ const STREAM_HIGH_WATER_MARK = parseInt(
 )
 const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10)
 
+// Filestore endpoint location
+const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1'
+const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009'
+
+async function fetchFromFilestore(projectId, fileId) {
+  const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}`
+  const response = await fetch(url)
+  if (!response.ok) {
+    if (response.status === 404) {
+      throw new NotFoundError('file not found in filestore', {
+        status: response.status,
+      })
+    }
+    const body = await response.text()
+    throw new OError('fetchFromFilestore failed', {
+      projectId,
+      fileId,
+      status: response.status,
+      body,
+    })
+  }
+  if (!response.body) {
+    throw new OError('fetchFromFilestore response has no body', {
+      projectId,
+      fileId,
+      status: response.status,
+    })
+  }
+  return response.body
+}
+
 /** @type {ProjectsCollection} */
 const projectsCollection = db.collection('projects')
 /** @type {DeletedProjectsCollection} */
@@ -302,19 +322,16 @@ async function setHashInMongo(projectId, fileId, hash) {
  * @return {Promise<void>}
  */
 async function importRestoredFilestoreFile(projectId, fileId, historyId) {
-  const filestoreKey = `${projectId}/${fileId}`
   const path = `${BUFFER_DIR}/${projectId}_${fileId}`
   try {
     let s
     try {
-      s = await filestorePersistor.getObjectStream(
-        USER_FILES_BUCKET_NAME,
-        filestoreKey
-      )
+      s = await fetchFromFilestore(projectId, fileId)
     } catch (err) {
       if (err instanceof NotFoundError) {
         throw new OError('missing blob, need to restore filestore file', {
-          filestoreKey,
+          projectId,
+          fileId,
         })
       }
       throw err
@@ -325,7 +342,6 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) {
     )
     const blobStore = new BlobStore(historyId)
     const blob = await blobStore.putFile(path)
-    await backupBlob(historyId, blob, path)
     await setHashInMongo(projectId, fileId, blob.getHash())
   } finally {
     await fs.promises.rm(path, { force: true })
@@ -339,13 +355,9 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) {
  * @return {Promise<Blob>}
  */
 async function bufferFilestoreFileToDisk(projectId, fileId, path) {
-  const filestoreKey = `${projectId}/${fileId}`
   try {
     await Stream.promises.pipeline(
-      await filestorePersistor.getObjectStream(
-        USER_FILES_BUCKET_NAME,
-        filestoreKey
-      ),
+      await fetchFromFilestore(projectId, fileId),
       fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK })
     )
     const blob = await makeBlobForFile(path)
@@ -356,7 +368,8 @@ async function bufferFilestoreFileToDisk(projectId, fileId, path) {
   } catch (err) {
     if (err instanceof NotFoundError) {
       throw new OError('missing blob, need to restore filestore file', {
-        filestoreKey,
+        projectId,
+        fileId,
       })
     }
     throw err
@@ -389,7 +402,7 @@ async function uploadFilestoreFile(projectId, fileId) {
     const blob = await bufferFilestoreFileToDisk(projectId, fileId, path)
     const hash = blob.getHash()
     try {
-      await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash)
+      await ensureBlobExistsForFile(projectId, fileId, hash)
     } catch (err) {
       if (!(err instanceof Blob.NotFoundError)) throw err
 
@@ -397,7 +410,7 @@ async function uploadFilestoreFile(projectId, fileId) {
       const historyId = project.overleaf.history.id.toString()
       const blobStore = new BlobStore(historyId)
       await blobStore.putBlob(path, blob)
-      await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash)
+      await ensureBlobExistsForFile(projectId, fileId, hash)
     }
   } finally {
     await fs.promises.rm(path, { force: true })
@@ -426,11 +439,7 @@ async function fixHashMismatch(line) {
     await importRestoredFilestoreFile(projectId, fileId, historyId)
     return true
   }
-  return await ensureBlobExistsForFileAndUploadToAWS(
-    projectId,
-    fileId,
-    computedHash
-  )
+  return await ensureBlobExistsForFile(projectId, fileId, computedHash)
 }
 
 /**
@@ -444,30 +453,19 @@ async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) {
   return fileRef.hash === hash
 }
 
-/**
- * @param {string} projectId
- * @param {string} hash
- * @return {Promise<boolean>}
- */
-async function needsBackingUpToAWS(projectId, hash) {
-  if (GLOBAL_BLOBS.has(hash)) return false
-  return !(await _blobIsBackedUp(projectId, hash))
-}
-
 /**
  * @param {string} projectId
  * @param {string} fileId
  * @param {string} hash
  * @return {Promise<boolean>}
  */
-async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) {
+async function ensureBlobExistsForFile(projectId, fileId, hash) {
   const { project } = await getProject(projectId)
   const historyId = project.overleaf.history.id.toString()
   const blobStore = new BlobStore(historyId)
   if (
     (await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) &&
-    (await blobStore.getBlob(hash)) &&
-    !(await needsBackingUpToAWS(projectId, hash))
+    (await blobStore.getBlob(hash))
   ) {
     return false // already processed
   }
@@ -488,7 +486,7 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) {
     )
     if (writtenBlob.getHash() !== hash) {
       // Double check download, better safe than sorry.
-      throw new OError('blob corrupted', { writtenBlob })
+      throw new OError('blob corrupted', { writtenBlob, hash })
     }
 
     let blob = await blobStore.getBlob(hash)
@@ -497,7 +495,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) {
       // HACK: Skip upload to GCS and finalize putBlob operation directly.
       await blobStore.backend.insertBlob(historyId, writtenBlob)
     }
-    await backupBlob(historyId, writtenBlob, path)
   } finally {
     await fs.promises.rm(path, { force: true })
   }
@@ -505,16 +502,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) {
   return true
 }
 
-/**
- * @param {string} line
- * @return {Promise<boolean>}
- */
-async function fixDeletePermission(line) {
-  let { projectId, fileId, hash } = JSON.parse(line)
-  if (!hash) hash = await computeFilestoreFileHash(projectId, fileId)
-  return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash)
-}
-
 /**
  * @param {string} line
  * @return {Promise<boolean>}
@@ -526,7 +513,7 @@ async function fixMissingHash(line) {
   } = await findFile(projectId, fileId)
   if (hash) {
     // processed, double check
-    return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash)
+    return await ensureBlobExistsForFile(projectId, fileId, hash)
   }
   await uploadFilestoreFile(projectId, fileId)
   return true
@@ -543,11 +530,6 @@ const CASES = {
     flag: FIX_HASH_MISMATCH,
     action: fixHashMismatch,
   },
-  'delete permission': {
-    match: 'storage.objects.delete',
-    flag: FIX_DELETE_PERMISSION,
-    action: fixDeletePermission,
-  },
   'missing file hash': {
     match: '"bad file hash"',
     flag: FIX_MISSING_HASH,

services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs

@@ -20,7 +20,7 @@ import {
   makeProjectKey,
 } from '../../../../storage/lib/blob_store/index.js'
 
-import express from 'express'
+import { mockFilestore } from './support/MockFilestore.mjs'
 
 chai.use(chaiExclude)
 const TIMEOUT = 20 * 1_000
@@ -28,59 +28,6 @@ const TIMEOUT = 20 * 1_000
 const projectsCollection = db.collection('projects')
 const deletedProjectsCollection = db.collection('deletedProjects')
 
-class MockFilestore {
-  constructor() {
-    this.host = process.env.FILESTORE_HOST || '127.0.0.1'
-    this.port = process.env.FILESTORE_PORT || 3009
-    // create a server listening on this.host and this.port
-    this.files = {}
-
-    this.app = express()
-
-    this.app.get('/project/:projectId/file/:fileId', (req, res) => {
-      const { projectId, fileId } = req.params
-      const content = this.files[projectId]?.[fileId]
-      if (!content) return res.status(404).end()
-      res.status(200).end(content)
-    })
-  }
-
-  start() {
-    // reset stored files
-    this.files = {}
-    // start the server
-    if (this.serverPromise) {
-      return this.serverPromise
-    } else {
-      this.serverPromise = new Promise((resolve, reject) => {
-        this.server = this.app.listen(this.port, this.host, err => {
-          if (err) return reject(err)
-          resolve()
-        })
-      })
-      return this.serverPromise
-    }
-  }
-
-  addFile(projectId, fileId, fileContent) {
-    if (!this.files[projectId]) {
-      this.files[projectId] = {}
-    }
-    this.files[projectId][fileId] = fileContent
-  }
-
-  deleteObject(projectId, fileId) {
-    if (this.files[projectId]) {
-      delete this.files[projectId][fileId]
-      if (Object.keys(this.files[projectId]).length === 0) {
-        delete this.files[projectId]
-      }
-    }
-  }
-}
-
-const mockFilestore = new MockFilestore()
-
 /**
  * @param {ObjectId} objectId
  * @return {string}

services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs

@@ -1,48 +1,24 @@
 import fs from 'node:fs'
 import Crypto from 'node:crypto'
-import Stream from 'node:stream'
 import { promisify } from 'node:util'
 import { Binary, ObjectId } from 'mongodb'
 import { Blob } from 'overleaf-editor-core'
-import { backedUpBlobs, blobs, db } from '../../../../storage/lib/mongodb.js'
+import { db } from '../../../../storage/lib/mongodb.js'
 import cleanup from './support/cleanup.js'
 import testProjects from '../api/support/test_projects.js'
 import { execFile } from 'node:child_process'
 import chai, { expect } from 'chai'
 import chaiExclude from 'chai-exclude'
-import config from 'config'
-import { WritableBuffer } from '@overleaf/stream-utils'
-import {
-  backupPersistor,
-  projectBlobsBucket,
-} from '../../../../storage/lib/backupPersistor.mjs'
-import projectKey from '../../../../storage/lib/project_key.js'
-import {
-  BlobStore,
-  makeProjectKey,
-} from '../../../../storage/lib/blob_store/index.js'
-import ObjectPersistor from '@overleaf/object-persistor'
+import { BlobStore } from '../../../../storage/lib/blob_store/index.js'
+import { mockFilestore } from './support/MockFilestore.mjs'
 
 chai.use(chaiExclude)
 
 const TIMEOUT = 20 * 1_000
 
-const { deksBucket } = config.get('backupStore')
-const { tieringStorageClass } = config.get('backupPersistor')
-
 const projectsCollection = db.collection('projects')
 const deletedProjectsCollection = db.collection('deletedProjects')
 
-const FILESTORE_PERSISTOR = ObjectPersistor({
-  backend: 'gcs',
-  gcs: {
-    endpoint: {
-      apiEndpoint: process.env.GCS_API_ENDPOINT,
-      projectId: process.env.GCS_PROJECT_ID,
-    },
-  },
-})
-
 /**
  * @param {ObjectId} objectId
  * @return {string}
@@ -70,17 +46,6 @@ function binaryForGitBlobHash(gitBlobHash) {
   return new Binary(Buffer.from(gitBlobHash, 'hex'))
 }
 
-async function listS3Bucket(bucket, wantStorageClass) {
-  const client = backupPersistor._getClientForBucket(bucket)
-  const response = await client.listObjectsV2({ Bucket: bucket }).promise()
-
-  for (const object of response.Contents || []) {
-    expect(object).to.have.property('StorageClass', wantStorageClass)
-  }
-
-  return (response.Contents || []).map(item => item.Key || '')
-}
-
 function objectIdFromTime(timestamp) {
   return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000)
 }
@@ -97,7 +62,6 @@ describe('back_fill_file_hash_fix_up script', function () {
   const historyIdDeleted0 = projectIdDeleted0.toString()
   const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z')
   const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z')
-  const fileIdBlobExistsInGCS0 = objectIdFromTime('2017-02-01T00:02:00Z')
   const fileIdWithDifferentHashNotFound0 = objectIdFromTime(
     '2017-02-01T00:03:00Z'
   )
@@ -112,9 +76,6 @@ describe('back_fill_file_hash_fix_up script', function () {
   const fileIdWithDifferentHashRestore = objectIdFromTime(
     '2017-02-01T00:08:00Z'
   )
-  const fileIdBlobExistsInGCS1 = objectIdFromTime('2017-02-01T00:09:00Z')
-  const fileIdRestoreFromFilestore0 = objectIdFromTime('2017-02-01T00:10:00Z')
-  const fileIdRestoreFromFilestore1 = objectIdFromTime('2017-02-01T00:11:00Z')
   const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z')
   const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z')
   const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z')
@@ -125,31 +86,11 @@ describe('back_fill_file_hash_fix_up script', function () {
   )
   const deleteProjectsRecordId0 = new ObjectId()
   const writtenBlobs = [
-    {
-      projectId: projectId0,
-      historyId: historyId0,
-      fileId: fileIdBlobExistsInGCS0,
-    },
-    {
-      projectId: projectId0,
-      historyId: historyId0,
-      fileId: fileIdBlobExistsInGCS1,
-    },
     {
       projectId: projectId0,
       historyId: historyId0,
       fileId: fileIdWithDifferentHashNotFound0,
     },
-    {
-      projectId: projectId0,
-      historyId: historyId0,
-      fileId: fileIdRestoreFromFilestore0,
-    },
-    {
-      projectId: projectId0,
-      historyId: historyId0,
-      fileId: fileIdRestoreFromFilestore1,
-    },
     {
       projectId: projectId0,
       historyId: historyId0,
@@ -200,17 +141,6 @@ describe('back_fill_file_hash_fix_up script', function () {
       },
       msg: 'failed to process file',
     },
-    {
-      projectId: projectId0,
-      fileId: fileIdRestoreFromFilestore0,
-      err: { message: 'OError: hash mismatch' },
-      hash: gitBlobHash(fileIdRestoreFromFilestore0),
-      entry: {
-        ctx: { historyId: historyId0.toString() },
-        hash: hashDoesNotExistAsBlob,
-      },
-      msg: 'failed to process file',
-    },
     {
       projectId: projectIdDeleted0,
       fileId: fileIdWithDifferentHashNotFound1,
@@ -236,33 +166,6 @@ describe('back_fill_file_hash_fix_up script', function () {
       err: { message: 'NotFoundError' },
       msg: 'failed to process file',
     },
-    {
-      projectId: projectId0,
-      fileId: fileIdBlobExistsInGCS0,
-      hash: gitBlobHash(fileIdBlobExistsInGCS0),
-      err: { message: 'storage.objects.delete' },
-      msg: 'failed to process file',
-    },
-    {
-      projectId: projectId0,
-      fileId: fileIdBlobExistsInGCSCorrupted,
-      hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
-      err: { message: 'storage.objects.delete' },
-      msg: 'failed to process file',
-    },
-    {
-      projectId: projectId0,
-      fileId: fileIdBlobExistsInGCS1,
-      hash: gitBlobHash(fileIdBlobExistsInGCS1),
-      err: { message: 'storage.objects.delete' },
-      msg: 'failed to process file',
-    },
-    {
-      projectId: projectId0,
-      fileId: fileIdRestoreFromFilestore1,
-      err: { message: 'storage.objects.delete' },
-      msg: 'failed to process file',
-    },
     {
       projectId: projectIdDeleted0,
       fileId: fileIdMissing1,
@@ -291,22 +194,23 @@ describe('back_fill_file_hash_fix_up script', function () {
       reason: 'bad file hash',
       msg: 'bad file-tree path',
     },
+    {
+      projectId: projectId0,
+      _id: fileIdBlobExistsInGCSCorrupted,
+      reason: 'bad file hash',
+      msg: 'bad file-tree path',
+    },
   ]
   if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) {
     const fileIds = {
       fileIdWithDifferentHashFound,
       fileIdInGoodState,
-      fileIdBlobExistsInGCS0,
-      fileIdBlobExistsInGCS1,
       fileIdWithDifferentHashNotFound0,
       fileIdWithDifferentHashNotFound1,
-      fileIdBlobExistsInGCSCorrupted,
       fileIdMissing0,
       fileIdMissing1,
       fileIdMissing2,
       fileIdWithDifferentHashRestore,
-      fileIdRestoreFromFilestore0,
-      fileIdRestoreFromFilestore1,
       fileIdHashMissing0,
       fileIdHashMissing1,
     }
@@ -330,38 +234,25 @@ describe('back_fill_file_hash_fix_up script', function () {
   before(cleanup.everything)
 
   before('populate blobs/GCS', async function () {
-    await FILESTORE_PERSISTOR.sendStream(
-      USER_FILES_BUCKET_NAME,
-      `${projectId0}/${fileIdRestoreFromFilestore0}`,
-      Stream.Readable.from([fileIdRestoreFromFilestore0.toString()])
+    await mockFilestore.start()
+    mockFilestore.addFile(
+      projectId0,
+      fileIdHashMissing0,
+      fileIdHashMissing0.toString()
     )
-    await FILESTORE_PERSISTOR.sendStream(
-      USER_FILES_BUCKET_NAME,
-      `${projectId0}/${fileIdRestoreFromFilestore1}`,
-      Stream.Readable.from([fileIdRestoreFromFilestore1.toString()])
+    mockFilestore.addFile(
+      projectId0,
+      fileIdHashMissing1,
+      fileIdHashMissing1.toString()
     )
-    await FILESTORE_PERSISTOR.sendStream(
-      USER_FILES_BUCKET_NAME,
-      `${projectId0}/${fileIdHashMissing0}`,
-      Stream.Readable.from([fileIdHashMissing0.toString()])
-    )
-    await FILESTORE_PERSISTOR.sendStream(
-      USER_FILES_BUCKET_NAME,
-      `${projectId0}/${fileIdHashMissing1}`,
-      Stream.Readable.from([fileIdHashMissing1.toString()])
+    mockFilestore.addFile(
+      projectId0,
+      fileIdBlobExistsInGCSCorrupted,
+      fileIdBlobExistsInGCSCorrupted.toString()
     )
     await new BlobStore(historyId0.toString()).putString(
       fileIdHashMissing1.toString() // partially processed
     )
-    await new BlobStore(historyId0.toString()).putString(
-      fileIdBlobExistsInGCS0.toString()
-    )
-    await new BlobStore(historyId0.toString()).putString(
-      fileIdBlobExistsInGCS1.toString()
-    )
-    await new BlobStore(historyId0.toString()).putString(
-      fileIdRestoreFromFilestore1.toString()
-    )
     const path = '/tmp/test-blob-corrupted'
     try {
       await fs.promises.writeFile(path, contentCorruptedBlob)
@@ -426,22 +317,10 @@ describe('back_fill_file_hash_fix_up script', function () {
                     _id: fileIdWithDifferentHashNotFound0,
                     hash: hashDoesNotExistAsBlob,
                   },
-                  {
-                    _id: fileIdRestoreFromFilestore0,
-                    hash: hashDoesNotExistAsBlob,
-                  },
-                  {
-                    _id: fileIdRestoreFromFilestore1,
-                  },
-                  {
-                    _id: fileIdBlobExistsInGCS0,
-                    hash: gitBlobHash(fileIdBlobExistsInGCS0),
-                  },
                   {
                     _id: fileIdBlobExistsInGCSCorrupted,
                     hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
                   },
-                  { _id: fileIdBlobExistsInGCS1 },
                 ],
                 folders: [],
               },
@@ -546,8 +425,8 @@ describe('back_fill_file_hash_fix_up script', function () {
   })
   it('should print stats', function () {
     expect(stats).to.contain({
-      processedLines: 16,
-      success: 11,
+      processedLines: 12,
+      success: 7,
       alreadyProcessed: 0,
       fileDeleted: 0,
       skipped: 0,
@@ -558,9 +437,9 @@ describe('back_fill_file_hash_fix_up script', function () {
   it('should handle re-run on same logs', async function () {
     ;({ stats } = await runScriptWithLogs())
     expect(stats).to.contain({
-      processedLines: 16,
+      processedLines: 12,
       success: 0,
-      alreadyProcessed: 8,
+      alreadyProcessed: 4,
       fileDeleted: 3,
       skipped: 0,
       failed: 3,
@@ -663,31 +542,11 @@ describe('back_fill_file_hash_fix_up script', function () {
                       _id: fileIdWithDifferentHashNotFound0,
                       hash: gitBlobHash(fileIdWithDifferentHashNotFound0),
                     },
-                    // Updated hash
-                    {
-                      _id: fileIdRestoreFromFilestore0,
-                      hash: gitBlobHash(fileIdRestoreFromFilestore0),
-                    },
-                    // Added hash
-                    {
-                      _id: fileIdRestoreFromFilestore1,
-                      hash: gitBlobHash(fileIdRestoreFromFilestore1),
-                    },
-                    // No change, blob created
-                    {
-                      _id: fileIdBlobExistsInGCS0,
-                      hash: gitBlobHash(fileIdBlobExistsInGCS0),
-                    },
                     // No change, flagged
                     {
                       _id: fileIdBlobExistsInGCSCorrupted,
                       hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted),
                     },
-                    // Added hash
-                    {
-                      _id: fileIdBlobExistsInGCS1,
-                      hash: gitBlobHash(fileIdBlobExistsInGCS1),
-                    },
                   ],
                   folders: [],
                 },
@@ -696,7 +555,7 @@ describe('back_fill_file_hash_fix_up script', function () {
           ],
           overleaf: { history: { id: historyId0 } },
           // Incremented when removing file/updating hash
-          version: 8,
+          version: 5,
         },
       ])
     expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([
@@ -745,62 +604,6 @@ describe('back_fill_file_hash_fix_up script', function () {
         (writtenBlobsByProject.get(projectId) || []).concat([fileId])
       )
     }
-    expect(
-      (await backedUpBlobs.find({}, { sort: { _id: 1 } }).toArray()).map(
-        entry => {
-          // blobs are pushed unordered into mongo. Sort the list for consistency.
-          entry.blobs.sort()
-          return entry
-        }
-      )
-    ).to.deep.equal(
-      Array.from(writtenBlobsByProject.entries()).map(
-        ([projectId, fileIds]) => {
-          return {
-            _id: projectId,
-            blobs: fileIds
-              .map(fileId => binaryForGitBlobHash(gitBlobHash(fileId)))
-              .sort(),
-          }
-        }
-      )
-    )
-  })
-  it('should have backed up all the files', async function () {
-    expect(tieringStorageClass).to.exist
-    const objects = await listS3Bucket(projectBlobsBucket, tieringStorageClass)
-    expect(objects.sort()).to.deep.equal(
-      writtenBlobs
-        .map(({ historyId, fileId, hash }) =>
-          makeProjectKey(historyId, hash || gitBlobHash(fileId))
-        )
-        .sort()
-    )
-    for (let { historyId, fileId } of writtenBlobs) {
-      const hash = gitBlobHash(fileId.toString())
-      const s = await backupPersistor.getObjectStream(
-        projectBlobsBucket,
-        makeProjectKey(historyId, hash),
-        { autoGunzip: true }
-      )
-      const buf = new WritableBuffer()
-      await Stream.promises.pipeline(s, buf)
-      expect(gitBlobHashBuffer(buf.getContents())).to.equal(hash)
-      const id = buf.getContents().toString('utf-8')
-      expect(id).to.equal(fileId.toString())
-      // double check we are not comparing 'undefined' or '[object Object]' above
-      expect(id).to.match(/^[a-f0-9]{24}$/)
-    }
-    const deks = await listS3Bucket(deksBucket, 'STANDARD')
-    expect(deks.sort()).to.deep.equal(
-      Array.from(
-        new Set(
-          writtenBlobs.map(
-            ({ historyId }) => projectKey.format(historyId) + '/dek'
-          )
-        )
-      ).sort()
-    )
   })
   it('should have written the back filled files to history v1', async function () {
     for (const { historyId, fileId } of writtenBlobs) {

services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs

@@ -0,0 +1,54 @@
+import express from 'express'
+
+class MockFilestore {
+  constructor() {
+    this.host = process.env.FILESTORE_HOST || '127.0.0.1'
+    this.port = process.env.FILESTORE_PORT || 3009
+    // create a server listening on this.host and this.port
+    this.files = {}
+
+    this.app = express()
+
+    this.app.get('/project/:projectId/file/:fileId', (req, res) => {
+      const { projectId, fileId } = req.params
+      const content = this.files[projectId]?.[fileId]
+      if (!content) return res.status(404).end()
+      res.status(200).end(content)
+    })
+  }
+
+  start() {
+    // reset stored files
+    this.files = {}
+    // start the server
+    if (this.serverPromise) {
+      return this.serverPromise
+    } else {
+      this.serverPromise = new Promise((resolve, reject) => {
+        this.server = this.app.listen(this.port, this.host, err => {
+          if (err) return reject(err)
+          resolve()
+        })
+      })
+      return this.serverPromise
+    }
+  }
+
+  addFile(projectId, fileId, fileContent) {
+    if (!this.files[projectId]) {
+      this.files[projectId] = {}
+    }
+    this.files[projectId][fileId] = fileContent
+  }
+
+  deleteObject(projectId, fileId) {
+    if (this.files[projectId]) {
+      delete this.files[projectId][fileId]
+      if (Object.keys(this.files[projectId]).length === 0) {
+        delete this.files[projectId]
+      }
+    }
+  }
+}
+
+export const mockFilestore = new MockFilestore()