Merge pull request #30045 from overleaf/bg-object-persistor-make-list-directory-safer

Improve safety of object persistor

GitOrigin-RevId: bced9814de6613b388ca288a5f72cd42cff6c1d3

services/history-v1/storage/lib/backupArchiver.mjs

@@ -400,28 +400,22 @@ export async function archiveLatestChunk(
  * @return {Promise<void>}
  */
 async function addRawBlobsToArchive(historyId, archive, projectCache) {
-  const key = projectKey.format(historyId)
-  const { contents } = await projectCache.listDirectory(projectBlobsBucket, key)
-  for (const blobRecord of contents) {
-    if (!blobRecord.Key) {
-      logger.debug({ blobRecord }, 'no key')
-      continue
-    }
-    const blobKey = blobRecord.Key
+  const blobKeys = await projectCache.listDirectoryKeys(
+    projectBlobsBucket,
+    projectKey.format(historyId)
+  )
+  for (const key of blobKeys) {
     try {
       const stream = await projectCache.getObjectStream(
         projectBlobsBucket,
-        blobKey,
+        key,
         { autoGunzip: true }
       )
       archive.append(stream, {
-        name: path.join(historyId, 'blobs', blobKey),
+        name: path.join(historyId, 'blobs', key),
       })
     } catch (err) {
-      logger.warn(
-        { err, path: blobRecord.Key },
-        'Failed to append blob to archive'
-      )
+      logger.warn({ err, path: key }, 'Failed to append blob to archive')
     }
   }
 }
@@ -445,24 +439,20 @@ export async function archiveRawProject(
 ) {
   const projectCache = await getProjectPersistor(historyId)
 
-  const { contents: chunks } = await projectCache.listDirectory(
+  const chunkKeys = await projectCache.listDirectoryKeys(
     chunksBucket,
     projectKey.format(historyId)
   )
 
-  if (chunks.length === 0) {
+  if (chunkKeys.length === 0) {
     throw new Error('No chunks found')
   }
 
-  for (const chunkRecord of chunks) {
-    if (!chunkRecord.Key) {
-      logger.debug({ chunkRecord }, 'no key')
-      continue
-    }
-    const chunkId = chunkRecord.Key.split('/').pop()
-    logger.debug({ chunkId, key: chunkRecord.Key }, 'Processing chunk')
+  for (const key of chunkKeys) {
+    const chunkId = key.split('/').pop()
+    logger.debug({ chunkId, key }, 'Processing chunk')
 
-    const { buffer } = await loadChunkByKey(projectCache, chunkRecord.Key)
+    const { buffer } = await loadChunkByKey(projectCache, key)
 
     archive.append(buffer, {
       name: `${historyId}/chunks/${chunkId}/chunk.json`,

services/history-v1/storage/scripts/backup.mjs

@@ -900,6 +900,11 @@ class BlobComparator {
 
 const SHA1_HEX_REGEX = /^[a-f0-9]{40}$/
 
+/**
+ * Get a listing of all blobs for a project
+ * @param {string} historyId - The history ID
+ * @returns {Promise<Map<string, {key: string, size: number}>>} Map of blob hash to blob metadata
+ */
 async function getBlobListing(historyId) {
   const backupPersistorForProject = await backupPersistor.forProject(
     projectBlobsBucket,
@@ -909,7 +914,7 @@ async function getBlobListing(historyId) {
   // get the blob listing
   const projectBlobsPath = projectKey.format(historyId)
 
-  const { contents: blobList } = await backupPersistorForProject.listDirectory(
+  const blobList = await backupPersistorForProject.listDirectoryStats(
     projectBlobsBucket,
     projectBlobsPath
   )
@@ -918,21 +923,22 @@ async function getBlobListing(historyId) {
     return new Map()
   }
 
+  /** @type {Map<string, {key: string, size: number}>} */
   const remoteBlobs = new Map()
 
   for (const blobRecord of blobList) {
-    if (!blobRecord.Key) {
-      logger.debug({ blobRecord }, 'no key')
+    if (!blobRecord.key || typeof blobRecord.size !== 'number') {
+      logger.debug({ blobRecord }, 'invalid blob record')
       continue
     }
-    const parts = blobRecord.Key.split('/')
+    const parts = blobRecord.key.split('/')
     const hash = parts[3] + parts[4]
 
     if (!SHA1_HEX_REGEX.test(hash)) {
       console.warn(`Invalid SHA1 hash for project ${historyId}: ${hash}`)
       continue
     }
-    remoteBlobs.set(hash, blobRecord)
+    remoteBlobs.set(hash, { key: blobRecord.key, size: blobRecord.size })
   }
   return remoteBlobs
 }
@@ -1075,26 +1081,26 @@ async function compareBackups(projectId, options, log = console.log) {
           const blobListEntry = blobsFromListing.get(blob.hash)
           if (options.fast) {
             if (blobListEntry) {
-              if (blob.byteLength === blobListEntry.Size) {
+              if (blob.byteLength === blobListEntry.size) {
                 // Size matches exactly
                 log(
                   `  ✓ Blob ${blob.hash} exists on remote with expected size (${blob.byteLength} bytes)`
                 )
                 totalBlobMatches++
                 continue
-              } else if (blob.stringLength > 0 && blobListEntry.Size > 0) {
+              } else if (blob.stringLength > 0 && blobListEntry.size > 0) {
                 // Text file present with compressed size, assume valid as we are in --fast comparison mode
                 const compressionRatio = (
-                  blobListEntry.Size / blob.byteLength
+                  blobListEntry.size / blob.byteLength
                 ).toFixed(2)
                 log(
-                  `  ✓ Blob ${blob.hash} consistent with compressed data on remote (${blob.byteLength} bytes => ${blobListEntry.Size} bytes, ratio=${compressionRatio})`
+                  `  ✓ Blob ${blob.hash} consistent with compressed data on remote (${blob.byteLength} bytes => ${blobListEntry.size} bytes, ratio=${compressionRatio})`
                 )
                 totalBlobMatches++
                 continue
               } else {
                 log(
-                  `  ✗ Blob ${blob.hash} size mismatch (original: ${blob.byteLength} bytes, stringLength: ${blob.stringLength}, backup: ${blobListEntry.Size} bytes)`
+                  `  ✗ Blob ${blob.hash} size mismatch (original: ${blob.byteLength} bytes, stringLength: ${blob.stringLength}, backup: ${blobListEntry.size} bytes)`
                 )
                 totalBlobMismatches++
                 errors.push({