Merge pull request #21375 from overleaf/jpa-if-none-match

[object-persistor] add support for ifNoneMatch=* in sendStream

GitOrigin-RevId: 268f054ac1b6452105b02757cdec32bad00702fd

libraries/object-persistor/src/Errors.js

@@ -5,6 +5,7 @@ class WriteError extends OError {}
 class ReadError extends OError {}
 class SettingsError extends OError {}
 class NotImplementedError extends OError {}
+class AlreadyWrittenError extends OError {}
 
 module.exports = {
   NotFoundError,
@@ -12,4 +13,5 @@ module.exports = {
   ReadError,
   SettingsError,
   NotImplementedError,
+  AlreadyWrittenError,
 }

libraries/object-persistor/src/FSPersistor.js

@@ -8,7 +8,7 @@ const { pipeline } = require('stream/promises')
 const { promisify } = require('util')
 
 const AbstractPersistor = require('./AbstractPersistor')
-const { ReadError, WriteError } = require('./Errors')
+const { ReadError, WriteError, NotImplementedError } = require('./Errors')
 const PersistorHelper = require('./PersistorHelper')
 
 const glob = promisify(globCallbacks)
@@ -36,6 +36,14 @@ module.exports = class FSPersistor extends AbstractPersistor {
   }
 
   async sendStream(location, target, sourceStream, opts = {}) {
+    if (opts.ifNoneMatch === '*') {
+      // The standard library only has fs.rename(), which does not support exclusive flags.
+      // Refuse to act on this write operation.
+      throw new NotImplementedError(
+        'Overwrite protection required by caller, but it is not available is FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
+      )
+    }
+
     const targetPath = this._getFsPath(location, target)
 
     try {
@@ -55,7 +63,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'failed to write stream',
-        { location, target },
+        { location, target, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }

libraries/object-persistor/src/GcsPersistor.js

@@ -78,10 +78,14 @@ module.exports = class GcsPersistor extends AbstractPersistor {
         writeOptions.metadata = writeOptions.metadata || {}
         writeOptions.metadata.contentEncoding = opts.contentEncoding
       }
+      const fileOptions = {}
+      if (opts.ifNoneMatch === '*') {
+        fileOptions.generation = 0
+      }
 
       const uploadStream = this.storage
         .bucket(bucketName)
-        .file(key)
+        .file(key, fileOptions)
         .createWriteStream(writeOptions)
 
       await pipeline(readStream, observer, uploadStream)
@@ -97,7 +101,7 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'upload to GCS failed',
-        { bucketName, key },
+        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }

libraries/object-persistor/src/PersistorHelper.js

@@ -3,7 +3,7 @@ const Stream = require('stream')
 const { pipeline } = require('stream/promises')
 const Logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
-const { WriteError, NotFoundError } = require('./Errors')
+const { WriteError, NotFoundError, AlreadyWrittenError } = require('./Errors')
 
 const _128KiB = 128 * 1024
 const TIMING_BUCKETS = [
@@ -146,6 +146,13 @@ function wrapError(error, message, params, ErrorType) {
     (error.response && error.response.statusCode === 404)
   ) {
     return new NotFoundError('no such file', params, error)
+  } else if (
+    params.ifNoneMatch === '*' &&
+    (error.code === 'PreconditionFailed' ||
+      error.response?.statusCode === 412 ||
+      error instanceof AlreadyWrittenError)
+  ) {
+    return new AlreadyWrittenError(message, params, error)
   } else {
     return new ErrorType(message, params, error)
   }

libraries/object-persistor/src/S3Persistor.js

@@ -52,6 +52,9 @@ module.exports = class S3Persistor extends AbstractPersistor {
       if (opts.contentEncoding) {
         uploadOptions.ContentEncoding = opts.contentEncoding
       }
+      if (opts.ifNoneMatch === '*') {
+        uploadOptions.IfNoneMatch = '*'
+      }
 
       // if we have an md5 hash, pass this to S3 to verify the upload - otherwise
       // we rely on the S3 client's checksum calculation to validate the upload
@@ -69,7 +72,7 @@ module.exports = class S3Persistor extends AbstractPersistor {
       throw PersistorHelper.wrapError(
         err,
         'upload to S3 failed',
-        { bucketName, key },
+        { bucketName, key, ifNoneMatch: opts.ifNoneMatch },
         WriteError
       )
     }