Merge pull request #21892 from overleaf/jpa-auto-gunzip

[object-persistor] add autoGunzip option to getObjectStream

GitOrigin-RevId: 21cd6c9ab349017ddb28d165238371d967ab9a59

libraries/object-persistor/src/FSPersistor.js

@@ -71,6 +71,11 @@ module.exports = class FSPersistor extends AbstractPersistor {
 
   // opts may be {start: Number, end: Number}
   async getObjectStream(location, name, opts = {}) {
+    if (opts.autoGunzip) {
+      throw new NotImplementedError(
+        'opts.autoGunzip is not supported by FS backend. Configure GCS or S3 backend instead, get in touch with support for further information.'
+      )
+    }
     const observer = new PersistorHelper.ObserverStream({
       metric: 'fs.ingress', // ingress to us from disk
       bucket: location,

libraries/object-persistor/src/GcsPersistor.js

@@ -7,6 +7,7 @@ const asyncPool = require('tiny-async-pool')
 const AbstractPersistor = require('./AbstractPersistor')
 const PersistorHelper = require('./PersistorHelper')
 const Logger = require('@overleaf/logger')
+const zlib = require('node:zlib')
 
 module.exports = class GcsPersistor extends AbstractPersistor {
   constructor(settings) {
@@ -117,12 +118,14 @@ module.exports = class GcsPersistor extends AbstractPersistor {
       .file(key)
       .createReadStream({ decompress: false, ...opts })
 
+    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
         stream.on('response', res => {
           switch (res.statusCode) {
             case 200: // full response
             case 206: // partial response
+              contentEncoding = res.headers['content-encoding']
               return resolve()
             case 404:
               return reject(new NotFoundError())
@@ -143,7 +146,11 @@ module.exports = class GcsPersistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    pipeline(stream, observer, pass).catch(() => {})
+    const transformer = []
+    if (contentEncoding === 'gzip' && opts.autoGunzip) {
+      transformer.push(zlib.createGunzip())
+    }
+    pipeline(stream, observer, ...transformer, pass).catch(() => {})
     return pass
   }
 

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -412,7 +412,7 @@ class CachedPerProjectEncryptedS3Persistor {
    * @param {Object} opts
    * @param {number} [opts.start]
    * @param {number} [opts.end]
-   * @param {string} [opts.contentEncoding]
+   * @param {boolean} [opts.autoGunzip]
    * @param {SSECOptions} [opts.ssecOptions]
    * @return {Promise<NodeJS.ReadableStream>}
    */

libraries/object-persistor/src/S3Persistor.js

@@ -18,6 +18,7 @@ const fs = require('node:fs')
 const S3 = require('aws-sdk/clients/s3')
 const { URL } = require('node:url')
 const { WriteError, ReadError, NotFoundError } = require('./Errors')
+const zlib = require('node:zlib')
 
 /**
  * Wrapper with private fields to avoid revealing them on console, JSON.stringify or similar.
@@ -147,7 +148,7 @@ class S3Persistor extends AbstractPersistor {
    * @param {Object} opts
    * @param {number} [opts.start]
    * @param {number} [opts.end]
-   * @param {string} [opts.contentEncoding]
+   * @param {boolean} [opts.autoGunzip]
    * @param {SSECOptions} [opts.ssecOptions]
    * @return {Promise<NodeJS.ReadableStream>}
    */
@@ -172,12 +173,14 @@ class S3Persistor extends AbstractPersistor {
     const req = this._getClientForBucket(bucketName).getObject(params)
     const stream = req.createReadStream()
 
+    let contentEncoding
     try {
       await new Promise((resolve, reject) => {
-        req.on('httpHeaders', statusCode => {
+        req.on('httpHeaders', (statusCode, headers) => {
           switch (statusCode) {
             case 200: // full response
             case 206: // partial response
+              contentEncoding = headers['content-encoding']
               return resolve(undefined)
             case 403: // AccessDenied
               return // handled by stream.on('error') handler below
@@ -202,7 +205,11 @@ class S3Persistor extends AbstractPersistor {
     }
     // Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
     const pass = new PassThrough()
-    pipeline(stream, observer, pass, err => {
+    const transformer = []
+    if (contentEncoding === 'gzip' && opts.autoGunzip) {
+      transformer.push(zlib.createGunzip())
+    }
+    pipeline(stream, observer, ...transformer, pass, err => {
       if (err) req.abort()
     })
     return pass

libraries/object-persistor/test/unit/GcsPersistorTests.js

@@ -63,7 +63,7 @@ describe('GcsPersistorTests', function () {
 
       read() {
         if (this.err) return this.emit('error', this.err)
-        this.emit('response', { statusCode: this.statusCode })
+        this.emit('response', { statusCode: this.statusCode, headers: {} })
       }
     }
 

libraries/object-persistor/test/unit/S3PersistorTests.js

@@ -93,14 +93,14 @@ describe('S3PersistorTests', function () {
         setTimeout(() => {
           if (this.notFoundSSEC) {
             // special case for AWS S3: 404 NoSuchKey wrapped in a 400. A single request received a single response, and multiple httpHeaders events are triggered. Don't ask.
-            this.emit('httpHeaders', 400)
-            this.emit('httpHeaders', 404)
+            this.emit('httpHeaders', 400, {})
+            this.emit('httpHeaders', 404, {})
             ReadStream.emit('error', S3NotFoundError)
             return
           }
 
           if (this.err) return ReadStream.emit('error', this.err)
-          this.emit('httpHeaders', this.statusCode)
+          this.emit('httpHeaders', this.statusCode, {})
           if (this.statusCode === 403) {
             ReadStream.emit('error', S3AccessDeniedError)
           }