[clsi-cache] backend (#24388)

* [clsi-cache] initial revision of the clsi-cache service

* [clsi] send output files to clsi-cache and import from clsi-cache

* [web] pass editorId to clsi

* [web] clear clsi-cache when clearing clsi cache

* [web] add split-tests for controlling clsi-cache rollout

* [web] populate clsi-cache when cloning/creating project from template

* [clsi-cache] produce less noise when populating cache hits 404

* [clsi-cache] push docker image to AR

* [clsi-cache] push docker image to AR

* [clsi-cache] allow compileGroup in job payload

* [clsi-cache] set X-Zone header from latest endpoint

* [clsi-cache] use method POST for /enqueue endpoint

* [web] populate clsi-cache in zone b with template data

* [clsi-cache] limit number of editors per project/user folder to 10

* [web] clone: populate the clsi-cache unless the TeXLive release changed

* [clsi-cache] keep user folder when clearing cache as anonymous user

* [clsi] download old output.tar.gz when synctex finds empty compile dir

* [web] fix lint

* [clsi-cache] multi-zonal lookup of single build output

* [clsi-cache] add more validation and limits

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [clsi] do not include clsi-cache tar-ball in output.zip

* [clsi-cache] fix reference after remaining constant

Co-authored-by: Alf Eaton <alf.eaton@overleaf.com>

* [web] consolidate validation of filename into ClsiCacheHandler

* [clsi-cache] extend metrics and event tracking

- break down most of the clsi metrics by label
  - compile=initial - new compile dir without previous output files
  - compile=recompile - recompile in existing compile dir
  - compile=from-cache - compile using previous clsi-cache
- extend segmentation on compile-result-backend event
  - isInitialCompile=true - found new compile dir at start of request
  - restoredClsiCache=true - restored compile dir from clsi-cache

* [clsi] rename metrics labels for download of clsi-cache

This is in preparation for synctex changes.

* [clsi] use constant for limit of entries in output.tar.gz

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>

* [clsi-cache] fix cloning of project cache

---------

Co-authored-by: Brian Gough <brian.gough@overleaf.com>
Co-authored-by: Alf Eaton <alf.eaton@overleaf.com>
Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
GitOrigin-RevId: 4901a65497af13be1549af7f38ceee3188fcf881

services/web/test/unit/src/Compile/ClsiManagerTests.js

@@ -144,6 +144,9 @@ describe('ClsiManager', function () {
       enablePdfCaching: true,
       clsiCookie: { key: 'clsiserver' },
     }
+    this.ClsiCacheHandler = {
+      clearCache: sinon.stub().resolves(),
+    }
     this.Features = {
       hasFeature: sinon.stub().withArgs('project-history-blobs').returns(true),
     }
@@ -172,6 +175,7 @@ describe('ClsiManager', function () {
           this.DocumentUpdaterHandler,
         './ClsiCookieManager': () => this.ClsiCookieManager,
         './ClsiStateManager': this.ClsiStateManager,
+        './ClsiCacheHandler': this.ClsiCacheHandler,
         '@overleaf/fetch-utils': this.FetchUtils,
         './ClsiFormatChecker': this.ClsiFormatChecker,
         '@overleaf/metrics': this.Metrics,
@@ -390,6 +394,8 @@ describe('ClsiManager', function () {
             incrementalCompilesEnabled: true,
             compileBackendClass: 'e2',
             compileGroup: 'priority',
+            compileFromClsiCache: true,
+            populateClsiCache: true,
             enablePdfCaching: true,
             pdfCachingMinChunkSize: 1337,
           }
@@ -448,6 +454,8 @@ describe('ClsiManager', function () {
                   syncType: 'incremental',
                   syncState: '01234567890abcdef',
                   compileGroup: 'priority',
+                  compileFromClsiCache: true,
+                  populateClsiCache: true,
                   enablePdfCaching: true,
                   pdfCachingMinChunkSize: 1337,
                   metricsMethod: 'priority',
@@ -945,6 +953,12 @@ describe('ClsiManager', function () {
         )
       })
 
+      it('should clear the output.tar.gz files in clsi-cache', function () {
+        this.ClsiCacheHandler.clearCache
+          .calledWith(this.project._id, this.user_id)
+          .should.equal(true)
+      })
+
       it('should clear the project state from the docupdater', function () {
         this.DocumentUpdaterHandler.promises.clearProjectState
           .calledWith(this.project._id)

services/web/test/unit/src/Compile/CompileControllerTests.js

@@ -244,9 +244,12 @@ describe('CompileController', function () {
           this.user_id,
           {
             isAutoCompile: false,
+            compileFromClsiCache: false,
+            populateClsiCache: false,
             enablePdfCaching: false,
             fileLineErrors: false,
             stopOnFirstError: false,
+            editorId: undefined,
           }
         )
       })
@@ -284,9 +287,12 @@ describe('CompileController', function () {
           this.user_id,
           {
             isAutoCompile: true,
+            compileFromClsiCache: false,
+            populateClsiCache: false,
             enablePdfCaching: false,
             fileLineErrors: false,
             stopOnFirstError: false,
+            editorId: undefined,
           }
         )
       })
@@ -305,10 +311,37 @@ describe('CompileController', function () {
           this.user_id,
           {
             isAutoCompile: false,
+            compileFromClsiCache: false,
+            populateClsiCache: false,
             enablePdfCaching: false,
             draft: true,
             fileLineErrors: false,
             stopOnFirstError: false,
+            editorId: undefined,
+          }
+        )
+      })
+    })
+
+    describe('with an editor id', function () {
+      beforeEach(function (done) {
+        this.res.callback = done
+        this.req.body = { editorId: 'the-editor-id' }
+        this.CompileController.compile(this.req, this.res, this.next)
+      })
+
+      it('should pass the editor id to the compiler', function () {
+        this.CompileManager.compile.should.have.been.calledWith(
+          this.projectId,
+          this.user_id,
+          {
+            isAutoCompile: false,
+            compileFromClsiCache: false,
+            populateClsiCache: false,
+            enablePdfCaching: false,
+            fileLineErrors: false,
+            stopOnFirstError: false,
+            editorId: 'the-editor-id',
           }
         )
       })
@@ -542,14 +575,16 @@ describe('CompileController', function () {
     })
   })
   describe('proxySyncCode', function () {
-    let file, line, column, imageName
+    let file, line, column, imageName, editorId, buildId
 
     beforeEach(function (done) {
       this.req.params = { Project_id: this.projectId }
       file = 'main.tex'
       line = String(Date.now())
       column = String(Date.now() + 1)
-      this.req.query = { file, line, column }
+      editorId = '172977cb-361e-4854-a4dc-a71cf11512e5'
+      buildId = '195b4a3f9e7-03e5be430a9e7796'
+      this.req.query = { file, line, column, editorId, buildId }
 
       imageName = 'foo/bar:tag-0'
       this.ProjectGetter.getProject = sinon.stub().yields(null, { imageName })
@@ -566,7 +601,15 @@ describe('CompileController', function () {
         this.projectId,
         'sync-to-code',
         `/project/${this.projectId}/user/${this.user_id}/sync/code`,
-        { file, line, column, imageName },
+        {
+          file,
+          line,
+          column,
+          imageName,
+          editorId,
+          buildId,
+          compileFromClsiCache: false,
+        },
         this.req,
         this.res,
         this.next
@@ -575,14 +618,16 @@ describe('CompileController', function () {
   })
 
   describe('proxySyncPdf', function () {
-    let page, h, v, imageName
+    let page, h, v, imageName, editorId, buildId
 
     beforeEach(function (done) {
       this.req.params = { Project_id: this.projectId }
       page = String(Date.now())
       h = String(Math.random())
       v = String(Math.random())
-      this.req.query = { page, h, v }
+      editorId = '172977cb-361e-4854-a4dc-a71cf11512e5'
+      buildId = '195b4a3f9e7-03e5be430a9e7796'
+      this.req.query = { page, h, v, editorId, buildId }
 
       imageName = 'foo/bar:tag-1'
       this.ProjectGetter.getProject = sinon.stub().yields(null, { imageName })
@@ -599,7 +644,15 @@ describe('CompileController', function () {
         this.projectId,
         'sync-to-pdf',
         `/project/${this.projectId}/user/${this.user_id}/sync/pdf`,
-        { page, h, v, imageName },
+        {
+          page,
+          h,
+          v,
+          imageName,
+          editorId,
+          buildId,
+          compileFromClsiCache: false,
+        },
         this.req,
         this.res,
         this.next

services/web/test/unit/src/Project/ProjectDuplicatorTests.js

@@ -245,6 +245,9 @@ describe('ProjectDuplicator', function () {
         '../Tags/TagsHandler': this.TagsHandler,
         '../History/HistoryManager': this.HistoryManager,
         '../../infrastructure/Features': this.Features,
+        '../Compile/ClsiCacheManager': {
+          prepareClsiCache: sinon.stub().rejects(new Error('ignore this')),
+        },
       },
     })
   })

services/web/test/unit/src/Templates/TemplatesManagerTests.js

@@ -121,6 +121,9 @@ describe('TemplatesManager', function () {
         fs: this.fs,
         '../../models/Project': { Project: this.Project },
         'stream/promises': { pipeline: this.pipeline },
+        '../Compile/ClsiCacheManager': {
+          prepareClsiCache: sinon.stub().rejects(new Error('ignore this')),
+        },
       },
     }).promises
     return (this.zipUrl =