[clsi-cache] backend (#24388)

* [clsi-cache] initial revision of the clsi-cache service * [clsi] send output files to clsi-cache and import from clsi-cache * [web] pass editorId to clsi * [web] clear clsi-cache when clearing clsi cache * [web] add split-tests for controlling clsi-cache rollout * [web] populate clsi-cache when cloning/creating project from template * [clsi-cache] produce less noise when populating cache hits 404 * [clsi-cache] push docker image to AR * [clsi-cache] push docker image to AR * [clsi-cache] allow compileGroup in job payload * [clsi-cache] set X-Zone header from latest endpoint * [clsi-cache] use method POST for /enqueue endpoint * [web] populate clsi-cache in zone b with template data * [clsi-cache] limit number of editors per project/user folder to 10 * [web] clone: populate the clsi-cache unless the TeXLive release changed * [clsi-cache] keep user folder when clearing cache as anonymous user * [clsi] download old output.tar.gz when synctex finds empty compile dir * [web] fix lint * [clsi-cache] multi-zonal lookup of single build output * [clsi-cache] add more validation and limits Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [clsi] do not include clsi-cache tar-ball in output.zip * [clsi-cache] fix reference after remaining constant Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> * [web] consolidate validation of filename into ClsiCacheHandler * [clsi-cache] extend metrics and event tracking - break down most of the clsi metrics by label - compile=initial - new compile dir without previous output files - compile=recompile - recompile in existing compile dir - compile=from-cache - compile using previous clsi-cache - extend segmentation on compile-result-backend event - isInitialCompile=true - found new compile dir at start of request - restoredClsiCache=true - restored compile dir from clsi-cache * [clsi] rename metrics labels for download of clsi-cache This is in preparation for synctex changes. * [clsi] use constant for limit of entries in output.tar.gz Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> * [clsi-cache] fix cloning of project cache --------- Co-authored-by: Brian Gough <brian.gough@overleaf.com> Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com> GitOrigin-RevId: 4901a65497af13be1549af7f38ceee3188fcf881
2025-04-10 08:05:17 +00:00
parent 7920cd9d3d
commit b538d56591
30 changed files with 1004 additions and 149 deletions
@@ -12,6 +12,7 @@
        "services/analytics",
        "services/chat",
        "services/clsi",
+        "services/clsi-cache",
        "services/clsi-perf",
        "services/contacts",
        "services/docstore",
@@ -8708,6 +8709,10 @@
      "resolved": "services/clsi",
      "link": true
    },
+    "node_modules/@overleaf/clsi-cache": {
+      "resolved": "services/clsi-cache",
+      "link": true
+    },
    "node_modules/@overleaf/clsi-perf": {
      "resolved": "services/clsi-perf",
      "link": true
@@ -15467,6 +15472,12 @@
        "deep-equal": "^2.0.5"
      }
    },
+    "node_modules/b4a": {
+      "version": "1.6.7",
+      "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz",
+      "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==",
+      "license": "Apache-2.0"
+    },
    "node_modules/babel-core": {
      "version": "7.0.0-bridge.0",
      "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz",
@@ -15829,6 +15840,70 @@
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
    },
+    "node_modules/bare-events": {
+      "version": "2.5.4",
+      "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz",
+      "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==",
+      "license": "Apache-2.0",
+      "optional": true
+    },
+    "node_modules/bare-fs": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.0.1.tgz",
+      "integrity": "sha512-ilQs4fm/l9eMfWY2dY0WCIUplSUp7U0CT1vrqMg1MUdeZl4fypu5UP0XcDBK5WBQPJAKP1b7XEodISmekH/CEg==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "dependencies": {
+        "bare-events": "^2.0.0",
+        "bare-path": "^3.0.0",
+        "bare-stream": "^2.0.0"
+      },
+      "engines": {
+        "bare": ">=1.7.0"
+      }
+    },
+    "node_modules/bare-os": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.0.tgz",
+      "integrity": "sha512-BUrFS5TqSBdA0LwHop4OjPJwisqxGy6JsWVqV6qaFoe965qqtaKfDzHY5T2YA1gUL0ZeeQeA+4BBc1FJTcHiPw==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "engines": {
+        "bare": ">=1.14.0"
+      }
+    },
+    "node_modules/bare-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz",
+      "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "dependencies": {
+        "bare-os": "^3.0.1"
+      }
+    },
+    "node_modules/bare-stream": {
+      "version": "2.6.5",
+      "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz",
+      "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "dependencies": {
+        "streamx": "^2.21.0"
+      },
+      "peerDependencies": {
+        "bare-buffer": "*",
+        "bare-events": "*"
+      },
+      "peerDependenciesMeta": {
+        "bare-buffer": {
+          "optional": true
+        },
+        "bare-events": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/base": {
      "version": "0.11.2",
      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
@@ -22529,8 +22604,7 @@
    "node_modules/fast-fifo": {
      "version": "1.3.2",
      "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
-      "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==",
-      "dev": true
+      "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="
    },
    "node_modules/fast-glob": {
      "version": "3.3.3",
@@ -34466,12 +34540,6 @@
        }
      ]
    },
-    "node_modules/queue-tick": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
-      "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==",
-      "dev": true
-    },
    "node_modules/quick-lru": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz",
@@ -37523,13 +37591,16 @@
      }
    },
    "node_modules/streamx": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz",
-      "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==",
-      "dev": true,
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz",
+      "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==",
+      "license": "MIT",
      "dependencies": {
-        "fast-fifo": "^1.1.0",
-        "queue-tick": "^1.0.1"
+        "fast-fifo": "^1.3.2",
+        "text-decoder": "^1.1.0"
+      },
+      "optionalDependencies": {
+        "bare-events": "^2.2.0"
      }
    },
    "node_modules/string_decoder": {
@@ -38605,6 +38676,31 @@
        "node": ">=10"
      }
    },
+    "node_modules/tar-fs": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz",
+      "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==",
+      "license": "MIT",
+      "dependencies": {
+        "pump": "^3.0.0",
+        "tar-stream": "^3.1.5"
+      },
+      "optionalDependencies": {
+        "bare-fs": "^4.0.1",
+        "bare-path": "^3.0.0"
+      }
+    },
+    "node_modules/tar-fs/node_modules/tar-stream": {
+      "version": "3.1.7",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz",
+      "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==",
+      "license": "MIT",
+      "dependencies": {
+        "b4a": "^1.6.4",
+        "fast-fifo": "^1.2.0",
+        "streamx": "^2.15.0"
+      }
+    },
    "node_modules/tar-stream": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
@@ -38864,6 +38960,15 @@
        "node": ">=8"
      }
    },
+    "node_modules/text-decoder": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz",
+      "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "b4a": "^1.6.4"
+      }
+    },
    "node_modules/text-table": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -41897,6 +42002,7 @@
        "p-limit": "^3.1.0",
        "request": "^2.88.2",
        "send": "^0.19.0",
+        "tar-fs": "^3.0.4",
        "workerpool": "^6.1.5"
      },
      "devDependencies": {
@@ -41913,6 +42019,27 @@
        "typescript": "^5.0.4"
      }
    },
+    "services/clsi-cache": {
+      "name": "@overleaf/clsi-cache",
+      "dependencies": {
+        "@overleaf/fetch-utils": "*",
+        "@overleaf/logger": "*",
+        "@overleaf/metrics": "*",
+        "@overleaf/o-error": "*",
+        "@overleaf/promise-utils": "*",
+        "@overleaf/settings": "*",
+        "body-parser": "^1.20.3",
+        "bunyan": "^1.8.15",
+        "celebrate": "^15.0.3",
+        "diskusage": "^1.1.3",
+        "express": "^4.21.2"
+      },
+      "devDependencies": {
+        "chai": "^4.3.6",
+        "chai-as-promised": "^7.1.1",
+        "mocha": "^11.1.0"
+      }
+    },
    "services/clsi-perf": {
      "name": "@overleaf/clsi-perf",
      "dependencies": {
@@ -42011,6 +42138,18 @@
        "node": ">= 8.0"
      }
    },
+    "services/clsi/node_modules/dockerode/node_modules/tar-fs": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
+      "license": "MIT",
+      "dependencies": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      }
+    },
    "services/clsi/node_modules/nan": {
      "version": "2.22.2",
      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz",
@@ -42090,18 +42229,6 @@
        "node": ">=8"
      }
    },
-    "services/clsi/node_modules/tar-fs": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
-      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
-      "license": "MIT",
-      "dependencies": {
-        "chownr": "^1.1.1",
-        "mkdirp-classic": "^0.5.2",
-        "pump": "^3.0.0",
-        "tar-stream": "^2.1.4"
-      }
-    },
    "services/clsi/node_modules/uuid": {
      "version": "10.0.0",
      "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",